You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/08/12 20:18:38 UTC
[01/23] hadoop git commit: HDFS-8805. Archival Storage:
getStoragePolicy should not need superuser privilege. Contributed by Brahma
Reddy Battula.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285-merge dc0a6173f -> bc2564afe (forced update)
HDFS-8805. Archival Storage: getStoragePolicy should not need superuser privilege. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1fc3c779
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1fc3c779
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1fc3c779
Branch: refs/heads/HDFS-7285-merge
Commit: 1fc3c779a422bafdb86ad1a5b2349802dda1cb62
Parents: fa1d84a
Author: Jing Zhao <ji...@apache.org>
Authored: Tue Aug 11 10:28:18 2015 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Tue Aug 11 10:28:18 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java | 2 +-
.../hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java | 9 +++------
.../hadoop/hdfs/server/namenode/FSDirWriteFileOp.java | 2 +-
4 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fc3c779/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 15c9df5..61def02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -775,6 +775,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8818. Changes the global moveExecutor to per datanode executors and
changes MAX_SIZE_TO_MOVE to be configurable. (szetszwo)
+ HDFS-8805. Archival Storage: getStoragePolicy should not need superuser privilege.
+ (Brahma Reddy Battula via jing9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fc3c779/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index abb2dc8..3d79d09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -135,7 +135,7 @@ final class FSDirAppendOp {
}
HdfsFileStatus stat = FSDirStatAndListingOp.getFileInfo(fsd, src, false,
- FSDirectory.isReservedRawName(srcArg), true);
+ FSDirectory.isReservedRawName(srcArg));
if (lb != null) {
NameNode.stateChangeLog.debug(
"DIR* NameSystem.appendFile: file {} for {} at {} block {} block"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fc3c779/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 14f4d66..4a45074 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -105,13 +105,11 @@ class FSDirStatAndListingOp {
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsd.resolvePath(pc, src, pathComponents);
final INodesInPath iip = fsd.getINodesInPath(src, resolveLink);
- boolean isSuperUser = true;
if (fsd.isPermissionEnabled()) {
fsd.checkPermission(pc, iip, false, null, null, null, null, false);
- isSuperUser = pc.isSuperUser();
}
return getFileInfo(fsd, src, resolveLink,
- FSDirectory.isReservedRawName(srcArg), isSuperUser);
+ FSDirectory.isReservedRawName(srcArg));
}
/**
@@ -369,8 +367,7 @@ class FSDirStatAndListingOp {
}
static HdfsFileStatus getFileInfo(
- FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath,
- boolean includeStoragePolicy)
+ FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath)
throws IOException {
String srcs = FSDirectory.normalizePath(src);
if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
@@ -385,7 +382,7 @@ class FSDirStatAndListingOp {
fsd.readLock();
try {
final INodesInPath iip = fsd.getINodesInPath(srcs, resolveLink);
- return getFileInfo(fsd, src, iip, isRawPath, includeStoragePolicy);
+ return getFileInfo(fsd, src, iip, isRawPath, true);
} finally {
fsd.readUnlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fc3c779/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 732e9cf..3d30a19 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -432,7 +432,7 @@ class FSDirWriteFileOp {
NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: added " +
src + " inode " + newNode.getId() + " " + holder);
}
- return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath, true);
+ return FSDirStatAndListingOp.getFileInfo(fsd, src, false, isRawPath);
}
static EncryptionKeyInfo getEncryptionKeyInfo(FSNamesystem fsn,
[02/23] hadoop git commit: Adding release 2.6.1 to CHANGES.txt
Posted by zh...@apache.org.
Adding release 2.6.1 to CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c796fd1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c796fd1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c796fd1
Branch: refs/heads/HDFS-7285-merge
Commit: 7c796fd12ca5092d74e6cc7044f28496f9884983
Parents: 1fc3c77
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Tue Aug 11 13:20:23 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Tue Aug 11 13:20:23 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c796fd1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ada1056..a1f6861 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1712,6 +1712,18 @@ Release 2.7.0 - 2015-04-20
YARN-3055. Fixed ResourceManager's DelegationTokenRenewer to not stop token
renewal of applications part of a bigger workflow. (Daryn Sharp via vinodkv)
+Release 2.6.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
[17/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
index 7ca6419..ee6f441 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguousUnderConstruction.java
@@ -31,7 +31,8 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
* Represents a block that is currently being constructed.<br>
* This is usually the last block of a file opened for write or append.
*/
-public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
+public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous
+ implements BlockInfoUnderConstruction{
/** Block state. See {@link BlockUCState} */
private BlockUCState blockUCState;
@@ -60,101 +61,6 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
private Block truncateBlock;
/**
- * ReplicaUnderConstruction contains information about replicas while
- * they are under construction.
- * The GS, the length and the state of the replica is as reported by
- * the data-node.
- * It is not guaranteed, but expected, that data-nodes actually have
- * corresponding replicas.
- */
- static class ReplicaUnderConstruction extends Block {
- private final DatanodeStorageInfo expectedLocation;
- private ReplicaState state;
- private boolean chosenAsPrimary;
-
- ReplicaUnderConstruction(Block block,
- DatanodeStorageInfo target,
- ReplicaState state) {
- super(block);
- this.expectedLocation = target;
- this.state = state;
- this.chosenAsPrimary = false;
- }
-
- /**
- * Expected block replica location as assigned when the block was allocated.
- * This defines the pipeline order.
- * It is not guaranteed, but expected, that the data-node actually has
- * the replica.
- */
- private DatanodeStorageInfo getExpectedStorageLocation() {
- return expectedLocation;
- }
-
- /**
- * Get replica state as reported by the data-node.
- */
- ReplicaState getState() {
- return state;
- }
-
- /**
- * Whether the replica was chosen for recovery.
- */
- boolean getChosenAsPrimary() {
- return chosenAsPrimary;
- }
-
- /**
- * Set replica state.
- */
- void setState(ReplicaState s) {
- state = s;
- }
-
- /**
- * Set whether this replica was chosen for recovery.
- */
- void setChosenAsPrimary(boolean chosenAsPrimary) {
- this.chosenAsPrimary = chosenAsPrimary;
- }
-
- /**
- * Is data-node the replica belongs to alive.
- */
- boolean isAlive() {
- return expectedLocation.getDatanodeDescriptor().isAlive;
- }
-
- @Override // Block
- public int hashCode() {
- return super.hashCode();
- }
-
- @Override // Block
- public boolean equals(Object obj) {
- // Sufficient to rely on super's implementation
- return (this == obj) || super.equals(obj);
- }
-
- @Override
- public String toString() {
- final StringBuilder b = new StringBuilder(50);
- appendStringTo(b);
- return b.toString();
- }
-
- @Override
- public void appendStringTo(StringBuilder sb) {
- sb.append("ReplicaUC[")
- .append(expectedLocation)
- .append("|")
- .append(state)
- .append("]");
- }
- }
-
- /**
* Create block and set its state to
* {@link BlockUCState#UNDER_CONSTRUCTION}.
*/
@@ -169,7 +75,7 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
BlockUCState state, DatanodeStorageInfo[] targets) {
super(blk, replication);
assert getBlockUCState() != BlockUCState.COMPLETE :
- "BlockInfoUnderConstruction cannot be in COMPLETE state";
+ "BlockInfoContiguousUnderConstruction cannot be in COMPLETE state";
this.blockUCState = state;
setExpectedLocations(targets);
}
@@ -183,34 +89,34 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
* the client or it does not have at least a minimal number of replicas
* reported from data-nodes.
*/
- BlockInfo convertToCompleteBlock() throws IOException {
+ @Override
+ public BlockInfo convertToCompleteBlock() throws IOException {
assert getBlockUCState() != BlockUCState.COMPLETE :
"Trying to convert a COMPLETE block";
return new BlockInfoContiguous(this);
}
- /** Set expected locations */
+ @Override
public void setExpectedLocations(DatanodeStorageInfo[] targets) {
int numLocations = targets == null ? 0 : targets.length;
- this.replicas = new ArrayList<ReplicaUnderConstruction>(numLocations);
- for(int i = 0; i < numLocations; i++)
- replicas.add(
- new ReplicaUnderConstruction(this, targets[i], ReplicaState.RBW));
+ this.replicas = new ArrayList<>(numLocations);
+ for(int i = 0; i < numLocations; i++) {
+ replicas.add(new ReplicaUnderConstruction(this, targets[i],
+ ReplicaState.RBW));
+ }
}
- /**
- * Create array of expected replica locations
- * (as has been assigned by chooseTargets()).
- */
+ @Override
public DatanodeStorageInfo[] getExpectedStorageLocations() {
int numLocations = replicas == null ? 0 : replicas.size();
DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations];
- for(int i = 0; i < numLocations; i++)
+ for (int i = 0; i < numLocations; i++) {
storages[i] = replicas.get(i).getExpectedStorageLocation();
+ }
return storages;
}
- /** Get the number of expected locations */
+ @Override
public int getNumExpectedLocations() {
return replicas == null ? 0 : replicas.size();
}
@@ -228,25 +134,26 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
blockUCState = s;
}
- /** Get block recovery ID */
+ @Override
public long getBlockRecoveryId() {
return blockRecoveryId;
}
- /** Get recover block */
+ @Override
public Block getTruncateBlock() {
return truncateBlock;
}
+ @Override
+ public Block toBlock(){
+ return this;
+ }
+
public void setTruncateBlock(Block recoveryBlock) {
this.truncateBlock = recoveryBlock;
}
- /**
- * Process the recorded replicas. When about to commit or finish the
- * pipeline recovery sort out bad replicas.
- * @param genStamp The final generation stamp for the block.
- */
+ @Override
public void setGenerationStampAndVerifyReplicas(long genStamp) {
// Set the generation stamp for the block.
setGenerationStamp(genStamp);
@@ -264,13 +171,8 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
}
}
- /**
- * Commit block's length and generation stamp as reported by the client.
- * Set block state to {@link BlockUCState#COMMITTED}.
- * @param block - contains client reported block length and generation
- * @throws IOException if block ids are inconsistent.
- */
- void commitBlock(Block block) throws IOException {
+ @Override
+ public void commitBlock(Block block) throws IOException {
if(getBlockId() != block.getBlockId())
throw new IOException("Trying to commit inconsistent block: id = "
+ block.getBlockId() + ", expected id = " + getBlockId());
@@ -280,31 +182,27 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
setGenerationStampAndVerifyReplicas(block.getGenerationStamp());
}
- /**
- * Initialize lease recovery for this block.
- * Find the first alive data-node starting from the previous primary and
- * make it primary.
- */
+ @Override
public void initializeBlockRecovery(long recoveryId) {
setBlockUCState(BlockUCState.UNDER_RECOVERY);
blockRecoveryId = recoveryId;
if (replicas.size() == 0) {
NameNode.blockStateChangeLog.warn("BLOCK*"
- + " BlockInfoUnderConstruction.initLeaseRecovery:"
+ + " BlockInfoContiguousUnderConstruction.initLeaseRecovery:"
+ " No blocks found, lease removed.");
}
boolean allLiveReplicasTriedAsPrimary = true;
- for (int i = 0; i < replicas.size(); i++) {
+ for (ReplicaUnderConstruction replica : replicas) {
// Check if all replicas have been tried or not.
- if (replicas.get(i).isAlive()) {
- allLiveReplicasTriedAsPrimary =
- (allLiveReplicasTriedAsPrimary && replicas.get(i).getChosenAsPrimary());
+ if (replica.isAlive()) {
+ allLiveReplicasTriedAsPrimary = (allLiveReplicasTriedAsPrimary &&
+ replica.getChosenAsPrimary());
}
}
if (allLiveReplicasTriedAsPrimary) {
// Just set all the replicas to be chosen whether they are alive or not.
- for (int i = 0; i < replicas.size(); i++) {
- replicas.get(i).setChosenAsPrimary(false);
+ for (ReplicaUnderConstruction replica : replicas) {
+ replica.setChosenAsPrimary(false);
}
}
long mostRecentLastUpdate = 0;
@@ -325,16 +223,17 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
}
}
if (primary != null) {
- primary.getExpectedStorageLocation().getDatanodeDescriptor().addBlockToBeRecovered(this);
+ primary.getExpectedStorageLocation().getDatanodeDescriptor()
+ .addBlockToBeRecovered(this);
primary.setChosenAsPrimary(true);
NameNode.blockStateChangeLog.debug(
"BLOCK* {} recovery started, primary={}", this, primary);
}
}
- void addReplicaIfNotPresent(DatanodeStorageInfo storage,
- Block block,
- ReplicaState rState) {
+ @Override
+ public void addReplicaIfNotPresent(DatanodeStorageInfo storage,
+ Block block, ReplicaState rState) {
Iterator<ReplicaUnderConstruction> it = replicas.iterator();
while (it.hasNext()) {
ReplicaUnderConstruction r = it.next();
@@ -358,18 +257,6 @@ public class BlockInfoContiguousUnderConstruction extends BlockInfoContiguous {
replicas.add(new ReplicaUnderConstruction(block, storage, rState));
}
- @Override // BlockInfo
- // BlockInfoUnderConstruction participates in maps the same way as BlockInfo
- public int hashCode() {
- return super.hashCode();
- }
-
- @Override // BlockInfo
- public boolean equals(Object obj) {
- // Sufficient to rely on super's implementation
- return (this == obj) || super.equals(obj);
- }
-
@Override
public String toString() {
final StringBuilder b = new StringBuilder(100);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
new file mode 100644
index 0000000..6674510
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStriped.java
@@ -0,0 +1,286 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+
+/**
+ * Subclass of {@link BlockInfo}, presenting a block group in erasure coding.
+ *
+ * We still use triplets to store DatanodeStorageInfo for each block in the
+ * block group, as well as the previous/next block in the corresponding
+ * DatanodeStorageInfo. For a (m+k) block group, the first (m+k) triplet units
+ * are sorted and strictly mapped to the corresponding block.
+ *
+ * Normally each block belonging to group is stored in only one DataNode.
+ * However, it is possible that some block is over-replicated. Thus the triplet
+ * array's size can be larger than (m+k). Thus currently we use an extra byte
+ * array to record the block index for each triplet.
+ */
+public class BlockInfoStriped extends BlockInfo {
+ private final ECSchema schema;
+ private final int cellSize;
+ /**
+ * Always the same size with triplets. Record the block index for each triplet
+ * TODO: actually this is only necessary for over-replicated block. Thus can
+ * be further optimized to save memory usage.
+ */
+ private byte[] indices;
+
+ public BlockInfoStriped(Block blk, ECSchema schema, int cellSize) {
+ super(blk, (short) (schema.getNumDataUnits() + schema.getNumParityUnits()));
+ indices = new byte[schema.getNumDataUnits() + schema.getNumParityUnits()];
+ initIndices();
+ this.schema = schema;
+ this.cellSize = cellSize;
+ }
+
+ BlockInfoStriped(BlockInfoStriped b) {
+ this(b, b.getSchema(), b.getCellSize());
+ this.setBlockCollection(b.getBlockCollection());
+ }
+
+ public short getTotalBlockNum() {
+ return (short) (this.schema.getNumDataUnits()
+ + this.schema.getNumParityUnits());
+ }
+
+ public short getDataBlockNum() {
+ return (short) this.schema.getNumDataUnits();
+ }
+
+ public short getParityBlockNum() {
+ return (short) this.schema.getNumParityUnits();
+ }
+
+ /**
+ * If the block is committed/completed and its length is less than a full
+ * stripe, it returns the the number of actual data blocks.
+ * Otherwise it returns the number of data units specified by schema.
+ */
+ public short getRealDataBlockNum() {
+ if (isComplete() || getBlockUCState() == BlockUCState.COMMITTED) {
+ return (short) Math.min(getDataBlockNum(),
+ (getNumBytes() - 1) / BLOCK_STRIPED_CELL_SIZE + 1);
+ } else {
+ return getDataBlockNum();
+ }
+ }
+
+ public short getRealTotalBlockNum() {
+ return (short) (getRealDataBlockNum() + getParityBlockNum());
+ }
+
+ public ECSchema getSchema() {
+ return schema;
+ }
+
+ public int getCellSize() {
+ return cellSize;
+ }
+
+ private void initIndices() {
+ for (int i = 0; i < indices.length; i++) {
+ indices[i] = -1;
+ }
+ }
+
+ private int findSlot() {
+ int i = getTotalBlockNum();
+ for (; i < getCapacity(); i++) {
+ if (getStorageInfo(i) == null) {
+ return i;
+ }
+ }
+ // need to expand the triplet size
+ ensureCapacity(i + 1, true);
+ return i;
+ }
+
+ @Override
+ boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
+ int blockIndex = BlockIdManager.getBlockIndex(reportedBlock);
+ int index = blockIndex;
+ DatanodeStorageInfo old = getStorageInfo(index);
+ if (old != null && !old.equals(storage)) { // over replicated
+ // check if the storage has been stored
+ int i = findStorageInfo(storage);
+ if (i == -1) {
+ index = findSlot();
+ } else {
+ return true;
+ }
+ }
+ addStorage(storage, index, blockIndex);
+ return true;
+ }
+
+ private void addStorage(DatanodeStorageInfo storage, int index,
+ int blockIndex) {
+ setStorageInfo(index, storage);
+ setNext(index, null);
+ setPrevious(index, null);
+ indices[index] = (byte) blockIndex;
+ }
+
+ private int findStorageInfoFromEnd(DatanodeStorageInfo storage) {
+ final int len = getCapacity();
+ for(int idx = len - 1; idx >= 0; idx--) {
+ DatanodeStorageInfo cur = getStorageInfo(idx);
+ if (storage.equals(cur)) {
+ return idx;
+ }
+ }
+ return -1;
+ }
+
+ int getStorageBlockIndex(DatanodeStorageInfo storage) {
+ int i = this.findStorageInfo(storage);
+ return i == -1 ? -1 : indices[i];
+ }
+
+ /**
+ * Identify the block stored in the given datanode storage. Note that
+ * the returned block has the same block Id with the one seen/reported by the
+ * DataNode.
+ */
+ Block getBlockOnStorage(DatanodeStorageInfo storage) {
+ int index = getStorageBlockIndex(storage);
+ if (index < 0) {
+ return null;
+ } else {
+ Block block = new Block(this);
+ block.setBlockId(this.getBlockId() + index);
+ return block;
+ }
+ }
+
+ @Override
+ boolean removeStorage(DatanodeStorageInfo storage) {
+ int dnIndex = findStorageInfoFromEnd(storage);
+ if (dnIndex < 0) { // the node is not found
+ return false;
+ }
+ assert getPrevious(dnIndex) == null && getNext(dnIndex) == null :
+ "Block is still in the list and must be removed first.";
+ // set the triplet to null
+ setStorageInfo(dnIndex, null);
+ setNext(dnIndex, null);
+ setPrevious(dnIndex, null);
+ indices[dnIndex] = -1;
+ return true;
+ }
+
+ private void ensureCapacity(int totalSize, boolean keepOld) {
+ if (getCapacity() < totalSize) {
+ Object[] old = triplets;
+ byte[] oldIndices = indices;
+ triplets = new Object[totalSize * 3];
+ indices = new byte[totalSize];
+ initIndices();
+
+ if (keepOld) {
+ System.arraycopy(old, 0, triplets, 0, old.length);
+ System.arraycopy(oldIndices, 0, indices, 0, oldIndices.length);
+ }
+ }
+ }
+
+ @Override
+ void replaceBlock(BlockInfo newBlock) {
+ assert newBlock instanceof BlockInfoStriped;
+ BlockInfoStriped newBlockGroup = (BlockInfoStriped) newBlock;
+ final int size = getCapacity();
+ newBlockGroup.ensureCapacity(size, false);
+ for (int i = 0; i < size; i++) {
+ final DatanodeStorageInfo storage = this.getStorageInfo(i);
+ if (storage != null) {
+ final int blockIndex = indices[i];
+ final boolean removed = storage.removeBlock(this);
+ assert removed : "currentBlock not found.";
+
+ newBlockGroup.addStorage(storage, i, blockIndex);
+ storage.insertToList(newBlockGroup);
+ }
+ }
+ }
+
+ public long spaceConsumed() {
+ // In case striped blocks, total usage by this striped blocks should
+ // be the total of data blocks and parity blocks because
+ // `getNumBytes` is the total of actual data block size.
+ return StripedBlockUtil.spaceConsumedByStripedBlock(getNumBytes(),
+ this.schema.getNumDataUnits(), this.schema.getNumParityUnits(),
+ BLOCK_STRIPED_CELL_SIZE);
+ }
+
+ @Override
+ public final boolean isStriped() {
+ return true;
+ }
+
+ @Override
+ public int numNodes() {
+ assert this.triplets != null : "BlockInfo is not initialized";
+ assert triplets.length % 3 == 0 : "Malformed BlockInfo";
+ int num = 0;
+ for (int idx = getCapacity()-1; idx >= 0; idx--) {
+ if (getStorageInfo(idx) != null) {
+ num++;
+ }
+ }
+ return num;
+ }
+
+ /**
+ * Convert a complete block to an under construction block.
+ * @return BlockInfoUnderConstruction - an under construction block.
+ */
+ public BlockInfoStripedUnderConstruction convertToBlockUnderConstruction(
+ BlockUCState s, DatanodeStorageInfo[] targets) {
+ final BlockInfoStripedUnderConstruction ucBlock;
+ if(isComplete()) {
+ ucBlock = new BlockInfoStripedUnderConstruction(this, schema, cellSize,
+ s, targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ } else {
+ // the block is already under construction
+ ucBlock = (BlockInfoStripedUnderConstruction) this;
+ ucBlock.setBlockUCState(s);
+ ucBlock.setExpectedLocations(targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ }
+ return ucBlock;
+ }
+
+ @Override
+ final boolean hasNoStorage() {
+ final int len = getCapacity();
+ for(int idx = 0; idx < len; idx++) {
+ if (getStorageInfo(idx) != null) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java
new file mode 100644
index 0000000..5f78096
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoStripedUnderConstruction.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.COMPLETE;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION;
+
+/**
+ * Represents a striped block that is currently being constructed.
+ * This is usually the last block of a file opened for write or append.
+ */
+public class BlockInfoStripedUnderConstruction extends BlockInfoStriped
+ implements BlockInfoUnderConstruction{
+ private BlockUCState blockUCState;
+
+ /**
+ * Block replicas as assigned when the block was allocated.
+ */
+ private ReplicaUnderConstruction[] replicas;
+
+ /**
+ * Index of the primary data node doing the recovery. Useful for log
+ * messages.
+ */
+ private int primaryNodeIndex = -1;
+
+ /**
+ * The new generation stamp, which this block will have
+ * after the recovery succeeds. Also used as a recovery id to identify
+ * the right recovery if any of the abandoned recoveries re-appear.
+ */
+ private long blockRecoveryId = 0;
+
+ /**
+ * Constructor with null storage targets.
+ */
+ public BlockInfoStripedUnderConstruction(Block blk, ECSchema schema,
+ int cellSize) {
+ this(blk, schema, cellSize, UNDER_CONSTRUCTION, null);
+ }
+
+ /**
+ * Create a striped block that is currently being constructed.
+ */
+ public BlockInfoStripedUnderConstruction(Block blk, ECSchema schema,
+ int cellSize, BlockUCState state, DatanodeStorageInfo[] targets) {
+ super(blk, schema, cellSize);
+ assert getBlockUCState() != COMPLETE :
+ "BlockInfoStripedUnderConstruction cannot be in COMPLETE state";
+ this.blockUCState = state;
+ setExpectedLocations(targets);
+ }
+
+ @Override
+ public BlockInfoStriped convertToCompleteBlock() throws IOException {
+ assert getBlockUCState() != COMPLETE :
+ "Trying to convert a COMPLETE block";
+ return new BlockInfoStriped(this);
+ }
+
+ /** Set expected locations */
+ @Override
+ public void setExpectedLocations(DatanodeStorageInfo[] targets) {
+ int numLocations = targets == null ? 0 : targets.length;
+ this.replicas = new ReplicaUnderConstruction[numLocations];
+ for(int i = 0; i < numLocations; i++) {
+ // when creating a new block we simply sequentially assign block index to
+ // each storage
+ Block blk = new Block(this.getBlockId() + i, 0, this.getGenerationStamp());
+ replicas[i] = new ReplicaUnderConstruction(blk, targets[i],
+ ReplicaState.RBW);
+ }
+ }
+
+ /**
+ * Create array of expected replica locations
+ * (as has been assigned by chooseTargets()).
+ */
+ @Override
+ public DatanodeStorageInfo[] getExpectedStorageLocations() {
+ int numLocations = getNumExpectedLocations();
+ DatanodeStorageInfo[] storages = new DatanodeStorageInfo[numLocations];
+ for (int i = 0; i < numLocations; i++) {
+ storages[i] = replicas[i].getExpectedStorageLocation();
+ }
+ return storages;
+ }
+
+ /** @return the index array indicating the block index in each storage */
+ public int[] getBlockIndices() {
+ int numLocations = getNumExpectedLocations();
+ int[] indices = new int[numLocations];
+ for (int i = 0; i < numLocations; i++) {
+ indices[i] = BlockIdManager.getBlockIndex(replicas[i]);
+ }
+ return indices;
+ }
+
+ @Override
+ public int getNumExpectedLocations() {
+ return replicas == null ? 0 : replicas.length;
+ }
+
+ /**
+ * Return the state of the block under construction.
+ * @see BlockUCState
+ */
+ @Override // BlockInfo
+ public BlockUCState getBlockUCState() {
+ return blockUCState;
+ }
+
+ void setBlockUCState(BlockUCState s) {
+ blockUCState = s;
+ }
+
+ @Override
+ public long getBlockRecoveryId() {
+ return blockRecoveryId;
+ }
+
+ @Override
+ public Block getTruncateBlock() {
+ return null;
+ }
+
+ @Override
+ public Block toBlock(){
+ return this;
+ }
+
+ @Override
+ public void setGenerationStampAndVerifyReplicas(long genStamp) {
+ // Set the generation stamp for the block.
+ setGenerationStamp(genStamp);
+ if (replicas == null)
+ return;
+
+ // Remove the replicas with wrong gen stamp.
+ // The replica list is unchanged.
+ for (ReplicaUnderConstruction r : replicas) {
+ if (genStamp != r.getGenerationStamp()) {
+ r.getExpectedStorageLocation().removeBlock(this);
+ NameNode.blockStateChangeLog.info("BLOCK* Removing stale replica "
+ + "from location: {}", r.getExpectedStorageLocation());
+ }
+ }
+ }
+
+ @Override
+ public void commitBlock(Block block) throws IOException {
+ if (getBlockId() != block.getBlockId()) {
+ throw new IOException("Trying to commit inconsistent block: id = "
+ + block.getBlockId() + ", expected id = " + getBlockId());
+ }
+ blockUCState = BlockUCState.COMMITTED;
+ this.set(getBlockId(), block.getNumBytes(), block.getGenerationStamp());
+ // Sort out invalid replicas.
+ setGenerationStampAndVerifyReplicas(block.getGenerationStamp());
+ }
+
+ @Override
+ public void initializeBlockRecovery(long recoveryId) {
+ setBlockUCState(BlockUCState.UNDER_RECOVERY);
+ blockRecoveryId = recoveryId;
+ if (replicas == null || replicas.length == 0) {
+ NameNode.blockStateChangeLog.warn("BLOCK*" +
+ " BlockInfoStripedUnderConstruction.initLeaseRecovery:" +
+ " No blocks found, lease removed.");
+ // sets primary node index and return.
+ primaryNodeIndex = -1;
+ return;
+ }
+ boolean allLiveReplicasTriedAsPrimary = true;
+ for (ReplicaUnderConstruction replica : replicas) {
+ // Check if all replicas have been tried or not.
+ if (replica.isAlive()) {
+ allLiveReplicasTriedAsPrimary = (allLiveReplicasTriedAsPrimary &&
+ replica.getChosenAsPrimary());
+ }
+ }
+ if (allLiveReplicasTriedAsPrimary) {
+ // Just set all the replicas to be chosen whether they are alive or not.
+ for (ReplicaUnderConstruction replica : replicas) {
+ replica.setChosenAsPrimary(false);
+ }
+ }
+ long mostRecentLastUpdate = 0;
+ ReplicaUnderConstruction primary = null;
+ primaryNodeIndex = -1;
+ for(int i = 0; i < replicas.length; i++) {
+ // Skip alive replicas which have been chosen for recovery.
+ if (!(replicas[i].isAlive() && !replicas[i].getChosenAsPrimary())) {
+ continue;
+ }
+ final ReplicaUnderConstruction ruc = replicas[i];
+ final long lastUpdate = ruc.getExpectedStorageLocation()
+ .getDatanodeDescriptor().getLastUpdateMonotonic();
+ if (lastUpdate > mostRecentLastUpdate) {
+ primaryNodeIndex = i;
+ primary = ruc;
+ mostRecentLastUpdate = lastUpdate;
+ }
+ }
+ if (primary != null) {
+ primary.getExpectedStorageLocation().getDatanodeDescriptor()
+ .addBlockToBeRecovered(this);
+ primary.setChosenAsPrimary(true);
+ NameNode.blockStateChangeLog.info(
+ "BLOCK* {} recovery started, primary={}", this, primary);
+ }
+ }
+
+ @Override
+ public void addReplicaIfNotPresent(DatanodeStorageInfo storage,
+ Block reportedBlock, ReplicaState rState) {
+ if (replicas == null) {
+ replicas = new ReplicaUnderConstruction[1];
+ replicas[0] = new ReplicaUnderConstruction(reportedBlock, storage, rState);
+ } else {
+ for (int i = 0; i < replicas.length; i++) {
+ DatanodeStorageInfo expected = replicas[i].getExpectedStorageLocation();
+ if (expected == storage) {
+ replicas[i].setBlockId(reportedBlock.getBlockId());
+ replicas[i].setGenerationStamp(reportedBlock.getGenerationStamp());
+ return;
+ } else if (expected != null && expected.getDatanodeDescriptor() ==
+ storage.getDatanodeDescriptor()) {
+ // The Datanode reported that the block is on a different storage
+ // than the one chosen by BlockPlacementPolicy. This can occur as
+ // we allow Datanodes to choose the target storage. Update our
+ // state by removing the stale entry and adding a new one.
+ replicas[i] = new ReplicaUnderConstruction(reportedBlock, storage,
+ rState);
+ return;
+ }
+ }
+ ReplicaUnderConstruction[] newReplicas =
+ new ReplicaUnderConstruction[replicas.length + 1];
+ System.arraycopy(replicas, 0, newReplicas, 0, replicas.length);
+ newReplicas[newReplicas.length - 1] = new ReplicaUnderConstruction(
+ reportedBlock, storage, rState);
+ replicas = newReplicas;
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder b = new StringBuilder(100);
+ appendStringTo(b);
+ return b.toString();
+ }
+
+ @Override
+ public void appendStringTo(StringBuilder sb) {
+ super.appendStringTo(sb);
+ appendUCParts(sb);
+ }
+
+ private void appendUCParts(StringBuilder sb) {
+ sb.append("{UCState=").append(blockUCState).
+ append(", primaryNodeIndex=").append(primaryNodeIndex).
+ append(", replicas=[");
+ if (replicas != null) {
+ int i = 0;
+ for (ReplicaUnderConstruction r : replicas) {
+ r.appendStringTo(sb);
+ if (++i < replicas.length) {
+ sb.append(", ");
+ }
+ }
+ }
+ sb.append("]}");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
new file mode 100644
index 0000000..10a8cae
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoUnderConstruction.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+
+public interface BlockInfoUnderConstruction {
+ /**
+ * Create array of expected replica locations
+ * (as has been assigned by chooseTargets()).
+ */
+ public DatanodeStorageInfo[] getExpectedStorageLocations();
+
+ /** Get recover block */
+ public Block getTruncateBlock();
+
+ /** Convert to a Block object */
+ public Block toBlock();
+
+ /** Get block recovery ID */
+ public long getBlockRecoveryId();
+
+ /** Get the number of expected locations */
+ public int getNumExpectedLocations();
+
+ /** Set expected locations */
+ public void setExpectedLocations(DatanodeStorageInfo[] targets);
+
+ /**
+ * Process the recorded replicas. When about to commit or finish the
+ * pipeline recovery sort out bad replicas.
+ * @param genStamp The final generation stamp for the block.
+ */
+ public void setGenerationStampAndVerifyReplicas(long genStamp);
+
+ /**
+ * Initialize lease recovery for this block.
+ * Find the first alive data-node starting from the previous primary and
+ * make it primary.
+ */
+ public void initializeBlockRecovery(long recoveryId);
+
+ /** Add the reported replica if it is not already in the replica list. */
+ public void addReplicaIfNotPresent(DatanodeStorageInfo storage,
+ Block reportedBlock, ReplicaState rState);
+
+ /**
+ * Commit block's length and generation stamp as reported by the client.
+ * Set block state to {@link BlockUCState#COMMITTED}.
+ * @param block - contains client reported block length and generation
+ * @throws IOException if block ids are inconsistent.
+ */
+ public void commitBlock(Block block) throws IOException;
+
+ /**
+ * Convert an under construction block to a complete block.
+ *
+ * @return a complete block.
+ * @throws IOException
+ * if the state of the block (the generation stamp and the length)
+ * has not been committed by the client or it does not have at least
+ * a minimal number of replicas reported from data-nodes.
+ */
+ public BlockInfo convertToCompleteBlock() throws IOException;
+}
[07/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
new file mode 100644
index 0000000..52626e1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
@@ -0,0 +1,433 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSStripedOutputStream;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
+import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS;
+import static org.junit.Assert.assertEquals;
+
+public class TestAddStripedBlocks {
+ private final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS +
+ HdfsConstants.NUM_PARITY_BLOCKS;
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem dfs;
+
+ @Before
+ public void setup() throws IOException {
+ cluster = new MiniDFSCluster.Builder(new HdfsConfiguration())
+ .numDataNodes(GROUP_SIZE).build();
+ cluster.waitActive();
+ dfs = cluster.getFileSystem();
+ dfs.getClient().createErasureCodingZone("/", null, 0);
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Make sure the IDs of striped blocks do not conflict
+ */
+ @Test
+ public void testAllocateBlockId() throws Exception {
+ Path testPath = new Path("/testfile");
+ // create a file while allocates a new block
+ DFSTestUtil.writeFile(dfs, testPath, "hello, world!");
+ LocatedBlocks lb = dfs.getClient().getLocatedBlocks(testPath.toString(), 0);
+ final long firstId = lb.get(0).getBlock().getBlockId();
+ // delete the file
+ dfs.delete(testPath, true);
+
+ // allocate a new block, and make sure the new block's id does not conflict
+ // with the previous one
+ DFSTestUtil.writeFile(dfs, testPath, "hello again");
+ lb = dfs.getClient().getLocatedBlocks(testPath.toString(), 0);
+ final long secondId = lb.get(0).getBlock().getBlockId();
+ Assert.assertEquals(firstId + HdfsServerConstants.MAX_BLOCKS_IN_GROUP, secondId);
+ }
+
+ private static void writeAndFlushStripedOutputStream(
+ DFSStripedOutputStream out, int chunkSize) throws IOException {
+ // FSOutputSummer.BUFFER_NUM_CHUNKS == 9
+ byte[] toWrite = new byte[chunkSize * 9 + 1];
+ out.write(toWrite);
+ DFSTestUtil.flushInternal(out);
+ }
+
+ @Test (timeout=60000)
+ public void testAddStripedBlock() throws Exception {
+ final Path file = new Path("/file1");
+ // create an empty file
+ FSDataOutputStream out = null;
+ try {
+ out = dfs.create(file, (short) 1);
+ writeAndFlushStripedOutputStream(
+ (DFSStripedOutputStream) out.getWrappedStream(),
+ DFS_BYTES_PER_CHECKSUM_DEFAULT);
+
+ FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+ INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+
+ BlockInfo[] blocks = fileNode.getBlocks();
+ assertEquals(1, blocks.length);
+ Assert.assertTrue(blocks[0].isStriped());
+
+ checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), true);
+
+ // restart NameNode to check editlog
+ cluster.restartNameNode(true);
+ fsdir = cluster.getNamesystem().getFSDirectory();
+ fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ blocks = fileNode.getBlocks();
+ assertEquals(1, blocks.length);
+ Assert.assertTrue(blocks[0].isStriped());
+ checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), false);
+
+ // save namespace, restart namenode, and check
+ dfs = cluster.getFileSystem();
+ dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ dfs.saveNamespace();
+ dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ cluster.restartNameNode(true);
+ fsdir = cluster.getNamesystem().getFSDirectory();
+ fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ blocks = fileNode.getBlocks();
+ assertEquals(1, blocks.length);
+ Assert.assertTrue(blocks[0].isStriped());
+ checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), false);
+ } finally {
+ IOUtils.cleanup(null, out);
+ }
+ }
+
+ private void checkStripedBlockUC(BlockInfoStriped block,
+ boolean checkReplica) {
+ assertEquals(0, block.numNodes());
+ Assert.assertFalse(block.isComplete());
+ Assert.assertEquals(HdfsConstants.NUM_DATA_BLOCKS, block.getDataBlockNum());
+ Assert.assertEquals(HdfsConstants.NUM_PARITY_BLOCKS,
+ block.getParityBlockNum());
+ Assert.assertEquals(0,
+ block.getBlockId() & HdfsServerConstants.BLOCK_GROUP_INDEX_MASK);
+
+ final BlockInfoStripedUnderConstruction blockUC =
+ (BlockInfoStripedUnderConstruction) block;
+ Assert.assertEquals(HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION,
+ blockUC.getBlockUCState());
+ if (checkReplica) {
+ Assert.assertEquals(GROUP_SIZE, blockUC.getNumExpectedLocations());
+ DatanodeStorageInfo[] storages = blockUC.getExpectedStorageLocations();
+ for (DataNode dn : cluster.getDataNodes()) {
+ Assert.assertTrue(includeDataNode(dn.getDatanodeId(), storages));
+ }
+ }
+ }
+
+ private boolean includeDataNode(DatanodeID dn, DatanodeStorageInfo[] storages) {
+ for (DatanodeStorageInfo storage : storages) {
+ if (storage.getDatanodeDescriptor().equals(dn)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Test
+ public void testGetLocatedStripedBlocks() throws Exception {
+ final Path file = new Path("/file1");
+ // create an empty file
+ FSDataOutputStream out = null;
+ try {
+ out = dfs.create(file, (short) 1);
+ writeAndFlushStripedOutputStream(
+ (DFSStripedOutputStream) out.getWrappedStream(),
+ DFS_BYTES_PER_CHECKSUM_DEFAULT);
+
+ FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+ INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ BlockInfoStripedUnderConstruction lastBlk =
+ (BlockInfoStripedUnderConstruction) fileNode.getLastBlock();
+ DatanodeInfo[] expectedDNs = DatanodeStorageInfo
+ .toDatanodeInfos(lastBlk.getExpectedStorageLocations());
+ int[] indices = lastBlk.getBlockIndices();
+
+ LocatedBlocks blks = dfs.getClient().getLocatedBlocks(file.toString(), 0L);
+ Assert.assertEquals(1, blks.locatedBlockCount());
+ LocatedBlock lblk = blks.get(0);
+
+ Assert.assertTrue(lblk instanceof LocatedStripedBlock);
+ DatanodeInfo[] datanodes = lblk.getLocations();
+ int[] blockIndices = ((LocatedStripedBlock) lblk).getBlockIndices();
+ Assert.assertEquals(GROUP_SIZE, datanodes.length);
+ Assert.assertEquals(GROUP_SIZE, blockIndices.length);
+ Assert.assertArrayEquals(indices, blockIndices);
+ Assert.assertArrayEquals(expectedDNs, datanodes);
+ } finally {
+ IOUtils.cleanup(null, out);
+ }
+ }
+
+ /**
+ * Test BlockInfoStripedUnderConstruction#addReplicaIfNotPresent in different
+ * scenarios.
+ */
+ @Test
+ public void testAddUCReplica() throws Exception {
+ final Path file = new Path("/file1");
+ final List<String> storageIDs = new ArrayList<>();
+ // create an empty file
+ FSDataOutputStream out = null;
+ try {
+ out = dfs.create(file, (short) 1);
+
+ // 1. create the UC striped block
+ FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+ INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ cluster.getNamesystem().getAdditionalBlock(file.toString(),
+ fileNode.getId(), dfs.getClient().getClientName(), null, null, null);
+ BlockInfo lastBlock = fileNode.getLastBlock();
+ BlockInfoStripedUnderConstruction ucBlock =
+ (BlockInfoStripedUnderConstruction) lastBlock;
+
+ DatanodeStorageInfo[] locs = ucBlock.getExpectedStorageLocations();
+ int[] indices = ucBlock.getBlockIndices();
+ Assert.assertEquals(GROUP_SIZE, locs.length);
+ Assert.assertEquals(GROUP_SIZE, indices.length);
+
+ // 2. mimic incremental block reports and make sure the uc-replica list in
+ // the BlockInfoUCStriped is correct
+ int i = 0;
+ for (DataNode dn : cluster.getDataNodes()) {
+ final Block block = new Block(lastBlock.getBlockId() + i++,
+ 0, lastBlock.getGenerationStamp());
+ DatanodeStorage storage = new DatanodeStorage(UUID.randomUUID().toString());
+ storageIDs.add(storage.getStorageID());
+ StorageReceivedDeletedBlocks[] reports = DFSTestUtil
+ .makeReportForReceivedBlock(block, BlockStatus.RECEIVING_BLOCK,
+ storage);
+ for (StorageReceivedDeletedBlocks report : reports) {
+ cluster.getNamesystem().processIncrementalBlockReport(
+ dn.getDatanodeId(), report);
+ }
+ }
+
+ // make sure lastBlock is correct and the storages have been updated
+ locs = ucBlock.getExpectedStorageLocations();
+ indices = ucBlock.getBlockIndices();
+ Assert.assertEquals(GROUP_SIZE, locs.length);
+ Assert.assertEquals(GROUP_SIZE, indices.length);
+ for (DatanodeStorageInfo newstorage : locs) {
+ Assert.assertTrue(storageIDs.contains(newstorage.getStorageID()));
+ }
+ } finally {
+ IOUtils.cleanup(null, out);
+ }
+
+ // 3. restart the namenode. mimic the full block reports and check the
+ // uc-replica list again
+ cluster.restartNameNode(true);
+ final String bpId = cluster.getNamesystem().getBlockPoolId();
+ INodeFile fileNode = cluster.getNamesystem().getFSDirectory()
+ .getINode4Write(file.toString()).asFile();
+ BlockInfo lastBlock = fileNode.getLastBlock();
+ int i = GROUP_SIZE - 1;
+ for (DataNode dn : cluster.getDataNodes()) {
+ String storageID = storageIDs.get(i);
+ final Block block = new Block(lastBlock.getBlockId() + i--,
+ lastBlock.getGenerationStamp(), 0);
+ DatanodeStorage storage = new DatanodeStorage(storageID);
+ List<ReplicaBeingWritten> blocks = new ArrayList<>();
+ ReplicaBeingWritten replica = new ReplicaBeingWritten(block, null, null,
+ null);
+ blocks.add(replica);
+ BlockListAsLongs bll = BlockListAsLongs.encode(blocks);
+ StorageBlockReport[] reports = {new StorageBlockReport(storage,
+ bll)};
+ cluster.getNameNodeRpc().blockReport(dn.getDNRegistrationForBP(bpId),
+ bpId, reports, null);
+ }
+
+ BlockInfoStripedUnderConstruction ucBlock =
+ (BlockInfoStripedUnderConstruction) lastBlock;
+ DatanodeStorageInfo[] locs = ucBlock.getExpectedStorageLocations();
+ int[] indices = ucBlock.getBlockIndices();
+ Assert.assertEquals(GROUP_SIZE, locs.length);
+ Assert.assertEquals(GROUP_SIZE, indices.length);
+ for (i = 0; i < GROUP_SIZE; i++) {
+ Assert.assertEquals(storageIDs.get(i),
+ locs[GROUP_SIZE - 1 - i].getStorageID());
+ Assert.assertEquals(GROUP_SIZE - i - 1, indices[i]);
+ }
+ }
+
+ @Test
+ public void testCheckStripedReplicaCorrupt() throws Exception {
+ final int numBlocks = 4;
+ final int numStripes = 4;
+ final Path filePath = new Path("/corrupt");
+ final FSNamesystem ns = cluster.getNameNode().getNamesystem();
+ final BlockManager bm = ns.getBlockManager();
+ DFSTestUtil.createStripedFile(cluster, filePath, null,
+ numBlocks, numStripes, false);
+
+ INodeFile fileNode = ns.getFSDirectory().getINode(filePath.toString()).
+ asFile();
+ Assert.assertTrue(fileNode.isStriped());
+ BlockInfo stored = fileNode.getBlocks()[0];
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(0, ns.getCorruptReplicaBlocks());
+
+ // Now send a block report with correct size
+ DatanodeStorage storage = new DatanodeStorage(UUID.randomUUID().toString());
+ final Block reported = new Block(stored);
+ reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE);
+ StorageReceivedDeletedBlocks[] reports = DFSTestUtil
+ .makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(0).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(0, ns.getCorruptReplicaBlocks());
+
+ // Now send a block report with wrong size
+ reported.setBlockId(stored.getBlockId() + 1);
+ reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE - 1);
+ reports = DFSTestUtil.makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(1).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
+
+ // Now send a parity block report with correct size
+ reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS);
+ reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE);
+ reports = DFSTestUtil.makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(2).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
+
+ // Now send a parity block report with wrong size
+ reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS);
+ reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE + 1);
+ reports = DFSTestUtil.makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(3).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ // the total number of corrupted block info is still 1
+ Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
+ // 2 internal blocks corrupted
+ Assert.assertEquals(2, bm.getCorruptReplicas(stored).size());
+
+ // Now change the size of stored block, and test verifying the last
+ // block size
+ stored.setNumBytes(stored.getNumBytes() + 10);
+ reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS + 2);
+ reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE);
+ reports = DFSTestUtil.makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(4).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
+ Assert.assertEquals(3, bm.getCorruptReplicas(stored).size());
+
+ // Now send a parity block report with correct size based on adjusted
+ // size of stored block
+ /** Now stored block has {@link numStripes} full stripes + a cell + 10 */
+ stored.setNumBytes(stored.getNumBytes() + BLOCK_STRIPED_CELL_SIZE);
+ reported.setBlockId(stored.getBlockId());
+ reported.setNumBytes((numStripes + 1) * BLOCK_STRIPED_CELL_SIZE);
+ reports = DFSTestUtil.makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(0).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
+ Assert.assertEquals(3, bm.getCorruptReplicas(stored).size());
+
+ reported.setBlockId(stored.getBlockId() + 1);
+ reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE + 10);
+ reports = DFSTestUtil.makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(5).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
+ Assert.assertEquals(3, bm.getCorruptReplicas(stored).size());
+
+ reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS);
+ reported.setNumBytes((numStripes + 1) * BLOCK_STRIPED_CELL_SIZE);
+ reports = DFSTestUtil.makeReportForReceivedBlock(reported,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ ns.processIncrementalBlockReport(
+ cluster.getDataNodes().get(2).getDatanodeId(), reports[0]);
+ BlockManagerTestUtil.updateState(ns.getBlockManager());
+ Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
+ Assert.assertEquals(3, bm.getCorruptReplicas(stored).size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index c5262d4..ec3d924 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -162,7 +162,8 @@ public class TestDeadDatanode {
// choose the targets, but local node should not get selected as this is not
// part of the cluster anymore
DatanodeStorageInfo[] results = bm.chooseTarget4NewBlock("/hello", 3,
- clientNode, new HashSet<Node>(), 256 * 1024 * 1024L, null, (byte) 7);
+ clientNode, new HashSet<Node>(), 256 * 1024 * 1024L, null, (byte) 7,
+ false);
for (DatanodeStorageInfo datanodeStorageInfo : results) {
assertFalse("Dead node should not be choosen", datanodeStorageInfo
.getDatanodeDescriptor().equals(clientNode));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
index 55ba379..6426b23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.junit.Assert.assertFalse;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.spy;
@@ -39,15 +40,24 @@ import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.test.PathUtils;
import org.apache.log4j.Level;
import org.junit.Test;
@@ -65,6 +75,9 @@ public class TestFSEditLogLoader {
private static final File TEST_DIR = PathUtils.getTestDir(TestFSEditLogLoader.class);
private static final int NUM_DATA_NODES = 0;
+
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
@Test
public void testDisplayRecentEditLogOpCodes() throws IOException {
@@ -414,4 +427,264 @@ public class TestFSEditLogLoader {
fromByte(code), FSEditLogOpCodes.fromByte(code));
}
}
+
+ @Test
+ public void testAddNewStripedBlock() throws IOException{
+ // start a cluster
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSNamesystem fns = cluster.getNamesystem();
+
+ String testDir = "/ec";
+ String testFile = "testfile_001";
+ String testFilePath = testDir + "/" + testFile;
+ String clientName = "testUser1";
+ String clientMachine = "testMachine1";
+ long blkId = 1;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+ short blockNum = HdfsConstants.NUM_DATA_BLOCKS;
+ short parityNum = HdfsConstants.NUM_PARITY_BLOCKS;
+ int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+
+ //set the storage policy of the directory
+ fs.mkdir(new Path(testDir), new FsPermission("755"));
+ fs.getClient().getNamenode().createErasureCodingZone(testDir, null, 0);
+
+ // Create a file with striped block
+ Path p = new Path(testFilePath);
+ DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
+
+ fns.enterSafeMode(false);
+ fns.saveNamespace(0, 0);
+ fns.leaveSafeMode();
+
+ // Add a striped block to the file
+ BlockInfoStriped stripedBlk = new BlockInfoStriped(
+ new Block(blkId, blkNumBytes, timestamp), testSchema, cellSize);
+ INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
+ file.toUnderConstruction(clientName, clientMachine);
+ file.addBlock(stripedBlk);
+ fns.getEditLog().logAddBlock(testFilePath, file);
+ file.toCompleteFile(System.currentTimeMillis());
+
+ //If the block by loaded is the same as above it means that
+ //we have successfully applied the edit log to the fsimage.
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+
+ INodeFile inodeLoaded = (INodeFile)fns.getFSDirectory()
+ .getINode(testFilePath);
+
+ assertTrue(inodeLoaded.isStriped());
+
+ BlockInfo[] blks = inodeLoaded.getBlocks();
+ assertEquals(1, blks.length);
+ assertEquals(blkId, blks[0].getBlockId());
+ assertEquals(blkNumBytes, blks[0].getNumBytes());
+ assertEquals(timestamp, blks[0].getGenerationStamp());
+ assertEquals(blockNum, ((BlockInfoStriped)blks[0]).getDataBlockNum());
+ assertEquals(parityNum, ((BlockInfoStriped)blks[0]).getParityBlockNum());
+ assertEquals(cellSize, ((BlockInfoStriped)blks[0]).getCellSize());
+
+ cluster.shutdown();
+ cluster = null;
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testUpdateStripedBlocks() throws IOException{
+ // start a cluster
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSNamesystem fns = cluster.getNamesystem();
+
+ String testDir = "/ec";
+ String testFile = "testfile_002";
+ String testFilePath = testDir + "/" + testFile;
+ String clientName = "testUser2";
+ String clientMachine = "testMachine2";
+ long blkId = 1;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+ short blockNum = HdfsConstants.NUM_DATA_BLOCKS;
+ short parityNum = HdfsConstants.NUM_PARITY_BLOCKS;
+ int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+
+ //set the storage policy of the directory
+ fs.mkdir(new Path(testDir), new FsPermission("755"));
+ fs.getClient().getNamenode().createErasureCodingZone(testDir, null, 0);
+
+ //create a file with striped blocks
+ Path p = new Path(testFilePath);
+ DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
+ BlockInfoStriped stripedBlk = new BlockInfoStriped(
+ new Block(blkId, blkNumBytes, timestamp), testSchema, cellSize);
+ INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
+ file.toUnderConstruction(clientName, clientMachine);
+ file.addBlock(stripedBlk);
+ fns.getEditLog().logAddBlock(testFilePath, file);
+ file.toCompleteFile(System.currentTimeMillis());
+ fns.enterSafeMode(false);
+ fns.saveNamespace(0, 0);
+ fns.leaveSafeMode();
+
+ //update the last block
+ long newBlkNumBytes = 1024*8;
+ long newTimestamp = 1426222918+3600;
+ file.toUnderConstruction(clientName, clientMachine);
+ file.getLastBlock().setNumBytes(newBlkNumBytes);
+ file.getLastBlock().setGenerationStamp(newTimestamp);
+ fns.getEditLog().logUpdateBlocks(testFilePath, file, true);
+ file.toCompleteFile(System.currentTimeMillis());
+
+ //After the namenode restarts if the block by loaded is the same as above
+ //(new block size and timestamp) it means that we have successfully
+ //applied the edit log to the fsimage.
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+
+ INodeFile inodeLoaded = (INodeFile)fns.getFSDirectory()
+ .getINode(testFilePath);
+
+ assertTrue(inodeLoaded.isStriped());
+
+ BlockInfo[] blks = inodeLoaded.getBlocks();
+ assertEquals(1, blks.length);
+ assertTrue(blks[0].isStriped());
+ assertEquals(blkId, blks[0].getBlockId());
+ assertEquals(newBlkNumBytes, blks[0].getNumBytes());
+ assertEquals(newTimestamp, blks[0].getGenerationStamp());
+ assertEquals(blockNum, ((BlockInfoStriped)blks[0]).getDataBlockNum());
+ assertEquals(parityNum, ((BlockInfoStriped)blks[0]).getParityBlockNum());
+ assertEquals(cellSize, ((BlockInfoStriped)blks[0]).getCellSize());
+
+ cluster.shutdown();
+ cluster = null;
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testHasNonEcBlockUsingStripedIDForAddBlock() throws IOException{
+ // start a cluster
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSNamesystem fns = cluster.getNamesystem();
+
+ String testDir = "/test_block_manager";
+ String testFile = "testfile_addblock";
+ String testFilePath = testDir + "/" + testFile;
+ String clientName = "testUser_addblock";
+ String clientMachine = "testMachine_addblock";
+ long blkId = -1;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+
+ fs.mkdir(new Path(testDir), new FsPermission("755"));
+ Path p = new Path(testFilePath);
+
+ //check whether the hasNonEcBlockUsingStripedID is set
+ //after loading a addblock-editlog
+ DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
+ BlockInfoContiguous cBlk = new BlockInfoContiguous(
+ new Block(blkId, blkNumBytes, timestamp), (short)3);
+ INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
+ file.toUnderConstruction(clientName, clientMachine);
+ file.addBlock(cBlk);
+ fns.getEditLog().logAddBlock(testFilePath, file);
+ file.toCompleteFile(System.currentTimeMillis());
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+ assertTrue(fns.getBlockManager().hasNonEcBlockUsingStripedID());
+
+ cluster.shutdown();
+ cluster = null;
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testHasNonEcBlockUsingStripedIDForUpdateBlocks()
+ throws IOException{
+ // start a cluster
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSNamesystem fns = cluster.getNamesystem();
+
+ String testDir = "/test_block_manager";
+ String testFile = "testfile_002";
+ String testFilePath = testDir + "/" + testFile;
+ String clientName = "testUser2";
+ String clientMachine = "testMachine1";
+ long blkId = 100;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+
+ fs.mkdir(new Path(testDir), new FsPermission("755"));
+ Path p = new Path(testFilePath);
+
+ DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
+ BlockInfoContiguous cBlk = new BlockInfoContiguous(
+ new Block(blkId, blkNumBytes, timestamp), (short)3);
+ INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
+ file.toUnderConstruction(clientName, clientMachine);
+ file.addBlock(cBlk);
+ file.toCompleteFile(System.currentTimeMillis());
+
+ long newBlkNumBytes = 1024*8;
+ long newTimestamp = 1426222918+3600;
+ file.toUnderConstruction(clientName, clientMachine);
+ file.getLastBlock().setBlockId(-100);
+ file.getLastBlock().setNumBytes(newBlkNumBytes);
+ file.getLastBlock().setGenerationStamp(newTimestamp);
+ fns.getEditLog().logUpdateBlocks(testFilePath, file, true);
+ file.toCompleteFile(System.currentTimeMillis());
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+ assertTrue(fns.getBlockManager().hasNonEcBlockUsingStripedID());
+
+ cluster.shutdown();
+ cluster = null;
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index df20fd6..350ef1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -22,11 +22,25 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.File;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.EnumSet;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.junit.Assert;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -34,6 +48,7 @@ import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSOutputStream;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -42,7 +57,6 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
@@ -50,10 +64,16 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+
public class TestFSImage {
private static final String HADOOP_2_7_ZER0_BLOCK_SIZE_TGZ =
"image-with-zero-block-size.tar.gz";
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+
@Test
public void testPersist() throws IOException {
Configuration conf = new Configuration();
@@ -159,6 +179,123 @@ public class TestFSImage {
}
}
+ private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf,
+ boolean isUC) throws IOException{
+ // contruct a INode with StripedBlock for saving and loading
+ fsn.createErasureCodingZone("/", null, 0, false);
+ long id = 123456789;
+ byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes();
+ PermissionStatus permissionStatus = new PermissionStatus("testuser_a",
+ "testuser_groups", new FsPermission((short)0x755));
+ long mtime = 1426222916-3600;
+ long atime = 1426222916;
+ BlockInfoContiguous[] blks = new BlockInfoContiguous[0];
+ short replication = 3;
+ long preferredBlockSize = 128*1024*1024;
+ INodeFile file = new INodeFile(id, name, permissionStatus, mtime, atime,
+ blks, replication, preferredBlockSize, (byte) 0, true);
+ ByteArrayOutputStream bs = new ByteArrayOutputStream();
+
+ //construct StripedBlocks for the INode
+ BlockInfoStriped[] stripedBlks = new BlockInfoStriped[3];
+ long stripedBlkId = 10000001;
+ long timestamp = mtime+3600;
+ for (int i = 0; i < stripedBlks.length; i++) {
+ stripedBlks[i] = new BlockInfoStriped(
+ new Block(stripedBlkId + i, preferredBlockSize, timestamp),
+ testSchema, cellSize);
+ file.addBlock(stripedBlks[i]);
+ }
+
+ final String client = "testClient";
+ final String clientMachine = "testClientMachine";
+ final String path = "testUnderConstructionPath";
+
+ //save the INode to byte array
+ DataOutput out = new DataOutputStream(bs);
+ if (isUC) {
+ file.toUnderConstruction(client, clientMachine);
+ FSImageSerialization.writeINodeUnderConstruction((DataOutputStream) out,
+ file, path);
+ } else {
+ FSImageSerialization.writeINodeFile(file, out, false);
+ }
+ DataInput in = new DataInputStream(
+ new ByteArrayInputStream(bs.toByteArray()));
+
+ // load the INode from the byte array
+ INodeFile fileByLoaded;
+ if (isUC) {
+ fileByLoaded = FSImageSerialization.readINodeUnderConstruction(in,
+ fsn, fsn.getFSImage().getLayoutVersion());
+ } else {
+ fileByLoaded = (INodeFile) new FSImageFormat.Loader(conf, fsn)
+ .loadINodeWithLocalName(false, in, false);
+ }
+
+ assertEquals(id, fileByLoaded.getId() );
+ assertArrayEquals(isUC ? path.getBytes() : name,
+ fileByLoaded.getLocalName().getBytes());
+ assertEquals(permissionStatus.getUserName(),
+ fileByLoaded.getPermissionStatus().getUserName());
+ assertEquals(permissionStatus.getGroupName(),
+ fileByLoaded.getPermissionStatus().getGroupName());
+ assertEquals(permissionStatus.getPermission(),
+ fileByLoaded.getPermissionStatus().getPermission());
+ assertEquals(mtime, fileByLoaded.getModificationTime());
+ assertEquals(isUC ? mtime : atime, fileByLoaded.getAccessTime());
+ // TODO for striped blocks, we currently save and load them as contiguous
+ // blocks to/from legacy fsimage
+ assertEquals(3, fileByLoaded.getBlocks().length);
+ assertEquals(preferredBlockSize, fileByLoaded.getPreferredBlockSize());
+
+ if (isUC) {
+ assertEquals(client,
+ fileByLoaded.getFileUnderConstructionFeature().getClientName());
+ assertEquals(clientMachine,
+ fileByLoaded.getFileUnderConstructionFeature().getClientMachine());
+ }
+ }
+
+ /**
+ * Test if a INodeFile with BlockInfoStriped can be saved by
+ * FSImageSerialization and loaded by FSImageFormat#Loader.
+ */
+ @Test
+ public void testSaveAndLoadStripedINodeFile() throws IOException{
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ testSaveAndLoadStripedINodeFile(cluster.getNamesystem(), conf, false);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * Test if a INodeFileUnderConstruction with BlockInfoStriped can be
+ * saved and loaded by FSImageSerialization
+ */
+ @Test
+ public void testSaveAndLoadStripedINodeFileUC() throws IOException{
+ // construct a INode with StripedBlock for saving and loading
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ testSaveAndLoadStripedINodeFile(cluster.getNamesystem(), conf, true);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
/**
* Ensure that the digest written by the saver equals to the digest of the
* file.
@@ -260,8 +397,7 @@ public class TestFSImage {
.format(false)
.manageDataDfsDirs(false)
.manageNameDfsDirs(false)
- .waitSafeMode(false)
- .startupOption(StartupOption.UPGRADE)
+ .waitSafeMode(false).startupOption(StartupOption.UPGRADE)
.build();
try {
FileSystem fs = cluster.getFileSystem();
@@ -274,4 +410,207 @@ public class TestFSImage {
FileUtil.fullyDelete(dfsDir);
}
}
+
+ /**
+ * Ensure that FSImage supports BlockGroup.
+ */
+ @Test
+ public void testSupportBlockGroup() throws IOException {
+ final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS +
+ HdfsConstants.NUM_PARITY_BLOCKS;
+ final int BLOCK_SIZE = 8 * 1024 * 1024;
+ Configuration conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ fs.getClient().getNamenode().createErasureCodingZone("/", null, 0);
+ Path file = new Path("/striped");
+ FSDataOutputStream out = fs.create(file);
+ byte[] bytes = DFSTestUtil.generateSequentialBytes(0, BLOCK_SIZE);
+ out.write(bytes);
+ out.close();
+
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.restartNameNodes();
+ fs = cluster.getFileSystem();
+ assertTrue(fs.exists(file));
+
+ // check the information of striped blocks
+ FSNamesystem fsn = cluster.getNamesystem();
+ INodeFile inode = fsn.dir.getINode(file.toString()).asFile();
+ assertTrue(inode.isStriped());
+ BlockInfo[] blks = inode.getBlocks();
+ assertEquals(1, blks.length);
+ assertTrue(blks[0].isStriped());
+ assertEquals(HdfsConstants.NUM_DATA_BLOCKS, ((BlockInfoStriped)blks[0]).getDataBlockNum());
+ assertEquals(HdfsConstants.NUM_PARITY_BLOCKS, ((BlockInfoStriped)blks[0]).getParityBlockNum());
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testHasNonEcBlockUsingStripedIDForLoadFile() throws IOException{
+ // start a cluster
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSNamesystem fns = cluster.getNamesystem();
+
+ String testDir = "/test_block_manager";
+ String testFile = "testfile_loadfile";
+ String testFilePath = testDir + "/" + testFile;
+ String clientName = "testUser_loadfile";
+ String clientMachine = "testMachine_loadfile";
+ long blkId = -1;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+
+ fs.mkdir(new Path(testDir), new FsPermission("755"));
+ Path p = new Path(testFilePath);
+
+ DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
+ BlockInfoContiguous cBlk = new BlockInfoContiguous(
+ new Block(blkId, blkNumBytes, timestamp), (short)3);
+ INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
+ file.toUnderConstruction(clientName, clientMachine);
+ file.addBlock(cBlk);
+ file.toCompleteFile(System.currentTimeMillis());
+ fns.enterSafeMode(false);
+ fns.saveNamespace(0, 0);
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+ assertTrue(fns.getBlockManager().hasNonEcBlockUsingStripedID());
+
+ //after nonEcBlockUsingStripedID is deleted
+ //the hasNonEcBlockUsingStripedID is set to false
+ fs = cluster.getFileSystem();
+ fs.delete(p,false);
+ fns.enterSafeMode(false);
+ fns.saveNamespace(0, 0);
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+ assertFalse(fns.getBlockManager().hasNonEcBlockUsingStripedID());
+
+ cluster.shutdown();
+ cluster = null;
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testHasNonEcBlockUsingStripedIDForLoadUCFile()
+ throws IOException{
+ // start a cluster
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSNamesystem fns = cluster.getNamesystem();
+
+ String testDir = "/test_block_manager";
+ String testFile = "testfile_loaducfile";
+ String testFilePath = testDir + "/" + testFile;
+ String clientName = "testUser_loaducfile";
+ String clientMachine = "testMachine_loaducfile";
+ long blkId = -1;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+
+ fs.mkdir(new Path(testDir), new FsPermission("755"));
+ Path p = new Path(testFilePath);
+
+ DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
+ BlockInfoContiguous cBlk = new BlockInfoContiguous(
+ new Block(blkId, blkNumBytes, timestamp), (short)3);
+ INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
+ file.toUnderConstruction(clientName, clientMachine);
+ file.addBlock(cBlk);
+ fns.enterSafeMode(false);
+ fns.saveNamespace(0, 0);
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+ assertTrue(fns.getBlockManager().hasNonEcBlockUsingStripedID());
+
+ cluster.shutdown();
+ cluster = null;
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ @Test
+ public void testHasNonEcBlockUsingStripedIDForLoadSnapshot()
+ throws IOException{
+ // start a cluster
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(9)
+ .build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ FSNamesystem fns = cluster.getNamesystem();
+
+ String testDir = "/test_block_manager";
+ String testFile = "testfile_loadSnapshot";
+ String testFilePath = testDir + "/" + testFile;
+ String clientName = "testUser_loadSnapshot";
+ String clientMachine = "testMachine_loadSnapshot";
+ long blkId = -1;
+ long blkNumBytes = 1024;
+ long timestamp = 1426222918;
+
+ Path d = new Path(testDir);
+ fs.mkdir(d, new FsPermission("755"));
+ fs.allowSnapshot(d);
+
+ Path p = new Path(testFilePath);
+ DFSTestUtil.createFile(fs, p, 0, (short) 1, 1);
+ BlockInfoContiguous cBlk = new BlockInfoContiguous(
+ new Block(blkId, blkNumBytes, timestamp), (short)3);
+ INodeFile file = (INodeFile)fns.getFSDirectory().getINode(testFilePath);
+ file.toUnderConstruction(clientName, clientMachine);
+ file.addBlock(cBlk);
+ file.toCompleteFile(System.currentTimeMillis());
+
+ fs.createSnapshot(d,"testHasNonEcBlockUsingStripeID");
+ fs.truncate(p,0);
+ fns.enterSafeMode(false);
+ fns.saveNamespace(0, 0);
+ cluster.restartNameNodes();
+ cluster.waitActive();
+ fns = cluster.getNamesystem();
+ assertTrue(fns.getBlockManager().hasNonEcBlockUsingStripedID());
+
+ cluster.shutdown();
+ cluster = null;
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index 767f4de..f6aae22 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
@@ -1032,7 +1033,8 @@ public class TestFileTruncate {
iip = fsn.getFSDirectory().getINodesInPath(src, true);
file = iip.getLastINode().asFile();
file.recordModification(iip.getLatestSnapshotId(), true);
- assertThat(file.isBlockInLatestSnapshot(file.getLastBlock()), is(true));
+ assertThat(file.isBlockInLatestSnapshot(
+ (BlockInfoContiguous) file.getLastBlock()), is(true));
initialGenStamp = file.getLastBlock().getGenerationStamp();
// Test that prepareFileForTruncate sets up copy-on-write truncate
fsn.writeLock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
index 8818f17..04e83a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
@@ -84,6 +84,8 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck.Result;
+import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck.ReplicationResult;
+import org.apache.hadoop.hdfs.server.namenode.NamenodeFsck.ErasureCodingResult;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.hdfs.tools.DFSck;
import org.apache.hadoop.io.IOUtils;
@@ -1058,13 +1060,14 @@ public class TestFsck {
final HdfsFileStatus file =
namenode.getRpcServer().getFileInfo(pathString);
assertNotNull(file);
- Result res = new Result(conf);
- fsck.check(pathString, file, res);
+ Result replRes = new ReplicationResult(conf);
+ Result ecRes = new ErasureCodingResult(conf);
+ fsck.check(pathString, file, replRes, ecRes);
// Also print the output from the fsck, for ex post facto sanity checks
System.out.println(result.toString());
- assertEquals(res.missingReplicas,
+ assertEquals(replRes.missingReplicas,
(NUM_BLOCKS*REPL_FACTOR) - (NUM_BLOCKS*NUM_REPLICAS));
- assertEquals(res.numExpectedReplicas, NUM_BLOCKS*REPL_FACTOR);
+ assertEquals(replRes.numExpectedReplicas, NUM_BLOCKS*REPL_FACTOR);
} finally {
if(dfs != null) {
dfs.close();
@@ -1135,10 +1138,11 @@ public class TestFsck {
final HdfsFileStatus file =
namenode.getRpcServer().getFileInfo(pathString);
assertNotNull(file);
- Result res = new Result(conf);
- fsck.check(pathString, file, res);
+ Result replRes = new ReplicationResult(conf);
+ Result ecRes = new ErasureCodingResult(conf);
+ fsck.check(pathString, file, replRes, ecRes);
// check misReplicatedBlock number.
- assertEquals(res.numMisReplicatedBlocks, NUM_BLOCKS);
+ assertEquals(replRes.numMisReplicatedBlocks, NUM_BLOCKS);
} finally {
if(dfs != null) {
dfs.close();
@@ -1198,15 +1202,16 @@ public class TestFsck {
HdfsFileStatus file = new HdfsFileStatus(length, isDir, blockReplication,
blockSize, modTime, accessTime, perms, owner, group, symlink,
- path, fileId, numChildren, null, storagePolicy);
- Result res = new Result(conf);
+ path, fileId, numChildren, null, storagePolicy, null, 0);
+ Result replRes = new ReplicationResult(conf);
+ Result ecRes = new ErasureCodingResult(conf);
try {
- fsck.check(pathString, file, res);
+ fsck.check(pathString, file, replRes, ecRes);
} catch (Exception e) {
fail("Unexpected exception " + e.getMessage());
}
- assertTrue(res.toString().contains("HEALTHY"));
+ assertTrue(replRes.isHealthy());
}
/** Test fsck with symlinks in the filesystem */
@@ -1629,4 +1634,60 @@ public class TestFsck {
}
}
}
+
+ @Test
+ public void testECFsck() throws Exception {
+ MiniDFSCluster cluster = null;
+ FileSystem fs = null;
+ try {
+ Configuration conf = new HdfsConfiguration();
+ final long precision = 1L;
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY, precision);
+ conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10000L);
+ int totalSize = ErasureCodingSchemaManager.getSystemDefaultSchema().getNumDataUnits()
+ + ErasureCodingSchemaManager.getSystemDefaultSchema().getNumParityUnits();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(totalSize).build();
+ fs = cluster.getFileSystem();
+
+ // create a contiguous file
+ Path replDirPath = new Path("/replicated");
+ Path replFilePath = new Path(replDirPath, "replfile");
+ final short factor = 3;
+ DFSTestUtil.createFile(fs, replFilePath, 1024, factor, 0);
+ DFSTestUtil.waitReplication(fs, replFilePath, factor);
+
+ // create a large striped file
+ Path ecDirPath = new Path("/striped");
+ Path largeFilePath = new Path(ecDirPath, "largeFile");
+ DFSTestUtil.createStripedFile(cluster, largeFilePath, ecDirPath, 1, 2, true);
+
+ // create a small striped file
+ Path smallFilePath = new Path(ecDirPath, "smallFile");
+ DFSTestUtil.writeFile(fs, smallFilePath, "hello world!");
+
+ long replTime = fs.getFileStatus(replFilePath).getAccessTime();
+ long ecTime = fs.getFileStatus(largeFilePath).getAccessTime();
+ Thread.sleep(precision);
+ setupAuditLogs();
+ String outStr = runFsck(conf, 0, true, "/");
+ verifyAuditLogs();
+ assertEquals(replTime, fs.getFileStatus(replFilePath).getAccessTime());
+ assertEquals(ecTime, fs.getFileStatus(largeFilePath).getAccessTime());
+ System.out.println(outStr);
+ assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
+ if (fs != null) {try{fs.close();} catch(Exception e){}}
+ cluster.shutdown();
+
+ // restart the cluster; bring up namenode but not the data nodes
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(0).format(false).build();
+ outStr = runFsck(conf, 1, true, "/");
+ // expect the result is corrupt
+ assertTrue(outStr.contains(NamenodeFsck.CORRUPT_STATUS));
+ System.out.println(outStr);
+ } finally {
+ if (fs != null) {try{fs.close();} catch(Exception e){}}
+ if (cluster != null) { cluster.shutdown(); }
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
index b8db998..b877181 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
@@ -94,7 +94,7 @@ public class TestINodeFile {
private static INodeFile createINodeFile(byte storagePolicyID) {
return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
- null, (short)3, 1024L, storagePolicyID);
+ null, (short)3, 1024L, storagePolicyID, false);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
new file mode 100644
index 0000000..10b77df
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+/**
+ * Make sure we correctly update the quota usage with the striped blocks.
+ */
+public class TestQuotaWithStripedBlocks {
+ private static final int BLOCK_SIZE = 1024 * 1024;
+ private static final long DISK_QUOTA = BLOCK_SIZE * 10;
+ private static final ECSchema ecSchema =
+ ErasureCodingSchemaManager.getSystemDefaultSchema();
+ private static final int NUM_DATA_BLOCKS = ecSchema.getNumDataUnits();
+ private static final int NUM_PARITY_BLOCKS = ecSchema.getNumParityUnits();
+ private static final int GROUP_SIZE = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ private static final Path ecDir = new Path("/ec");
+
+ private MiniDFSCluster cluster;
+ private FSDirectory dir;
+ private DistributedFileSystem dfs;
+
+ @Before
+ public void setUp() throws IOException {
+ final Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
+ cluster.waitActive();
+
+ dir = cluster.getNamesystem().getFSDirectory();
+ dfs = cluster.getFileSystem();
+
+ dfs.mkdirs(ecDir);
+ dfs.getClient().createErasureCodingZone(ecDir.toString(), ecSchema, 0);
+ dfs.setQuota(ecDir, Long.MAX_VALUE - 1, DISK_QUOTA);
+ dfs.setQuotaByStorageType(ecDir, StorageType.DISK, DISK_QUOTA);
+ dfs.setStoragePolicy(ecDir, HdfsConstants.HOT_STORAGE_POLICY_NAME);
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testUpdatingQuotaCount() throws Exception {
+ final Path file = new Path(ecDir, "file");
+ FSDataOutputStream out = null;
+
+ try {
+ out = dfs.create(file, (short) 1);
+
+ INodeFile fileNode = dir.getINode4Write(file.toString()).asFile();
+ ExtendedBlock previous = null;
+ // Create striped blocks which have a cell in each block.
+ Block newBlock = DFSTestUtil.addStripedBlockToFile(cluster.getDataNodes(),
+ dfs, cluster.getNamesystem(), file.toString(), fileNode,
+ dfs.getClient().getClientName(), previous, 1);
+ previous = new ExtendedBlock(cluster.getNamesystem().getBlockPoolId(),
+ newBlock);
+
+ final INodeDirectory dirNode = dir.getINode4Write(ecDir.toString())
+ .asDirectory();
+ final long spaceUsed = dirNode.getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getStorageSpace();
+ final long diskUsed = dirNode.getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.DISK);
+ // When we add a new block we update the quota using the full block size.
+ Assert.assertEquals(BLOCK_SIZE * GROUP_SIZE, spaceUsed);
+ Assert.assertEquals(BLOCK_SIZE * GROUP_SIZE, diskUsed);
+
+ dfs.getClient().getNamenode().complete(file.toString(),
+ dfs.getClient().getClientName(), previous, fileNode.getId());
+
+ final long actualSpaceUsed = dirNode.getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getStorageSpace();
+ final long actualDiskUsed = dirNode.getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.DISK);
+ // In this case the file's real size is cell size * block group size.
+ Assert.assertEquals(HdfsConstants.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE,
+ actualSpaceUsed);
+ Assert.assertEquals(HdfsConstants.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE,
+ actualDiskUsed);
+ } finally {
+ IOUtils.cleanup(null, out);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java
new file mode 100644
index 0000000..3134373
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestRecoverStripedBlocks.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
+import org.junit.Test;
+import java.util.List;
+
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_PARITY_BLOCKS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestRecoverStripedBlocks {
+ private final short GROUP_SIZE =
+ NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ private MiniDFSCluster cluster;
+ private final Path dirPath = new Path("/dir");
+ private Path filePath = new Path(dirPath, "file");
+ private int maxReplicationStreams =
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_DEFAULT;
+
+ private void initConf(Configuration conf) {
+ // Large value to make sure the pending replication request can stay in
+ // DatanodeDescriptor.replicateBlocks before test timeout.
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 100);
+ // Make sure BlockManager can pull all blocks from UnderReplicatedBlocks via
+ // chooseUnderReplicatedBlocks at once.
+ conf.setInt(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_WORK_MULTIPLIER_PER_ITERATION, 5);
+ }
+
+ @Test
+ public void testMissingStripedBlock() throws Exception {
+ doTestMissingStripedBlock(1, 0);
+ }
+
+ @Test
+ public void testMissingStripedBlockWithBusyNode1() throws Exception {
+ doTestMissingStripedBlock(2, 1);
+ }
+
+ @Test
+ public void testMissingStripedBlockWithBusyNode2() throws Exception {
+ doTestMissingStripedBlock(3, 1);
+ }
+
+ /**
+ * Start GROUP_SIZE + 1 datanodes.
+ * Inject striped blocks to first GROUP_SIZE datanodes.
+ * Then make numOfBusy datanodes busy, make numOfMissed datanodes missed.
+ * Then trigger BlockManager to compute recovery works. (so all recovery work
+ * will be scheduled to the last datanode)
+ * Finally, verify the recovery work of the last datanode.
+ */
+ private void doTestMissingStripedBlock(int numOfMissed, int numOfBusy)
+ throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ initConf(conf);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE + 1)
+ .build();
+
+ try {
+ cluster.waitActive();
+ final int numBlocks = 4;
+ DFSTestUtil.createStripedFile(cluster, filePath,
+ dirPath, numBlocks, 1, true);
+ // all blocks will be located at first GROUP_SIZE DNs, the last DN is
+ // empty because of the util function createStripedFile
+
+ // make sure the file is complete in NN
+ final INodeFile fileNode = cluster.getNamesystem().getFSDirectory()
+ .getINode4Write(filePath.toString()).asFile();
+ assertFalse(fileNode.isUnderConstruction());
+ assertTrue(fileNode.isStriped());
+ BlockInfo[] blocks = fileNode.getBlocks();
+ assertEquals(numBlocks, blocks.length);
+ for (BlockInfo blk : blocks) {
+ assertTrue(blk.isStriped());
+ assertTrue(blk.isComplete());
+ assertEquals(BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS,
+ blk.getNumBytes());
+ final BlockInfoStriped sb = (BlockInfoStriped) blk;
+ assertEquals(GROUP_SIZE, sb.numNodes());
+ }
+
+ final BlockManager bm = cluster.getNamesystem().getBlockManager();
+ BlockInfo firstBlock = fileNode.getBlocks()[0];
+ DatanodeStorageInfo[] storageInfos = bm.getStorages(firstBlock);
+
+ // make numOfBusy nodes busy
+ int i = 0;
+ for (; i < numOfBusy; i++) {
+ DatanodeDescriptor busyNode = storageInfos[i].getDatanodeDescriptor();
+ for (int j = 0; j < maxReplicationStreams + 1; j++) {
+ BlockManagerTestUtil.addBlockToBeReplicated(busyNode, new Block(j),
+ new DatanodeStorageInfo[]{storageInfos[0]});
+ }
+ }
+
+ // make numOfMissed internal blocks missed
+ for (; i < numOfBusy + numOfMissed; i++) {
+ DatanodeDescriptor missedNode = storageInfos[i].getDatanodeDescriptor();
+ assertEquals(numBlocks, missedNode.numBlocks());
+ bm.getDatanodeManager().removeDatanode(missedNode);
+ }
+
+ BlockManagerTestUtil.getComputedDatanodeWork(bm);
+
+ // all the recovery work will be scheduled on the last DN
+ DataNode lastDn = cluster.getDataNodes().get(GROUP_SIZE);
+ DatanodeDescriptor last =
+ bm.getDatanodeManager().getDatanode(lastDn.getDatanodeId());
+ assertEquals("Counting the number of outstanding EC tasks", numBlocks,
+ last.getNumberOfBlocksToBeErasureCoded());
+ List<BlockECRecoveryInfo> recovery =
+ last.getErasureCodeCommand(numBlocks);
+ for (BlockECRecoveryInfo info : recovery) {
+ assertEquals(1, info.getTargetDnInfos().length);
+ assertEquals(last, info.getTargetDnInfos()[0]);
+ assertEquals(info.getSourceDnInfos().length,
+ info.getLiveBlockIndices().length);
+ if (GROUP_SIZE - numOfMissed == NUM_DATA_BLOCKS) {
+ // It's a QUEUE_HIGHEST_PRIORITY block, so the busy DNs will be chosen
+ // to make sure we have NUM_DATA_BLOCKS DNs to do recovery work.
+ assertEquals(NUM_DATA_BLOCKS, info.getSourceDnInfos().length);
+ } else {
+ // The block has no highest priority, so we don't use the busy DNs as
+ // sources
+ assertEquals(GROUP_SIZE - numOfMissed - numOfBusy,
+ info.getSourceDnInfos().length);
+ }
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
new file mode 100644
index 0000000..9c6a590
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -0,0 +1,284 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import org.junit.Test;
+
+/**
+ * This class tests INodeFile with striped feature.
+ */
+public class TestStripedINodeFile {
+ public static final Log LOG = LogFactory.getLog(TestINodeFile.class);
+
+ private static final PermissionStatus perm = new PermissionStatus(
+ "userName", null, FsPermission.getDefault());
+
+ private final BlockStoragePolicySuite defaultSuite =
+ BlockStoragePolicySuite.createDefaultSuite();
+ private final BlockStoragePolicy defaultPolicy =
+ defaultSuite.getDefaultPolicy();
+
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+
+ private static INodeFile createStripedINodeFile() {
+ return new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
+ null, (short)0, 1024L, HdfsServerConstants.COLD_STORAGE_POLICY_ID, true);
+ }
+
+ @Test
+ public void testBlockStripedFeature()
+ throws IOException, InterruptedException{
+ INodeFile inf = createStripedINodeFile();
+ assertTrue(inf.isStriped());
+ }
+
+ @Test
+ public void testBlockStripedTotalBlockCount() {
+ Block blk = new Block(1);
+ BlockInfoStriped blockInfoStriped
+ = new BlockInfoStriped(blk, testSchema, cellSize);
+ assertEquals(9, blockInfoStriped.getTotalBlockNum());
+ }
+
+ @Test
+ public void testBlockStripedLength()
+ throws IOException, InterruptedException {
+ INodeFile inf = createStripedINodeFile();
+ Block blk = new Block(1);
+ BlockInfoStriped blockInfoStriped
+ = new BlockInfoStriped(blk, testSchema, cellSize);
+ inf.addBlock(blockInfoStriped);
+ assertEquals(1, inf.getBlocks().length);
+ }
+
+ @Test
+ public void testBlockStripedConsumedSpace()
+ throws IOException, InterruptedException {
+ INodeFile inf = createStripedINodeFile();
+ Block blk = new Block(1);
+ BlockInfoStriped blockInfoStriped
+ = new BlockInfoStriped(blk, testSchema, cellSize);
+ blockInfoStriped.setNumBytes(1);
+ inf.addBlock(blockInfoStriped);
+ // 0. Calculate the total bytes per stripes <Num Bytes per Stripes>
+ // 1. Calculate the number of stripes in this block group. <Num Stripes>
+ // 2. Calculate the last remaining length which does not make a stripe. <Last Stripe Length>
+ // 3. Total consumed space is the total of
+ // a. The total of the full cells of data blocks and parity blocks.
+ // b. The remaining of data block which does not make a stripe.
+ // c. The last parity block cells. These size should be same
+ // to the first cell in this stripe.
+ // So the total consumed space is the sum of
+ // a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
+ // b. <Num Bytes> % <Num Bytes per Stripes> = 1
+ // c. <Last Stripe Length> * <Parity Block Num> = 1 * 3
+ assertEquals(4, inf.storagespaceConsumedStriped().getStorageSpace());
+ assertEquals(4, inf.storagespaceConsumed(defaultPolicy).getStorageSpace());
+ }
+
+ @Test
+ public void testMultipleBlockStripedConsumedSpace()
+ throws IOException, InterruptedException {
+ INodeFile inf = createStripedINodeFile();
+ Block blk1 = new Block(1);
+ BlockInfoStriped blockInfoStriped1
+ = new BlockInfoStriped(blk1, testSchema, cellSize);
+ blockInfoStriped1.setNumBytes(1);
+ Block blk2 = new Block(2);
+ BlockInfoStriped blockInfoStriped2
+ = new BlockInfoStriped(blk2, testSchema, cellSize);
+ blockInfoStriped2.setNumBytes(1);
+ inf.addBlock(blockInfoStriped1);
+ inf.addBlock(blockInfoStriped2);
+ // This is the double size of one block in above case.
+ assertEquals(4 * 2, inf.storagespaceConsumedStriped().getStorageSpace());
+ assertEquals(4 * 2, inf.storagespaceConsumed(defaultPolicy).getStorageSpace());
+ }
+
+ @Test
+ public void testBlockStripedFileSize()
+ throws IOException, InterruptedException {
+ INodeFile inf = createStripedINodeFile();
+ Block blk = new Block(1);
+ BlockInfoStriped blockInfoStriped
+ = new BlockInfoStriped(blk, testSchema, cellSize);
+ blockInfoStriped.setNumBytes(100);
+ inf.addBlock(blockInfoStriped);
+ // Compute file size should return actual data
+ // size which is retained by this file.
+ assertEquals(100, inf.computeFileSize());
+ assertEquals(100, inf.computeFileSize(false, false));
+ }
+
+ @Test
+ public void testBlockUCStripedFileSize()
+ throws IOException, InterruptedException {
+ INodeFile inf = createStripedINodeFile();
+ Block blk = new Block(1);
+ BlockInfoStripedUnderConstruction bInfoUCStriped
+ = new BlockInfoStripedUnderConstruction(blk, testSchema, cellSize);
+ bInfoUCStriped.setNumBytes(100);
+ inf.addBlock(bInfoUCStriped);
+ assertEquals(100, inf.computeFileSize());
+ assertEquals(0, inf.computeFileSize(false, false));
+ }
+
+ @Test
+ public void testBlockStripedComputeQuotaUsage()
+ throws IOException, InterruptedException {
+ INodeFile inf = createStripedINodeFile();
+ Block blk = new Block(1);
+ BlockInfoStriped blockInfoStriped
+ = new BlockInfoStriped(blk, testSchema, cellSize);
+ blockInfoStriped.setNumBytes(100);
+ inf.addBlock(blockInfoStriped);
+
+ QuotaCounts counts =
+ inf.computeQuotaUsageWithStriped(defaultPolicy,
+ new QuotaCounts.Builder().build());
+ assertEquals(1, counts.getNameSpace());
+ // The total consumed space is the sum of
+ // a. <Cell Size> * (<Num Stripes> - 1) * <Total Block Num> = 0
+ // b. <Num Bytes> % <Num Bytes per Stripes> = 100
+ // c. <Last Stripe Length> * <Parity Block Num> = 100 * 3
+ assertEquals(400, counts.getStorageSpace());
+ }
+
+ @Test
+ public void testBlockUCStripedComputeQuotaUsage()
+ throws IOException, InterruptedException {
+ INodeFile inf = createStripedINodeFile();
+ Block blk = new Block(1);
+ BlockInfoStripedUnderConstruction bInfoUCStriped
+ = new BlockInfoStripedUnderConstruction(blk, testSchema, cellSize);
+ bInfoUCStriped.setNumBytes(100);
+ inf.addBlock(bInfoUCStriped);
+
+ QuotaCounts counts
+ = inf.computeQuotaUsageWithStriped(defaultPolicy,
+ new QuotaCounts.Builder().build());
+ assertEquals(1024, inf.getPreferredBlockSize());
+ assertEquals(1, counts.getNameSpace());
+ // Consumed space in the case of BlockInfoUCStriped can be calculated
+ // by using preferred block size. This is 1024 and total block num
+ // is 9(= 3 + 6). Consumed storage space should be 1024 * 9 = 9216.
+ assertEquals(9216, counts.getStorageSpace());
+ }
+
+ /**
+ * Test the behavior of striped and contiguous block deletions.
+ */
+ @Test(timeout = 60000)
+ public void testDeleteOp() throws Exception {
+ MiniDFSCluster cluster = null;
+ try {
+ final int len = 1024;
+ final Path parentDir = new Path("/parentDir");
+ final Path zone = new Path(parentDir, "zone");
+ final Path zoneFile = new Path(zone, "zoneFile");
+ final Path contiguousFile = new Path(parentDir, "someFile");
+ final DistributedFileSystem dfs;
+ final Configuration conf = new Configuration();
+ final short GROUP_SIZE = HdfsConstants.NUM_DATA_BLOCKS
+ + HdfsConstants.NUM_PARITY_BLOCKS;
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 2);
+
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE)
+ .build();
+ cluster.waitActive();
+
+ FSNamesystem fsn = cluster.getNamesystem();
+ dfs = cluster.getFileSystem();
+ dfs.mkdirs(zone);
+
+ // create erasure zone
+ dfs.createErasureCodingZone(zone, null, 0);
+ DFSTestUtil.createFile(dfs, zoneFile, len, (short) 1, 0xFEED);
+ DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED);
+ final FSDirectory fsd = fsn.getFSDirectory();
+
+ // Case-1: Verify the behavior of striped blocks
+ // Get blocks of striped file
+ INode inodeStriped = fsd.getINode("/parentDir/zone/zoneFile");
+ assertTrue("Failed to get INodeFile for /parentDir/zone/zoneFile",
+ inodeStriped instanceof INodeFile);
+ INodeFile inodeStripedFile = (INodeFile) inodeStriped;
+ BlockInfo[] stripedBlks = inodeStripedFile.getBlocks();
+ for (BlockInfo blockInfo : stripedBlks) {
+ assertFalse("Mistakenly marked the block as deleted!",
+ blockInfo.isDeleted());
+ }
+
+ // delete erasure zone directory
+ dfs.delete(zone, true);
+ for (BlockInfo blockInfo : stripedBlks) {
+ assertTrue("Didn't mark the block as deleted!", blockInfo.isDeleted());
+ }
+
+ // Case-2: Verify the behavior of contiguous blocks
+ // Get blocks of contiguous file
+ INode inode = fsd.getINode("/parentDir/someFile");
+ assertTrue("Failed to get INodeFile for /parentDir/someFile",
+ inode instanceof INodeFile);
+ INodeFile inodeFile = (INodeFile) inode;
+ BlockInfo[] contiguousBlks = inodeFile.getBlocks();
+ for (BlockInfo blockInfo : contiguousBlks) {
+ assertFalse("Mistakenly marked the block as deleted!",
+ blockInfo.isDeleted());
+ }
+
+ // delete parent directory
+ dfs.delete(parentDir, true);
+ for (BlockInfo blockInfo : contiguousBlks) {
+ assertTrue("Didn't mark the block as deleted!", blockInfo.isDeleted());
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
index a1abd08..57026cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotTestHelper.java
@@ -175,7 +175,7 @@ public class SnapshotTestHelper {
* localName (className@hashCode) parent permission group user
*
* Specific information for different types of INode:
- * {@link INodeDirectory}:childrenSize
+ * {@link INodeDirectory}:childrenSize
* {@link INodeFile}: fileSize, block list. Check {@link BlockInfo#toString()}
* and {@link BlockInfoContiguousUnderConstruction#toString()} for detailed information.
* {@link FileWithSnapshot}: next link
[04/23] hadoop git commit: HDFS-8887. Expose storage type and storage
ID in BlockLocation.
Posted by zh...@apache.org.
HDFS-8887. Expose storage type and storage ID in BlockLocation.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ea1a833
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ea1a833
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ea1a833
Branch: refs/heads/HDFS-7285-merge
Commit: 1ea1a8334ea01814121490a5bfd2a0205c66d6e4
Parents: 3ae716f
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Aug 11 23:25:33 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Aug 11 23:25:33 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/BlockLocation.java | 55 ++++++++++++++++-
.../org/apache/hadoop/fs/TestBlockLocation.java | 23 ++++++--
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 2 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../apache/hadoop/fs/BlockStorageLocation.java | 1 +
.../hadoop/hdfs/DistributedFileSystem.java | 6 ++
.../hadoop/hdfs/TestDistributedFileSystem.java | 62 +++++++++++++++++++-
7 files changed, 145 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea1a833/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
index 286d851..7811ef5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/BlockLocation.java
@@ -34,11 +34,15 @@ public class BlockLocation {
private String[] cachedHosts; // Datanode hostnames with a cached replica
private String[] names; // Datanode IP:xferPort for accessing the block
private String[] topologyPaths; // Full path name in network topology
+ private String[] storageIds; // Storage ID of each replica
+ private StorageType[] storageTypes; // Storage type of each replica
private long offset; // Offset of the block in the file
private long length;
private boolean corrupt;
private static final String[] EMPTY_STR_ARRAY = new String[0];
+ private static final StorageType[] EMPTY_STORAGE_TYPE_ARRAY =
+ new StorageType[0];
/**
* Default Constructor
@@ -58,6 +62,8 @@ public class BlockLocation {
this.offset = that.offset;
this.length = that.length;
this.corrupt = that.corrupt;
+ this.storageIds = that.storageIds;
+ this.storageTypes = that.storageTypes;
}
/**
@@ -95,6 +101,13 @@ public class BlockLocation {
public BlockLocation(String[] names, String[] hosts, String[] cachedHosts,
String[] topologyPaths, long offset, long length, boolean corrupt) {
+ this(names, hosts, cachedHosts, topologyPaths, null, null, offset, length,
+ corrupt);
+ }
+
+ public BlockLocation(String[] names, String[] hosts, String[] cachedHosts,
+ String[] topologyPaths, String[] storageIds, StorageType[] storageTypes,
+ long offset, long length, boolean corrupt) {
if (names == null) {
this.names = EMPTY_STR_ARRAY;
} else {
@@ -115,6 +128,16 @@ public class BlockLocation {
} else {
this.topologyPaths = topologyPaths;
}
+ if (storageIds == null) {
+ this.storageIds = EMPTY_STR_ARRAY;
+ } else {
+ this.storageIds = storageIds;
+ }
+ if (storageTypes == null) {
+ this.storageTypes = EMPTY_STORAGE_TYPE_ARRAY;
+ } else {
+ this.storageTypes = storageTypes;
+ }
this.offset = offset;
this.length = length;
this.corrupt = corrupt;
@@ -148,7 +171,21 @@ public class BlockLocation {
public String[] getTopologyPaths() throws IOException {
return topologyPaths;
}
-
+
+ /**
+ * Get the storageID of each replica of the block.
+ */
+ public String[] getStorageIds() {
+ return storageIds;
+ }
+
+ /**
+ * Get the storage type of each replica of the block.
+ */
+ public StorageType[] getStorageTypes() {
+ return storageTypes;
+ }
+
/**
* Get the start offset of file associated with this block
*/
@@ -235,6 +272,22 @@ public class BlockLocation {
}
}
+ public void setStorageIds(String[] storageIds) {
+ if (storageIds == null) {
+ this.storageIds = EMPTY_STR_ARRAY;
+ } else {
+ this.storageIds = storageIds;
+ }
+ }
+
+ public void setStorageTypes(StorageType[] storageTypes) {
+ if (storageTypes == null) {
+ this.storageTypes = EMPTY_STORAGE_TYPE_ARRAY;
+ } else {
+ this.storageTypes = storageTypes;
+ }
+ }
+
@Override
public String toString() {
StringBuilder result = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea1a833/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestBlockLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestBlockLocation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestBlockLocation.java
index 3cb608a..8569ea7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestBlockLocation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestBlockLocation.java
@@ -26,6 +26,8 @@ import org.junit.Test;
public class TestBlockLocation {
private static final String[] EMPTY_STR_ARRAY = new String[0];
+ private static final StorageType[] EMPTY_STORAGE_TYPE_ARRAY =
+ new StorageType[0];
private static void checkBlockLocation(final BlockLocation loc)
throws Exception {
@@ -36,22 +38,29 @@ public class TestBlockLocation {
final long offset, final long length, final boolean corrupt)
throws Exception {
checkBlockLocation(loc, EMPTY_STR_ARRAY, EMPTY_STR_ARRAY, EMPTY_STR_ARRAY,
- EMPTY_STR_ARRAY, offset, length, corrupt);
+ EMPTY_STR_ARRAY, EMPTY_STR_ARRAY, EMPTY_STORAGE_TYPE_ARRAY, offset,
+ length, corrupt);
}
private static void checkBlockLocation(final BlockLocation loc,
String[] names, String[] hosts, String[] cachedHosts,
- String[] topologyPaths, final long offset, final long length,
+ String[] topologyPaths,
+ String[] storageIds, StorageType[] storageTypes,
+ final long offset, final long length,
final boolean corrupt) throws Exception {
assertNotNull(loc.getHosts());
assertNotNull(loc.getCachedHosts());
assertNotNull(loc.getNames());
assertNotNull(loc.getTopologyPaths());
+ assertNotNull(loc.getStorageIds());
+ assertNotNull(loc.getStorageTypes());
assertArrayEquals(hosts, loc.getHosts());
assertArrayEquals(cachedHosts, loc.getCachedHosts());
assertArrayEquals(names, loc.getNames());
assertArrayEquals(topologyPaths, loc.getTopologyPaths());
+ assertArrayEquals(storageIds, loc.getStorageIds());
+ assertArrayEquals(storageTypes, loc.getStorageTypes());
assertEquals(offset, loc.getOffset());
assertEquals(length, loc.getLength());
@@ -75,6 +84,8 @@ public class TestBlockLocation {
checkBlockLocation(loc, 1, 2, true);
loc = new BlockLocation(null, null, null, null, 1, 2, true);
checkBlockLocation(loc, 1, 2, true);
+ loc = new BlockLocation(null, null, null, null, null, null, 1, 2, true);
+ checkBlockLocation(loc, 1, 2, true);
}
/**
@@ -95,14 +106,18 @@ public class TestBlockLocation {
String[] hosts = new String[] { "host" };
String[] cachedHosts = new String[] { "cachedHost" };
String[] topologyPaths = new String[] { "path" };
+ String[] storageIds = new String[] { "storageId" };
+ StorageType[] storageTypes = new StorageType[] { StorageType.DISK };
loc.setNames(names);
loc.setHosts(hosts);
loc.setCachedHosts(cachedHosts);
loc.setTopologyPaths(topologyPaths);
+ loc.setStorageIds(storageIds);
+ loc.setStorageTypes(storageTypes);
loc.setOffset(1);
loc.setLength(2);
loc.setCorrupt(true);
- checkBlockLocation(loc, names, hosts, cachedHosts, topologyPaths, 1, 2,
- true);
+ checkBlockLocation(loc, names, hosts, cachedHosts, topologyPaths,
+ storageIds, storageTypes, 1, 2, true);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea1a833/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index 0e72b98..fa1f5e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -181,6 +181,8 @@ public class DFSUtilClient {
}
blkLocations[idx] = new BlockLocation(xferAddrs, hosts, cachedHosts,
racks,
+ blk.getStorageIDs(),
+ blk.getStorageTypes(),
blk.getStartOffset(),
blk.getBlockSize(),
blk.isCorrupt());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea1a833/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 61def02..6c2e0f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -778,6 +778,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8805. Archival Storage: getStoragePolicy should not need superuser privilege.
(Brahma Reddy Battula via jing9)
+ HDFS-8887. Expose storage type and storage ID in BlockLocation. (wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea1a833/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java
index abf3e38..2200994 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/BlockStorageLocation.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
*/
@InterfaceStability.Unstable
@InterfaceAudience.Public
+@Deprecated
public class BlockStorageLocation extends BlockLocation {
private final VolumeId[] volumeIds;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea1a833/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 4d5c0f6..0197cfb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -234,6 +234,11 @@ public class DistributedFileSystem extends FileSystem {
}
/**
+ * This API has been deprecated since the NameNode now tracks datanode
+ * storages separately. Storage IDs can be gotten from {@link
+ * BlockLocation#getStorageIds()}, which are functionally equivalent to
+ * the volume IDs returned here (although a String rather than a byte[]).
+ *
* Used to query storage location information for a list of blocks. This list
* of blocks is normally constructed via a series of calls to
* {@link DistributedFileSystem#getFileBlockLocations(Path, long, long)} to
@@ -257,6 +262,7 @@ public class DistributedFileSystem extends FileSystem {
* information for each replica of each block.
*/
@InterfaceStability.Unstable
+ @Deprecated
public BlockStorageLocation[] getFileBlockStorageLocations(
List<BlockLocation> blocks) throws IOException,
UnsupportedOperationException, InvalidBlockTokenException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea1a833/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 6012c5d..4b76c3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -40,8 +40,10 @@ import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.List;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.impl.Log4JLogger;
@@ -71,7 +73,10 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeFaultInjector;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.web.WebHdfsConstants;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -824,7 +829,62 @@ public class TestDistributedFileSystem {
noXmlDefaults = false;
}
}
-
+
+ @Test(timeout=120000)
+ public void testLocatedFileStatusStorageIdsTypes() throws Exception {
+ final Configuration conf = getTestConfiguration();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(3).build();
+ try {
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ final Path testFile = new Path("/testListLocatedStatus");
+ final int blockSize = 4096;
+ final int numBlocks = 10;
+ // Create a test file
+ final int repl = 2;
+ DFSTestUtil.createFile(fs, testFile, blockSize, numBlocks * blockSize,
+ blockSize, (short) repl, 0xADDED);
+ // Get the listing
+ RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(testFile);
+ assertTrue("Expected file to be present", it.hasNext());
+ LocatedFileStatus stat = it.next();
+ BlockLocation[] locs = stat.getBlockLocations();
+ assertEquals("Unexpected number of locations", numBlocks, locs.length);
+
+ Set<String> dnStorageIds = new HashSet<>();
+ for (DataNode d : cluster.getDataNodes()) {
+ try (FsDatasetSpi.FsVolumeReferences volumes = d.getFSDataset()
+ .getFsVolumeReferences()) {
+ for (FsVolumeSpi vol : volumes) {
+ dnStorageIds.add(vol.getStorageID());
+ }
+ }
+ }
+
+ for (BlockLocation loc : locs) {
+ String[] ids = loc.getStorageIds();
+ // Run it through a set to deduplicate, since there should be no dupes
+ Set<String> storageIds = new HashSet<>();
+ for (String id: ids) {
+ storageIds.add(id);
+ }
+ assertEquals("Unexpected num storage ids", repl, storageIds.size());
+ // Make sure these are all valid storage IDs
+ assertTrue("Unknown storage IDs found!", dnStorageIds.containsAll
+ (storageIds));
+ // Check storage types are the default, since we didn't set any
+ StorageType[] types = loc.getStorageTypes();
+ assertEquals("Unexpected num storage types", repl, types.length);
+ for (StorageType t: types) {
+ assertEquals("Unexpected storage type", StorageType.DEFAULT, t);
+ }
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
/**
* Tests the normal path of batching up BlockLocation[]s to be passed to a
[14/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
new file mode 100644
index 0000000..3c9adc4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -0,0 +1,988 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.erasurecode;
+
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.BlockReader;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSPacket;
+import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.RemoteBlockReader2;
+import org.apache.hadoop.hdfs.net.Peer;
+import org.apache.hadoop.hdfs.net.TcpPeerServer;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
+import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
+import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
+import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
+import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunkReadResult;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.DataChecksum;
+
+import com.google.common.base.Preconditions;
+
+import static org.apache.hadoop.hdfs.util.StripedBlockUtil.convertIndex4Decode;
+
+/**
+ * ErasureCodingWorker handles the erasure coding recovery work commands. These
+ * commands would be issued from Namenode as part of Datanode's heart beat
+ * response. BPOfferService delegates the work to this class for handling EC
+ * commands.
+ */
+public final class ErasureCodingWorker {
+ private static final Log LOG = DataNode.LOG;
+
+ private final DataNode datanode;
+ private final Configuration conf;
+
+ private ThreadPoolExecutor STRIPED_READ_THREAD_POOL;
+ private final int STRIPED_READ_THRESHOLD_MILLIS;
+ private final int STRIPED_READ_BUFFER_SIZE;
+
+ public ErasureCodingWorker(Configuration conf, DataNode datanode) {
+ this.datanode = datanode;
+ this.conf = conf;
+
+ STRIPED_READ_THRESHOLD_MILLIS = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY,
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT);
+ initializeStripedReadThreadPool(conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_KEY,
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT));
+ STRIPED_READ_BUFFER_SIZE = conf.getInt(
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY,
+ DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT);
+ }
+
+ private RawErasureDecoder newDecoder(int numDataUnits, int numParityUnits) {
+ return CodecUtil.createRSRawDecoder(conf, numDataUnits, numParityUnits);
+ }
+
+ private void initializeStripedReadThreadPool(int num) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using striped reads; pool threads=" + num);
+ }
+ STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60,
+ TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ new Daemon.DaemonFactory() {
+ private final AtomicInteger threadIndex = new AtomicInteger(0);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = super.newThread(r);
+ t.setName("stripedRead-" + threadIndex.getAndIncrement());
+ return t;
+ }
+ }, new ThreadPoolExecutor.CallerRunsPolicy() {
+ @Override
+ public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) {
+ LOG.info("Execution for striped reading rejected, "
+ + "Executing in current thread");
+ // will run in the current thread
+ super.rejectedExecution(runnable, e);
+ }
+ });
+ STRIPED_READ_THREAD_POOL.allowCoreThreadTimeOut(true);
+ }
+
+ /**
+ * Handles the Erasure Coding recovery work commands.
+ *
+ * @param ecTasks
+ * BlockECRecoveryInfo
+ */
+ public void processErasureCodingTasks(Collection<BlockECRecoveryInfo> ecTasks) {
+ for (BlockECRecoveryInfo recoveryInfo : ecTasks) {
+ try {
+ new Daemon(new ReconstructAndTransferBlock(recoveryInfo)).start();
+ } catch (Throwable e) {
+ LOG.warn("Failed to recover striped block " +
+ recoveryInfo.getExtendedBlock().getLocalBlock(), e);
+ }
+ }
+ }
+
+ /**
+ * ReconstructAndTransferBlock recover one or more missed striped block in the
+ * striped block group, the minimum number of live striped blocks should be
+ * no less than data block number.
+ *
+ * | <- Striped Block Group -> |
+ * blk_0 blk_1 blk_2(*) blk_3 ... <- A striped block group
+ * | | | |
+ * v v v v
+ * +------+ +------+ +------+ +------+
+ * |cell_0| |cell_1| |cell_2| |cell_3| ...
+ * +------+ +------+ +------+ +------+
+ * |cell_4| |cell_5| |cell_6| |cell_7| ...
+ * +------+ +------+ +------+ +------+
+ * |cell_8| |cell_9| |cell10| |cell11| ...
+ * +------+ +------+ +------+ +------+
+ * ... ... ... ...
+ *
+ *
+ * We use following steps to recover striped block group, in each round, we
+ * recover <code>bufferSize</code> data until finish, the
+ * <code>bufferSize</code> is configurable and may be less or larger than
+ * cell size:
+ * step1: read <code>bufferSize</code> data from minimum number of sources
+ * required by recovery.
+ * step2: decode data for targets.
+ * step3: transfer data to targets.
+ *
+ * In step1, try to read <code>bufferSize</code> data from minimum number
+ * of sources , if there is corrupt or stale sources, read from new source
+ * will be scheduled. The best sources are remembered for next round and
+ * may be updated in each round.
+ *
+ * In step2, typically if source blocks we read are all data blocks, we
+ * need to call encode, and if there is one parity block, we need to call
+ * decode. Notice we only read once and recover all missed striped block
+ * if they are more than one.
+ *
+ * In step3, send the recovered data to targets by constructing packet
+ * and send them directly. Same as continuous block replication, we
+ * don't check the packet ack. Since the datanode doing the recovery work
+ * are one of the source datanodes, so the recovered data are sent
+ * remotely.
+ *
+ * There are some points we can do further improvements in next phase:
+ * 1. we can read the block file directly on the local datanode,
+ * currently we use remote block reader. (Notice short-circuit is not
+ * a good choice, see inline comments).
+ * 2. We need to check the packet ack for EC recovery? Since EC recovery
+ * is more expensive than continuous block replication, it needs to
+ * read from several other datanodes, should we make sure the
+ * recovered result received by targets?
+ */
+ private class ReconstructAndTransferBlock implements Runnable {
+ private final int dataBlkNum;
+ private final int parityBlkNum;
+ private final int cellSize;
+
+ private RawErasureDecoder decoder;
+
+ // Striped read buffer size
+ private int bufferSize;
+
+ private final ExtendedBlock blockGroup;
+ private final int minRequiredSources;
+ // position in striped internal block
+ private long positionInBlock;
+
+ // sources
+ private final short[] liveIndices;
+ private final DatanodeInfo[] sources;
+
+ private final List<StripedReader> stripedReaders;
+
+ // The buffers and indices for striped blocks whose length is 0
+ private ByteBuffer[] zeroStripeBuffers;
+ private short[] zeroStripeIndices;
+
+ // targets
+ private final DatanodeInfo[] targets;
+ private final StorageType[] targetStorageTypes;
+
+ private final short[] targetIndices;
+ private final ByteBuffer[] targetBuffers;
+
+ private final Socket[] targetSockets;
+ private final DataOutputStream[] targetOutputStreams;
+ private final DataInputStream[] targetInputStreams;
+
+ private final long[] blockOffset4Targets;
+ private final long[] seqNo4Targets;
+
+ private final static int WRITE_PACKET_SIZE = 64 * 1024;
+ private DataChecksum checksum;
+ private int maxChunksPerPacket;
+ private byte[] packetBuf;
+ private byte[] checksumBuf;
+ private int bytesPerChecksum;
+ private int checksumSize;
+
+ private final CachingStrategy cachingStrategy;
+
+ private final Map<Future<Void>, Integer> futures = new HashMap<>();
+ private final CompletionService<Void> readService =
+ new ExecutorCompletionService<>(STRIPED_READ_THREAD_POOL);
+
+ ReconstructAndTransferBlock(BlockECRecoveryInfo recoveryInfo) {
+ ECSchema schema = recoveryInfo.getECSchema();
+ dataBlkNum = schema.getNumDataUnits();
+ parityBlkNum = schema.getNumParityUnits();
+ cellSize = recoveryInfo.getCellSize();
+
+ blockGroup = recoveryInfo.getExtendedBlock();
+ final int cellsNum = (int)((blockGroup.getNumBytes() - 1) / cellSize + 1);
+ minRequiredSources = Math.min(cellsNum, dataBlkNum);
+
+ liveIndices = recoveryInfo.getLiveBlockIndices();
+ sources = recoveryInfo.getSourceDnInfos();
+ stripedReaders = new ArrayList<>(sources.length);
+
+ Preconditions.checkArgument(liveIndices.length >= minRequiredSources,
+ "No enough live striped blocks.");
+ Preconditions.checkArgument(liveIndices.length == sources.length,
+ "liveBlockIndices and source dns should match");
+
+ if (minRequiredSources < dataBlkNum) {
+ zeroStripeBuffers =
+ new ByteBuffer[dataBlkNum - minRequiredSources];
+ zeroStripeIndices = new short[dataBlkNum - minRequiredSources];
+ }
+
+ targets = recoveryInfo.getTargetDnInfos();
+ targetStorageTypes = recoveryInfo.getTargetStorageTypes();
+ targetIndices = new short[targets.length];
+ targetBuffers = new ByteBuffer[targets.length];
+
+ Preconditions.checkArgument(targetIndices.length <= parityBlkNum,
+ "Too much missed striped blocks.");
+
+ targetSockets = new Socket[targets.length];
+ targetOutputStreams = new DataOutputStream[targets.length];
+ targetInputStreams = new DataInputStream[targets.length];
+
+ blockOffset4Targets = new long[targets.length];
+ seqNo4Targets = new long[targets.length];
+
+ for (int i = 0; i < targets.length; i++) {
+ blockOffset4Targets[i] = 0;
+ seqNo4Targets[i] = 0;
+ }
+
+ getTargetIndices();
+ cachingStrategy = CachingStrategy.newDefaultStrategy();
+ }
+
+ private ByteBuffer allocateBuffer(int length) {
+ return ByteBuffer.allocate(length);
+ }
+
+ private ExtendedBlock getBlock(ExtendedBlock blockGroup, int i) {
+ return StripedBlockUtil.constructInternalBlock(blockGroup, cellSize,
+ dataBlkNum, i);
+ }
+
+ private long getBlockLen(ExtendedBlock blockGroup, int i) {
+ return StripedBlockUtil.getInternalBlockLength(blockGroup.getNumBytes(),
+ cellSize, dataBlkNum, i);
+ }
+
+ /**
+ * StripedReader is used to read from one source DN, it contains a block
+ * reader, buffer and striped block index.
+ * Only allocate StripedReader once for one source, and the StripedReader
+ * has the same array order with sources. Typically we only need to allocate
+ * minimum number (minRequiredSources) of StripedReader, and allocate
+ * new for new source DN if some existing DN invalid or slow.
+ * If some source DN is corrupt, set the corresponding blockReader to
+ * null and will never read from it again.
+ *
+ * @param i the array index of sources
+ * @param offsetInBlock offset for the internal block
+ * @return StripedReader
+ */
+ private StripedReader addStripedReader(int i, long offsetInBlock) {
+ StripedReader reader = new StripedReader(liveIndices[i]);
+ stripedReaders.add(reader);
+
+ BlockReader blockReader = newBlockReader(
+ getBlock(blockGroup, liveIndices[i]), offsetInBlock, sources[i]);
+ if (blockReader != null) {
+ initChecksumAndBufferSizeIfNeeded(blockReader);
+ reader.blockReader = blockReader;
+ }
+ reader.buffer = allocateBuffer(bufferSize);
+ return reader;
+ }
+
+ @Override
+ public void run() {
+ datanode.incrementXmitsInProgress();
+ try {
+ // Store the array indices of source DNs we have read successfully.
+ // In each iteration of read, the success list may be updated if
+ // some source DN is corrupted or slow. And use the updated success
+ // list of DNs for next iteration read.
+ int[] success = new int[minRequiredSources];
+
+ int nsuccess = 0;
+ for (int i = 0;
+ i < sources.length && nsuccess < minRequiredSources; i++) {
+ StripedReader reader = addStripedReader(i, 0);
+ if (reader.blockReader != null) {
+ success[nsuccess++] = i;
+ }
+ }
+
+ if (nsuccess < minRequiredSources) {
+ String error = "Can't find minimum sources required by "
+ + "recovery, block id: " + blockGroup.getBlockId();
+ throw new IOException(error);
+ }
+
+ if (zeroStripeBuffers != null) {
+ for (int i = 0; i < zeroStripeBuffers.length; i++) {
+ zeroStripeBuffers[i] = allocateBuffer(bufferSize);
+ }
+ }
+
+ for (int i = 0; i < targets.length; i++) {
+ targetBuffers[i] = allocateBuffer(bufferSize);
+ }
+
+ checksumSize = checksum.getChecksumSize();
+ int chunkSize = bytesPerChecksum + checksumSize;
+ maxChunksPerPacket = Math.max(
+ (WRITE_PACKET_SIZE - PacketHeader.PKT_MAX_HEADER_LEN)/chunkSize, 1);
+ int maxPacketSize = chunkSize * maxChunksPerPacket
+ + PacketHeader.PKT_MAX_HEADER_LEN;
+
+ packetBuf = new byte[maxPacketSize];
+ checksumBuf = new byte[checksumSize * (bufferSize / bytesPerChecksum)];
+
+ // targetsStatus store whether some target is success, it will record
+ // any failed target once, if some target failed (invalid DN or transfer
+ // failed), will not transfer data to it any more.
+ boolean[] targetsStatus = new boolean[targets.length];
+ if (initTargetStreams(targetsStatus) == 0) {
+ String error = "All targets are failed.";
+ throw new IOException(error);
+ }
+
+ long firstStripedBlockLength = getBlockLen(blockGroup, 0);
+ while (positionInBlock < firstStripedBlockLength) {
+ int toRead = Math.min(
+ bufferSize, (int)(firstStripedBlockLength - positionInBlock));
+ // step1: read from minimum source DNs required for reconstruction.
+ // The returned success list is the source DNs we do real read from
+ success = readMinimumStripedData4Recovery(success);
+
+ // step2: decode to reconstruct targets
+ long remaining = firstStripedBlockLength - positionInBlock;
+ int toRecoverLen = remaining < bufferSize ?
+ (int)remaining : bufferSize;
+ recoverTargets(success, targetsStatus, toRecoverLen);
+
+ // step3: transfer data
+ if (transferData2Targets(targetsStatus) == 0) {
+ String error = "Transfer failed for all targets.";
+ throw new IOException(error);
+ }
+
+ clearBuffers();
+ positionInBlock += toRead;
+ }
+
+ endTargetBlocks(targetsStatus);
+
+ // Currently we don't check the acks for packets, this is similar as
+ // block replication.
+ } catch (Throwable e) {
+ LOG.warn("Failed to recover striped block: " + blockGroup, e);
+ } finally {
+ datanode.decrementXmitsInProgress();
+ // close block readers
+ for (StripedReader stripedReader : stripedReaders) {
+ closeBlockReader(stripedReader.blockReader);
+ }
+ for (int i = 0; i < targets.length; i++) {
+ IOUtils.closeStream(targetOutputStreams[i]);
+ IOUtils.closeStream(targetInputStreams[i]);
+ IOUtils.closeStream(targetSockets[i]);
+ }
+ }
+ }
+
+ // init checksum from block reader
+ private void initChecksumAndBufferSizeIfNeeded(BlockReader blockReader) {
+ if (checksum == null) {
+ checksum = blockReader.getDataChecksum();
+ bytesPerChecksum = checksum.getBytesPerChecksum();
+ // The bufferSize is flat to divide bytesPerChecksum
+ int readBufferSize = STRIPED_READ_BUFFER_SIZE;
+ bufferSize = readBufferSize < bytesPerChecksum ? bytesPerChecksum :
+ readBufferSize - readBufferSize % bytesPerChecksum;
+ } else {
+ assert blockReader.getDataChecksum().equals(checksum);
+ }
+ }
+
+ private void getTargetIndices() {
+ BitSet bitset = new BitSet(dataBlkNum + parityBlkNum);
+ for (int i = 0; i < sources.length; i++) {
+ bitset.set(liveIndices[i]);
+ }
+ int m = 0;
+ int k = 0;
+ for (int i = 0; i < dataBlkNum + parityBlkNum; i++) {
+ if (!bitset.get(i)) {
+ if (getBlockLen(blockGroup, i) > 0) {
+ if (m < targets.length) {
+ targetIndices[m++] = (short)i;
+ }
+ } else {
+ zeroStripeIndices[k++] = (short)i;
+ }
+ }
+ }
+ }
+
+ private long getReadLength(int index) {
+ long blockLen = getBlockLen(blockGroup, index);
+ long remaining = blockLen - positionInBlock;
+ return remaining > bufferSize ? bufferSize : remaining;
+ }
+
+ /**
+ * Read from minimum source DNs required for reconstruction in the iteration.
+ * First try the success list which we think they are the best DNs
+ * If source DN is corrupt or slow, try to read some other source DN,
+ * and will update the success list.
+ *
+ * Remember the updated success list and return it for following
+ * operations and next iteration read.
+ *
+ * @param success the initial success list of source DNs we think best
+ * @return updated success list of source DNs we do real read
+ * @throws IOException
+ */
+ private int[] readMinimumStripedData4Recovery(final int[] success)
+ throws IOException {
+ int nsuccess = 0;
+ int[] newSuccess = new int[minRequiredSources];
+ BitSet used = new BitSet(sources.length);
+ /*
+ * Read from minimum source DNs required, the success list contains
+ * source DNs which we think best.
+ */
+ for (int i = 0; i < minRequiredSources; i++) {
+ StripedReader reader = stripedReaders.get(success[i]);
+ if (getReadLength(liveIndices[success[i]]) > 0) {
+ Callable<Void> readCallable = readFromBlock(
+ reader.blockReader, reader.buffer);
+ Future<Void> f = readService.submit(readCallable);
+ futures.put(f, success[i]);
+ } else {
+ // If the read length is 0, we don't need to do real read
+ reader.buffer.position(0);
+ newSuccess[nsuccess++] = success[i];
+ }
+ used.set(success[i]);
+ }
+
+ while (!futures.isEmpty()) {
+ try {
+ StripingChunkReadResult result =
+ StripedBlockUtil.getNextCompletedStripedRead(
+ readService, futures, STRIPED_READ_THRESHOLD_MILLIS);
+ int resultIndex = -1;
+ if (result.state == StripingChunkReadResult.SUCCESSFUL) {
+ resultIndex = result.index;
+ } else if (result.state == StripingChunkReadResult.FAILED) {
+ // If read failed for some source DN, we should not use it anymore
+ // and schedule read from another source DN.
+ StripedReader failedReader = stripedReaders.get(result.index);
+ closeBlockReader(failedReader.blockReader);
+ failedReader.blockReader = null;
+ resultIndex = scheduleNewRead(used);
+ } else if (result.state == StripingChunkReadResult.TIMEOUT) {
+ // If timeout, we also schedule a new read.
+ resultIndex = scheduleNewRead(used);
+ }
+ if (resultIndex >= 0) {
+ newSuccess[nsuccess++] = resultIndex;
+ if (nsuccess >= minRequiredSources) {
+ // cancel remaining reads if we read successfully from minimum
+ // number of source DNs required by reconstruction.
+ cancelReads(futures.keySet());
+ futures.clear();
+ break;
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.info("Read data interrupted.", e);
+ break;
+ }
+ }
+
+ if (nsuccess < minRequiredSources) {
+ String error = "Can't read data from minimum number of sources "
+ + "required by reconstruction, block id: " + blockGroup.getBlockId();
+ throw new IOException(error);
+ }
+
+ return newSuccess;
+ }
+
+ private void paddingBufferToLen(ByteBuffer buffer, int len) {
+ int toPadding = len - buffer.position();
+ for (int i = 0; i < toPadding; i++) {
+ buffer.put((byte) 0);
+ }
+ }
+
+ // Initialize decoder
+ private void initDecoderIfNecessary() {
+ if (decoder == null) {
+ decoder = newDecoder(dataBlkNum, parityBlkNum);
+ }
+ }
+
+ private int[] getErasedIndices(boolean[] targetsStatus) {
+ int[] result = new int[targets.length];
+ int m = 0;
+ for (int i = 0; i < targets.length; i++) {
+ if (targetsStatus[i]) {
+ result[m++] = convertIndex4Decode(targetIndices[i],
+ dataBlkNum, parityBlkNum);
+ }
+ }
+ return Arrays.copyOf(result, m);
+ }
+
+ private void recoverTargets(int[] success, boolean[] targetsStatus,
+ int toRecoverLen) {
+ initDecoderIfNecessary();
+ ByteBuffer[] inputs = new ByteBuffer[dataBlkNum + parityBlkNum];
+ for (int i = 0; i < success.length; i++) {
+ StripedReader reader = stripedReaders.get(success[i]);
+ ByteBuffer buffer = reader.buffer;
+ paddingBufferToLen(buffer, toRecoverLen);
+ inputs[convertIndex4Decode(reader.index, dataBlkNum, parityBlkNum)] =
+ (ByteBuffer)buffer.flip();
+ }
+ if (success.length < dataBlkNum) {
+ for (int i = 0; i < zeroStripeBuffers.length; i++) {
+ ByteBuffer buffer = zeroStripeBuffers[i];
+ paddingBufferToLen(buffer, toRecoverLen);
+ int index = convertIndex4Decode(zeroStripeIndices[i], dataBlkNum,
+ parityBlkNum);
+ inputs[index] = (ByteBuffer)buffer.flip();
+ }
+ }
+ int[] erasedIndices = getErasedIndices(targetsStatus);
+ ByteBuffer[] outputs = new ByteBuffer[erasedIndices.length];
+ int m = 0;
+ for (int i = 0; i < targetBuffers.length; i++) {
+ if (targetsStatus[i]) {
+ outputs[m++] = targetBuffers[i];
+ outputs[i].limit(toRecoverLen);
+ }
+ }
+ decoder.decode(inputs, erasedIndices, outputs);
+
+ for (int i = 0; i < targets.length; i++) {
+ if (targetsStatus[i]) {
+ long blockLen = getBlockLen(blockGroup, targetIndices[i]);
+ long remaining = blockLen - positionInBlock;
+ if (remaining < 0) {
+ targetBuffers[i].limit(0);
+ } else if (remaining < toRecoverLen) {
+ targetBuffers[i].limit((int)remaining);
+ }
+ }
+ }
+ }
+
+ /**
+ * Schedule a read from some new source DN if some DN is corrupted
+ * or slow, this is called from the read iteration.
+ * Initially we may only have <code>minRequiredSources</code> number of
+ * StripedReader.
+ * If the position is at the end of target block, don't need to do
+ * real read, and return the array index of source DN, otherwise -1.
+ *
+ * @param used the used source DNs in this iteration.
+ * @return the array index of source DN if don't need to do real read.
+ */
+ private int scheduleNewRead(BitSet used) {
+ StripedReader reader = null;
+ // step1: initially we may only have <code>minRequiredSources</code>
+ // number of StripedReader, and there may be some source DNs we never
+ // read before, so will try to create StripedReader for one new source DN
+ // and try to read from it. If found, go to step 3.
+ int m = stripedReaders.size();
+ while (reader == null && m < sources.length) {
+ reader = addStripedReader(m, positionInBlock);
+ if (getReadLength(liveIndices[m]) > 0) {
+ if (reader.blockReader == null) {
+ reader = null;
+ m++;
+ }
+ } else {
+ used.set(m);
+ return m;
+ }
+ }
+
+ // step2: if there is no new source DN we can use, try to find a source
+ // DN we ever read from but because some reason, e.g., slow, it
+ // is not in the success DN list at the begin of this iteration, so
+ // we have not tried it in this iteration. Now we have a chance to
+ // revisit it again.
+ for (int i = 0; reader == null && i < stripedReaders.size(); i++) {
+ if (!used.get(i)) {
+ StripedReader r = stripedReaders.get(i);
+ if (getReadLength(liveIndices[i]) > 0) {
+ closeBlockReader(r.blockReader);
+ r.blockReader = newBlockReader(
+ getBlock(blockGroup, liveIndices[i]), positionInBlock,
+ sources[i]);
+ if (r.blockReader != null) {
+ m = i;
+ reader = r;
+ }
+ } else {
+ used.set(i);
+ r.buffer.position(0);
+ return i;
+ }
+ }
+ }
+
+ // step3: schedule if find a correct source DN and need to do real read.
+ if (reader != null) {
+ Callable<Void> readCallable = readFromBlock(
+ reader.blockReader, reader.buffer);
+ Future<Void> f = readService.submit(readCallable);
+ futures.put(f, m);
+ used.set(m);
+ }
+
+ return -1;
+ }
+
+ // cancel all reads.
+ private void cancelReads(Collection<Future<Void>> futures) {
+ for (Future<Void> future : futures) {
+ future.cancel(true);
+ }
+ }
+
+ private Callable<Void> readFromBlock(final BlockReader reader,
+ final ByteBuffer buf) {
+ return new Callable<Void>() {
+
+ @Override
+ public Void call() throws Exception {
+ try {
+ actualReadFromBlock(reader, buf);
+ return null;
+ } catch (IOException e) {
+ LOG.info(e.getMessage());
+ throw e;
+ }
+ }
+
+ };
+ }
+
+ /**
+ * Read bytes from block
+ */
+ private void actualReadFromBlock(BlockReader reader, ByteBuffer buf)
+ throws IOException {
+ int len = buf.remaining();
+ int n = 0;
+ while (n < len) {
+ int nread = reader.read(buf);
+ if (nread <= 0) {
+ break;
+ }
+ n += nread;
+ }
+ }
+
+ // close block reader
+ private void closeBlockReader(BlockReader blockReader) {
+ try {
+ if (blockReader != null) {
+ blockReader.close();
+ }
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+
+ private InetSocketAddress getSocketAddress4Transfer(DatanodeInfo dnInfo) {
+ return NetUtils.createSocketAddr(dnInfo.getXferAddr(
+ datanode.getDnConf().getConnectToDnViaHostname()));
+ }
+
+ private BlockReader newBlockReader(final ExtendedBlock block,
+ long offsetInBlock, DatanodeInfo dnInfo) {
+ if (offsetInBlock >= block.getNumBytes()) {
+ return null;
+ }
+ try {
+ InetSocketAddress dnAddr = getSocketAddress4Transfer(dnInfo);
+ Token<BlockTokenIdentifier> blockToken = datanode.getBlockAccessToken(
+ block, EnumSet.of(BlockTokenIdentifier.AccessMode.READ));
+ /*
+ * This can be further improved if the replica is local, then we can
+ * read directly from DN and need to check the replica is FINALIZED
+ * state, notice we should not use short-circuit local read which
+ * requires config for domain-socket in UNIX or legacy config in Windows.
+ */
+ return RemoteBlockReader2.newBlockReader(
+ "dummy", block, blockToken, offsetInBlock,
+ block.getNumBytes() - offsetInBlock, true,
+ "", newConnectedPeer(block, dnAddr, blockToken, dnInfo), dnInfo,
+ null, cachingStrategy);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ private Peer newConnectedPeer(ExtendedBlock b, InetSocketAddress addr,
+ Token<BlockTokenIdentifier> blockToken, DatanodeID datanodeId)
+ throws IOException {
+ Peer peer = null;
+ boolean success = false;
+ Socket sock = null;
+ final int socketTimeout = datanode.getDnConf().getSocketTimeout();
+ try {
+ sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
+ NetUtils.connect(sock, addr, socketTimeout);
+ peer = TcpPeerServer.peerFromSocketAndKey(datanode.getSaslClient(),
+ sock, datanode.getDataEncryptionKeyFactoryForBlock(b),
+ blockToken, datanodeId);
+ peer.setReadTimeout(socketTimeout);
+ success = true;
+ return peer;
+ } finally {
+ if (!success) {
+ IOUtils.cleanup(LOG, peer);
+ IOUtils.closeSocket(sock);
+ }
+ }
+ }
+
+ /**
+ * Send data to targets
+ */
+ private int transferData2Targets(boolean[] targetsStatus) {
+ int nsuccess = 0;
+ for (int i = 0; i < targets.length; i++) {
+ if (targetsStatus[i]) {
+ boolean success = false;
+ try {
+ ByteBuffer buffer = targetBuffers[i];
+
+ if (buffer.remaining() == 0) {
+ continue;
+ }
+
+ checksum.calculateChunkedSums(
+ buffer.array(), 0, buffer.remaining(), checksumBuf, 0);
+
+ int ckOff = 0;
+ while (buffer.remaining() > 0) {
+ DFSPacket packet = new DFSPacket(packetBuf, maxChunksPerPacket,
+ blockOffset4Targets[i], seqNo4Targets[i]++, checksumSize, false);
+ int maxBytesToPacket = maxChunksPerPacket * bytesPerChecksum;
+ int toWrite = buffer.remaining() > maxBytesToPacket ?
+ maxBytesToPacket : buffer.remaining();
+ int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * checksumSize;
+ packet.writeChecksum(checksumBuf, ckOff, ckLen);
+ ckOff += ckLen;
+ packet.writeData(buffer, toWrite);
+
+ // Send packet
+ packet.writeTo(targetOutputStreams[i]);
+
+ blockOffset4Targets[i] += toWrite;
+ nsuccess++;
+ success = true;
+ }
+ } catch (IOException e) {
+ LOG.warn(e.getMessage());
+ }
+ targetsStatus[i] = success;
+ }
+ }
+ return nsuccess;
+ }
+
+ /**
+ * clear all buffers
+ */
+ private void clearBuffers() {
+ for (StripedReader stripedReader : stripedReaders) {
+ if (stripedReader.buffer != null) {
+ stripedReader.buffer.clear();
+ }
+ }
+
+ if (zeroStripeBuffers != null) {
+ for (int i = 0; i < zeroStripeBuffers.length; i++) {
+ zeroStripeBuffers[i].clear();
+ }
+ }
+
+ for (int i = 0; i < targetBuffers.length; i++) {
+ if (targetBuffers[i] != null) {
+ cleanBuffer(targetBuffers[i]);
+ }
+ }
+ }
+
+ private ByteBuffer cleanBuffer(ByteBuffer buffer) {
+ Arrays.fill(buffer.array(), (byte) 0);
+ return (ByteBuffer)buffer.clear();
+ }
+
+ // send an empty packet to mark the end of the block
+ private void endTargetBlocks(boolean[] targetsStatus) {
+ for (int i = 0; i < targets.length; i++) {
+ if (targetsStatus[i]) {
+ try {
+ DFSPacket packet = new DFSPacket(packetBuf, 0,
+ blockOffset4Targets[i], seqNo4Targets[i]++, checksumSize, true);
+ packet.writeTo(targetOutputStreams[i]);
+ targetOutputStreams[i].flush();
+ } catch (IOException e) {
+ LOG.warn(e.getMessage());
+ }
+ }
+ }
+ }
+
+ /**
+ * Initialize output/input streams for transferring data to target
+ * and send create block request.
+ */
+ private int initTargetStreams(boolean[] targetsStatus) {
+ int nsuccess = 0;
+ for (int i = 0; i < targets.length; i++) {
+ Socket socket = null;
+ DataOutputStream out = null;
+ DataInputStream in = null;
+ boolean success = false;
+ try {
+ InetSocketAddress targetAddr =
+ getSocketAddress4Transfer(targets[i]);
+ socket = datanode.newSocket();
+ NetUtils.connect(socket, targetAddr,
+ datanode.getDnConf().getSocketTimeout());
+ socket.setSoTimeout(datanode.getDnConf().getSocketTimeout());
+
+ ExtendedBlock block = getBlock(blockGroup, targetIndices[i]);
+ Token<BlockTokenIdentifier> blockToken =
+ datanode.getBlockAccessToken(block,
+ EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE));
+
+ long writeTimeout = datanode.getDnConf().getSocketWriteTimeout();
+ OutputStream unbufOut = NetUtils.getOutputStream(socket, writeTimeout);
+ InputStream unbufIn = NetUtils.getInputStream(socket);
+ DataEncryptionKeyFactory keyFactory =
+ datanode.getDataEncryptionKeyFactoryForBlock(block);
+ IOStreamPair saslStreams = datanode.getSaslClient().socketSend(
+ socket, unbufOut, unbufIn, keyFactory, blockToken, targets[i]);
+
+ unbufOut = saslStreams.out;
+ unbufIn = saslStreams.in;
+
+ out = new DataOutputStream(new BufferedOutputStream(unbufOut,
+ DFSUtil.getSmallBufferSize(conf)));
+ in = new DataInputStream(unbufIn);
+
+ DatanodeInfo source = new DatanodeInfo(datanode.getDatanodeId());
+ new Sender(out).writeBlock(block, targetStorageTypes[i],
+ blockToken, "", new DatanodeInfo[]{targets[i]},
+ new StorageType[]{targetStorageTypes[i]}, source,
+ BlockConstructionStage.PIPELINE_SETUP_CREATE, 0, 0, 0, 0,
+ checksum, cachingStrategy, false, false, null);
+
+ targetSockets[i] = socket;
+ targetOutputStreams[i] = out;
+ targetInputStreams[i] = in;
+ nsuccess++;
+ success = true;
+ } catch (Throwable e) {
+ LOG.warn(e.getMessage());
+ } finally {
+ if (!success) {
+ IOUtils.closeStream(out);
+ IOUtils.closeStream(in);
+ IOUtils.closeStream(socket);
+ }
+ }
+ targetsStatus[i] = success;
+ }
+ return nsuccess;
+ }
+ }
+
+ private static class StripedReader {
+ private final short index; // internal block index
+ private BlockReader blockReader;
+ private ByteBuffer buffer;
+
+ private StripedReader(short index) {
+ this.index = index;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index afacebb..ea6ba54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
@@ -174,8 +175,20 @@ public class Mover {
}
}
- DBlock newDBlock(Block block, List<MLocation> locations) {
- final DBlock db = new DBlock(block);
+ DBlock newDBlock(LocatedBlock lb, List<MLocation> locations,
+ ECSchema ecSchema) {
+ Block blk = lb.getBlock().getLocalBlock();
+ DBlock db;
+ if (lb.isStriped()) {
+ LocatedStripedBlock lsb = (LocatedStripedBlock) lb;
+ byte[] indices = new byte[lsb.getBlockIndices().length];
+ for (int i = 0; i < indices.length; i++) {
+ indices[i] = (byte) lsb.getBlockIndices()[i];
+ }
+ db = new DBlockStriped(blk, indices, (short) ecSchema.getNumDataUnits());
+ } else {
+ db = new DBlock(blk);
+ }
for(MLocation ml : locations) {
StorageGroup source = storages.getSource(ml);
if (source != null) {
@@ -358,9 +371,10 @@ public class Mover {
LOG.warn("Failed to get the storage policy of file " + fullPath);
return;
}
- final List<StorageType> types = policy.chooseStorageTypes(
+ List<StorageType> types = policy.chooseStorageTypes(
status.getReplication());
+ final ECSchema ecSchema = status.getECSchema();
final LocatedBlocks locatedBlocks = status.getBlockLocations();
final boolean lastBlkComplete = locatedBlocks.isLastBlockComplete();
List<LocatedBlock> lbs = locatedBlocks.getLocatedBlocks();
@@ -370,10 +384,13 @@ public class Mover {
continue;
}
LocatedBlock lb = lbs.get(i);
+ if (lb.isStriped()) {
+ types = policy.chooseStorageTypes((short) lb.getLocations().length);
+ }
final StorageTypeDiff diff = new StorageTypeDiff(types,
lb.getStorageTypes());
if (!diff.removeOverlap(true)) {
- if (scheduleMoves4Block(diff, lb)) {
+ if (scheduleMoves4Block(diff, lb, ecSchema)) {
result.updateHasRemaining(diff.existing.size() > 1
&& diff.expected.size() > 1);
// One block scheduled successfully, set noBlockMoved to false
@@ -385,10 +402,13 @@ public class Mover {
}
}
- boolean scheduleMoves4Block(StorageTypeDiff diff, LocatedBlock lb) {
+ boolean scheduleMoves4Block(StorageTypeDiff diff, LocatedBlock lb,
+ ECSchema ecSchema) {
final List<MLocation> locations = MLocation.toLocations(lb);
- Collections.shuffle(locations);
- final DBlock db = newDBlock(lb.getBlock().getLocalBlock(), locations);
+ if (!(lb instanceof LocatedStripedBlock)) {
+ Collections.shuffle(locations);
+ }
+ final DBlock db = newDBlock(lb, locations, ecSchema);
for (final StorageType t : diff.existing) {
for (final MLocation ml : locations) {
@@ -781,4 +801,4 @@ public class Mover {
System.exit(-1);
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingSchemaManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingSchemaManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingSchemaManager.java
new file mode 100644
index 0000000..4c4aae9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingSchemaManager.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ * This manages EC schemas predefined and activated in the system.
+ * It loads customized schemas and syncs with persisted ones in
+ * NameNode image.
+ *
+ * This class is instantiated by the FSNamesystem.
+ */
+@InterfaceAudience.LimitedPrivate({"HDFS"})
+public final class ErasureCodingSchemaManager {
+
+ /**
+ * TODO: HDFS-8095
+ */
+ private static final int DEFAULT_DATA_BLOCKS = 6;
+ private static final int DEFAULT_PARITY_BLOCKS = 3;
+ private static final String DEFAULT_CODEC_NAME = "rs";
+ private static final String DEFAULT_SCHEMA_NAME = "RS-6-3";
+ private static final ECSchema SYS_DEFAULT_SCHEMA =
+ new ECSchema(DEFAULT_SCHEMA_NAME,
+ DEFAULT_CODEC_NAME, DEFAULT_DATA_BLOCKS, DEFAULT_PARITY_BLOCKS);
+
+ //We may add more later.
+ private static ECSchema[] SYS_SCHEMAS = new ECSchema[] {
+ SYS_DEFAULT_SCHEMA
+ };
+
+ /**
+ * All active EC activeSchemas maintained in NN memory for fast querying,
+ * identified and sorted by its name.
+ */
+ private final Map<String, ECSchema> activeSchemas;
+
+ ErasureCodingSchemaManager() {
+
+ this.activeSchemas = new TreeMap<String, ECSchema>();
+ for (ECSchema schema : SYS_SCHEMAS) {
+ activeSchemas.put(schema.getSchemaName(), schema);
+ }
+
+ /**
+ * TODO: HDFS-7859 persist into NameNode
+ * load persistent schemas from image and editlog, which is done only once
+ * during NameNode startup. This can be done here or in a separate method.
+ */
+ }
+
+ /**
+ * Get system defined schemas.
+ * @return system schemas
+ */
+ public static ECSchema[] getSystemSchemas() {
+ return SYS_SCHEMAS;
+ }
+
+ /**
+ * Get system-wide default EC schema, which can be used by default when no
+ * schema is specified for an EC zone.
+ * @return schema
+ */
+ public static ECSchema getSystemDefaultSchema() {
+ return SYS_DEFAULT_SCHEMA;
+ }
+
+ /**
+ * Tell the specified schema is the system default one or not.
+ * @param schema
+ * @return true if it's the default false otherwise
+ */
+ public static boolean isSystemDefault(ECSchema schema) {
+ if (schema == null) {
+ throw new IllegalArgumentException("Invalid schema parameter");
+ }
+
+ // schema name is the identifier.
+ return SYS_DEFAULT_SCHEMA.getSchemaName().equals(schema.getSchemaName());
+ }
+
+ /**
+ * Get all EC schemas that's available to use.
+ * @return all EC schemas
+ */
+ public ECSchema[] getSchemas() {
+ ECSchema[] results = new ECSchema[activeSchemas.size()];
+ return activeSchemas.values().toArray(results);
+ }
+
+ /**
+ * Get the EC schema specified by the schema name.
+ * @param schemaName
+ * @return EC schema specified by the schema name
+ */
+ public ECSchema getSchema(String schemaName) {
+ return activeSchemas.get(schemaName);
+ }
+
+ /**
+ * Clear and clean up
+ */
+ public void clear() {
+ activeSchemas.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
new file mode 100644
index 0000000..2638126
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.hdfs.XAttrHelper;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.XATTR_ERASURECODING_ZONE;
+
+/**
+ * Manages the list of erasure coding zones in the filesystem.
+ * <p/>
+ * The ErasureCodingZoneManager has its own lock, but relies on the FSDirectory
+ * lock being held for many operations. The FSDirectory lock should not be
+ * taken if the manager lock is already held.
+ * TODO: consolidate zone logic w/ encrypt. zones {@link EncryptionZoneManager}
+ */
+public class ErasureCodingZoneManager {
+ private final FSDirectory dir;
+
+ /**
+ * Construct a new ErasureCodingZoneManager.
+ *
+ * @param dir Enclosing FSDirectory
+ */
+ public ErasureCodingZoneManager(FSDirectory dir) {
+ this.dir = dir;
+ }
+
+ ECSchema getErasureCodingSchema(INodesInPath iip) throws IOException {
+ ErasureCodingZone ecZone = getErasureCodingZone(iip);
+ return ecZone == null ? null : ecZone.getSchema();
+ }
+
+ ErasureCodingZone getErasureCodingZone(INodesInPath iip) throws IOException {
+ assert dir.hasReadLock();
+ Preconditions.checkNotNull(iip, "INodes cannot be null");
+ List<INode> inodes = iip.getReadOnlyINodes();
+ for (int i = inodes.size() - 1; i >= 0; i--) {
+ final INode inode = inodes.get(i);
+ if (inode == null) {
+ continue;
+ }
+ // We don't allow symlinks in an EC zone, or pointing to a file/dir in
+ // an EC. Therefore if a symlink is encountered, the dir shouldn't have
+ // EC
+ // TODO: properly support symlinks in EC zones
+ if (inode.isSymlink()) {
+ return null;
+ }
+ final List<XAttr> xAttrs = inode.getXAttrFeature() == null ?
+ new ArrayList<XAttr>(0)
+ : inode.getXAttrFeature().getXAttrs();
+ for (XAttr xAttr : xAttrs) {
+ if (XATTR_ERASURECODING_ZONE.equals(XAttrHelper.getPrefixName(xAttr))) {
+ ByteArrayInputStream bIn=new ByteArrayInputStream(xAttr.getValue());
+ DataInputStream dIn=new DataInputStream(bIn);
+ int cellSize = WritableUtils.readVInt(dIn);
+ String schemaName = WritableUtils.readString(dIn);
+ ECSchema schema = dir.getFSNamesystem()
+ .getErasureCodingSchemaManager().getSchema(schemaName);
+ return new ErasureCodingZone(dir.getInode(inode.getId())
+ .getFullPathName(), schema, cellSize);
+ }
+ }
+ }
+ return null;
+ }
+
+ List<XAttr> createErasureCodingZone(final INodesInPath srcIIP,
+ ECSchema schema, int cellSize) throws IOException {
+ assert dir.hasWriteLock();
+ Preconditions.checkNotNull(srcIIP, "INodes cannot be null");
+ String src = srcIIP.getPath();
+ if (dir.isNonEmptyDirectory(srcIIP)) {
+ throw new IOException(
+ "Attempt to create an erasure coding zone for a " +
+ "non-empty directory " + src);
+ }
+ if (srcIIP.getLastINode() != null &&
+ !srcIIP.getLastINode().isDirectory()) {
+ throw new IOException("Attempt to create an erasure coding zone " +
+ "for a file " + src);
+ }
+ if (getErasureCodingSchema(srcIIP) != null) {
+ throw new IOException("Directory " + src + " is already in an " +
+ "erasure coding zone.");
+ }
+
+ // System default schema will be used since no specified.
+ if (schema == null) {
+ schema = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ }
+
+ if (cellSize <= 0) {
+ cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ }
+
+ // Write the cellsize first and then schema name
+ final XAttr ecXAttr;
+ DataOutputStream dOut = null;
+ try {
+ ByteArrayOutputStream bOut = new ByteArrayOutputStream();
+ dOut = new DataOutputStream(bOut);
+ WritableUtils.writeVInt(dOut, cellSize);
+ // Now persist the schema name in xattr
+ WritableUtils.writeString(dOut, schema.getSchemaName());
+ ecXAttr = XAttrHelper.buildXAttr(XATTR_ERASURECODING_ZONE,
+ bOut.toByteArray());
+ } finally {
+ IOUtils.closeStream(dOut);
+ }
+ final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ xattrs.add(ecXAttr);
+ FSDirXAttrOp.unprotectedSetXAttrs(dir, src, xattrs,
+ EnumSet.of(XAttrSetFlag.CREATE));
+ return xattrs;
+ }
+
+ void checkMoveValidity(INodesInPath srcIIP, INodesInPath dstIIP, String src)
+ throws IOException {
+ assert dir.hasReadLock();
+ final ErasureCodingZone srcZone = getErasureCodingZone(srcIIP);
+ final ErasureCodingZone dstZone = getErasureCodingZone(dstIIP);
+ if (srcZone != null && srcZone.getDir().equals(src) && dstZone == null) {
+ return;
+ }
+ final ECSchema srcSchema = (srcZone != null) ? srcZone.getSchema() : null;
+ final ECSchema dstSchema = (dstZone != null) ? dstZone.getSchema() : null;
+ if ((srcSchema != null && !srcSchema.equals(dstSchema)) ||
+ (dstSchema != null && !dstSchema.equals(srcSchema))) {
+ throw new IOException(
+ src + " can't be moved because the source and destination have " +
+ "different erasure coding policies.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 3d79d09..07a513c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -106,6 +106,11 @@ final class FSDirAppendOp {
+ clientMachine);
}
final INodeFile file = INodeFile.valueOf(inode, path, true);
+ // not support appending file with striped blocks
+ if (file.isStriped()) {
+ throw new UnsupportedOperationException(
+ "Cannot append file with striped block " + src);
+ }
BlockManager blockManager = fsd.getBlockManager();
final BlockStoragePolicy lpPolicy = blockManager
.getStoragePolicy("LAZY_PERSIST");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index d624f84..9b08092 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -401,7 +401,7 @@ public class FSDirAttrOp {
static BlockInfo[] unprotectedSetReplication(
FSDirectory fsd, String src, short replication, short[] blockRepls)
throws QuotaExceededException, UnresolvedLinkException,
- SnapshotAccessControlException {
+ SnapshotAccessControlException, UnsupportedActionException {
assert fsd.hasWriteLock();
final INodesInPath iip = fsd.getINodesInPath4Write(src, true);
@@ -410,6 +410,11 @@ public class FSDirAttrOp {
return null;
}
INodeFile file = inode.asFile();
+ if (file.isStriped()) {
+ throw new UnsupportedActionException(
+ "Cannot set replication to a file with striped blocks");
+ }
+
final short oldBR = file.getPreferredBlockReplication();
// before setFileReplication, check for increasing block replication.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
index bb00130..7503272 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
@@ -143,6 +143,7 @@ class FSDirConcatOp {
throw new HadoopIllegalArgumentException("concat: source file " + src
+ " is invalid or empty or underConstruction");
}
+
// source file's preferred block size cannot be greater than the target
// file
if (srcINodeFile.getPreferredBlockSize() >
@@ -152,6 +153,11 @@ class FSDirConcatOp {
+ " which is greater than the target file's preferred block size "
+ targetINode.getPreferredBlockSize());
}
+ // TODO currently we do not support concatenating EC files
+ if (srcINodeFile.isStriped()) {
+ throw new HadoopIllegalArgumentException("concat: the src file " + src
+ + " is with striped blocks");
+ }
si.add(srcINodeFile);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
new file mode 100644
index 0000000..fd7ef33
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+/**
+ * Helper class to perform erasure coding related operations.
+ */
+final class FSDirErasureCodingOp {
+
+ /**
+ * Private constructor for preventing FSDirErasureCodingOp object
+ * creation. Static-only class.
+ */
+ private FSDirErasureCodingOp() {}
+
+ /**
+ * Create an erasure coding zone on directory src.
+ *
+ * @param fsn namespace
+ * @param srcArg the path of a directory which will be the root of the
+ * erasure coding zone. The directory must be empty.
+ * @param schema ECSchema for the erasure coding zone
+ * @param cellSize Cell size of stripe
+ * @param logRetryCache whether to record RPC ids in editlog for retry
+ * cache rebuilding
+ * @return {@link HdfsFileStatus}
+ * @throws IOException
+ */
+ static HdfsFileStatus createErasureCodingZone(final FSNamesystem fsn,
+ final String srcArg, final ECSchema schema, final int cellSize,
+ final boolean logRetryCache) throws IOException {
+ assert fsn.hasWriteLock();
+
+ String src = srcArg;
+ FSPermissionChecker pc = null;
+ byte[][] pathComponents = null;
+ pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ pc = fsn.getPermissionChecker();
+ FSDirectory fsd = fsn.getFSDirectory();
+ src = fsd.resolvePath(pc, src, pathComponents);
+ final INodesInPath iip;
+ List<XAttr> xAttrs;
+ fsd.writeLock();
+ try {
+ iip = fsd.getINodesInPath4Write(src, false);
+ xAttrs = fsn.getErasureCodingZoneManager().createErasureCodingZone(
+ iip, schema, cellSize);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsn.getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
+ return fsd.getAuditFileInfo(iip);
+ }
+
+ /**
+ * Get the erasure coding zone information for specified path.
+ *
+ * @param fsn namespace
+ * @param src path
+ * @return {@link ErasureCodingZone}
+ * @throws IOException
+ */
+ static ErasureCodingZone getErasureCodingZone(final FSNamesystem fsn,
+ final String src) throws IOException {
+ assert fsn.hasReadLock();
+
+ final INodesInPath iip = getINodesInPath(fsn, src);
+ return getErasureCodingZoneForPath(fsn, iip);
+ }
+
+ /**
+ * Get erasure coding zone information for specified path.
+ *
+ * @param fsn namespace
+ * @param iip inodes in the path containing the file
+ * @return {@link ErasureCodingZone}
+ * @throws IOException
+ */
+ static ErasureCodingZone getErasureCodingZone(final FSNamesystem fsn,
+ final INodesInPath iip) throws IOException {
+ assert fsn.hasReadLock();
+
+ return getErasureCodingZoneForPath(fsn, iip);
+ }
+
+ /**
+ * Check if the file is in erasure coding zone.
+ *
+ * @param fsn namespace
+ * @param srcArg path
+ * @return true represents the file is in erasure coding zone, false otw
+ * @throws IOException
+ */
+ static boolean isInErasureCodingZone(final FSNamesystem fsn,
+ final String srcArg) throws IOException {
+ assert fsn.hasReadLock();
+
+ final INodesInPath iip = getINodesInPath(fsn, srcArg);
+ return getErasureCodingSchemaForPath(fsn, iip) != null;
+ }
+
+ /**
+ * Check if the file is in erasure coding zone.
+ *
+ * @param fsn namespace
+ * @param iip inodes in the path containing the file
+ * @return true represents the file is in erasure coding zone, false otw
+ * @throws IOException
+ */
+ static boolean isInErasureCodingZone(final FSNamesystem fsn,
+ final INodesInPath iip) throws IOException {
+ return getErasureCodingSchema(fsn, iip) != null;
+ }
+
+ /**
+ * Get erasure coding schema.
+ *
+ * @param fsn namespace
+ * @param iip inodes in the path containing the file
+ * @return {@link ECSchema}
+ * @throws IOException
+ */
+ static ECSchema getErasureCodingSchema(final FSNamesystem fsn,
+ final INodesInPath iip) throws IOException {
+ assert fsn.hasReadLock();
+
+ return getErasureCodingSchemaForPath(fsn, iip);
+ }
+
+ /**
+ * Get available erasure coding schemas.
+ *
+ * @param fsn namespace
+ * @return {@link ECSchema} array
+ */
+ static ECSchema[] getErasureCodingSchemas(final FSNamesystem fsn)
+ throws IOException {
+ assert fsn.hasReadLock();
+
+ return fsn.getErasureCodingSchemaManager().getSchemas();
+ }
+
+ /**
+ * Get the ECSchema specified by the name.
+ *
+ * @param fsn namespace
+ * @param schemaName schema name
+ * @return {@link ECSchema}
+ */
+ static ECSchema getErasureCodingSchema(final FSNamesystem fsn,
+ final String schemaName) throws IOException {
+ assert fsn.hasReadLock();
+
+ return fsn.getErasureCodingSchemaManager().getSchema(schemaName);
+ }
+
+ private static INodesInPath getINodesInPath(final FSNamesystem fsn,
+ final String srcArg) throws IOException {
+ String src = srcArg;
+ final byte[][] pathComponents = FSDirectory
+ .getPathComponentsForReservedPath(src);
+ final FSDirectory fsd = fsn.getFSDirectory();
+ final FSPermissionChecker pc = fsn.getPermissionChecker();
+ src = fsd.resolvePath(pc, src, pathComponents);
+ INodesInPath iip = fsd.getINodesInPath(src, true);
+ if (fsn.isPermissionEnabled()) {
+ fsn.getFSDirectory().checkPathAccess(pc, iip, FsAction.READ);
+ }
+ return iip;
+ }
+
+ private static ErasureCodingZone getErasureCodingZoneForPath(
+ final FSNamesystem fsn, final INodesInPath iip) throws IOException {
+ final FSDirectory fsd = fsn.getFSDirectory();
+ fsd.readLock();
+ try {
+ return fsn.getErasureCodingZoneManager().getErasureCodingZone(iip);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ private static ECSchema getErasureCodingSchemaForPath(final FSNamesystem fsn,
+ final INodesInPath iip) throws IOException {
+ final FSDirectory fsd = fsn.getFSDirectory();
+ fsd.readLock();
+ try {
+ return fsn.getErasureCodingZoneManager().getErasureCodingSchema(iip);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index b69bb42..127474c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -185,6 +185,7 @@ class FSDirRenameOp {
}
fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src);
+ fsd.ecZoneManager.checkMoveValidity(srcIIP, dstIIP, src);
// Ensure dst has quota to accommodate rename
verifyFsLimitsForRename(fsd, srcIIP, dstIIP);
verifyQuotaForRename(fsd, srcIIP, dstIIP);
@@ -357,6 +358,7 @@ class FSDirRenameOp {
BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src);
+ fsd.ecZoneManager.checkMoveValidity(srcIIP, dstIIP, src);
final INode dstInode = dstIIP.getLastINode();
List<INodeDirectory> snapshottableDirs = new ArrayList<>();
if (dstInode != null) { // Destination exists
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index 4a45074..c4cfd34 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
+
import org.apache.commons.io.Charsets;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.DirectoryListingStartAfterNotFoundException;
@@ -29,6 +30,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -40,6 +42,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -179,10 +182,12 @@ class FSDirStatAndListingOp {
final FileEncryptionInfo feInfo = isReservedName ? null
: fsd.getFileEncryptionInfo(inode, iip.getPathSnapshotId(), iip);
+ final ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsd.getFSNamesystem(), iip);
final LocatedBlocks blocks = bm.createLocatedBlocks(
inode.getBlocks(iip.getPathSnapshotId()), fileSize, isUc, offset,
- length, needBlockToken, iip.isSnapshot(), feInfo);
+ length, needBlockToken, iip.isSnapshot(), feInfo, ecZone);
// Set caching information for the located blocks.
for (LocatedBlock lb : blocks.getLocatedBlocks()) {
@@ -374,7 +379,7 @@ class FSDirStatAndListingOp {
if (fsd.getINode4DotSnapshot(srcs) != null) {
return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
- HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED);
+ HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null, 0);
}
return null;
}
@@ -442,6 +447,11 @@ class FSDirStatAndListingOp {
final FileEncryptionInfo feInfo = isRawPath ? null :
fsd.getFileEncryptionInfo(node, snapshot, iip);
+ final ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsd.getFSNamesystem(), iip);
+ final ECSchema schema = ecZone != null ? ecZone.getSchema() : null;
+ final int cellSize = ecZone != null ? ecZone.getCellSize() : 0;
+
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
@@ -471,7 +481,9 @@ class FSDirStatAndListingOp {
node.getId(),
childrenNum,
feInfo,
- storagePolicy);
+ storagePolicy,
+ schema,
+ cellSize);
}
private static INodeAttributes getINodeAttributes(
@@ -494,6 +506,8 @@ class FSDirStatAndListingOp {
final boolean isEncrypted;
final FileEncryptionInfo feInfo = isRawPath ? null :
fsd.getFileEncryptionInfo(node, snapshot, iip);
+ final ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsd.getFSNamesystem(), iip);
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
@@ -507,7 +521,7 @@ class FSDirStatAndListingOp {
loc = fsd.getBlockManager().createLocatedBlocks(
fileNode.getBlocks(snapshot), fileSize, isUc, 0L, size, false,
- inSnapshot, feInfo);
+ inSnapshot, feInfo, ecZone);
if (loc == null) {
loc = new LocatedBlocks();
}
@@ -518,6 +532,8 @@ class FSDirStatAndListingOp {
}
int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0;
+ final ECSchema schema = ecZone != null ? ecZone.getSchema() : null;
+ final int cellSize = ecZone != null ? ecZone.getCellSize() : 0;
HdfsLocatedFileStatus status =
new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
@@ -526,7 +542,8 @@ class FSDirStatAndListingOp {
getPermissionForFileStatus(nodeAttrs, isEncrypted),
nodeAttrs.getUserName(), nodeAttrs.getGroupName(),
node.isSymlink() ? node.asSymlink().getSymlink() : null, path,
- node.getId(), loc, childrenNum, feInfo, storagePolicy);
+ node.getId(), loc, childrenNum, feInfo, storagePolicy, schema,
+ cellSize);
// Set caching information for the located blocks.
if (loc != null) {
CacheManager cacheManager = fsd.getFSNamesystem().getCacheManager();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
index 474c257..5d2bf09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import java.io.IOException;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.FsAction;
@@ -86,6 +87,12 @@ final class FSDirTruncateOp {
final BlockStoragePolicy lpPolicy = fsd.getBlockManager()
.getStoragePolicy("LAZY_PERSIST");
+ // not support truncating file with striped blocks
+ if (file.isStriped()) {
+ throw new UnsupportedOperationException(
+ "Cannot truncate file with striped block " + src);
+ }
+
if (lpPolicy != null && lpPolicy.getId() == file.getStoragePolicyID()) {
throw new UnsupportedOperationException(
"Cannot truncate lazy persist file " + src);
@@ -214,10 +221,12 @@ final class FSDirTruncateOp {
file.getFileUnderConstructionFeature().getClientName(), file.getId());
boolean shouldRecoverNow = (newBlock == null);
BlockInfo oldBlock = file.getLastBlock();
+ Preconditions.checkState(!oldBlock.isStriped());
boolean shouldCopyOnTruncate = shouldCopyOnTruncate(fsn, file, oldBlock);
if (newBlock == null) {
- newBlock = (shouldCopyOnTruncate) ? fsn.createNewBlock() : new Block(
- oldBlock.getBlockId(), oldBlock.getNumBytes(),
+ newBlock = (shouldCopyOnTruncate) ?
+ fsn.createNewBlock(file.isStriped()) :
+ new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
fsn.nextGenerationStamp(fsn.getBlockIdManager().isLegacyBlock(
oldBlock)));
}
@@ -231,7 +240,8 @@ final class FSDirTruncateOp {
file.getPreferredBlockReplication());
truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
truncatedBlockUC.setTruncateBlock(oldBlock);
- file.setLastBlock(truncatedBlockUC, blockManager.getStorages(oldBlock));
+ file.convertLastBlockToUC(truncatedBlockUC,
+ blockManager.getStorages(oldBlock));
blockManager.addBlockCollection(truncatedBlockUC, file);
NameNode.stateChangeLog.debug(
[05/23] hadoop git commit: YARN-4023. Publish Application Priority to
TimelineServer. (Sunil G via rohithsharmaks)
Posted by zh...@apache.org.
YARN-4023. Publish Application Priority to TimelineServer. (Sunil G via rohithsharmaks)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c12adb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c12adb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c12adb7
Branch: refs/heads/HDFS-7285-merge
Commit: 1c12adb71f32f24ec525313ff8dfcd2b738fc8e2
Parents: 1ea1a83
Author: rohithsharmaks <ro...@apache.org>
Authored: Wed Aug 12 14:45:41 2015 +0530
Committer: rohithsharmaks <ro...@apache.org>
Committed: Wed Aug 12 14:45:41 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/api/records/ApplicationReport.java | 3 ++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../hadoop/yarn/api/TestApplicatonReport.java | 3 ++-
...pplicationHistoryManagerOnTimelineStore.java | 21 ++++++++++++++------
...pplicationHistoryManagerOnTimelineStore.java | 2 ++
.../metrics/ApplicationMetricsConstants.java | 3 +++
.../hadoop/yarn/server/webapp/AppsBlock.java | 12 ++++++-----
.../metrics/ApplicationCreatedEvent.java | 10 +++++++++-
.../metrics/SystemMetricsPublisher.java | 5 ++++-
.../metrics/TestSystemMetricsPublisher.java | 6 ++++++
.../src/site/markdown/TimelineServer.md | 19 ++++++++++++++----
12 files changed, 69 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f31dd4b..199a930 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -167,6 +167,9 @@ Release 2.8.0 - UNRELEASED
YARN-3887. Support changing Application priority during runtime. (Sunil G
via jianhe)
+ YARN-4023. Publish Application Priority to TimelineServer. (Sunil G
+ via rohithsharmaks)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index 258b991..fa3b1e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -92,7 +92,7 @@ public abstract class ApplicationReport {
long startTime, long finishTime, FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken,
- Set<String> tags, boolean unmanagedApplication) {
+ Set<String> tags, boolean unmanagedApplication, Priority priority) {
ApplicationReport report =
newInstance(applicationId, applicationAttemptId, user, queue, name,
host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
@@ -100,6 +100,7 @@ public abstract class ApplicationReport {
applicationType, amRmToken);
report.setApplicationTags(tags);
report.setUnmanagedApp(unmanagedApplication);
+ report.setPriority(priority);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index eb50e00..8f17c8f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -104,7 +104,7 @@ public class TestYarnCLI {
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
- null, null, false);
+ null, null, false, Priority.newInstance(0));
newApplicationReport.setLogAggregationStatus(LogAggregationStatus.SUCCEEDED);
newApplicationReport.setPriority(Priority.newInstance(0));
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
index e2fa648..d0d1d40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Assert;
@@ -59,7 +60,7 @@ public class TestApplicatonReport {
"appname", "host", 124, null, YarnApplicationState.FINISHED,
"diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
"N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
- null,false);
+ null, false, Priority.newInstance(0));
return appReport;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 9ae99c5..f02e83c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -251,6 +251,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
long createdTime = 0;
long finishedTime = 0;
float progress = 0.0f;
+ int applicationPriority = 0;
ApplicationAttemptId latestApplicationAttemptId = null;
String diagnosticsInfo = null;
FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
@@ -276,10 +277,11 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
if (field == ApplicationReportField.USER_AND_ACLS) {
return new ApplicationReportExt(ApplicationReport.newInstance(
ConverterUtils.toApplicationId(entity.getEntityId()),
- latestApplicationAttemptId, user, queue, name, null, -1, null, state,
- diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
- null, progress, type, null, appTags,
- unmanagedApplication), appViewACLs);
+ latestApplicationAttemptId, user, queue, name, null, -1, null,
+ state, diagnosticsInfo, null, createdTime, finishedTime,
+ finalStatus, null, null, progress, type, null, appTags,
+ unmanagedApplication, Priority.newInstance(applicationPriority)),
+ appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@@ -303,6 +305,11 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)
.toString());
}
+ if (entityInfo
+ .containsKey(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)) {
+ applicationPriority = Integer.parseInt(entityInfo.get(
+ ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString());
+ }
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_CPU_METRICS).toString());
@@ -373,8 +380,10 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
return new ApplicationReportExt(ApplicationReport.newInstance(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
- diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
- null, progress, type, null, appTags, unmanagedApplication), appViewACLs);
+ diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
+ appResources, null, progress, type, null, appTags,
+ unmanagedApplication, Priority.newInstance(applicationPriority)),
+ appViewACLs);
}
private static ApplicationAttemptReport convertToApplicationAttemptReport(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index fb4af85..e24c11c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -477,6 +477,8 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
entityInfo.put(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false");
+ entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+ Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index 26b113f..481b84c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -77,4 +77,7 @@ public class ApplicationMetricsConstants {
public static final String UNMANAGED_APPLICATION_ENTITY_INFO =
"YARN_APPLICATION_UNMANAGED_APPLICATION";
+
+ public static final String APPLICATION_PRIORITY_INFO =
+ "YARN_APPLICATION_PRIORITY";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 19ab1ce..7f42343 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -144,10 +144,10 @@ public class AppsBlock extends HtmlBlock {
TBODY<TABLE<Hamlet>> tbody =
html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
.th(".name", "Name").th(".type", "Application Type")
- .th(".queue", "Queue").th(".starttime", "StartTime")
- .th(".finishtime", "FinishTime").th(".state", "State")
- .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
- .th(".ui", "Tracking UI")._()._().tbody();
+ .th(".queue", "Queue").th(".priority", "Application Priority")
+ .th(".starttime", "StartTime").th(".finishtime", "FinishTime")
+ .th(".state", "State").th(".finalstatus", "FinalStatus")
+ .th(".progress", "Progress").th(".ui", "Tracking UI")._()._().tbody();
StringBuilder appsTableData = new StringBuilder("[\n");
for (ApplicationReport appReport : appReports) {
@@ -180,7 +180,9 @@ public class AppsBlock extends HtmlBlock {
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
- .getQueue()))).append("\",\"").append(app.getStartedTime())
+ .getQueue()))).append("\",\"").append(String
+ .valueOf(app.getPriority()))
+ .append("\",\"").append(app.getStartedTime())
.append("\",\"").append(app.getFinishedTime())
.append("\",\"")
.append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
index df0e396..3436413 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationCreatedEvent.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.metrics;
import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Priority;
public class ApplicationCreatedEvent extends
SystemMetricsEvent {
@@ -33,6 +34,7 @@ public class ApplicationCreatedEvent extends
private long submittedTime;
private Set<String> appTags;
private boolean unmanagedApplication;
+ private Priority applicationPriority;
public ApplicationCreatedEvent(ApplicationId appId,
String name,
@@ -42,7 +44,8 @@ public class ApplicationCreatedEvent extends
long submittedTime,
long createdTime,
Set<String> appTags,
- boolean unmanagedApplication) {
+ boolean unmanagedApplication,
+ Priority applicationPriority) {
super(SystemMetricsEventType.APP_CREATED, createdTime);
this.appId = appId;
this.name = name;
@@ -52,6 +55,7 @@ public class ApplicationCreatedEvent extends
this.submittedTime = submittedTime;
this.appTags = appTags;
this.unmanagedApplication = unmanagedApplication;
+ this.applicationPriority = applicationPriority;
}
@Override
@@ -90,4 +94,8 @@ public class ApplicationCreatedEvent extends
public boolean isUnmanagedApp() {
return unmanagedApplication;
}
+
+ public Priority getApplicationPriority() {
+ return applicationPriority;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 07553a7..3d7ac9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -107,7 +107,8 @@ public class SystemMetricsPublisher extends CompositeService {
app.getQueue(),
app.getSubmitTime(),
createdTime, app.getApplicationTags(),
- app.getApplicationSubmissionContext().getUnmanagedAM()));
+ app.getApplicationSubmissionContext().getUnmanagedAM(),
+ app.getApplicationSubmissionContext().getPriority()));
}
}
@@ -257,6 +258,8 @@ public class SystemMetricsPublisher extends CompositeService {
entityInfo.put(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
event.isUnmanagedApp());
+ entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+ event.getApplicationPriority().getPriority());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 890dfb1..eb48cc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -141,6 +141,11 @@ public class TestSystemMetricsPublisher {
entity.getOtherInfo().get(
ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO));
+ Assert.assertEquals(
+ app.getApplicationSubmissionContext().getPriority().getPriority(),
+ entity.getOtherInfo().get(
+ ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO));
+
Assert
.assertEquals(
app.getUser(),
@@ -371,6 +376,7 @@ public class TestSystemMetricsPublisher {
when(app.getApplicationTags()).thenReturn(appTags);
ApplicationSubmissionContext asc = mock(ApplicationSubmissionContext.class);
when(asc.getUnmanagedAM()).thenReturn(false);
+ when(asc.getPriority()).thenReturn(Priority.newInstance(0));
when(app.getApplicationSubmissionContext()).thenReturn(asc);
return app;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c12adb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index bbf3e59..72b813a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -1088,7 +1088,8 @@ Response Body:
"startedTime":1430425001004,
"finishedTime":1430425008861,
"elapsedTime":7857,
- "unmanagedApplication":"false"},
+ "unmanagedApplication":"false",
+ "applicationPriority":0},
{
"appId":"application_1430424020775_0003",
"currentAppAttemptId":"appattempt_1430424020775_0003_000001",
@@ -1108,7 +1109,8 @@ Response Body:
"startedTime":1430424956650,
"finishedTime":1430424963907,
"elapsedTime":7257,
- "unmanagedApplication":"false"},
+ "unmanagedApplication":"false",
+ "applicationPriority":0},
{
"appId":"application_1430424020775_0002",
"currentAppAttemptId":"appattempt_1430424020775_0002_000001",
@@ -1128,7 +1130,8 @@ Response Body:
"startedTime":1430424769395,
"finishedTime":1430424776594,
"elapsedTime":7199,
- "unmanagedApplication":"false"},
+ "unmanagedApplication":"false",
+ "applicationPriority":0},
{
"appId":"application_1430424020775_0001",
"currentAppAttemptId":"appattempt_1430424020775_0001_000001",
@@ -1149,7 +1152,8 @@ Response Body:
"finishedTime":1430424776594,
"elapsedTime":18344,
"applicationTags":"mrapplication,ta-example",
- "unmanagedApplication":"false"
+ "unmanagedApplication":"false",
+ "applicationPriority":0
}
]
}
@@ -1192,6 +1196,7 @@ Response Body:
<finishedTime>1430425008861</finishedTime>
<elapsedTime>7857</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
+ <applicationPriority>0</applicationPriority>
</app>
<app>
<appId>application_1430424020775_0003</appId>
@@ -1213,6 +1218,7 @@ Response Body:
<finishedTime>1430424963907</finishedTime>
<elapsedTime>7257</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
+ <applicationPriority>0</applicationPriority>
</app>
<app>
<appId>application_1430424020775_0002</appId>
@@ -1234,6 +1240,7 @@ Response Body:
<finishedTime>1430424776594</finishedTime>
<elapsedTime>7199</elapsedTime>
<unmanagedApplication>false</unmanagedApplication>
+ <applicationPriority>0</applicationPriority>
</app>
<app>
<appId>application_1430424020775_0001</appId>
@@ -1256,6 +1263,7 @@ Response Body:
<elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication>
+ <applicationPriority>0</applicationPriority>
</app>
</apps>
@@ -1307,6 +1315,7 @@ None
| `rpcPort` | int | The RPC port of the ApplicationMaster; zero if no IPC service declared |
| `applicationTags` | string | The application tags. |
| `unmanagedApplication` | boolean | Is the application unmanaged. |
+| `applicationPriority` | int | Priority of the submitted application. |
### Response Examples:
@@ -1345,6 +1354,7 @@ Response Body:
"elapsedTime": 18344,
"applicationTags": mrapplication,tag-example,
"unmanagedApplication":"false"
+ "applicationPriority":0
}
#### XML response
@@ -1384,6 +1394,7 @@ Response Body:
<elapsedTime>18344</elapsedTime>
<applicationTags>mrapplication,ta-example</applicationTags>
<unmanagedApplication>false</unmanagedApplication>
+ <applicationPriority>0</applicationPriority>
</app>
## <a name="REST_API_APPLICATION_ATTEMPT_LIST"></a>Application Attempt List
[12/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 3f242e0..bfb8dcc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION;
import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.CURRENT_STATE_ID;
import static org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.NO_SNAPSHOT_ID;
@@ -34,13 +35,16 @@ import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
@@ -79,12 +83,14 @@ public class INodeFile extends INodeWithAdditionalFields
/**
* Bit format:
- * [4-bit storagePolicyID][12-bit replication][48-bit preferredBlockSize]
+ * [4-bit storagePolicyID][1-bit isStriped]
+ * [11-bit replication][48-bit preferredBlockSize]
*/
static enum HeaderFormat {
PREFERRED_BLOCK_SIZE(null, 48, 1),
- REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 12, 1),
- STORAGE_POLICY_ID(REPLICATION.BITS, BlockStoragePolicySuite.ID_BIT_LENGTH,
+ REPLICATION(PREFERRED_BLOCK_SIZE.BITS, 11, 0),
+ IS_STRIPED(REPLICATION.BITS, 1, 0),
+ STORAGE_POLICY_ID(IS_STRIPED.BITS, BlockStoragePolicySuite.ID_BIT_LENGTH,
0);
private final LongBitFormat BITS;
@@ -105,14 +111,27 @@ public class INodeFile extends INodeWithAdditionalFields
return (byte)STORAGE_POLICY_ID.BITS.retrieve(header);
}
+ static boolean isStriped(long header) {
+ long isStriped = IS_STRIPED.BITS.retrieve(header);
+ Preconditions.checkState(isStriped == 0 || isStriped == 1);
+ return isStriped == 1;
+ }
+
static long toLong(long preferredBlockSize, short replication,
- byte storagePolicyID) {
+ boolean isStriped, byte storagePolicyID) {
long h = 0;
if (preferredBlockSize == 0) {
preferredBlockSize = PREFERRED_BLOCK_SIZE.BITS.getMin();
}
h = PREFERRED_BLOCK_SIZE.BITS.combine(preferredBlockSize, h);
- h = REPLICATION.BITS.combine(replication, h);
+ // Replication factor for striped files is zero
+ if (isStriped) {
+ h = REPLICATION.BITS.combine(0L, h);
+ h = IS_STRIPED.BITS.combine(1L, h);
+ } else {
+ h = REPLICATION.BITS.combine(replication, h);
+ h = IS_STRIPED.BITS.combine(0L, h);
+ }
h = STORAGE_POLICY_ID.BITS.combine(storagePolicyID, h);
return h;
}
@@ -127,15 +146,21 @@ public class INodeFile extends INodeWithAdditionalFields
long atime, BlockInfo[] blklist, short replication,
long preferredBlockSize) {
this(id, name, permissions, mtime, atime, blklist, replication,
- preferredBlockSize, (byte) 0);
+ preferredBlockSize, (byte) 0, false);
}
INodeFile(long id, byte[] name, PermissionStatus permissions, long mtime,
long atime, BlockInfo[] blklist, short replication,
- long preferredBlockSize, byte storagePolicyID) {
+ long preferredBlockSize, byte storagePolicyID, boolean isStriped) {
super(id, name, permissions, mtime, atime);
- header = HeaderFormat.toLong(preferredBlockSize, replication, storagePolicyID);
- this.blocks = blklist;
+ header = HeaderFormat.toLong(preferredBlockSize, replication, isStriped,
+ storagePolicyID);
+ if (blklist != null && blklist.length > 0) {
+ for (BlockInfo b : blklist) {
+ Preconditions.checkArgument(b.isStriped() == isStriped);
+ }
+ }
+ setBlocks(blklist);
}
public INodeFile(INodeFile that) {
@@ -227,31 +252,37 @@ public class INodeFile extends INodeWithAdditionalFields
@Override // BlockCollection
public void setBlock(int index, BlockInfo blk) {
+ Preconditions.checkArgument(blk.isStriped() == this.isStriped());
this.blocks[index] = blk;
}
@Override // BlockCollection, the file should be under construction
- public BlockInfoContiguousUnderConstruction setLastBlock(
- BlockInfo lastBlock, DatanodeStorageInfo[] locations)
- throws IOException {
+ public void convertLastBlockToUC(BlockInfo lastBlock,
+ DatanodeStorageInfo[] locations) throws IOException {
Preconditions.checkState(isUnderConstruction(),
"file is no longer under construction");
-
if (numBlocks() == 0) {
throw new IOException("Failed to set last block: File is empty.");
}
- BlockInfoContiguousUnderConstruction ucBlock =
- lastBlock.convertToBlockUnderConstruction(
- BlockUCState.UNDER_CONSTRUCTION, locations);
+
+ final BlockInfo ucBlock;
+ if (isStriped()) {
+ Preconditions.checkState(lastBlock.isStriped());
+ ucBlock = ((BlockInfoStriped) lastBlock)
+ .convertToBlockUnderConstruction(UNDER_CONSTRUCTION, locations);
+ } else {
+ Preconditions.checkState(!lastBlock.isStriped());
+ ucBlock = ((BlockInfoContiguous) lastBlock)
+ .convertToBlockUnderConstruction(UNDER_CONSTRUCTION, locations);
+ }
setBlock(numBlocks() - 1, ucBlock);
- return ucBlock;
}
/**
* Remove a block from the block list. This block should be
* the last one on the list.
*/
- BlockInfoContiguousUnderConstruction removeLastBlock(Block oldblock) {
+ BlockInfoUnderConstruction removeLastBlock(Block oldblock) {
Preconditions.checkState(isUnderConstruction(),
"file is no longer under construction");
if (blocks == null || blocks.length == 0) {
@@ -262,8 +293,8 @@ public class INodeFile extends INodeWithAdditionalFields
return null;
}
- BlockInfoContiguousUnderConstruction uc =
- (BlockInfoContiguousUnderConstruction)blocks[size_1];
+ BlockInfoUnderConstruction uc =
+ (BlockInfoUnderConstruction)blocks[size_1];
//copy to a new list
BlockInfo[] newlist = new BlockInfo[size_1];
System.arraycopy(blocks, 0, newlist, 0, size_1);
@@ -350,6 +381,7 @@ public class INodeFile extends INodeWithAdditionalFields
/** The same as getFileReplication(null). */
@Override // INodeFileAttributes
+ // TODO properly handle striped files
public final short getFileReplication() {
return getFileReplication(CURRENT_STATE_ID);
}
@@ -365,7 +397,8 @@ public class INodeFile extends INodeWithAdditionalFields
}
max = maxInSnapshot > max ? maxInSnapshot : max;
}
- return max;
+ return isStriped() ?
+ HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS : max;
}
/** Set the replication factor of this file. */
@@ -413,6 +446,15 @@ public class INodeFile extends INodeWithAdditionalFields
setStoragePolicyID(storagePolicyId);
}
+ /**
+ * @return true if the file is in the striping layout.
+ */
+ @VisibleForTesting
+ @Override
+ public boolean isStriped() {
+ return HeaderFormat.isStriped(header);
+ }
+
@Override // INodeFileAttributes
public long getHeaderLong() {
return header;
@@ -429,7 +471,9 @@ public class INodeFile extends INodeWithAdditionalFields
if(snapshot == CURRENT_STATE_ID || getDiffs() == null) {
return getBlocks();
}
+ // find blocks stored in snapshot diffs (for truncate)
FileDiff diff = getDiffs().getDiffById(snapshot);
+ // note that currently FileDiff can only store contiguous blocks
BlockInfo[] snapshotBlocks = diff == null ? getBlocks() : diff.getBlocks();
if (snapshotBlocks != null) {
return snapshotBlocks;
@@ -456,6 +500,7 @@ public class INodeFile extends INodeWithAdditionalFields
int size = this.blocks.length;
int totalAddedBlocks = 0;
for(INodeFile f : inodes) {
+ Preconditions.checkState(f.isStriped() == this.isStriped());
totalAddedBlocks += f.blocks.length;
}
@@ -476,6 +521,7 @@ public class INodeFile extends INodeWithAdditionalFields
* add a block to the block list
*/
void addBlock(BlockInfo newblock) {
+ Preconditions.checkArgument(newblock.isStriped() == this.isStriped());
if (this.blocks == null) {
this.setBlocks(new BlockInfo[]{newblock});
} else {
@@ -589,6 +635,10 @@ public class INodeFile extends INodeWithAdditionalFields
final long ssDeltaNoReplication;
short replication;
+ if (isStriped()) {
+ return computeQuotaUsageWithStriped(bsp, counts);
+ }
+
if (last < lastSnapshotId) {
ssDeltaNoReplication = computeFileSize(true, false);
replication = getFileReplication();
@@ -611,6 +661,18 @@ public class INodeFile extends INodeWithAdditionalFields
return counts;
}
+ /**
+ * Compute quota of striped file. Note that currently EC files do not support
+ * append/hflush/hsync, thus the file length recorded in snapshots should be
+ * the same with the current file length.
+ */
+ public final QuotaCounts computeQuotaUsageWithStriped(
+ BlockStoragePolicy bsp, QuotaCounts counts) {
+ counts.addNameSpace(1);
+ counts.add(storagespaceConsumed(bsp));
+ return counts;
+ }
+
@Override
public final ContentSummaryComputationContext computeContentSummary(
final ContentSummaryComputationContext summary) {
@@ -695,16 +757,20 @@ public class INodeFile extends INodeWithAdditionalFields
}
final int last = blocks.length - 1;
//check if the last block is BlockInfoUnderConstruction
- long size = blocks[last].getNumBytes();
- if (blocks[last] instanceof BlockInfoContiguousUnderConstruction) {
+ BlockInfo lastBlk = blocks[last];
+ long size = lastBlk.getNumBytes();
+ if (lastBlk instanceof BlockInfoUnderConstruction) {
if (!includesLastUcBlock) {
size = 0;
} else if (usePreferredBlockSize4LastUcBlock) {
- size = getPreferredBlockSize();
+ size = isStriped()?
+ getPreferredBlockSize() *
+ ((BlockInfoStriped)lastBlk).getDataBlockNum() :
+ getPreferredBlockSize();
}
}
//sum other blocks
- for(int i = 0; i < last; i++) {
+ for (int i = 0; i < last; i++) {
size += blocks[i].getNumBytes();
}
return size;
@@ -716,6 +782,32 @@ public class INodeFile extends INodeWithAdditionalFields
* Use preferred block size for the last block if it is under construction.
*/
public final QuotaCounts storagespaceConsumed(BlockStoragePolicy bsp) {
+ if (isStriped()) {
+ return storagespaceConsumedStriped();
+ } else {
+ return storagespaceConsumedContiguous(bsp);
+ }
+ }
+
+ // TODO: support EC with heterogeneous storage
+ public final QuotaCounts storagespaceConsumedStriped() {
+ QuotaCounts counts = new QuotaCounts.Builder().build();
+ if (blocks == null || blocks.length == 0) {
+ return counts;
+ }
+
+ for (BlockInfo b : blocks) {
+ Preconditions.checkState(b.isStriped());
+ long blockSize = b.isComplete() ?
+ ((BlockInfoStriped)b).spaceConsumed() : getPreferredBlockSize() *
+ ((BlockInfoStriped)b).getTotalBlockNum();
+ counts.addStorageSpace(blockSize);
+ }
+ return counts;
+ }
+
+ public final QuotaCounts storagespaceConsumedContiguous(
+ BlockStoragePolicy bsp) {
QuotaCounts counts = new QuotaCounts.Builder().build();
final Iterable<BlockInfo> blocks;
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
@@ -819,6 +911,7 @@ public class INodeFile extends INodeWithAdditionalFields
/**
* compute the quota usage change for a truncate op
* @param newLength the length for truncation
+ * TODO: properly handle striped blocks (HDFS-7622)
**/
void computeQuotaDeltaForTruncate(
long newLength, BlockStoragePolicy bsps,
@@ -883,8 +976,15 @@ public class INodeFile extends INodeWithAdditionalFields
setBlocks(newBlocks);
}
+ /**
+ * This function is only called when block list is stored in snapshot
+ * diffs. Note that this can only happen when truncation happens with
+ * snapshots. Since we do not support truncation with striped blocks,
+ * we only need to handle contiguous blocks here.
+ */
public void collectBlocksBeyondSnapshot(BlockInfo[] snapshotBlocks,
BlocksMapUpdateInfo collectedBlocks) {
+ Preconditions.checkState(!isStriped());
BlockInfo[] oldBlocks = getBlocks();
if(snapshotBlocks == null || oldBlocks == null)
return;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
index 204c8ac..13bd9e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFileAttributes.java
@@ -29,6 +29,9 @@ public interface INodeFileAttributes extends INodeAttributes {
/** @return the file replication. */
public short getFileReplication();
+ /** @return whether the file is striped (instead of contiguous) */
+ public boolean isStriped();
+
/** @return preferred block size in bytes */
public long getPreferredBlockSize();
@@ -47,10 +50,10 @@ public interface INodeFileAttributes extends INodeAttributes {
public SnapshotCopy(byte[] name, PermissionStatus permissions,
AclFeature aclFeature, long modificationTime, long accessTime,
short replication, long preferredBlockSize,
- byte storagePolicyID, XAttrFeature xAttrsFeature) {
+ byte storagePolicyID, XAttrFeature xAttrsFeature, boolean isStriped) {
super(name, permissions, aclFeature, modificationTime, accessTime,
xAttrsFeature);
- header = HeaderFormat.toLong(preferredBlockSize, replication,
+ header = HeaderFormat.toLong(preferredBlockSize, replication, isStriped,
storagePolicyID);
}
@@ -70,6 +73,11 @@ public interface INodeFileAttributes extends INodeAttributes {
}
@Override
+ public boolean isStriped() {
+ return HeaderFormat.isStriped(header);
+ }
+
+ @Override
public long getPreferredBlockSize() {
return HeaderFormat.getPreferredBlockSize(header);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 1a1edaf..244dafe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -116,8 +116,8 @@ public class LeaseManager {
for(BlockInfo b : blocks) {
if(!b.isComplete())
numUCBlocks++;
+ }
}
- }
LOG.info("Number of blocks under construction: " + numUCBlocks);
return numUCBlocks;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 3a5dc12..2943fc2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -87,7 +87,8 @@ public class NameNodeLayoutVersion {
BLOCK_STORAGE_POLICY(-60, -60, "Block Storage policy"),
TRUNCATE(-61, -61, "Truncate"),
APPEND_NEW_BLOCK(-62, -61, "Support appending to new block"),
- QUOTA_BY_STORAGE_TYPE(-63, -61, "Support quota for specific storage types");
+ QUOTA_BY_STORAGE_TYPE(-63, -61, "Support quota for specific storage types"),
+ ERASURE_CODING(-64, -61, "Support erasure coding");
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 6b7e8cf..bee06a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.FSLimitException;
@@ -143,6 +144,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RetriableException;
@@ -794,7 +796,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
public LocatedBlock updateBlockForPipeline(ExtendedBlock block, String clientName)
throws IOException {
checkNNStartup();
- return namesystem.updateBlockForPipeline(block, clientName);
+ return namesystem.bumpBlockGenerationStamp(block, clientName);
}
@@ -1847,6 +1849,24 @@ class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
+ public void createErasureCodingZone(String src, ECSchema schema, int cellSize)
+ throws IOException {
+ checkNNStartup();
+ final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+ if (cacheEntry != null && cacheEntry.isSuccess()) {
+ return;
+ }
+ boolean success = false;
+ try {
+ namesystem.createErasureCodingZone(src, schema, cellSize,
+ cacheEntry != null);
+ success = true;
+ } finally {
+ RetryCache.setState(cacheEntry, success);
+ }
+ }
+
+ @Override // ClientProtocol
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
throws IOException {
checkNNStartup();
@@ -2041,4 +2061,16 @@ class NameNodeRpcServer implements NamenodeProtocols {
namesystem.checkSuperuserPrivilege();
nn.spanReceiverHost.removeSpanReceiver(id);
}
+
+ @Override // ClientProtocol
+ public ECSchema[] getECSchemas() throws IOException {
+ checkNNStartup();
+ return namesystem.getErasureCodingSchemas();
+ }
+
+ @Override // ClientProtocol
+ public ErasureCodingZone getErasureCodingZone(String src) throws IOException {
+ checkNNStartup();
+ return namesystem.getErasureCodingZone(src);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 7d4cd7e..21062e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -65,8 +65,9 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicies;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
@@ -123,6 +124,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
private final int totalDatanodes;
private final InetAddress remoteAddress;
+ private long totalDirs = 0L;
+ private long totalSymlinks = 0L;
+
private String lostFound = null;
private boolean lfInited = false;
private boolean lfInitedOk = false;
@@ -170,7 +174,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
private final PrintWriter out;
private List<String> snapshottableDirs = null;
- private final BlockPlacementPolicy bpPolicy;
+ private final BlockPlacementPolicies bpPolicies;
private StoragePolicySummary storageTypeSummary = null;
/**
@@ -192,7 +196,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
this.out = out;
this.totalDatanodes = totalDatanodes;
this.remoteAddress = remoteAddress;
- this.bpPolicy = BlockPlacementPolicy.getInstance(conf, null,
+ this.bpPolicies = new BlockPlacementPolicies(conf, null,
networktopology,
namenode.getNamesystem().getBlockManager().getDatanodeManager()
.getHost2DatanodeMap());
@@ -255,7 +259,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
out.println("Block Id: " + blockId);
out.println("Block belongs to: "+iNode.getFullPathName());
out.println("No. of Expected Replica: " +
- bc.getPreferredBlockReplication());
+ bm.getExpectedReplicaNum(bc, blockInfo));
out.println("No. of live Replica: " + numberReplicas.liveReplicas());
out.println("No. of excess Replica: " + numberReplicas.excessReplicas());
out.println("No. of stale Replica: " +
@@ -356,13 +360,21 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
namenode.getNamesystem().getBlockManager().getStoragePolicies());
}
- Result res = new Result(conf);
+ Result replRes = new ReplicationResult(conf);
+ Result ecRes = new ErasureCodingResult(conf);
- check(path, file, res);
+ check(path, file, replRes, ecRes);
- out.println(res);
- out.println(" Number of data-nodes:\t\t" + totalDatanodes);
+ out.print("\nStatus: ");
+ out.println(replRes.isHealthy() && ecRes.isHealthy() ? "HEALTHY" : "CORRUPT");
+ out.println(" Number of data-nodes:\t" + totalDatanodes);
out.println(" Number of racks:\t\t" + networktopology.getNumOfRacks());
+ out.println(" Total dirs:\t\t\t" + totalDirs);
+ out.println(" Total symlinks:\t\t" + totalSymlinks);
+ out.println("\nReplicated Blocks:");
+ out.println(replRes);
+ out.println("\nErasure Coded Block Groups:");
+ out.println(ecRes);
if (this.showStoragePolcies) {
out.print(storageTypeSummary.toString());
@@ -382,7 +394,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
// of file system and return appropriate code. Changing the output
// string might break testcases. Also note this must be the last line
// of the report.
- if (res.isHealthy()) {
+ if (replRes.isHealthy() && ecRes.isHealthy()) {
out.print("\n\nThe filesystem under path '" + path + "' " + HEALTHY_STATUS);
} else {
out.print("\n\nThe filesystem under path '" + path + "' " + CORRUPT_STATUS);
@@ -425,42 +437,49 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
@VisibleForTesting
- void check(String parent, HdfsFileStatus file, Result res) throws IOException {
+ void check(String parent, HdfsFileStatus file, Result replRes, Result ecRes)
+ throws IOException {
String path = file.getFullName(parent);
if (file.isDir()) {
- checkDir(path, res);
+ checkDir(path, replRes, ecRes);
return;
}
if (file.isSymlink()) {
if (showFiles) {
out.println(path + " <symlink>");
}
- res.totalSymlinks++;
+ totalSymlinks++;
return;
}
LocatedBlocks blocks = getBlockLocations(path, file);
if (blocks == null) { // the file is deleted
return;
}
- collectFileSummary(path, file, res, blocks);
- collectBlocksSummary(parent, file, res, blocks);
+
+ final Result r = file.getReplication() == 0? ecRes: replRes;
+ collectFileSummary(path, file, r, blocks);
+ if (showprogress && (replRes.totalFiles + ecRes.totalFiles) % 100 == 0) {
+ out.println();
+ out.flush();
+ }
+ collectBlocksSummary(parent, file, r, blocks);
}
- private void checkDir(String path, Result res) throws IOException {
+ private void checkDir(String path, Result replRes, Result ecRes) throws IOException {
if (snapshottableDirs != null && snapshottableDirs.contains(path)) {
String snapshotPath = (path.endsWith(Path.SEPARATOR) ? path : path
+ Path.SEPARATOR)
+ HdfsConstants.DOT_SNAPSHOT_DIR;
HdfsFileStatus snapshotFileInfo = namenode.getRpcServer().getFileInfo(
snapshotPath);
- check(snapshotPath, snapshotFileInfo, res);
+ check(snapshotPath, snapshotFileInfo, replRes, ecRes);
}
byte[] lastReturnedName = HdfsFileStatus.EMPTY_NAME;
DirectoryListing thisListing;
if (showFiles) {
out.println(path + " <dir>");
}
- res.totalDirs++;
+ totalDirs++;
do {
assert lastReturnedName != null;
thisListing = namenode.getRpcServer().getListing(
@@ -470,7 +489,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
HdfsFileStatus[] files = thisListing.getPartialListing();
for (int i = 0; i < files.length; i++) {
- check(path, files[i], res);
+ check(path, files[i], replRes, ecRes);
}
lastReturnedName = thisListing.getLastName();
} while (thisListing.hasMore());
@@ -518,10 +537,6 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
} else if (showprogress) {
out.print('.');
}
- if ((showprogress) && res.totalFiles % 100 == 0) {
- out.println();
- out.flush();
- }
}
private void collectBlocksSummary(String parent, HdfsFileStatus file, Result res,
@@ -542,9 +557,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
final BlockInfo storedBlock = bm.getStoredBlock(
block.getLocalBlock());
+ final int minReplication = bm.getMinStorageNum(storedBlock);
// count decommissionedReplicas / decommissioningReplicas
NumberReplicas numberReplicas = bm.countNodes(storedBlock);
- int decommissionedReplicas = numberReplicas.decommissioned();;
+ int decommissionedReplicas = numberReplicas.decommissioned();
int decommissioningReplicas = numberReplicas.decommissioning();
res.decommissionedReplicas += decommissionedReplicas;
res.decommissioningReplicas += decommissioningReplicas;
@@ -556,11 +572,18 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
res.totalReplicas += totalReplicasPerBlock;
// count expected replicas
- short targetFileReplication = file.getReplication();
+ short targetFileReplication;
+ if (file.getECSchema() != null) {
+ assert storedBlock instanceof BlockInfoStriped;
+ targetFileReplication = ((BlockInfoStriped) storedBlock)
+ .getRealTotalBlockNum();
+ } else {
+ targetFileReplication = file.getReplication();
+ }
res.numExpectedReplicas += targetFileReplication;
// count under min repl'd blocks
- if(totalReplicasPerBlock < res.minReplication){
+ if(totalReplicasPerBlock < minReplication){
res.numUnderMinReplicatedBlocks++;
}
@@ -581,7 +604,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
// count minimally replicated blocks
- if (totalReplicasPerBlock >= res.minReplication)
+ if (totalReplicasPerBlock >= minReplication)
res.numMinReplicatedBlocks++;
// count missing replicas / under replicated blocks
@@ -601,7 +624,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
}
// count mis replicated blocks
- BlockPlacementStatus blockPlacementStatus = bpPolicy
+ BlockPlacementStatus blockPlacementStatus = bpPolicies.getPolicy(false)
.verifyBlockPlacement(path, lBlk, targetFileReplication);
if (!blockPlacementStatus.isPlacementPolicySatisfied()) {
res.numMisReplicatedBlocks++;
@@ -636,9 +659,9 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
report.append(" Live_repl=" + liveReplicas);
if (showLocations || showRacks || showReplicaDetails) {
StringBuilder sb = new StringBuilder("[");
- Iterable<DatanodeStorageInfo> storages = bm.getStorages(block.getLocalBlock());
- for (Iterator<DatanodeStorageInfo> iterator = storages.iterator(); iterator.hasNext();) {
- DatanodeStorageInfo storage = iterator.next();
+ DatanodeStorageInfo[] storages = bm.getStorages(storedBlock);
+ for (int i = 0; i < storages.length; i++) {
+ DatanodeStorageInfo storage = storages[i];
DatanodeDescriptor dnDesc = storage.getDatanodeDescriptor();
if (showRacks) {
sb.append(NodeBase.getPath(dnDesc));
@@ -647,7 +670,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
.getStorageType()));
}
if (showReplicaDetails) {
- LightWeightLinkedSet<Block> blocksExcess =
+ LightWeightLinkedSet<BlockInfo> blocksExcess =
bm.excessReplicateMap.get(dnDesc.getDatanodeUuid());
Collection<DatanodeDescriptor> corruptReplicas =
bm.getCorruptReplicas(block.getLocalBlock());
@@ -668,7 +691,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
sb.append("LIVE)");
}
}
- if (iterator.hasNext()) {
+ if (i < storages.length - 1) {
sb.append(", ");
}
}
@@ -982,7 +1005,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
long missingReplicas = 0L;
long decommissionedReplicas = 0L;
long decommissioningReplicas = 0L;
- long numUnderMinReplicatedBlocks=0L;
+ long numUnderMinReplicatedBlocks = 0L;
long numOverReplicatedBlocks = 0L;
long numUnderReplicatedBlocks = 0L;
long numMisReplicatedBlocks = 0L; // blocks that do not satisfy block placement policy
@@ -992,22 +1015,10 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
long totalOpenFilesBlocks = 0L;
long totalFiles = 0L;
long totalOpenFiles = 0L;
- long totalDirs = 0L;
- long totalSymlinks = 0L;
long totalSize = 0L;
long totalOpenFilesSize = 0L;
long totalReplicas = 0L;
- final short replication;
- final int minReplication;
-
- Result(Configuration conf) {
- this.replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
- DFSConfigKeys.DFS_REPLICATION_DEFAULT);
- this.minReplication = (short)conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
- }
-
/**
* DFS is considered healthy if there are no missing blocks.
*/
@@ -1033,19 +1044,29 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
return 0.0f;
return (float) (totalReplicas) / (float) totalBlocks;
}
+ }
+
+ @VisibleForTesting
+ static class ReplicationResult extends Result {
+ final short replication;
+ final short minReplication;
+
+ ReplicationResult(Configuration conf) {
+ this.replication = (short)conf.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
+ DFSConfigKeys.DFS_REPLICATION_DEFAULT);
+ this.minReplication = (short)conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_DEFAULT);
+ }
@Override
public String toString() {
StringBuilder res = new StringBuilder();
- res.append("Status: ").append((isHealthy() ? "HEALTHY" : "CORRUPT"))
- .append("\n Total size:\t").append(totalSize).append(" B");
+ res.append(" Total size:\t").append(totalSize).append(" B");
if (totalOpenFilesSize != 0) {
res.append(" (Total open files size: ").append(totalOpenFilesSize)
.append(" B)");
}
- res.append("\n Total dirs:\t").append(totalDirs).append(
- "\n Total files:\t").append(totalFiles);
- res.append("\n Total symlinks:\t\t").append(totalSymlinks);
+ res.append("\n Total files:\t").append(totalFiles);
if (totalOpenFiles != 0) {
res.append(" (Files currently being written: ").append(totalOpenFiles)
.append(")");
@@ -1135,4 +1156,110 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
return res.toString();
}
}
+
+ @VisibleForTesting
+ static class ErasureCodingResult extends Result {
+ final String defaultSchema;
+
+ ErasureCodingResult(Configuration conf) {
+ defaultSchema = ErasureCodingSchemaManager.getSystemDefaultSchema()
+ .getSchemaName();
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder res = new StringBuilder();
+ res.append(" Total size:\t").append(totalSize).append(" B");
+ if (totalOpenFilesSize != 0) {
+ res.append(" (Total open files size: ").append(totalOpenFilesSize)
+ .append(" B)");
+ }
+ res.append("\n Total files:\t").append(totalFiles);
+ if (totalOpenFiles != 0) {
+ res.append(" (Files currently being written: ").append(totalOpenFiles)
+ .append(")");
+ }
+ res.append("\n Total block groups (validated):\t").append(totalBlocks);
+ if (totalBlocks > 0) {
+ res.append(" (avg. block group size ").append((totalSize / totalBlocks))
+ .append(" B)");
+ }
+ if (totalOpenFilesBlocks != 0) {
+ res.append(" (Total open file block groups (not validated): ").append(
+ totalOpenFilesBlocks).append(")");
+ }
+ if (corruptFiles > 0 || numUnderMinReplicatedBlocks > 0) {
+ res.append("\n ********************************");
+ if(numUnderMinReplicatedBlocks>0){
+ res.append("\n UNRECOVERABLE BLOCK GROUPS:\t").append(numUnderMinReplicatedBlocks);
+ if(totalBlocks>0){
+ res.append(" (").append(
+ ((float) (numUnderMinReplicatedBlocks * 100) / (float) totalBlocks))
+ .append(" %)");
+ }
+ }
+ if(corruptFiles>0) {
+ res.append(
+ "\n CORRUPT FILES:\t").append(corruptFiles);
+ if (missingSize > 0) {
+ res.append("\n MISSING BLOCK GROUPS:\t").append(missingIds.size()).append(
+ "\n MISSING SIZE:\t\t").append(missingSize).append(" B");
+ }
+ if (corruptBlocks > 0) {
+ res.append("\n CORRUPT BLOCK GROUPS: \t").append(corruptBlocks).append(
+ "\n CORRUPT SIZE:\t\t").append(corruptSize).append(" B");
+ }
+ }
+ res.append("\n ********************************");
+ }
+ res.append("\n Minimally erasure-coded block groups:\t").append(
+ numMinReplicatedBlocks);
+ if (totalBlocks > 0) {
+ res.append(" (").append(
+ ((float) (numMinReplicatedBlocks * 100) / (float) totalBlocks))
+ .append(" %)");
+ }
+ res.append("\n Over-erasure-coded block groups:\t")
+ .append(numOverReplicatedBlocks);
+ if (totalBlocks > 0) {
+ res.append(" (").append(
+ ((float) (numOverReplicatedBlocks * 100) / (float) totalBlocks))
+ .append(" %)");
+ }
+ res.append("\n Under-erasure-coded block groups:\t").append(
+ numUnderReplicatedBlocks);
+ if (totalBlocks > 0) {
+ res.append(" (").append(
+ ((float) (numUnderReplicatedBlocks * 100) / (float) totalBlocks))
+ .append(" %)");
+ }
+ res.append("\n Unsatisfactory placement block groups:\t")
+ .append(numMisReplicatedBlocks);
+ if (totalBlocks > 0) {
+ res.append(" (").append(
+ ((float) (numMisReplicatedBlocks * 100) / (float) totalBlocks))
+ .append(" %)");
+ }
+ res.append("\n Default schema:\t\t").append(defaultSchema)
+ .append("\n Average block group size:\t").append(
+ getReplicationFactor()).append("\n Missing block groups:\t\t").append(
+ missingIds.size()).append("\n Corrupt block groups:\t\t").append(
+ corruptBlocks).append("\n Missing internal blocks:\t").append(
+ missingReplicas);
+ if (totalReplicas > 0) {
+ res.append(" (").append(
+ ((float) (missingReplicas * 100) / (float) numExpectedReplicas)).append(
+ " %)");
+ }
+ if (decommissionedReplicas > 0) {
+ res.append("\n Decommissioned internal blocks:\t").append(
+ decommissionedReplicas);
+ }
+ if (decommissioningReplicas > 0) {
+ res.append("\n Decommissioning internal blocks:\t").append(
+ decommissioningReplicas);
+ }
+ return res.toString();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
index a5053bc..430853a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
@@ -17,8 +17,12 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.NameNode.OperationCategory;
import org.apache.hadoop.hdfs.util.RwLock;
@@ -45,7 +49,16 @@ public interface Namesystem extends RwLock, SafeMode {
void checkOperation(OperationCategory read) throws StandbyException;
- boolean isInSnapshot(BlockInfoContiguousUnderConstruction blockUC);
+ boolean isInSnapshot(BlockCollection bc);
CacheManager getCacheManager();
+
+ /**
+ * Gets the ECZone for path
+ * @param src the filesystem path
+ * @return {@link ErasureCodingZone}
+ * @throws IOException
+ */
+ ErasureCodingZone getErasureCodingZoneForPath(String src)
+ throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SafeMode.java
index 1428482..252844c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SafeMode.java
@@ -44,9 +44,10 @@ public interface SafeMode {
/**
* Increment number of blocks that reached minimal replication.
- * @param replication current replication
+ * @param replication current replication
+ * @param storedBlock current stored Block
*/
- public void incrementSafeBlockCount(int replication);
+ public void incrementSafeBlockCount(int replication, BlockInfo storedBlock);
/** Decrement number of blocks that reached minimal replication. */
public void decrementSafeBlockCount(BlockInfo b);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 64ad1f6..e684b41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -234,19 +234,23 @@ public class FSImageFormatPBSnapshot {
.toByteArray(), permission, acl, fileInPb.getModificationTime(),
fileInPb.getAccessTime(), (short) fileInPb.getReplication(),
fileInPb.getPreferredBlockSize(),
- (byte)fileInPb.getStoragePolicyID(), xAttrs);
+ (byte)fileInPb.getStoragePolicyID(), xAttrs,
+ fileInPb.getIsStriped());
}
FileDiff diff = new FileDiff(pbf.getSnapshotId(), copy, null,
pbf.getFileSize());
List<BlockProto> bpl = pbf.getBlocksList();
- BlockInfo[] blocks = new BlockInfo[bpl.size()];
+ // in file diff there can only be contiguous blocks
+ BlockInfoContiguous[] blocks = new BlockInfoContiguous[bpl.size()];
for(int j = 0, e = bpl.size(); j < e; ++j) {
Block blk = PBHelper.convert(bpl.get(j));
- BlockInfo storedBlock = fsn.getBlockManager().getStoredBlock(blk);
+ BlockInfoContiguous storedBlock =
+ (BlockInfoContiguous) fsn.getBlockManager().getStoredBlock(blk);
if(storedBlock == null) {
- storedBlock = fsn.getBlockManager().addBlockCollection(
- new BlockInfoContiguous(blk, copy.getFileReplication()), file);
+ storedBlock = (BlockInfoContiguous) fsn.getBlockManager()
+ .addBlockCollectionWithCheck(new BlockInfoContiguous(blk,
+ copy.getFileReplication()), file);
}
blocks[j] = storedBlock;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
index 6b8388e..0a64a38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
@@ -55,7 +56,9 @@ public class FileDiffList extends
final FileDiff diff =
super.saveSelf2Snapshot(latestSnapshotId, iNodeFile, snapshotCopy);
if (withBlocks) { // Store blocks if this is the first update
- diff.setBlocks(iNodeFile.getBlocks());
+ BlockInfo[] blks = iNodeFile.getBlocks();
+ assert blks != null;
+ diff.setBlocks(blks);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java
new file mode 100644
index 0000000..56a1546
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlockECRecoveryCommand.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.protocol;
+
+import com.google.common.base.Joiner;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * A BlockECRecoveryCommand is an instruction to a DataNode to reconstruct a
+ * striped block group with missing blocks.
+ *
+ * Upon receiving this command, the DataNode pulls data from other DataNodes
+ * hosting blocks in this group and reconstructs the lost blocks through codec
+ * calculation.
+ *
+ * After the reconstruction, the DataNode pushes the reconstructed blocks to
+ * their final destinations if necessary (e.g., the destination is different
+ * from the reconstruction node, or multiple blocks in a group are to be
+ * reconstructed).
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BlockECRecoveryCommand extends DatanodeCommand {
+ final Collection<BlockECRecoveryInfo> ecTasks;
+
+ /**
+ * Create BlockECRecoveryCommand from a collection of
+ * {@link BlockECRecoveryInfo}, each representing a recovery task
+ */
+ public BlockECRecoveryCommand(int action,
+ Collection<BlockECRecoveryInfo> blockECRecoveryInfoList) {
+ super(action);
+ this.ecTasks = blockECRecoveryInfoList;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("BlockECRecoveryCommand(\n ");
+ Joiner.on("\n ").appendTo(sb, ecTasks);
+ sb.append("\n)");
+ return sb.toString();
+ }
+
+ /** Block and targets pair */
+ @InterfaceAudience.Private
+ @InterfaceStability.Evolving
+ public static class BlockECRecoveryInfo {
+ private final ExtendedBlock block;
+ private final DatanodeInfo[] sources;
+ private DatanodeInfo[] targets;
+ private String[] targetStorageIDs;
+ private StorageType[] targetStorageTypes;
+ private final short[] liveBlockIndices;
+ private final ECSchema ecSchema;
+ private final int cellSize;
+
+ public BlockECRecoveryInfo(ExtendedBlock block, DatanodeInfo[] sources,
+ DatanodeStorageInfo[] targetDnStorageInfo, short[] liveBlockIndices,
+ ECSchema ecSchema, int cellSize) {
+ this(block, sources, DatanodeStorageInfo
+ .toDatanodeInfos(targetDnStorageInfo), DatanodeStorageInfo
+ .toStorageIDs(targetDnStorageInfo), DatanodeStorageInfo
+ .toStorageTypes(targetDnStorageInfo), liveBlockIndices, ecSchema,
+ cellSize);
+ }
+
+ public BlockECRecoveryInfo(ExtendedBlock block, DatanodeInfo[] sources,
+ DatanodeInfo[] targets, String[] targetStorageIDs,
+ StorageType[] targetStorageTypes, short[] liveBlockIndices,
+ ECSchema ecSchema, int cellSize) {
+ this.block = block;
+ this.sources = sources;
+ this.targets = targets;
+ this.targetStorageIDs = targetStorageIDs;
+ this.targetStorageTypes = targetStorageTypes;
+ this.liveBlockIndices = liveBlockIndices;
+ this.ecSchema = ecSchema;
+ this.cellSize = cellSize;
+ }
+
+ public ExtendedBlock getExtendedBlock() {
+ return block;
+ }
+
+ public DatanodeInfo[] getSourceDnInfos() {
+ return sources;
+ }
+
+ public DatanodeInfo[] getTargetDnInfos() {
+ return targets;
+ }
+
+ public String[] getTargetStorageIDs() {
+ return targetStorageIDs;
+ }
+
+ public StorageType[] getTargetStorageTypes() {
+ return targetStorageTypes;
+ }
+
+ public short[] getLiveBlockIndices() {
+ return liveBlockIndices;
+ }
+
+ public ECSchema getECSchema() {
+ return ecSchema;
+ }
+
+ public int getCellSize() {
+ return cellSize;
+ }
+
+ @Override
+ public String toString() {
+ return new StringBuilder().append("BlockECRecoveryInfo(\n ")
+ .append("Recovering ").append(block).append(" From: ")
+ .append(Arrays.asList(sources)).append(" To: [")
+ .append(Arrays.asList(targets)).append(")\n")
+ .append(" Block Indices: ").append(Arrays.asList(liveBlockIndices))
+ .toString();
+ }
+ }
+
+ public Collection<BlockECRecoveryInfo> getECTasks() {
+ return this.ecTasks;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
index a985dbd..0507faf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/BlocksWithLocations.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.protocol;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.StorageType;
@@ -91,6 +92,30 @@ public class BlocksWithLocations {
}
}
+ public static class StripedBlockWithLocations extends BlockWithLocations {
+ final byte[] indices;
+ final short dataBlockNum;
+
+ public StripedBlockWithLocations(BlockWithLocations blk, byte[] indices,
+ short dataBlockNum) {
+ super(blk.getBlock(), blk.getDatanodeUuids(), blk.getStorageIDs(),
+ blk.getStorageTypes());
+ Preconditions.checkArgument(
+ blk.getDatanodeUuids().length == indices.length);
+ this.indices = indices;
+ this.dataBlockNum = dataBlockNum;
+
+ }
+
+ public byte[] getIndices() {
+ return indices;
+ }
+
+ public short getDataBlockNum() {
+ return dataBlockNum;
+ }
+ }
+
private final BlockWithLocations[] blocks;
/** Constructor with one parameter */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
index dfe0813..add4e73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
@@ -76,6 +76,7 @@ public interface DatanodeProtocol {
final static int DNA_BALANCERBANDWIDTHUPDATE = 8; // update balancer bandwidth
final static int DNA_CACHE = 9; // cache blocks
final static int DNA_UNCACHE = 10; // uncache blocks
+ final static int DNA_ERASURE_CODING_RECOVERY = 11; // erasure coding recovery command
/**
* Register Datanode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
new file mode 100644
index 0000000..4ed9d0a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCli.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.tools.erasurecode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.shell.CommandFactory;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * CLI for the erasure code encoding operations.
+ */
+public class ECCli extends FsShell {
+
+ private final static String usagePrefix =
+ "Usage: hdfs erasurecode [generic options]";
+
+ @Override
+ protected String getUsagePrefix() {
+ return usagePrefix;
+ }
+
+ @Override
+ protected void registerCommands(CommandFactory factory) {
+ factory.registerCommands(ECCommand.class);
+ }
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ int res = ToolRunner.run(conf, new ECCli(), args);
+ System.exit(res);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
new file mode 100644
index 0000000..03026d8
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.tools.erasurecode;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.shell.Command;
+import org.apache.hadoop.fs.shell.CommandFactory;
+import org.apache.hadoop.fs.shell.PathData;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Erasure Coding CLI commands
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public abstract class ECCommand extends Command {
+
+ public static void registerCommands(CommandFactory factory) {
+ // Register all commands of Erasure CLI, with a '-' at the beginning in name
+ // of the command.
+ factory.addClass(CreateECZoneCommand.class, "-" + CreateECZoneCommand.NAME);
+ factory.addClass(GetECZoneCommand.class, "-"
+ + GetECZoneCommand.NAME);
+ factory.addClass(ListECSchemas.class, "-" + ListECSchemas.NAME);
+ }
+
+ @Override
+ public String getCommandName() {
+ return getName();
+ }
+
+ @Override
+ protected void run(Path path) throws IOException {
+ throw new RuntimeException("Not suppose to get here");
+ }
+
+ @Deprecated
+ @Override
+ public int runAll() {
+ return run(args);
+ }
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ if (!(item.fs instanceof DistributedFileSystem)) {
+ throw new UnsupportedActionException(
+ "Erasure commands are only supported for the HDFS paths");
+ }
+ }
+
+ /**
+ * Create EC encoding zone command. Zones are created to use specific EC
+ * encoding schema, other than default while encoding the files under some
+ * specific directory.
+ */
+ static class CreateECZoneCommand extends ECCommand {
+ public static final String NAME = "createZone";
+ public static final String USAGE = "[-s <schemaName>] [-c <cellSize>] <path>";
+ public static final String DESCRIPTION =
+ "Create a zone to encode files using a specified schema\n"
+ + "Options :\n"
+ + " -s <schemaName> : EC schema name to encode files. "
+ + "If not passed default schema will be used\n"
+ + " -c <cellSize> : cell size to use for striped encoding files."
+ + " If not passed default cellsize of "
+ + HdfsConstants.BLOCK_STRIPED_CELL_SIZE + " will be used\n"
+ + " <path> : Path to an empty directory. Under this directory "
+ + "files will be encoded using specified schema";
+ private String schemaName;
+ private int cellSize = 0;
+ private ECSchema schema = null;
+
+ @Override
+ protected void processOptions(LinkedList<String> args) throws IOException {
+ schemaName = StringUtils.popOptionWithArgument("-s", args);
+ String cellSizeStr = StringUtils.popOptionWithArgument("-c", args);
+ if (cellSizeStr != null) {
+ cellSize = (int) StringUtils.TraditionalBinaryPrefix
+ .string2long(cellSizeStr);
+ }
+ if (args.isEmpty()) {
+ throw new HadoopIllegalArgumentException("<path> is missing");
+ }
+ if (args.size() > 1) {
+ throw new HadoopIllegalArgumentException("Too many arguments");
+ }
+ }
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ super.processPath(item);
+ DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
+ try {
+ if (schemaName != null) {
+ ECSchema[] ecSchemas = dfs.getClient().getECSchemas();
+ for (ECSchema ecSchema : ecSchemas) {
+ if (schemaName.equals(ecSchema.getSchemaName())) {
+ schema = ecSchema;
+ break;
+ }
+ }
+ if (schema == null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Schema '");
+ sb.append(schemaName);
+ sb.append("' does not match any of the supported schemas.");
+ sb.append(" Please select any one of ");
+ List<String> schemaNames = new ArrayList<String>();
+ for (ECSchema ecSchema : ecSchemas) {
+ schemaNames.add(ecSchema.getSchemaName());
+ }
+ sb.append(schemaNames);
+ throw new HadoopIllegalArgumentException(sb.toString());
+ }
+ }
+ dfs.createErasureCodingZone(item.path, schema, cellSize);
+ out.println("EC Zone created successfully at " + item.path);
+ } catch (IOException e) {
+ throw new IOException("Unable to create EC zone for the path "
+ + item.path + ". " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * Get the information about the zone
+ */
+ static class GetECZoneCommand extends ECCommand {
+ public static final String NAME = "getZone";
+ public static final String USAGE = "<path>";
+ public static final String DESCRIPTION =
+ "Get information about the EC zone at specified path\n";
+
+ @Override
+ protected void processOptions(LinkedList<String> args) throws IOException {
+ if (args.isEmpty()) {
+ throw new HadoopIllegalArgumentException("<path> is missing");
+ }
+ if (args.size() > 1) {
+ throw new HadoopIllegalArgumentException("Too many arguments");
+ }
+ }
+
+ @Override
+ protected void processPath(PathData item) throws IOException {
+ super.processPath(item);
+ DistributedFileSystem dfs = (DistributedFileSystem) item.fs;
+ try {
+ ErasureCodingZone ecZone = dfs.getErasureCodingZone(item.path);
+ if (ecZone != null) {
+ out.println(ecZone.toString());
+ } else {
+ out.println("Path " + item.path + " is not in EC zone");
+ }
+ } catch (IOException e) {
+ throw new IOException("Unable to get EC zone for the path "
+ + item.path + ". " + e.getMessage());
+ }
+ }
+ }
+
+ /**
+ * List all supported EC Schemas
+ */
+ static class ListECSchemas extends ECCommand {
+ public static final String NAME = "listSchemas";
+ public static final String USAGE = "";
+ public static final String DESCRIPTION =
+ "Get the list of ECSchemas supported\n";
+
+ @Override
+ protected void processOptions(LinkedList<String> args) throws IOException {
+ if (!args.isEmpty()) {
+ throw new HadoopIllegalArgumentException("Too many parameters");
+ }
+
+ FileSystem fs = FileSystem.get(getConf());
+ if (fs instanceof DistributedFileSystem == false) {
+ throw new UnsupportedActionException(
+ "Erasure commands are only supported for the HDFS");
+ }
+ DistributedFileSystem dfs = (DistributedFileSystem) fs;
+
+ ECSchema[] ecSchemas = dfs.getClient().getECSchemas();
+ StringBuilder sb = new StringBuilder();
+ int i = 0;
+ while (i < ecSchemas.length) {
+ ECSchema ecSchema = ecSchemas[i];
+ sb.append(ecSchema.getSchemaName());
+ i++;
+ if (i < ecSchemas.length) {
+ sb.append(", ");
+ }
+ }
+ out.println(sb.toString());
+ }
+ }
+}
[20/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 443576d..fa816e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -78,6 +78,17 @@ public final class HdfsConstants {
public static final String CLIENT_NAMENODE_PROTOCOL_NAME =
"org.apache.hadoop.hdfs.protocol.ClientProtocol";
+ /*
+ * These values correspond to the values used by the system default erasure
+ * coding schema.
+ * TODO: to be removed once all places use schema.
+ */
+
+ public static final byte NUM_DATA_BLOCKS = 6;
+ public static final byte NUM_PARITY_BLOCKS = 3;
+ // The chunk size for striped block which is used by erasure coding
+ public static final int BLOCK_STRIPED_CELL_SIZE = 64 * 1024;
+
// SafeMode actions
public enum SafeModeAction {
SAFEMODE_LEAVE, SAFEMODE_ENTER, SAFEMODE_GET
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index 34f429a..8c902b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.io.erasurecode.ECSchema;
/** Interface that represents the over the wire information for a file.
*/
@@ -48,6 +49,9 @@ public class HdfsFileStatus {
private final FileEncryptionInfo feInfo;
+ private final ECSchema ecSchema;
+ private final int stripeCellSize;
+
// Used by dir, not including dot and dotdot. Always zero for a regular file.
private final int childrenNum;
private final byte storagePolicy;
@@ -73,7 +77,7 @@ public class HdfsFileStatus {
long blocksize, long modification_time, long access_time,
FsPermission permission, String owner, String group, byte[] symlink,
byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo,
- byte storagePolicy) {
+ byte storagePolicy, ECSchema ecSchema, int stripeCellSize) {
this.length = length;
this.isdir = isdir;
this.block_replication = (short)block_replication;
@@ -93,6 +97,8 @@ public class HdfsFileStatus {
this.childrenNum = childrenNum;
this.feInfo = feInfo;
this.storagePolicy = storagePolicy;
+ this.ecSchema = ecSchema;
+ this.stripeCellSize = stripeCellSize;
}
/**
@@ -250,6 +256,14 @@ public class HdfsFileStatus {
return feInfo;
}
+ public ECSchema getECSchema() {
+ return ecSchema;
+ }
+
+ public int getStripeCellSize() {
+ return stripeCellSize;
+ }
+
public final int getChildrenNum() {
return childrenNum;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
index cc13f10..a9596bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
@@ -49,14 +49,14 @@ public class LocatedBlock {
// else false. If block has few corrupt replicas, they are filtered and
// their locations are not part of this object
private boolean corrupt;
- private Token<BlockTokenIdentifier> blockToken = new Token<BlockTokenIdentifier>();
+ private Token<BlockTokenIdentifier> blockToken = new Token<>();
/**
* List of cached datanode locations
*/
private DatanodeInfo[] cachedLocs;
// Used when there are no locations
- private static final DatanodeInfoWithStorage[] EMPTY_LOCS =
+ static final DatanodeInfoWithStorage[] EMPTY_LOCS =
new DatanodeInfoWithStorage[0];
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) {
@@ -203,4 +203,8 @@ public class LocatedBlock {
+ "; locs=" + Arrays.asList(locs)
+ "}";
}
+
+ public boolean isStriped() {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
index e4896977..735e7b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlocks.java
@@ -24,6 +24,7 @@ import java.util.Comparator;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* Collection of blocks with their locations and the file length.
@@ -37,6 +38,8 @@ public class LocatedBlocks {
private final LocatedBlock lastLocatedBlock;
private final boolean isLastBlockComplete;
private final FileEncryptionInfo fileEncryptionInfo;
+ private final ECSchema ecSchema;
+ private final int stripeCellSize;
public LocatedBlocks() {
fileLength = 0;
@@ -45,17 +48,22 @@ public class LocatedBlocks {
lastLocatedBlock = null;
isLastBlockComplete = false;
fileEncryptionInfo = null;
+ ecSchema = null;
+ stripeCellSize = 0;
}
public LocatedBlocks(long flength, boolean isUnderConstuction,
- List<LocatedBlock> blks, LocatedBlock lastBlock,
- boolean isLastBlockCompleted, FileEncryptionInfo feInfo) {
+ List<LocatedBlock> blks, LocatedBlock lastBlock,
+ boolean isLastBlockCompleted, FileEncryptionInfo feInfo,
+ ECSchema ecSchema, int stripeCellSize) {
fileLength = flength;
blocks = blks;
underConstruction = isUnderConstuction;
this.lastLocatedBlock = lastBlock;
this.isLastBlockComplete = isLastBlockCompleted;
this.fileEncryptionInfo = feInfo;
+ this.ecSchema = ecSchema;
+ this.stripeCellSize = stripeCellSize;
}
/**
@@ -112,6 +120,20 @@ public class LocatedBlocks {
}
/**
+ * @return The ECSchema for ErasureCoded file, null otherwise.
+ */
+ public ECSchema getECSchema() {
+ return ecSchema;
+ }
+
+ /**
+ * @return Stripe Cell size for ErasureCoded file, 0 otherwise.
+ */
+ public int getStripeCellSize() {
+ return stripeCellSize;
+ }
+
+ /**
* Find block containing specified offset.
*
* @return block if found, or null otherwise.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java
new file mode 100644
index 0000000..a9a80c2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedStripedBlock.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.security.token.Token;
+
+import java.util.Arrays;
+
+/**
+ * {@link LocatedBlock} with striped block support. For a striped block, each
+ * datanode storage is associated with a block in the block group. We need to
+ * record the index (in the striped block group) for each of them.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class LocatedStripedBlock extends LocatedBlock {
+ private static final int[] EMPTY_INDICES = {};
+ private static final Token<BlockTokenIdentifier> EMPTY_TOKEN = new Token<>();
+
+ private int[] blockIndices;
+ private Token<BlockTokenIdentifier>[] blockTokens;
+
+ public LocatedStripedBlock(ExtendedBlock b, DatanodeInfo[] locs,
+ String[] storageIDs, StorageType[] storageTypes, int[] indices,
+ long startOffset, boolean corrupt, DatanodeInfo[] cachedLocs) {
+ super(b, locs, storageIDs, storageTypes, startOffset, corrupt, cachedLocs);
+
+ if (indices == null) {
+ this.blockIndices = EMPTY_INDICES;
+ } else {
+ this.blockIndices = new int[indices.length];
+ System.arraycopy(indices, 0, blockIndices, 0, indices.length);
+ }
+ blockTokens = new Token[blockIndices.length];
+ for (int i = 0; i < blockIndices.length; i++) {
+ blockTokens[i] = EMPTY_TOKEN;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "{" + getBlock()
+ + "; getBlockSize()=" + getBlockSize()
+ + "; corrupt=" + isCorrupt()
+ + "; offset=" + getStartOffset()
+ + "; locs=" + Arrays.asList(getLocations())
+ + "; indices=" + Arrays.toString(blockIndices)
+ + "}";
+ }
+
+ public int[] getBlockIndices() {
+ return this.blockIndices;
+ }
+
+ @Override
+ public boolean isStriped() {
+ return true;
+ }
+
+ public Token<BlockTokenIdentifier>[] getBlockTokens() {
+ return blockTokens;
+ }
+
+ public void setBlockTokens(Token<BlockTokenIdentifier>[] tokens) {
+ this.blockTokens = tokens;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
index ac19d44..a6c7b10 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SnapshottableDirectoryStatus.java
@@ -61,7 +61,7 @@ public class SnapshottableDirectoryStatus {
int snapshotNumber, int snapshotQuota, byte[] parentFullPath) {
this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
access_time, permission, owner, group, null, localName, inodeId,
- childrenNum, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED);
+ childrenNum, null, HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED, null, 0);
this.snapshotNumber = snapshotNumber;
this.snapshotQuota = snapshotQuota;
this.parentFullPath = parentFullPath;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 713836c..eeadd73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -132,7 +132,7 @@ class JsonUtilClient {
blockSize, mTime, aTime, permission, owner, group,
symlink, DFSUtilClient.string2Bytes(localName),
fileId, childrenNum, null,
- storagePolicy);
+ storagePolicy, null, 0);
}
/** Convert a Json map to an ExtendedBlock object. */
@@ -503,7 +503,7 @@ class JsonUtilClient {
(Map<?, ?>) m.get("lastLocatedBlock"));
final boolean isLastBlockComplete = (Boolean)m.get("isLastBlockComplete");
return new LocatedBlocks(fileLength, isUnderConstruction, locatedBlocks,
- lastLocatedBlock, isLastBlockComplete, null);
+ lastLocatedBlock, isLastBlockComplete, null, null, 0);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 7d32568..62db8ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -34,6 +34,7 @@ import "acl.proto";
import "xattr.proto";
import "encryption.proto";
import "inotify.proto";
+import "erasurecoding.proto";
/**
* The ClientNamenodeProtocol Service defines the interface between a client
@@ -866,8 +867,14 @@ service ClientNamenodeProtocol {
returns(ListEncryptionZonesResponseProto);
rpc getEZForPath(GetEZForPathRequestProto)
returns(GetEZForPathResponseProto);
+ rpc createErasureCodingZone(CreateErasureCodingZoneRequestProto)
+ returns(CreateErasureCodingZoneResponseProto);
rpc getCurrentEditLogTxid(GetCurrentEditLogTxidRequestProto)
returns(GetCurrentEditLogTxidResponseProto);
rpc getEditsFromTxid(GetEditsFromTxidRequestProto)
returns(GetEditsFromTxidResponseProto);
+ rpc getECSchemas(GetECSchemasRequestProto)
+ returns(GetECSchemasResponseProto);
+ rpc getErasureCodingZone(GetErasureCodingZoneRequestProto)
+ returns(GetErasureCodingZoneResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
new file mode 100644
index 0000000..56bb7a2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hdfs.protocol.proto";
+option java_outer_classname = "ErasureCodingProtos";
+option java_generate_equals_and_hash = true;
+package hadoop.hdfs;
+
+import "hdfs.proto";
+
+/**
+ * ErasureCodingZone
+ */
+message ErasureCodingZoneProto {
+ required string dir = 1;
+ required ECSchemaProto schema = 2;
+ required uint32 cellSize = 3;
+}
+
+message CreateErasureCodingZoneRequestProto {
+ required string src = 1;
+ optional ECSchemaProto schema = 2;
+ optional uint32 cellSize = 3;
+}
+
+message CreateErasureCodingZoneResponseProto {
+}
+
+message GetECSchemasRequestProto { // void request
+}
+
+message GetECSchemasResponseProto {
+ repeated ECSchemaProto schemas = 1;
+}
+
+message GetErasureCodingZoneRequestProto {
+ required string src = 1; // path to get the zone info
+}
+
+message GetErasureCodingZoneResponseProto {
+ optional ErasureCodingZoneProto ECZone = 1;
+}
+
+/**
+ * Block erasure coding recovery info
+ */
+message BlockECRecoveryInfoProto {
+ required ExtendedBlockProto block = 1;
+ required DatanodeInfosProto sourceDnInfos = 2;
+ required DatanodeInfosProto targetDnInfos = 3;
+ required StorageUuidsProto targetStorageUuids = 4;
+ required StorageTypesProto targetStorageTypes = 5;
+ repeated uint32 liveBlockIndices = 6;
+ required ECSchemaProto ecSchema = 7;
+ required uint32 cellSize = 8;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 86fb462..d2cb665 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -220,6 +220,10 @@ message LocatedBlockProto {
repeated bool isCached = 6 [packed=true]; // if a location in locs is cached
repeated StorageTypeProto storageTypes = 7;
repeated string storageIDs = 8;
+
+ // striped block related fields
+ repeated uint32 blockIndex = 9; // used for striped block to indicate block index for each storage
+ repeated hadoop.common.TokenProto blockTokens = 10; // each internal block has a block token
}
message DataEncryptionKeyProto {
@@ -300,6 +304,29 @@ message LocatedBlocksProto {
optional LocatedBlockProto lastBlock = 4;
required bool isLastBlockComplete = 5;
optional FileEncryptionInfoProto fileEncryptionInfo = 6;
+
+ // Optional field for erasure coding
+ optional ECSchemaProto eCSchema = 7;
+ optional uint32 stripeCellSize = 8;
+}
+
+/**
+ * ECSchema options entry
+ */
+message ECSchemaOptionEntryProto {
+ required string key = 1;
+ required string value = 2;
+}
+
+/**
+ * ECSchema for erasurecoding
+ */
+message ECSchemaProto {
+ required string schemaName = 1;
+ required string codecName = 2;
+ required uint32 dataUnits = 3;
+ required uint32 parityUnits = 4;
+ repeated ECSchemaOptionEntryProto options = 5;
}
/**
@@ -336,7 +363,11 @@ message HdfsFileStatusProto {
optional FileEncryptionInfoProto fileEncryptionInfo = 15;
optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id
-}
+
+ // Optional field for erasure coding
+ optional ECSchemaProto ecSchema = 17;
+ optional uint32 stripeCellSize = 18;
+}
/**
* Checksum algorithms/types used in HDFS
@@ -498,6 +529,9 @@ message BlockWithLocationsProto {
repeated string datanodeUuids = 2; // Datanodes with replicas of the block
repeated string storageUuids = 3; // Storages with replicas of the block
repeated StorageTypeProto storageTypes = 4;
+
+ optional bytes indices = 5;
+ optional uint32 dataBlockNum = 6;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
new file mode 100755
index 0000000..45afd2c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -0,0 +1,396 @@
+ BREAKDOWN OF HDFS-7285 SUBTASKS AND RELATED JIRAS
+
+ HDFS-7347. Configurable erasure coding policy for individual files and
+ directories ( Zhe Zhang via vinayakumarb )
+
+ HDFS-7339. Representing striped block groups in NameNode with hierarchical
+ naming protocol ( Zhe Zhang )
+
+ HDFS-7652. Process block reports for erasure coded blocks (Zhe Zhang)
+
+ HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info (Jing Zhao)
+
+ HDFS-7749. Erasure Coding: Add striped block support in INodeFile (Jing Zhao)
+
+ HDFS-7837. Erasure Coding: allocate and persist striped blocks in NameNode
+ (Jing Zhao via Zhe Zhang)
+
+ HDFS-7872. Erasure Coding: INodeFile.dumpTreeRecursively() supports to print
+ striped blocks (Takuya Fukudome via jing9)
+
+ HDFS-7853. Erasure coding: extend LocatedBlocks to support reading from
+ striped files (Jing Zhao)
+
+ HDFS-7826. Erasure Coding: Update INodeFile quota computation for striped
+ blocks ( Kai Sasaki via jing9 )
+
+ HDFS-7912. Erasure Coding: track BlockInfo instead of Block in
+ UnderReplicatedBlocks and PendingReplicationBlocks (Jing Zhao)
+
+ HDFS-7369. Erasure coding: distribute recovery work for striped blocks to
+ DataNode (Zhe Zhang)
+
+ HDFS-7864. Erasure Coding: Update safemode calculation for striped blocks
+ (GAO Rui via jing9)
+
+ HDFS-7827. Erasure Coding: support striped blocks in non-protobuf fsimage
+ ( Hui Zheng via jing9 )
+
+ HDFS-7616. Add a test for BlockGroup support in FSImage.
+ (Takuya Fukudome via szetszwo)
+
+ HDFS-7907. Erasure Coding: track invalid, corrupt, and under-recovery striped
+ blocks in NameNode (Jing Zhao)
+
+ HDFS-8005. Erasure Coding: simplify striped block recovery work computation
+ and add tests (Jing Zhao)
+
+ HDFS-7617. Add unit tests for editlog transactions for EC
+ (Hui Zheng via Zhe Zhang)
+
+ HDFS-7839. Erasure coding: implement facilities in NameNode to create and
+ manage EC zones (Zhe Zhang)
+
+ HDFS-7969. Erasure coding: NameNode support for lease recovery of striped
+ block groups. (Zhe Zhang)
+
+ HDFS-7782. Erasure coding: pread from files in striped layout.
+ (Zhe Zhang and Jing Zhao via Zhe Zhang)
+
+ HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from
+ NameNode (vinayakumarb)
+
+ HDFS-8074. Define a system-wide default EC schema. (Kai Zheng)
+
+ HDFS-8077. Erasure coding: fix bugs in EC zone and symlinks.
+ (Jing Zhao and Zhe Zhang via Jing Zhao)
+
+ HDFS-8104. Make hard-coded values consistent with the system default schema first before remove them. (Kai Zheng)
+
+ HDFS-7889. Subclass DFSOutputStream to support writing striping layout files. (Li Bo via Kai Zheng)
+
+ HDFS-8090. Erasure Coding: Add RPC to client-namenode to list all
+ ECSchemas loaded in Namenode. (vinayakumarb)
+
+ HDFS-8122. Erasure Coding: Support specifying ECSchema during creation of ECZone.
+ (Vinayakumar B via Zhe Zhang)
+
+ HDFS-8114. Erasure coding: Add auditlog FSNamesystem#createErasureCodingZone if this
+ operation fails. (Rakesh R via Zhe Zhang)
+
+ HDFS-8123. Erasure Coding: Better to move EC related proto messages to a
+ separate erasurecoding proto file (Rakesh R via vinayakumarb)
+
+ HDFS-7349. Support DFS command for the EC encoding (vinayakumarb)
+
+ HDFS-8120. Erasure coding: created util class to analyze striped block groups.
+ (Contributed by Zhe Zhang and Li Bo via Jing Zhao)
+
+ HDFS-7994. Detect if resevered EC Block ID is already used during namenode
+ startup. (Hui Zheng via szetszwo)
+
+ HDFS-8167. BlockManager.addBlockCollectionWithCheck should check if the block is a striped block. (Hui Zheng via zhz).
+
+ HDFS-8166. DFSStripedOutputStream should not create empty blocks. (Jing Zhao)
+
+ HDFS-7937. Erasure Coding: INodeFile quota computation unit tests.
+ (Kai Sasaki via Jing Zhao)
+
+ HDFS-8145. Fix the editlog corruption exposed by failed TestAddStripedBlocks.
+ (Jing Zhao)
+
+ HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for
+ making it ready for transfer to DN (Uma Maheswara Rao G via vinayakumarb)
+
+ HDFS-8181. createErasureCodingZone sets retryCache state as false always
+ (Uma Maheswara Rao G via vinayakumarb)
+
+ HDFS-8190. StripedBlockUtil.getInternalBlockLength may have overflow error.
+ (szetszwo)
+
+ HDFS-8216. TestDFSStripedOutputStream should use BlockReaderTestUtil to
+ create BlockReader. (szetszwo via Zhe Zhang)
+
+ HDFS-8212. DistributedFileSystem.createErasureCodingZone should pass schema
+ in FileSystemLinkResolver. (szetszwo via Zhe Zhang)
+
+ HDFS-8024. Erasure Coding: ECworker frame, basics, bootstraping and configuration.
+ (umamahesh)
+
+ HDFS-8156. Add/implement necessary APIs even we just have the system default
+ schema. (Kai Zheng via Zhe Zhang)
+
+ HDFS-8136. Client gets and uses EC schema when reads and writes a stripping
+ file. (Kai Sasaki via Kai Zheng)
+
+ HDFS-8233. Fix DFSStripedOutputStream#getCurrentBlockGroupBytes when the last
+ stripe is at the block group boundary. (jing9)
+
+ HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream.
+ (Yi Liu via jing9)
+
+ HDFS-8228. Erasure Coding: SequentialBlockGroupIdGenerator#nextValue may cause
+ block id conflicts (Jing Zhao via Zhe Zhang)
+
+ HDFS-8033. Erasure coding: stateful (non-positional) read from files in
+ striped layout (Zhe Zhang)
+
+ HDFS-8230. Erasure Coding: Ignore DatanodeProtocol#DNA_ERASURE_CODING_RECOVERY
+ commands from standbynode if any (vinayakumarb)
+
+ HDFS-8189. ClientProtocol#createErasureCodingZone API was wrongly annotated
+ as Idempotent (vinayakumarb)
+
+ HDFS-8235. Erasure Coding: Create DFSStripedInputStream in DFSClient#open.
+ (Kai Sasaki via jing9)
+
+ HDFS-8272. Erasure Coding: simplify the retry logic in DFSStripedInputStream
+ (stateful read). (Jing Zhao via Zhe Zhang)
+
+ HDFS-8282. Erasure coding: move striped reading logic to StripedBlockUtil.
+ (Zhe Zhang)
+
+ HDFS-8183. Erasure Coding: Improve DFSStripedOutputStream closing of
+ datastreamer threads. (Rakesh R via Zhe Zhang)
+
+ HDFS-8308. Erasure Coding: NameNode may get blocked in waitForLoadingFSImage()
+ when loading editlog. (jing9)
+
+ HDFS-7949. WebImageViewer need support file size calculation with striped
+ blocks. (Rakesh R via Zhe Zhang)
+
+ HDFS-8316. Erasure coding: refactor EC constants to be consistent with HDFS-8249.
+ (Zhe Zhang via jing9)
+
+ HDFS-8281. Erasure Coding: implement parallel stateful reading for striped layout.
+ (jing9)
+
+ HDFS-8137. Send the EC schema to DataNode via EC encoding/recovering command(umamahesh)
+
+ HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands
+ (Rakesh R via vinayakumarb)
+
+ HDFS-8324. Add trace info to DFSClient#getErasureCodingZoneInfo(..) (vinayakumarb via
+ umamahesh)
+
+ HDFS-7672. Handle write failure for stripping blocks and refactor the
+ existing code in DFSStripedOutputStream and StripedDataStreamer. (szetszwo)
+
+ HDFS-7348. Erasure Coding: DataNode reconstruct striped blocks.
+ (Yi Liu via Zhe Zhang)
+
+ HADOOP-11921. Enhance tests for erasure coders. (Kai Zheng)
+
+ HDFS-8334. Erasure coding: rename DFSStripedInputStream related test
+ classes. (Zhe Zhang)
+
+ HDFS-8129. Erasure Coding: Maintain consistent naming for Erasure Coding related classes - EC/ErasureCoding
+ (umamahesh)
+
+ HDFS-8203. Erasure Coding: Seek and other Ops in DFSStripedInputStream.
+ (Yi Liu via jing9)
+
+ HDFS-8289. Erasure Coding: add ECSchema to HdfsFileStatus. (Yong Zhang via
+ jing9)
+
+ HDFS-8355. Erasure Coding: Refactor BlockInfo and BlockInfoUnderConstruction.
+ (Tsz Wo Nicholas Sze via jing9)
+
+ HDFS-7678. Erasure coding: DFSInputStream with decode functionality (pread).
+ (Zhe Zhang)
+
+ HDFS-8372. Erasure coding: compute storage type quotas for striped files,
+ to be consistent with HDFS-8327. (Zhe Zhang via jing9)
+
+ HDFS-8368. Erasure Coding: DFS opening a non-existent file need to be
+ handled properly (Rakesh R via zhz)
+
+ HDFS-8363. Erasure Coding: DFSStripedInputStream#seekToNewSource. (yliu)
+
+ HDFS-8195. Erasure coding: Fix file quota change when we complete/commit
+ the striped blocks. (Takuya Fukudome via zhz)
+
+ HDFS-8364. Erasure coding: fix some minor bugs in EC CLI
+ (Walter Su via vinayakumarb)
+
+ HDFS-8391. NN should consider current EC tasks handling count from DN while
+ assigning new tasks. (umamahesh)
+
+ HDFS-8367. BlockInfoStriped uses EC schema. (Kai Sasaki via Kai Zheng)
+
+ HDFS-8352. Erasure Coding: test webhdfs read write stripe file. (waltersu4549)
+
+ HDFS-8417. Erasure Coding: Pread failed to read data starting from not-first stripe.
+ (Walter Su via jing9)
+
+ HDFS-8418. Fix the isNeededReplication calculation for Striped block in NN.
+ (Yi Liu via jing9)
+
+ HDFS-8320. Erasure coding: consolidate striping-related terminologies. (zhz)
+
+ HDFS-8366. Erasure Coding: Make the timeout parameter of polling blocking queue
+ configurable in DFSStripedOutputStream. (Li Bo)
+
+ HDFS-8378. Erasure Coding: Few improvements for the erasure coding worker.
+ (Rakesh R via waltersu4549)
+
+ HDFS-8375. Add cellSize as an XAttr to ECZone. ( Vinayakumar B via zhz).
+
+ HDFS-8428. Erasure Coding: Fix the NullPointerException when deleting file.
+ (Yi Liu via zhz).
+
+ HDFS-8323. Bump GenerationStamp for write faliure in DFSStripedOutputStream.
+ (Tsz Wo Nicholas Sze via jing9)
+
+ HDFS-8427. Remove dataBlockNum and parityBlockNum from BlockInfoStriped.
+ (Kai Sasaki via jing9)
+
+ HDFS-8186. Erasure coding: Make block placement policy for EC file configurable.
+ (Walter Su via zhz)
+
+ HDFS-8294. Erasure Coding: Fix Findbug warnings present in erasure coding.
+ (Rakesh R via zhz)
+
+ HDFS-8441. Erasure Coding: make condition check earlier for setReplication.
+ (waltersu4549)
+
+ HDFS-7768. Change fsck to support EC files. (Takanobu Asanuma via szetszwo)
+
+ HDFS-8382. Remove chunkSize and initialize from erasure coder. (Kai Zheng)
+
+ HDFS-8408. Revisit and refactor ErasureCodingInfo (vinayakumarb)
+
+ HDFS-8479. Erasure coding: fix striping related logic in FSDirWriteFileOp to
+ sync with HDFS-8421. (Zhe Zhang via jing9)
+
+ HDFS-8481. Erasure coding: remove workarounds in client side stripped blocks
+ recovering. (zhz)
+
+ HDFS-8336. Expose some administrative erasure coding operations to HdfsAdmin
+ (Uma Maheswara Rao G via vinayakumarb)
+
+ HDFS-8444. Erasure Coding: fix cannot rename a zone dir
+ (Walter Su via vinayakumarb)
+
+ HDFS-8517. Fix a decoding issue in stripped block recovering in client side.
+ (Kai Zheng via jing9)
+
+ HDFS-8453. Erasure coding: properly handle start offset for internal blocks
+ in a block group. (Zhe Zhang via jing9)
+
+ HDFS-7621. Erasure Coding: update the Balancer/Mover data migration logic.
+ (Walter Su via zhz)
+
+ HDFS-8328. Follow-on to update decode for DataNode striped blocks
+ reconstruction. (yliu)
+
+ HDFS-8319. Erasure Coding: support decoding for stateful read.
+ (Jing Zhao via zhz)
+
+ HDFS-8460. Erasure Coding: stateful read result doesn't match data
+ occasionally because of flawed test. (Walter Su via zhz)
+
+ HDFS-8556. Erasure Coding: Fix usage of 'createZone' (vinayakumarb)
+
+ HDFS-8571. Fix TestErasureCodingCli test (Vinayakumar B via waltersu4549)
+
+ HDFS-8450. Erasure Coding: Consolidate erasure coding zone related
+ implementation into a single class (Rakesh R via vinayakumarb)
+
+ HDFS-8585. Erasure Coding: Remove dataBlockNum and parityBlockNum from
+ StripedBlockProto. (Yi Liu via jing9)
+
+ HDFS-8559. Erasure Coding: fix non-protobuf fsimage for striped blocks.
+ (Jing Zhao via yliu)
+
+ HDFS-8580. Erasure coding: Persist cellSize in BlockInfoStriped and
+ StripedBlocksFeature. (Walter Su via jing9)
+
+ HDFS-8466. Refactor BlockInfoContiguous and fix NPE in
+ TestBlockInfo#testCopyConstructor() (vinayakumarb)
+
+ HDFS-8254. Avoid assigning a leading streamer in StripedDataStreamer to
+ tolerate datanode failure. (Tsz Wo Nicholas Sze via jing9)
+
+ HDFS-8543. Erasure Coding: processOverReplicatedBlock() handles striped block.
+ (Walter Su via jing9)
+
+ HDFS-8602. Erasure Coding: Client can't read(decode) the EC files which have
+ corrupt blocks. (jing9 and Kai Sasaki)
+
+ HDFS-8567. Erasure Coding: SafeMode handles file smaller than a full stripe.
+ (Walter Su via jing9)
+
+ HDFS-8253. DFSStripedOutputStream.closeThreads releases cellBuffers
+ multiple times. (Kai Sasaki via szetszwo)
+
+ HDFS-8468. 2 RPC calls for every file read in DFSClient#open(..) resulting in
+ double Audit log entries (vinayakumarb)
+
+ HDFS-8684. Erasure Coding: fix some block number calculation for striped
+ block. (yliu)
+
+ HDFS-8461. Erasure coding: fix priority level of UnderReplicatedBlocks for
+ striped block. (Walter Su via jing9)
+
+ HDFS-8719. Erasure Coding: client generates too many small packets when
+ writing parity data. (Li Bo via waltersu4549)
+
+ HDFS-8563. Erasure Coding: fsck handles file smaller than a full stripe.
+ (Walter Su via jing9)
+
+ HDFS-8484. Erasure coding: Two contiguous blocks occupy IDs belong to same
+ striped group. (Walter Su via jing9)
+
+ HDFS-8744. Erasure Coding: the number of chunks in packet is not updated
+ when writing parity data. (Li Bo)
+
+ HDFS-8669. Erasure Coding: handle missing internal block locations in
+ DFSStripedInputStream. (jing9)
+
+ HDFS-8702. Erasure coding: update BlockManager.blockHasEnoughRacks(..) logic
+ for striped block. (Kai Sasaki via jing9)
+
+ HDFS-8734. Erasure Coding: fix one cell need two packets. (Walter Su via
+ jing9)
+
+ HDFS-8619. Erasure Coding: revisit replica counting for striped blocks.
+ (Jing Zhao via yliu)
+
+ HDFS-8058. Erasure coding: use BlockInfo[] for both striped and contiguous
+ blocks in INodeFile. (Zhe Zhang and Yi Liu via zhz)
+
+ HDFS-8787. Erasure coding: rename BlockInfoContiguousUC and BlockInfoStripedUC
+ to be consistent with trunk. (zhz)
+
+ HDFS-8433. Erasure coding: set blockToken in LocatedStripedBlock.(waltersu4549)
+
+ HDFS-8760. Erasure Coding: reuse BlockReader when reading the same block in pread.
+ (jing9)
+
+ HDFS-8781. Erasure Coding: Correctly handle BlockManager#InvalidateBlocks for
+ striped block. (Yi Liu via jing9)
+
+ HDFS-8813. Erasure Coding: Client no need to decode missing parity blocks.
+ (Walter Su via jing9)
+
+ HDFS-8798. Erasure Coding: fix DFSStripedInputStream/DFSStripedOutputStream
+ re-fetch token when expired. (Walter Su via jing9)
+
+ HDFS-8769. Erasure coding: unit test for SequentialBlockGroupIdGenerator.
+ (Rakesh R via waltersu4549)
+
+ HDFS-8202. Improve end to end stirpping file test to add erasure recovering
+ test. (Xinwei Qin via zhz)
+
+ HDFS-8804. Erasure Coding: use DirectBufferPool in DFSStripedInputStream for
+ buffer allocation. (jing9)
+
+ HDFS-8399. Erasure Coding: unit test the behaviour of BlockManager recovery
+ work for the deleted blocks. (Rakesh R via zhz)
+
+ HDFS-8857. Erasure Coding: Fix ArrayIndexOutOfBoundsException in
+ TestWriteStripedFileWithFailure. (Li Bo)
+
+ HDFS-8827. Erasure Coding: Fix NPE when NameNode processes over-replicated
+ striped blocks. (Walter Su and Takuya Fukudome via jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 852b040..20c1302 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -34,6 +34,7 @@ function hadoop_usage
hadoop_add_subcommand "debug" "run a Debug Admin to execute HDFS debug commands"
hadoop_add_subcommand "dfs" "run a filesystem command on the file system"
hadoop_add_subcommand "dfsadmin" "run a DFS admin client"
+ hadoop_add_subcommand "erasurecode" "configure HDFS erasure coding zones"
hadoop_add_subcommand "fetchdt" "fetch a delegation token from the NameNode"
hadoop_add_subcommand "fsck" "run a DFS filesystem checking utility"
hadoop_add_subcommand "getconf" "get config values from configuration"
@@ -133,6 +134,11 @@ case ${COMMAND} in
hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
+ erasurecode)
+ CLASS=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ ;;
fetchdt)
CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java
index aa3e8ba..8f988af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReader.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs;
+import java.io.Closeable;
import java.io.IOException;
import java.util.EnumSet;
@@ -24,13 +25,14 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.ByteBufferReadable;
import org.apache.hadoop.fs.ReadOption;
import org.apache.hadoop.hdfs.shortcircuit.ClientMmap;
+import org.apache.hadoop.util.DataChecksum;
/**
* A BlockReader is responsible for reading a single block
* from a single datanode.
*/
@InterfaceAudience.Private
-public interface BlockReader extends ByteBufferReadable {
+public interface BlockReader extends ByteBufferReadable, Closeable {
/* same interface as inputStream java.io.InputStream#read()
@@ -62,6 +64,7 @@ public interface BlockReader extends ByteBufferReadable {
*
* @throws IOException
*/
+ @Override // java.io.Closeable
void close() throws IOException;
/**
@@ -99,4 +102,9 @@ public interface BlockReader extends ByteBufferReadable {
* supported.
*/
ClientMmap getClientMmap(EnumSet<ReadOption> opts);
+
+ /**
+ * @return The DataChecksum used by the read block
+ */
+ DataChecksum getDataChecksum();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
index d913f3a..0b2420d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
@@ -738,4 +738,9 @@ class BlockReaderLocal implements BlockReader {
void forceUnanchorable() {
replica.getSlot().makeUnanchorable();
}
+
+ @Override
+ public DataChecksum getDataChecksum() {
+ return checksum;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
index c16ffdf..04cf733 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
@@ -732,4 +732,9 @@ class BlockReaderLocalLegacy implements BlockReader {
public ClientMmap getClientMmap(EnumSet<ReadOption> opts) {
return null;
}
+
+ @Override
+ public DataChecksum getDataChecksum() {
+ return checksum;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 3f4621e..8bf1444 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -118,6 +118,7 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -163,6 +164,7 @@ import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.retry.LossyRetryInvocationHandler;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
@@ -237,6 +239,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
private static final DFSHedgedReadMetrics HEDGED_READ_METRIC =
new DFSHedgedReadMetrics();
private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
+ private static volatile ThreadPoolExecutor STRIPED_READ_THREAD_POOL;
private final Sampler<?> traceSampler;
private final int smallBufferSize;
@@ -374,8 +377,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
dfsClientConf);
if (dfsClientConf.getHedgedReadThreadpoolSize() > 0) {
- this.initThreadsNumForHedgedReads(dfsClientConf.getHedgedReadThreadpoolSize());
+ this.initThreadsNumForHedgedReads(dfsClientConf.
+ getHedgedReadThreadpoolSize());
}
+
+ this.initThreadsNumForStripedReads(dfsClientConf.
+ getStripedReadThreadpoolSize());
this.saslClient = new SaslDataTransferClient(
conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf),
TrustedChannelResolver.getInstance(conf), nnFallbackToSimpleAuth);
@@ -1167,7 +1174,17 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
// Get block info from namenode
TraceScope scope = getPathTraceScope("newDFSInputStream", src);
try {
- return new DFSInputStream(this, src, verifyChecksum, null);
+ LocatedBlocks locatedBlocks = getLocatedBlocks(src, 0);
+ if (locatedBlocks != null) {
+ ECSchema schema = locatedBlocks.getECSchema();
+ if (schema != null) {
+ return new DFSStripedInputStream(this, src, verifyChecksum, schema,
+ locatedBlocks.getStripeCellSize(), locatedBlocks);
+ }
+ return new DFSInputStream(this, src, verifyChecksum, locatedBlocks);
+ } else {
+ throw new IOException("Cannot open filename " + src);
+ }
} finally {
scope.close();
}
@@ -1299,7 +1316,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
Progressable progress,
int buffersize,
ChecksumOpt checksumOpt) throws IOException {
- return create(src, permission, flag, createParent, replication, blockSize,
+ return create(src, permission, flag, createParent, replication, blockSize,
progress, buffersize, checksumOpt, null);
}
@@ -2995,6 +3012,21 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return new EncryptionZoneIterator(namenode, traceSampler);
}
+ public void createErasureCodingZone(String src, ECSchema schema, int cellSize)
+ throws IOException {
+ checkOpen();
+ TraceScope scope = getPathTraceScope("createErasureCodingZone", src);
+ try {
+ namenode.createErasureCodingZone(src, schema, cellSize);
+ } catch (RemoteException re) {
+ throw re.unwrapRemoteException(AccessControlException.class,
+ SafeModeException.class,
+ UnresolvedPathException.class);
+ } finally {
+ scope.close();
+ }
+ }
+
public void setXAttr(String src, String name, byte[] value,
EnumSet<XAttrSetFlag> flag) throws IOException {
checkOpen();
@@ -3107,6 +3139,16 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
}
+ public ECSchema[] getECSchemas() throws IOException {
+ checkOpen();
+ TraceScope scope = Trace.startSpan("getECSchemas", traceSampler);
+ try {
+ return namenode.getECSchemas();
+ } finally {
+ scope.close();
+ }
+ }
+
public DFSInotifyEventInputStream getInotifyEventStream() throws IOException {
checkOpen();
return new DFSInotifyEventInputStream(traceSampler, namenode);
@@ -3181,10 +3223,51 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
}
+ /**
+ * Create thread pool for parallel reading in striped layout,
+ * STRIPED_READ_THREAD_POOL, if it does not already exist.
+ * @param num Number of threads for striped reads thread pool.
+ */
+ private void initThreadsNumForStripedReads(int num) {
+ assert num > 0;
+ if (STRIPED_READ_THREAD_POOL != null) {
+ return;
+ }
+ synchronized (DFSClient.class) {
+ if (STRIPED_READ_THREAD_POOL == null) {
+ STRIPED_READ_THREAD_POOL = new ThreadPoolExecutor(1, num, 60,
+ TimeUnit.SECONDS, new SynchronousQueue<Runnable>(),
+ new Daemon.DaemonFactory() {
+ private final AtomicInteger threadIndex = new AtomicInteger(0);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = super.newThread(r);
+ t.setName("stripedRead-" + threadIndex.getAndIncrement());
+ return t;
+ }
+ }, new ThreadPoolExecutor.CallerRunsPolicy() {
+ @Override
+ public void rejectedExecution(Runnable runnable, ThreadPoolExecutor e) {
+ LOG.info("Execution for striped reading rejected, "
+ + "Executing in current thread");
+ // will run in the current thread
+ super.rejectedExecution(runnable, e);
+ }
+ });
+ STRIPED_READ_THREAD_POOL.allowCoreThreadTimeOut(true);
+ }
+ }
+ }
+
ThreadPoolExecutor getHedgedReadsThreadPool() {
return HEDGED_READ_THREAD_POOL;
}
+ ThreadPoolExecutor getStripedReadsThreadPool() {
+ return STRIPED_READ_THREAD_POOL;
+ }
+
boolean isHedgedReadsEnabled() {
return (HEDGED_READ_THREAD_POOL != null) &&
HEDGED_READ_THREAD_POOL.getMaximumPoolSize() > 0;
@@ -3249,4 +3332,24 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
return scope;
}
+
+ /**
+ * Get the erasure coding zone information for the specified path
+ *
+ * @param src path to get the information for
+ * @return Returns the zone information if path is in EC Zone, null otherwise
+ * @throws IOException
+ */
+ public ErasureCodingZone getErasureCodingZone(String src) throws IOException {
+ checkOpen();
+ TraceScope scope = getPathTraceScope("getErasureCodingZone", src);
+ try {
+ return namenode.getErasureCodingZone(src);
+ } catch (RemoteException re) {
+ throw re.unwrapRemoteException(FileNotFoundException.class,
+ AccessControlException.class, UnresolvedPathException.class);
+ } finally {
+ scope.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 4ef7a4d..59f14d5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyRackFaultTolarent;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.RamDiskReplicaLruTracker;
import org.apache.hadoop.http.HttpConfig;
@@ -160,6 +161,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT = 3;
public static final String DFS_NAMENODE_REPLICATION_MIN_KEY = "dfs.namenode.replication.min";
public static final int DFS_NAMENODE_REPLICATION_MIN_DEFAULT = 1;
+ public static final String DFS_NAMENODE_STRIPE_MIN_KEY = "dfs.namenode.stripe.min";
+ public static final int DFS_NAMENODE_STRIPE_MIN_DEFAULT = 1;
public static final String DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_KEY = "dfs.namenode.replication.pending.timeout-sec";
public static final int DFS_NAMENODE_REPLICATION_PENDING_TIMEOUT_SEC_DEFAULT = -1;
public static final String DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY = "dfs.namenode.replication.max-streams";
@@ -371,6 +374,12 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT = 21600;
public static final String DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY = "dfs.datanode.directoryscan.threads";
public static final int DFS_DATANODE_DIRECTORYSCAN_THREADS_DEFAULT = 1;
+ public static final String DFS_DATANODE_STRIPED_READ_THREADS_KEY = "dfs.datanode.stripedread.threads";
+ public static final int DFS_DATANODE_STRIPED_READ_THREADS_DEFAULT = 20;
+ public static final String DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.stripedread.buffer.size";
+ public static final int DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_DEFAULT = 64 * 1024;
+ public static final String DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_KEY = "dfs.datanode.stripedread.threshold.millis";
+ public static final int DFS_DATANODE_STRIPED_READ_THRESHOLD_MILLIS_DEFAULT = 5000; //5s
public static final String DFS_DATANODE_DNS_INTERFACE_KEY = "dfs.datanode.dns.interface";
public static final String DFS_DATANODE_DNS_INTERFACE_DEFAULT = "default";
public static final String DFS_DATANODE_DNS_NAMESERVER_KEY = "dfs.datanode.dns.nameserver";
@@ -430,6 +439,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final Class<BlockPlacementPolicyDefault> DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT = BlockPlacementPolicyDefault.class;
public static final String DFS_REPLICATION_MAX_KEY = "dfs.replication.max";
public static final int DFS_REPLICATION_MAX_DEFAULT = 512;
+ public static final String DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY = "dfs.block.placement.ec.classname";
+ public static final Class<BlockPlacementPolicyRackFaultTolarent> DFS_BLOCK_PLACEMENT_EC_CLASSNAME_DEFAULT = BlockPlacementPolicyRackFaultTolarent.class;
public static final String DFS_DF_INTERVAL_KEY = "dfs.df.interval";
public static final int DFS_DF_INTERVAL_DEFAULT = 60000;
@@ -679,7 +690,13 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
"dfs.namenode.reject-unresolved-dn-topology-mapping";
public static final boolean DFS_REJECT_UNRESOLVED_DN_TOPOLOGY_MAPPING_DEFAULT =
false;
-
+
+ public static final String DFS_CLIENT_STRIPED_READ_THREADPOOL_MAX_SIZE =
+ "dfs.client.striped.read.threadpool.size";
+ // With default 3+2 schema, each normal read could span 3 DNs. So this
+ // default value accommodates 6 read streams
+ public static final int DFS_CLIENT_STRIPED_READ_THREADPOOL_MAX_DEFAULT_SIZE = 18;
+
// Slow io warning log threshold settings for dfsclient and datanode.
public static final String DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY =
"dfs.datanode.slow.io.warning.threshold.ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 7f3722f..7d83db6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -1140,41 +1140,25 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
/**
- * Used when reading contiguous blocks
- */
- private void actualGetFromOneDataNode(final DNAddrPair datanode,
- LocatedBlock block, final long start, final long end, byte[] buf,
- int offset, Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
- throws IOException {
- final int length = (int) (end - start + 1);
- actualGetFromOneDataNode(datanode, block, start, end, buf,
- new int[]{offset}, new int[]{length}, corruptedBlockMap);
- }
-
- /**
* Read data from one DataNode.
- * @param datanode the datanode from which to read data
- * @param block the located block containing the requested data
- * @param startInBlk the startInBlk offset of the block
- * @param endInBlk the endInBlk offset of the block
- * @param buf the given byte array into which the data is read
- * @param offsets the data may be read into multiple segments of the buf
- * (when reading a striped block). this array indicates the
- * offset of each buf segment.
- * @param lengths the length of each buf segment
+ *
+ * @param datanode the datanode from which to read data
+ * @param block the located block containing the requested data
+ * @param startInBlk the startInBlk offset of the block
+ * @param endInBlk the endInBlk offset of the block
+ * @param buf the given byte array into which the data is read
+ * @param offset the offset in buf
* @param corruptedBlockMap map recording list of datanodes with corrupted
* block replica
*/
- void actualGetFromOneDataNode(final DNAddrPair datanode,
- LocatedBlock block, final long startInBlk, final long endInBlk,
- byte[] buf, int[] offsets, int[] lengths,
+ void actualGetFromOneDataNode(final DNAddrPair datanode, LocatedBlock block,
+ final long startInBlk, final long endInBlk, byte[] buf, int offset,
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
throws IOException {
DFSClientFaultInjector.get().startFetchFromDatanode();
int refetchToken = 1; // only need to get a new access token once
int refetchEncryptionKey = 1; // only need to get a new encryption key once
final int len = (int) (endInBlk - startInBlk + 1);
- checkReadPortions(offsets, lengths, len);
while (true) {
// cached block locations may have been updated by chooseDataNode()
@@ -1186,13 +1170,11 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
DFSClientFaultInjector.get().fetchFromDatanodeException();
reader = getBlockReader(block, startInBlk, len, datanode.addr,
datanode.storageType, datanode.info);
- for (int i = 0; i < offsets.length; i++) {
- int nread = reader.readAll(buf, offsets[i], lengths[i]);
- updateReadStatistics(readStatistics, nread, reader);
- if (nread != lengths[i]) {
- throw new IOException("truncated return from reader.read(): " +
- "excpected " + lengths[i] + ", got " + nread);
- }
+ int nread = reader.readAll(buf, offset, len);
+ updateReadStatistics(readStatistics, nread, reader);
+ if (nread != len) {
+ throw new IOException("truncated return from reader.read(): " +
+ "excpected " + len + ", got " + nread);
}
DFSClientFaultInjector.get().readFromDatanodeDelay();
return;
@@ -1248,24 +1230,6 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
/**
- * This method verifies that the read portions are valid and do not overlap
- * with each other.
- */
- private void checkReadPortions(int[] offsets, int[] lengths, int totalLen) {
- Preconditions.checkArgument(offsets.length == lengths.length && offsets.length > 0);
- int sum = 0;
- for (int i = 0; i < lengths.length; i++) {
- if (i > 0) {
- int gap = offsets[i] - offsets[i - 1];
- // make sure read portions do not overlap with each other
- Preconditions.checkArgument(gap >= lengths[i - 1]);
- }
- sum += lengths[i];
- }
- Preconditions.checkArgument(sum == totalLen);
- }
-
- /**
* Like {@link #fetchBlockByteRange}except we start up a second, parallel,
* 'hedged' read if the first read is taking longer than configured amount of
* time. We then wait on which ever read returns first.
@@ -1485,7 +1449,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
long targetStart = position - blk.getStartOffset();
long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
try {
- if (dfsClient.isHedgedReadsEnabled()) {
+ if (dfsClient.isHedgedReadsEnabled() && !blk.isStriped()) {
hedgedFetchBlockByteRange(blk, targetStart,
targetStart + bytesToRead - 1, buffer, offset, corruptedBlockMap);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index c16aef2..404bbfc 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -24,6 +24,8 @@ import java.nio.channels.ClosedChannelException;
import java.util.EnumSet;
import java.util.concurrent.atomic.AtomicReference;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
@@ -110,7 +112,7 @@ public class DFSOutputStream extends FSOutputSummer
protected final int bytesPerChecksum;
protected DFSPacket currentPacket = null;
- private DataStreamer streamer;
+ protected DataStreamer streamer;
protected int packetSize = 0; // write packet size, not including the header.
protected int chunksPerPacket = 0;
protected long lastFlushOffset = 0; // offset when flush was invoked
@@ -269,8 +271,14 @@ public class DFSOutputStream extends FSOutputSummer
}
}
Preconditions.checkNotNull(stat, "HdfsFileStatus should not be null!");
- final DFSOutputStream out = new DFSOutputStream(dfsClient, src, stat,
- flag, progress, checksum, favoredNodes);
+ final DFSOutputStream out;
+ if(stat.getECSchema() != null) {
+ out = new DFSStripedOutputStream(dfsClient, src, stat,
+ flag, progress, checksum, favoredNodes);
+ } else {
+ out = new DFSOutputStream(dfsClient, src, stat,
+ flag, progress, checksum, favoredNodes);
+ }
out.start();
return out;
} finally {
@@ -350,6 +358,9 @@ public class DFSOutputStream extends FSOutputSummer
String[] favoredNodes) throws IOException {
TraceScope scope =
dfsClient.getPathTraceScope("newStreamForAppend", src);
+ if(stat.getReplication() == 0) {
+ throw new IOException("Not support appending to a striping layout file yet.");
+ }
try {
final DFSOutputStream out = new DFSOutputStream(dfsClient, src, flags,
progress, lastBlock, stat, checksum, favoredNodes);
@@ -413,7 +424,6 @@ public class DFSOutputStream extends FSOutputSummer
getStreamer().incBytesCurBlock(len);
// If packet is full, enqueue it for transmission
- //
if (currentPacket.getNumChunks() == currentPacket.getMaxChunks() ||
getStreamer().getBytesCurBlock() == blockSize) {
enqueueCurrentPacketFull();
@@ -907,4 +917,9 @@ public class DFSOutputStream extends FSOutputSummer
protected DataStreamer getStreamer() {
return streamer;
}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + ":" + streamer;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
index 22055c3..a26e35e 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSPacket.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
import java.util.Arrays;
@@ -36,7 +37,7 @@ import org.apache.htrace.Span;
****************************************************************/
@InterfaceAudience.Private
-class DFSPacket {
+public class DFSPacket {
public static final long HEART_BEAT_SEQNO = -1L;
private static long[] EMPTY = new long[0];
private final long seqno; // sequence number of buffer in block
@@ -79,7 +80,7 @@ class DFSPacket {
* @param checksumSize the size of checksum
* @param lastPacketInBlock if this is the last packet
*/
- DFSPacket(byte[] buf, int chunksPerPkt, long offsetInBlock, long seqno,
+ public DFSPacket(byte[] buf, int chunksPerPkt, long offsetInBlock, long seqno,
int checksumSize, boolean lastPacketInBlock) {
this.lastPacketInBlock = lastPacketInBlock;
this.numChunks = 0;
@@ -113,6 +114,19 @@ class DFSPacket {
dataPos += len;
}
+ public synchronized void writeData(ByteBuffer inBuffer, int len)
+ throws ClosedChannelException {
+ checkBuffer();
+ len = len > inBuffer.remaining() ? inBuffer.remaining() : len;
+ if (dataPos + len > buf.length) {
+ throw new BufferOverflowException();
+ }
+ for (int i = 0; i < len; i++) {
+ buf[dataPos + i] = inBuffer.get();
+ }
+ dataPos += len;
+ }
+
/**
* Write checksums to this packet
*
@@ -121,7 +135,7 @@ class DFSPacket {
* @param len the length of checksums to write
* @throws ClosedChannelException
*/
- synchronized void writeChecksum(byte[] inarray, int off, int len)
+ public synchronized void writeChecksum(byte[] inarray, int off, int len)
throws ClosedChannelException {
checkBuffer();
if (len == 0) {
@@ -140,7 +154,7 @@ class DFSPacket {
* @param stm
* @throws IOException
*/
- synchronized void writeTo(DataOutputStream stm) throws IOException {
+ public synchronized void writeTo(DataOutputStream stm) throws IOException {
checkBuffer();
final int dataLen = dataPos - dataStart;
@@ -222,7 +236,7 @@ class DFSPacket {
*
* @return true if the packet is the last packet
*/
- boolean isLastPacketInBlock(){
+ boolean isLastPacketInBlock() {
return lastPacketInBlock;
}
@@ -231,7 +245,7 @@ class DFSPacket {
*
* @return the sequence number of this packet
*/
- long getSeqno(){
+ long getSeqno() {
return seqno;
}
@@ -240,14 +254,14 @@ class DFSPacket {
*
* @return the number of chunks in this packet
*/
- synchronized int getNumChunks(){
+ synchronized int getNumChunks() {
return numChunks;
}
/**
* increase the number of chunks by one
*/
- synchronized void incNumChunks(){
+ synchronized void incNumChunks() {
numChunks++;
}
@@ -256,7 +270,7 @@ class DFSPacket {
*
* @return the maximum number of packets
*/
- int getMaxChunks(){
+ int getMaxChunks() {
return maxChunks;
}
@@ -265,7 +279,7 @@ class DFSPacket {
*
* @param syncBlock if to sync block
*/
- synchronized void setSyncBlock(boolean syncBlock){
+ synchronized void setSyncBlock(boolean syncBlock) {
this.syncBlock = syncBlock;
}
[10/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
new file mode 100644
index 0000000..baf6106
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.List;
+
+public class TestDFSStripedInputStream {
+
+ public static final Log LOG = LogFactory.getLog(TestDFSStripedInputStream.class);
+
+ private MiniDFSCluster cluster;
+ private Configuration conf = new Configuration();
+ private DistributedFileSystem fs;
+ private final Path dirPath = new Path("/striped");
+ private Path filePath = new Path(dirPath, "file");
+ private final ECSchema schema = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ private final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS;
+ private final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS;
+ private final int CELLSIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final int NUM_STRIPE_PER_BLOCK = 2;
+ private final int INTERNAL_BLOCK_SIZE = NUM_STRIPE_PER_BLOCK * CELLSIZE;
+ private final int BLOCK_GROUP_SIZE = DATA_BLK_NUM * INTERNAL_BLOCK_SIZE;
+
+ @Before
+ public void setup() throws IOException {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, INTERNAL_BLOCK_SIZE);
+ SimulatedFSDataset.setFactory(conf);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
+ DATA_BLK_NUM + PARITY_BLK_NUM).build();
+ cluster.waitActive();
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ }
+ fs = cluster.getFileSystem();
+ fs.mkdirs(dirPath);
+ fs.getClient().createErasureCodingZone(dirPath.toString(), null, CELLSIZE);
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Test {@link DFSStripedInputStream#getBlockAt(long)}
+ */
+ @Test
+ public void testRefreshBlock() throws Exception {
+ final int numBlocks = 4;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
+ filePath.toString(), 0, BLOCK_GROUP_SIZE * numBlocks);
+ final DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(),
+ filePath.toString(), false, schema, CELLSIZE, null);
+
+ List<LocatedBlock> lbList = lbs.getLocatedBlocks();
+ for (LocatedBlock aLbList : lbList) {
+ LocatedStripedBlock lsb = (LocatedStripedBlock) aLbList;
+ LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(lsb,
+ CELLSIZE, DATA_BLK_NUM, PARITY_BLK_NUM);
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ LocatedBlock refreshed = in.refreshLocatedBlock(blks[j]);
+ assertEquals(blks[j].getBlock(), refreshed.getBlock());
+ assertEquals(blks[j].getStartOffset(), refreshed.getStartOffset());
+ assertArrayEquals(blks[j].getLocations(), refreshed.getLocations());
+ }
+ }
+ }
+
+ @Test
+ public void testPread() throws Exception {
+ final int numBlocks = 2;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
+ filePath.toString(), 0, BLOCK_GROUP_SIZE * numBlocks);
+ int fileLen = BLOCK_GROUP_SIZE * numBlocks;
+
+ byte[] expected = new byte[fileLen];
+ assertEquals(numBlocks, lbs.getLocatedBlocks().size());
+ for (int bgIdx = 0; bgIdx < numBlocks; bgIdx++) {
+ LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(bgIdx));
+ for (int i = 0; i < DATA_BLK_NUM; i++) {
+ Block blk = new Block(bg.getBlock().getBlockId() + i,
+ NUM_STRIPE_PER_BLOCK * CELLSIZE,
+ bg.getBlock().getGenerationStamp());
+ blk.setGenerationStamp(bg.getBlock().getGenerationStamp());
+ cluster.injectBlocks(i, Arrays.asList(blk),
+ bg.getBlock().getBlockPoolId());
+ }
+
+ /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
+ for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ for (int k = 0; k < CELLSIZE; k++) {
+ int posInBlk = i * CELLSIZE + k;
+ int posInFile = i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE + k;
+ expected[bgIdx*BLOCK_GROUP_SIZE + posInFile] =
+ SimulatedFSDataset.simulatedByte(
+ new Block(bg.getBlock().getBlockId() + j), posInBlk);
+ }
+ }
+ }
+ }
+ DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(),
+ filePath.toString(), false, schema, CELLSIZE, null);
+
+ int[] startOffsets = {0, 1, CELLSIZE - 102, CELLSIZE, CELLSIZE + 102,
+ CELLSIZE*DATA_BLK_NUM, CELLSIZE*DATA_BLK_NUM + 102,
+ BLOCK_GROUP_SIZE - 102, BLOCK_GROUP_SIZE, BLOCK_GROUP_SIZE + 102,
+ fileLen - 1};
+ for (int startOffset : startOffsets) {
+ startOffset = Math.max(0, Math.min(startOffset, fileLen - 1));
+ int remaining = fileLen - startOffset;
+ byte[] buf = new byte[fileLen];
+ int ret = in.read(startOffset, buf, 0, fileLen);
+ assertEquals(remaining, ret);
+ for (int i = 0; i < remaining; i++) {
+ Assert.assertEquals("Byte at " + (startOffset + i) + " should be the " +
+ "same",
+ expected[startOffset + i], buf[i]);
+ }
+ }
+ in.close();
+ }
+
+ @Test
+ public void testPreadWithDNFailure() throws Exception {
+ final int numBlocks = 4;
+ final int failedDNIdx = 2;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
+ filePath.toString(), 0, BLOCK_GROUP_SIZE);
+
+ assert lbs.get(0) instanceof LocatedStripedBlock;
+ LocatedStripedBlock bg = (LocatedStripedBlock)(lbs.get(0));
+ for (int i = 0; i < DATA_BLK_NUM + PARITY_BLK_NUM; i++) {
+ Block blk = new Block(bg.getBlock().getBlockId() + i,
+ NUM_STRIPE_PER_BLOCK * CELLSIZE,
+ bg.getBlock().getGenerationStamp());
+ blk.setGenerationStamp(bg.getBlock().getGenerationStamp());
+ cluster.injectBlocks(i, Arrays.asList(blk),
+ bg.getBlock().getBlockPoolId());
+ }
+ DFSStripedInputStream in =
+ new DFSStripedInputStream(fs.getClient(), filePath.toString(), false,
+ ErasureCodingSchemaManager.getSystemDefaultSchema(), CELLSIZE, null);
+ int readSize = BLOCK_GROUP_SIZE;
+ byte[] readBuffer = new byte[readSize];
+ byte[] expected = new byte[readSize];
+ cluster.stopDataNode(failedDNIdx);
+ /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
+ for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ for (int k = 0; k < CELLSIZE; k++) {
+ int posInBlk = i * CELLSIZE + k;
+ int posInFile = i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE + k;
+ expected[posInFile] = SimulatedFSDataset.simulatedByte(
+ new Block(bg.getBlock().getBlockId() + j), posInBlk);
+ }
+ }
+ }
+
+ // Update the expected content for decoded data
+ for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
+ byte[][] decodeInputs = new byte[DATA_BLK_NUM + PARITY_BLK_NUM][CELLSIZE];
+ int[] missingBlkIdx = new int[]{failedDNIdx + PARITY_BLK_NUM, 1, 2};
+ byte[][] decodeOutputs = new byte[PARITY_BLK_NUM][CELLSIZE];
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ int posInBuf = i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE;
+ if (j != failedDNIdx) {
+ System.arraycopy(expected, posInBuf, decodeInputs[j + PARITY_BLK_NUM],
+ 0, CELLSIZE);
+ }
+ }
+ for (int k = 0; k < CELLSIZE; k++) {
+ int posInBlk = i * CELLSIZE + k;
+ decodeInputs[0][k] = SimulatedFSDataset.simulatedByte(
+ new Block(bg.getBlock().getBlockId() + DATA_BLK_NUM), posInBlk);
+ }
+ for (int m : missingBlkIdx) {
+ decodeInputs[m] = null;
+ }
+ RawErasureDecoder rawDecoder = CodecUtil.createRSRawDecoder(conf,
+ DATA_BLK_NUM, PARITY_BLK_NUM);
+ rawDecoder.decode(decodeInputs, missingBlkIdx, decodeOutputs);
+ int posInBuf = i * CELLSIZE * DATA_BLK_NUM + failedDNIdx * CELLSIZE;
+ System.arraycopy(decodeOutputs[0], 0, expected, posInBuf, CELLSIZE);
+ }
+ int delta = 10;
+ int done = 0;
+ // read a small delta, shouldn't trigger decode
+ // |cell_0 |
+ // |10 |
+ done += in.read(0, readBuffer, 0, delta);
+ assertEquals(delta, done);
+ // both head and trail cells are partial
+ // |c_0 |c_1 |c_2 |c_3 |c_4 |c_5 |
+ // |256K - 10|missing|256K|256K|256K - 10|not in range|
+ done += in.read(delta, readBuffer, delta,
+ CELLSIZE * (DATA_BLK_NUM - 1) - 2 * delta);
+ assertEquals(CELLSIZE * (DATA_BLK_NUM - 1) - delta, done);
+ // read the rest
+ done += in.read(done, readBuffer, done, readSize - done);
+ assertEquals(readSize, done);
+ assertArrayEquals(expected, readBuffer);
+ }
+
+ @Test
+ public void testStatefulRead() throws Exception {
+ testStatefulRead(false, false);
+ testStatefulRead(true, false);
+ testStatefulRead(true, true);
+ }
+
+ private void testStatefulRead(boolean useByteBuffer, boolean cellMisalignPacket)
+ throws Exception {
+ final int numBlocks = 2;
+ final int fileSize = numBlocks * BLOCK_GROUP_SIZE;
+ if (cellMisalignPacket) {
+ conf.setInt(IO_FILE_BUFFER_SIZE_KEY, IO_FILE_BUFFER_SIZE_DEFAULT + 1);
+ tearDown();
+ setup();
+ }
+ DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(filePath.toString(), 0, fileSize);
+
+ assert lbs.getLocatedBlocks().size() == numBlocks;
+ for (LocatedBlock lb : lbs.getLocatedBlocks()) {
+ assert lb instanceof LocatedStripedBlock;
+ LocatedStripedBlock bg = (LocatedStripedBlock)(lb);
+ for (int i = 0; i < DATA_BLK_NUM; i++) {
+ Block blk = new Block(bg.getBlock().getBlockId() + i,
+ NUM_STRIPE_PER_BLOCK * CELLSIZE,
+ bg.getBlock().getGenerationStamp());
+ blk.setGenerationStamp(bg.getBlock().getGenerationStamp());
+ cluster.injectBlocks(i, Arrays.asList(blk),
+ bg.getBlock().getBlockPoolId());
+ }
+ }
+
+ DFSStripedInputStream in =
+ new DFSStripedInputStream(fs.getClient(), filePath.toString(),
+ false, schema, CELLSIZE, null);
+
+ byte[] expected = new byte[fileSize];
+
+ for (LocatedBlock bg : lbs.getLocatedBlocks()) {
+ /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
+ for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ for (int k = 0; k < CELLSIZE; k++) {
+ int posInBlk = i * CELLSIZE + k;
+ int posInFile = (int) bg.getStartOffset() +
+ i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE + k;
+ expected[posInFile] = SimulatedFSDataset.simulatedByte(
+ new Block(bg.getBlock().getBlockId() + j), posInBlk);
+ }
+ }
+ }
+ }
+
+ if (useByteBuffer) {
+ ByteBuffer readBuffer = ByteBuffer.allocate(fileSize);
+ int done = 0;
+ while (done < fileSize) {
+ int ret = in.read(readBuffer);
+ assertTrue(ret > 0);
+ done += ret;
+ }
+ assertArrayEquals(expected, readBuffer.array());
+ } else {
+ byte[] readBuffer = new byte[fileSize];
+ int done = 0;
+ while (done < fileSize) {
+ int ret = in.read(readBuffer, done, fileSize - done);
+ assertTrue(ret > 0);
+ done += ret;
+ }
+ assertArrayEquals(expected, readBuffer);
+ }
+ fs.delete(filePath, true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
new file mode 100644
index 0000000..3f40dee
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -0,0 +1,293 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestDFSStripedOutputStream {
+ public static final Log LOG = LogFactory.getLog(
+ TestDFSStripedOutputStream.class);
+
+ static {
+ GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL);
+ }
+
+ private int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+ private int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private Configuration conf;
+ private final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final int stripesPerBlock = 4;
+ private final int blockSize = cellSize * stripesPerBlock;
+
+ @Before
+ public void setup() throws IOException {
+ int numDNs = dataBlocks + parityBlocks + 2;
+ conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.getFileSystem().getClient().createErasureCodingZone("/", null, 0);
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testFileEmpty() throws IOException {
+ testOneFile("/EmptyFile", 0);
+ }
+
+ @Test
+ public void testFileSmallerThanOneCell1() throws IOException {
+ testOneFile("/SmallerThanOneCell", 1);
+ }
+
+ @Test
+ public void testFileSmallerThanOneCell2() throws IOException {
+ testOneFile("/SmallerThanOneCell", cellSize - 1);
+ }
+
+ @Test
+ public void testFileEqualsWithOneCell() throws IOException {
+ testOneFile("/EqualsWithOneCell", cellSize);
+ }
+
+ @Test
+ public void testFileSmallerThanOneStripe1() throws IOException {
+ testOneFile("/SmallerThanOneStripe", cellSize * dataBlocks - 1);
+ }
+
+ @Test
+ public void testFileSmallerThanOneStripe2() throws IOException {
+ testOneFile("/SmallerThanOneStripe", cellSize + 123);
+ }
+
+ @Test
+ public void testFileEqualsWithOneStripe() throws IOException {
+ testOneFile("/EqualsWithOneStripe", cellSize * dataBlocks);
+ }
+
+ @Test
+ public void testFileMoreThanOneStripe1() throws IOException {
+ testOneFile("/MoreThanOneStripe1", cellSize * dataBlocks + 123);
+ }
+
+ @Test
+ public void testFileMoreThanOneStripe2() throws IOException {
+ testOneFile("/MoreThanOneStripe2", cellSize * dataBlocks
+ + cellSize * dataBlocks + 123);
+ }
+
+ @Test
+ public void testFileLessThanFullBlockGroup() throws IOException {
+ testOneFile("/LessThanFullBlockGroup",
+ cellSize * dataBlocks * (stripesPerBlock - 1) + cellSize);
+ }
+
+ @Test
+ public void testFileFullBlockGroup() throws IOException {
+ testOneFile("/FullBlockGroup", blockSize * dataBlocks);
+ }
+
+ @Test
+ public void testFileMoreThanABlockGroup1() throws IOException {
+ testOneFile("/MoreThanABlockGroup1", blockSize * dataBlocks + 123);
+ }
+
+ @Test
+ public void testFileMoreThanABlockGroup2() throws IOException {
+ testOneFile("/MoreThanABlockGroup2",
+ blockSize * dataBlocks + cellSize+ 123);
+ }
+
+
+ @Test
+ public void testFileMoreThanABlockGroup3() throws IOException {
+ testOneFile("/MoreThanABlockGroup3",
+ blockSize * dataBlocks * 3 + cellSize * dataBlocks
+ + cellSize + 123);
+ }
+
+ private byte[] generateBytes(int cnt) {
+ byte[] bytes = new byte[cnt];
+ for (int i = 0; i < cnt; i++) {
+ bytes[i] = getByte(i);
+ }
+ return bytes;
+ }
+
+ private byte getByte(long pos) {
+ int mod = 29;
+ return (byte) (pos % mod + 1);
+ }
+
+ private void testOneFile(String src, int writeBytes) throws IOException {
+ Path testPath = new Path(src);
+
+ byte[] bytes = generateBytes(writeBytes);
+ DFSTestUtil.writeFile(fs, testPath, new String(bytes));
+
+ // check file length
+ FileStatus status = fs.getFileStatus(testPath);
+ Assert.assertEquals(writeBytes, status.getLen());
+
+ checkData(src, writeBytes);
+ }
+
+ void checkData(String src, int writeBytes) throws IOException {
+ List<List<LocatedBlock>> blockGroupList = new ArrayList<>();
+ LocatedBlocks lbs = fs.getClient().getLocatedBlocks(src, 0L);
+
+ for (LocatedBlock firstBlock : lbs.getLocatedBlocks()) {
+ Assert.assertTrue(firstBlock instanceof LocatedStripedBlock);
+ LocatedBlock[] blocks = StripedBlockUtil.
+ parseStripedBlockGroup((LocatedStripedBlock) firstBlock,
+ cellSize, dataBlocks, parityBlocks);
+ List<LocatedBlock> oneGroup = Arrays.asList(blocks);
+ blockGroupList.add(oneGroup);
+ }
+
+ // test each block group
+ for (int group = 0; group < blockGroupList.size(); group++) {
+ //get the data of this block
+ List<LocatedBlock> blockList = blockGroupList.get(group);
+ byte[][] dataBlockBytes = new byte[dataBlocks][];
+ byte[][] parityBlockBytes = new byte[parityBlocks][];
+
+ // for each block, use BlockReader to read data
+ for (int i = 0; i < blockList.size(); i++) {
+ LocatedBlock lblock = blockList.get(i);
+ if (lblock == null) {
+ continue;
+ }
+ ExtendedBlock block = lblock.getBlock();
+ byte[] blockBytes = new byte[(int)block.getNumBytes()];
+ if (i < dataBlocks) {
+ dataBlockBytes[i] = blockBytes;
+ } else {
+ parityBlockBytes[i - dataBlocks] = blockBytes;
+ }
+
+ if (block.getNumBytes() == 0) {
+ continue;
+ }
+
+ final BlockReader blockReader = BlockReaderTestUtil.getBlockReader(
+ fs, lblock, 0, block.getNumBytes());
+ blockReader.readAll(blockBytes, 0, (int) block.getNumBytes());
+ blockReader.close();
+ }
+
+ // check if we write the data correctly
+ for (int blkIdxInGroup = 0; blkIdxInGroup < dataBlockBytes.length;
+ blkIdxInGroup++) {
+ final byte[] actualBlkBytes = dataBlockBytes[blkIdxInGroup];
+ if (actualBlkBytes == null) {
+ continue;
+ }
+ for (int posInBlk = 0; posInBlk < actualBlkBytes.length; posInBlk++) {
+ // calculate the position of this byte in the file
+ long posInFile = StripedBlockUtil.offsetInBlkToOffsetInBG(cellSize,
+ dataBlocks, posInBlk, blkIdxInGroup) +
+ group * blockSize * dataBlocks;
+ Assert.assertTrue(posInFile < writeBytes);
+ final byte expected = getByte(posInFile);
+
+ String s = "Unexpected byte " + actualBlkBytes[posInBlk]
+ + ", expect " + expected
+ + ". Block group index is " + group
+ + ", stripe index is " + posInBlk / cellSize
+ + ", cell index is " + blkIdxInGroup
+ + ", byte index is " + posInBlk % cellSize;
+ Assert.assertEquals(s, expected, actualBlkBytes[posInBlk]);
+ }
+ }
+
+ verifyParity(lbs.getLocatedBlocks().get(group).getBlockSize(),
+ cellSize, dataBlockBytes, parityBlockBytes);
+ }
+ }
+
+ void verifyParity(final long size, final int cellSize,
+ byte[][] dataBytes, byte[][] parityBytes) {
+ verifyParity(conf, size, cellSize, dataBytes, parityBytes, -1);
+ }
+
+ static void verifyParity(Configuration conf, final long size,
+ final int cellSize, byte[][] dataBytes,
+ byte[][] parityBytes, int killedDnIndex) {
+ // verify the parity blocks
+ int parityBlkSize = (int) StripedBlockUtil.getInternalBlockLength(
+ size, cellSize, dataBytes.length, dataBytes.length);
+ final byte[][] expectedParityBytes = new byte[parityBytes.length][];
+ for (int i = 0; i < parityBytes.length; i++) {
+ expectedParityBytes[i] = new byte[parityBlkSize];
+ }
+ for (int i = 0; i < dataBytes.length; i++) {
+ if (dataBytes[i] == null) {
+ dataBytes[i] = new byte[dataBytes[0].length];
+ } else if (dataBytes[i].length < dataBytes[0].length) {
+ final byte[] tmp = dataBytes[i];
+ dataBytes[i] = new byte[dataBytes[0].length];
+ System.arraycopy(tmp, 0, dataBytes[i], 0, tmp.length);
+ }
+ }
+ final RawErasureEncoder encoder =
+ CodecUtil.createRSRawEncoder(conf,
+ dataBytes.length, parityBytes.length);
+ encoder.encode(dataBytes, expectedParityBytes);
+ for (int i = 0; i < parityBytes.length; i++) {
+ if (i != killedDnIndex) {
+ Assert.assertArrayEquals("i=" + i + ", killedDnIndex=" + killedDnIndex,
+ expectedParityBytes[i], parityBytes[i]);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
new file mode 100644
index 0000000..6594ae1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -0,0 +1,329 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
+import org.apache.hadoop.hdfs.security.token.block.SecurityTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Preconditions;
+
+public class TestDFSStripedOutputStreamWithFailure {
+ public static final Log LOG = LogFactory.getLog(
+ TestDFSStripedOutputStreamWithFailure.class);
+ static {
+ GenericTestUtils.setLogLevel(DFSOutputStream.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(DataStreamer.LOG, Level.ALL);
+ }
+
+ private static final int NUM_DATA_BLOCKS = HdfsConstants.NUM_DATA_BLOCKS;
+ private static final int NUM_PARITY_BLOCKS = HdfsConstants.NUM_PARITY_BLOCKS;
+ private static final int CELL_SIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private static final int STRIPES_PER_BLOCK = 4;
+ private static final int BLOCK_SIZE = CELL_SIZE * STRIPES_PER_BLOCK;
+ private static final int BLOCK_GROUP_SIZE = BLOCK_SIZE * NUM_DATA_BLOCKS;
+
+ private static final int FLUSH_POS
+ = 9*DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT + 1;
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem dfs;
+ private final Path dir = new Path("/"
+ + TestDFSStripedOutputStreamWithFailure.class.getSimpleName());
+
+ private void setup(Configuration conf) throws IOException {
+ final int numDNs = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.waitActive();
+ dfs = cluster.getFileSystem();
+ dfs.mkdirs(dir);
+ dfs.createErasureCodingZone(dir, null, 0);
+ }
+
+ private void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ private static byte getByte(long pos) {
+ return (byte)pos;
+ }
+
+ private void initConf(Configuration conf){
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ }
+
+ private void initConfWithBlockToken(Configuration conf) {
+ conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+ conf.setInt("ipc.client.connect.max.retries", 0);
+ // Set short retry timeouts so this test runs faster
+ conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 10);
+ }
+
+ @Test(timeout=240000)
+ public void testDatanodeFailure() throws Exception {
+ final int length = NUM_DATA_BLOCKS*(BLOCK_SIZE - CELL_SIZE);
+ HdfsConfiguration conf = new HdfsConfiguration();
+ initConf(conf);
+ for (int dn = 0; dn < 9; dn++) {
+ try {
+ setup(conf);
+ cluster.startDataNodes(conf, 1, true, null, null);
+ cluster.waitActive();
+ runTest(new Path(dir, "file" + dn), length, length / 2, dn, false);
+ } catch (Exception e) {
+ LOG.error("failed, dn=" + dn + ", length=" + length);
+ throw e;
+ } finally {
+ tearDown();
+ }
+ }
+ }
+
+ @Test(timeout=240000)
+ public void testBlockTokenExpired() throws Exception {
+ final int length = NUM_DATA_BLOCKS * (BLOCK_SIZE - CELL_SIZE);
+ HdfsConfiguration conf = new HdfsConfiguration();
+ initConf(conf);
+ initConfWithBlockToken(conf);
+ for (int dn = 0; dn < 9; dn += 2) {
+ try {
+ setup(conf);
+ cluster.startDataNodes(conf, 1, true, null, null);
+ cluster.waitActive();
+ runTest(new Path(dir, "file" + dn), length, length / 2, dn, true);
+ } catch (Exception e) {
+ LOG.error("failed, dn=" + dn + ", length=" + length);
+ throw e;
+ } finally {
+ tearDown();
+ }
+ }
+ }
+
+ private void runTest(final Path p, final int length, final int killPos,
+ final int dnIndex, final boolean tokenExpire) throws Exception {
+ LOG.info("p=" + p + ", length=" + length + ", killPos=" + killPos
+ + ", dnIndex=" + dnIndex);
+ Preconditions.checkArgument(killPos < length);
+ Preconditions.checkArgument(killPos > FLUSH_POS);
+ final String fullPath = p.toString();
+
+ final NameNode nn = cluster.getNameNode();
+ final BlockManager bm = nn.getNamesystem().getBlockManager();
+ final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
+
+ if (tokenExpire) {
+ // set a short token lifetime (1 second)
+ SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
+ }
+
+ final AtomicInteger pos = new AtomicInteger();
+ final FSDataOutputStream out = dfs.create(p);
+ final DFSStripedOutputStream stripedOut
+ = (DFSStripedOutputStream)out.getWrappedStream();
+
+ long oldGS = -1;
+ boolean killed = false;
+ for(; pos.get() < length; ) {
+ final int i = pos.getAndIncrement();
+ if (i == killPos) {
+ final long gs = getGenerationStamp(stripedOut);
+ Assert.assertTrue(oldGS != -1);
+ Assert.assertEquals(oldGS, gs);
+
+ if (tokenExpire) {
+ DFSTestUtil.flushInternal(stripedOut);
+ waitTokenExpires(out);
+ }
+
+ StripedFileTestUtil.killDatanode(cluster, stripedOut, dnIndex, pos);
+ killed = true;
+ }
+
+ write(out, i);
+
+ if (i == FLUSH_POS) {
+ oldGS = getGenerationStamp(stripedOut);
+ }
+ }
+ out.close();
+ Assert.assertTrue(killed);
+
+ // check file length
+ final FileStatus status = dfs.getFileStatus(p);
+ Assert.assertEquals(length, status.getLen());
+
+ checkData(dfs, fullPath, length, dnIndex, oldGS);
+ }
+
+ static void write(FSDataOutputStream out, int i) throws IOException {
+ try {
+ out.write(getByte(i));
+ } catch(IOException ioe) {
+ throw new IOException("Failed at i=" + i, ioe);
+ }
+ }
+
+ static long getGenerationStamp(DFSStripedOutputStream out)
+ throws IOException {
+ final long gs = DFSTestUtil.flushInternal(out).getGenerationStamp();
+ LOG.info("getGenerationStamp returns " + gs);
+ return gs;
+
+ }
+
+ static void checkData(DistributedFileSystem dfs, String src, int length,
+ int killedDnIndex, long oldGS) throws IOException {
+ List<List<LocatedBlock>> blockGroupList = new ArrayList<>();
+ LocatedBlocks lbs = dfs.getClient().getLocatedBlocks(src, 0L);
+ final int expectedNumGroup = (length - 1)/BLOCK_GROUP_SIZE + 1;
+ Assert.assertEquals(expectedNumGroup, lbs.getLocatedBlocks().size());
+
+ for (LocatedBlock firstBlock : lbs.getLocatedBlocks()) {
+ Assert.assertTrue(firstBlock instanceof LocatedStripedBlock);
+
+ final long gs = firstBlock.getBlock().getGenerationStamp();
+ final String s = "gs=" + gs + ", oldGS=" + oldGS;
+ LOG.info(s);
+ Assert.assertTrue(s, gs > oldGS);
+
+ LocatedBlock[] blocks = StripedBlockUtil.parseStripedBlockGroup(
+ (LocatedStripedBlock) firstBlock,
+ CELL_SIZE, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+ blockGroupList.add(Arrays.asList(blocks));
+ }
+
+ // test each block group
+ for (int group = 0; group < blockGroupList.size(); group++) {
+ final boolean isLastGroup = group == blockGroupList.size() - 1;
+ final int groupSize = !isLastGroup? BLOCK_GROUP_SIZE
+ : length - (blockGroupList.size() - 1)*BLOCK_GROUP_SIZE;
+ final int numCellInGroup = (groupSize - 1)/CELL_SIZE + 1;
+ final int lastCellIndex = (numCellInGroup - 1) % NUM_DATA_BLOCKS;
+ final int lastCellSize = groupSize - (numCellInGroup - 1)*CELL_SIZE;
+
+ //get the data of this block
+ List<LocatedBlock> blockList = blockGroupList.get(group);
+ byte[][] dataBlockBytes = new byte[NUM_DATA_BLOCKS][];
+ byte[][] parityBlockBytes = new byte[NUM_PARITY_BLOCKS][];
+
+ // for each block, use BlockReader to read data
+ for (int i = 0; i < blockList.size(); i++) {
+ final int j = i >= NUM_DATA_BLOCKS? 0: i;
+ final int numCellInBlock = (numCellInGroup - 1)/NUM_DATA_BLOCKS
+ + (j <= lastCellIndex? 1: 0);
+ final int blockSize = numCellInBlock*CELL_SIZE
+ + (isLastGroup && i == lastCellIndex? lastCellSize - CELL_SIZE: 0);
+
+ final byte[] blockBytes = new byte[blockSize];
+ if (i < NUM_DATA_BLOCKS) {
+ dataBlockBytes[i] = blockBytes;
+ } else {
+ parityBlockBytes[i - NUM_DATA_BLOCKS] = blockBytes;
+ }
+
+ final LocatedBlock lb = blockList.get(i);
+ LOG.info("XXX i=" + i + ", lb=" + lb);
+ if (lb == null) {
+ continue;
+ }
+ final ExtendedBlock block = lb.getBlock();
+ Assert.assertEquals(blockSize, block.getNumBytes());
+
+
+ if (block.getNumBytes() == 0) {
+ continue;
+ }
+
+ if (i != killedDnIndex) {
+ final BlockReader blockReader = BlockReaderTestUtil.getBlockReader(
+ dfs, lb, 0, block.getNumBytes());
+ blockReader.readAll(blockBytes, 0, (int) block.getNumBytes());
+ blockReader.close();
+ }
+ }
+
+ // check data
+ final int groupPosInFile = group*BLOCK_GROUP_SIZE;
+ for (int i = 0; i < dataBlockBytes.length; i++) {
+ final byte[] actual = dataBlockBytes[i];
+ for (int posInBlk = 0; posInBlk < actual.length; posInBlk++) {
+ final long posInFile = StripedBlockUtil.offsetInBlkToOffsetInBG(
+ CELL_SIZE, NUM_DATA_BLOCKS, posInBlk, i) + groupPosInFile;
+ Assert.assertTrue(posInFile < length);
+ final byte expected = getByte(posInFile);
+
+ if (i == killedDnIndex) {
+ actual[posInBlk] = expected;
+ } else {
+ String s = "expected=" + expected + " but actual=" + actual[posInBlk]
+ + ", posInFile=" + posInFile + ", posInBlk=" + posInBlk
+ + ". group=" + group + ", i=" + i;
+ Assert.assertEquals(s, expected, actual[posInBlk]);
+ }
+ }
+ }
+
+ // check parity
+ TestDFSStripedOutputStream.verifyParity(dfs.getConf(),
+ lbs.getLocatedBlocks().get(group).getBlockSize(),
+ CELL_SIZE, dataBlockBytes, parityBlockBytes,
+ killedDnIndex - dataBlockBytes.length);
+ }
+ }
+
+ private void waitTokenExpires(FSDataOutputStream out) throws IOException {
+ Token<BlockTokenIdentifier> token = DFSTestUtil.getBlockToken(out);
+ while (!SecurityTestUtil.isBlockTokenExpired(token)) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException ignored) {
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index a821c30..4233a1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -110,7 +110,7 @@ public class TestDFSUtil {
l2.setCorrupt(true);
List<LocatedBlock> ls = Arrays.asList(l1, l2);
- LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null);
+ LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null, null, 0);
BlockLocation[] bs = DFSUtilClient.locatedBlocks2Locations(lbs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestECSchemas.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestECSchemas.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestECSchemas.java
new file mode 100644
index 0000000..88198c9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestECSchemas.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestECSchemas {
+ private MiniDFSCluster cluster;
+
+ @Before
+ public void before() throws IOException {
+ cluster = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(0)
+ .build();
+ cluster.waitActive();
+ }
+
+ @After
+ public void after() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testGetECSchemas() throws Exception {
+ ECSchema[] ecSchemas = cluster.getFileSystem().getClient().getECSchemas();
+ assertNotNull(ecSchemas);
+ assertTrue("Should have at least one schema", ecSchemas.length > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 567a70a..8724ed5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -743,7 +743,7 @@ public class TestEncryptionZones {
version, new byte[suite.getAlgorithmBlockSize()],
new byte[suite.getAlgorithmBlockSize()],
"fakeKey", "fakeVersion"),
- (byte) 0))
+ (byte) 0, null, 0))
.when(mcp)
.create(anyString(), (FsPermission) anyObject(), anyString(),
(EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
new file mode 100644
index 0000000..1a10ebf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INode;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
+import static org.junit.Assert.*;
+
+public class TestErasureCodingZones {
+ private Configuration conf;
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private static final int BLOCK_SIZE = 1024;
+ private FSNamesystem namesystem;
+
+ @Before
+ public void setupCluster() throws IOException {
+ conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ cluster = new MiniDFSCluster.Builder(conf).
+ numDataNodes(1).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ namesystem = cluster.getNamesystem();
+ }
+
+ @After
+ public void shutdownCluster() throws IOException {
+ cluster.shutdown();
+ }
+
+ @Test
+ public void testCreateECZone()
+ throws IOException, InterruptedException {
+ final Path testDir = new Path("/ec");
+ fs.mkdir(testDir, FsPermission.getDirDefault());
+
+ /* Normal creation of an erasure coding zone */
+ fs.getClient().createErasureCodingZone(testDir.toString(), null, 0);
+
+ /* Verify files under the zone are striped */
+ final Path ECFilePath = new Path(testDir, "foo");
+ fs.create(ECFilePath);
+ INode inode = namesystem.getFSDirectory().getINode(ECFilePath.toString());
+ assertTrue(inode.asFile().isStriped());
+
+ /* Verify that EC zone cannot be created on non-empty dir */
+ final Path notEmpty = new Path("/nonEmpty");
+ fs.mkdir(notEmpty, FsPermission.getDirDefault());
+ fs.create(new Path(notEmpty, "foo"));
+ try {
+ fs.getClient().createErasureCodingZone(notEmpty.toString(), null, 0);
+ fail("Erasure coding zone on non-empty dir");
+ } catch (IOException e) {
+ assertExceptionContains("erasure coding zone for a non-empty directory", e);
+ }
+
+ /* Verify that nested EC zones cannot be created */
+ final Path zone1 = new Path("/zone1");
+ final Path zone2 = new Path(zone1, "zone2");
+ fs.mkdir(zone1, FsPermission.getDirDefault());
+ fs.getClient().createErasureCodingZone(zone1.toString(), null, 0);
+ fs.mkdir(zone2, FsPermission.getDirDefault());
+ try {
+ fs.getClient().createErasureCodingZone(zone2.toString(), null, 0);
+ fail("Nested erasure coding zones");
+ } catch (IOException e) {
+ assertExceptionContains("already in an erasure coding zone", e);
+ }
+
+ /* Verify that EC zone cannot be created on a file */
+ final Path fPath = new Path("/file");
+ fs.create(fPath);
+ try {
+ fs.getClient().createErasureCodingZone(fPath.toString(), null, 0);
+ fail("Erasure coding zone on file");
+ } catch (IOException e) {
+ assertExceptionContains("erasure coding zone for a file", e);
+ }
+ }
+
+ @Test
+ public void testMoveValidity() throws IOException, InterruptedException {
+ final Path srcECDir = new Path("/srcEC");
+ final Path dstECDir = new Path("/dstEC");
+ fs.mkdir(srcECDir, FsPermission.getDirDefault());
+ fs.mkdir(dstECDir, FsPermission.getDirDefault());
+ fs.getClient().createErasureCodingZone(srcECDir.toString(), null, 0);
+ fs.getClient().createErasureCodingZone(dstECDir.toString(), null, 0);
+ final Path srcFile = new Path(srcECDir, "foo");
+ fs.create(srcFile);
+
+ // Test move dir
+ // Move EC dir under non-EC dir
+ final Path newDir = new Path("/srcEC_new");
+ fs.rename(srcECDir, newDir);
+ fs.rename(newDir, srcECDir); // move back
+
+ // Move EC dir under another EC dir
+ fs.rename(srcECDir, dstECDir);
+ fs.rename(new Path("/dstEC/srcEC"), srcECDir); // move back
+
+ // Test move file
+ /* Verify that a file can be moved between 2 EC zones */
+ fs.rename(srcFile, dstECDir);
+ fs.rename(new Path(dstECDir, "foo"), srcECDir); // move back
+
+ /* Verify that a file cannot be moved from a non-EC dir to an EC zone */
+ final Path nonECDir = new Path("/nonEC");
+ fs.mkdir(nonECDir, FsPermission.getDirDefault());
+ try {
+ fs.rename(srcFile, nonECDir);
+ fail("A file shouldn't be able to move from a non-EC dir to an EC zone");
+ } catch (IOException e) {
+ assertExceptionContains("can't be moved because the source and " +
+ "destination have different erasure coding policies", e);
+ }
+
+ /* Verify that a file cannot be moved from an EC zone to a non-EC dir */
+ final Path nonECFile = new Path(nonECDir, "nonECFile");
+ fs.create(nonECFile);
+ try {
+ fs.rename(nonECFile, dstECDir);
+ } catch (IOException e) {
+ assertExceptionContains("can't be moved because the source and " +
+ "destination have different erasure coding policies", e);
+ }
+ }
+
+ @Test
+ public void testReplication() throws IOException {
+ final Path testDir = new Path("/ec");
+ fs.mkdir(testDir, FsPermission.getDirDefault());
+ fs.createErasureCodingZone(testDir, null, 0);
+ final Path fooFile = new Path(testDir, "foo");
+ // create ec file with replication=0
+ fs.create(fooFile, FsPermission.getFileDefault(), true,
+ conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
+ (short)0, fs.getDefaultBlockSize(fooFile), null);
+
+ try {
+ fs.setReplication(fooFile, (short) 3);
+ fail("Shouldn't allow to set replication to a file with striped blocks");
+ } catch (IOException e) {
+ assertExceptionContains(
+ "Cannot set replication to a file with striped blocks", e);
+ }
+ }
+
+ @Test
+ public void testGetErasureCodingInfoWithSystemDefaultSchema() throws Exception {
+ String src = "/ec";
+ final Path ecDir = new Path(src);
+ fs.mkdir(ecDir, FsPermission.getDirDefault());
+ // dir ECInfo before creating ec zone
+ assertNull(fs.getClient().getFileInfo(src).getECSchema());
+ // dir ECInfo after creating ec zone
+ fs.getClient().createErasureCodingZone(src, null, 0); //Default one will be used.
+ ECSchema sysDefaultSchema = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ verifyErasureCodingInfo(src, sysDefaultSchema);
+ fs.create(new Path(ecDir, "child1")).close();
+ // verify for the files in ec zone
+ verifyErasureCodingInfo(src + "/child1", sysDefaultSchema);
+ }
+
+ @Test
+ public void testGetErasureCodingInfo() throws Exception {
+ ECSchema[] sysSchemas = ErasureCodingSchemaManager.getSystemSchemas();
+ assertTrue("System schemas should be of only 1 for now",
+ sysSchemas.length == 1);
+
+ ECSchema usingSchema = sysSchemas[0];
+ String src = "/ec2";
+ final Path ecDir = new Path(src);
+ fs.mkdir(ecDir, FsPermission.getDirDefault());
+ // dir ECInfo before creating ec zone
+ assertNull(fs.getClient().getFileInfo(src).getECSchema());
+ // dir ECInfo after creating ec zone
+ fs.getClient().createErasureCodingZone(src, usingSchema, 0);
+ verifyErasureCodingInfo(src, usingSchema);
+ fs.create(new Path(ecDir, "child1")).close();
+ // verify for the files in ec zone
+ verifyErasureCodingInfo(src + "/child1", usingSchema);
+ }
+
+ private void verifyErasureCodingInfo(
+ String src, ECSchema usingSchema) throws IOException {
+ HdfsFileStatus hdfsFileStatus = fs.getClient().getFileInfo(src);
+ ECSchema schema = hdfsFileStatus.getECSchema();
+ assertNotNull(schema);
+ assertEquals("Actually used schema should be equal with target schema",
+ usingSchema, schema);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java
new file mode 100644
index 0000000..3c400b7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECschema.java
@@ -0,0 +1,65 @@
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestFileStatusWithECschema {
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private DFSClient client;
+
+ @Before
+ public void before() throws IOException {
+ cluster =
+ new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ client = fs.getClient();
+ }
+
+ @After
+ public void after() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testFileStatusWithECschema() throws Exception {
+ // test directory not in EC zone
+ final Path dir = new Path("/foo");
+ assertTrue(fs.mkdir(dir, FsPermission.getDirDefault()));
+ assertNull(client.getFileInfo(dir.toString()).getECSchema());
+ // test file not in EC zone
+ final Path file = new Path(dir, "foo");
+ fs.create(file).close();
+ assertNull(client.getFileInfo(file.toString()).getECSchema());
+ fs.delete(file, true);
+
+ final ECSchema schema1 = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ // create EC zone on dir
+ fs.createErasureCodingZone(dir, schema1, 0);
+ final ECSchema schame2 = client.getFileInfo(dir.toUri().getPath()).getECSchema();
+ assertNotNull(schame2);
+ assertTrue(schema1.equals(schame2));
+
+ // test file in EC zone
+ fs.create(file).close();
+ final ECSchema schame3 =
+ fs.getClient().getFileInfo(file.toUri().getPath()).getECSchema();
+ assertNotNull(schame3);
+ assertTrue(schema1.equals(schame3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
index 985f43e..d0cd335 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLease.java
@@ -354,12 +354,12 @@ public class TestLease {
Mockito.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
- 1010, 0, null, (byte) 0)).when(mcp).getFileInfo(anyString());
+ 1010, 0, null, (byte) 0, null, 0)).when(mcp).getFileInfo(anyString());
Mockito
.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
- 1010, 0, null, (byte) 0))
+ 1010, 0, null, (byte) 0, null, 0))
.when(mcp)
.create(anyString(), (FsPermission) anyObject(), anyString(),
(EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
new file mode 100644
index 0000000..1719d3f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java
@@ -0,0 +1,358 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.cellSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.parityBlocks;
+
+public class TestReadStripedFileWithDecoding {
+ static final Log LOG = LogFactory.getLog(TestReadStripedFileWithDecoding.class);
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private final int smallFileLength = blockSize * dataBlocks - 123;
+ private final int largeFileLength = blockSize * dataBlocks + 123;
+ private final int[] fileLengths = {smallFileLength, largeFileLength};
+ private final int[] dnFailureNums = {1, 2, 3};
+
+ @Before
+ public void setup() throws IOException {
+ cluster = new MiniDFSCluster.Builder(new HdfsConfiguration())
+ .numDataNodes(numDNs).build();
+ cluster.getFileSystem().getClient().createErasureCodingZone("/",
+ null, cellSize);
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Shutdown tolerable number of Datanode before reading.
+ * Verify the decoding works correctly.
+ */
+ @Test(timeout=300000)
+ public void testReadWithDNFailure() throws IOException {
+ for (int fileLength : fileLengths) {
+ for (int dnFailureNum : dnFailureNums) {
+ try {
+ // setup a new cluster with no dead datanode
+ setup();
+ testReadWithDNFailure(fileLength, dnFailureNum);
+ } catch (IOException ioe) {
+ String fileType = fileLength < (blockSize * dataBlocks) ?
+ "smallFile" : "largeFile";
+ LOG.error("Failed to read file with DN failure:"
+ + " fileType = "+ fileType
+ + ", dnFailureNum = " + dnFailureNum);
+ } finally {
+ // tear down the cluster
+ tearDown();
+ }
+ }
+ }
+ }
+
+ /**
+ * Corrupt tolerable number of block before reading.
+ * Verify the decoding works correctly.
+ */
+ @Test(timeout=300000)
+ public void testReadCorruptedData() throws IOException {
+ for (int fileLength : fileLengths) {
+ for (int dataDelNum = 1; dataDelNum < 4; dataDelNum++) {
+ for (int parityDelNum = 0; (dataDelNum+parityDelNum) < 4; parityDelNum++) {
+ String src = "/corrupted_" + dataDelNum + "_" + parityDelNum;
+ testReadWithBlockCorrupted(src, fileLength,
+ dataDelNum, parityDelNum, false);
+ }
+ }
+ }
+ }
+
+ /**
+ * Delete tolerable number of block before reading.
+ * Verify the decoding works correctly.
+ */
+ @Test(timeout=300000)
+ public void testReadCorruptedDataByDeleting() throws IOException {
+ for (int fileLength : fileLengths) {
+ for (int dataDelNum = 1; dataDelNum < 4; dataDelNum++) {
+ for (int parityDelNum = 0; (dataDelNum+parityDelNum) < 4; parityDelNum++) {
+ String src = "/deleted_" + dataDelNum + "_" + parityDelNum;
+ testReadWithBlockCorrupted(src, fileLength,
+ dataDelNum, parityDelNum, true);
+ }
+ }
+ }
+ }
+
+ private int findFirstDataNode(Path file, long length) throws IOException {
+ BlockLocation[] locs = fs.getFileBlockLocations(file, 0, length);
+ String name = (locs[0].getNames())[0];
+ int dnIndex = 0;
+ for (DataNode dn : cluster.getDataNodes()) {
+ int port = dn.getXferPort();
+ if (name.contains(Integer.toString(port))) {
+ return dnIndex;
+ }
+ dnIndex++;
+ }
+ return -1;
+ }
+
+ private void verifyRead(Path testPath, int length, byte[] expected)
+ throws IOException {
+ byte[] buffer = new byte[length + 100];
+ StripedFileTestUtil.verifyLength(fs, testPath, length);
+ StripedFileTestUtil.verifyPread(fs, testPath, length, expected, buffer);
+ StripedFileTestUtil.verifyStatefulRead(fs, testPath, length, expected, buffer);
+ StripedFileTestUtil.verifyStatefulRead(fs, testPath, length, expected,
+ ByteBuffer.allocate(length + 100));
+ StripedFileTestUtil.verifySeek(fs, testPath, length);
+ }
+
+ private void testReadWithDNFailure(int fileLength, int dnFailureNum)
+ throws IOException {
+ String fileType = fileLength < (blockSize * dataBlocks) ?
+ "smallFile" : "largeFile";
+ String src = "/dnFailure_" + dnFailureNum + "_" + fileType;
+ LOG.info("testReadWithDNFailure: file = " + src
+ + ", fileSize = " + fileLength
+ + ", dnFailureNum = " + dnFailureNum);
+
+ Path testPath = new Path(src);
+ final byte[] bytes = StripedFileTestUtil.generateBytes(fileLength);
+ DFSTestUtil.writeFile(fs, testPath, bytes);
+
+ // shut down the DN that holds an internal data block
+ BlockLocation[] locs = fs.getFileBlockLocations(testPath, cellSize * 5,
+ cellSize);
+ for (int failedDnIdx = 0; failedDnIdx < dnFailureNum; failedDnIdx++) {
+ String name = (locs[0].getNames())[failedDnIdx];
+ for (DataNode dn : cluster.getDataNodes()) {
+ int port = dn.getXferPort();
+ if (name.contains(Integer.toString(port))) {
+ dn.shutdown();
+ }
+ }
+ }
+
+ // check file length, pread, stateful read and seek
+ verifyRead(testPath, fileLength, bytes);
+ }
+
+ /**
+ * After reading a corrupted block, make sure the client can correctly report
+ * the corruption to the NameNode.
+ */
+ @Test
+ public void testReportBadBlock() throws IOException {
+ // create file
+ final Path file = new Path("/corrupted");
+ final int length = 10; // length of "corruption"
+ final byte[] bytes = StripedFileTestUtil.generateBytes(length);
+ DFSTestUtil.writeFile(fs, file, bytes);
+
+ // corrupt the first data block
+ int dnIndex = findFirstDataNode(file, cellSize * dataBlocks);
+ Assert.assertNotEquals(-1, dnIndex);
+ LocatedStripedBlock slb = (LocatedStripedBlock)fs.getClient()
+ .getLocatedBlocks(file.toString(), 0, cellSize * dataBlocks).get(0);
+ final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(slb,
+ cellSize, dataBlocks, parityBlocks);
+ // find the first block file
+ File storageDir = cluster.getInstanceStorageDir(dnIndex, 0);
+ File blkFile = MiniDFSCluster.getBlockFile(storageDir, blks[0].getBlock());
+ Assert.assertTrue("Block file does not exist", blkFile.exists());
+ // corrupt the block file
+ LOG.info("Deliberately corrupting file " + blkFile.getName());
+ try (FileOutputStream out = new FileOutputStream(blkFile)) {
+ out.write("corruption".getBytes());
+ }
+
+ // disable the heartbeat from DN so that the corrupted block record is kept
+ // in NameNode
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+ }
+
+ try {
+ // do stateful read
+ StripedFileTestUtil.verifyStatefulRead(fs, file, length, bytes,
+ ByteBuffer.allocate(1024));
+
+ // check whether the corruption has been reported to the NameNode
+ final FSNamesystem ns = cluster.getNamesystem();
+ final BlockManager bm = ns.getBlockManager();
+ BlockInfo blockInfo = (ns.getFSDirectory().getINode4Write(file.toString())
+ .asFile().getBlocks())[0];
+ Assert.assertEquals(1, bm.getCorruptReplicas(blockInfo).size());
+ } finally {
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
+ }
+ }
+ }
+
+ @Test
+ public void testInvalidateBlock() throws IOException {
+ final Path file = new Path("/invalidate");
+ final int length = 10;
+ final byte[] bytes = StripedFileTestUtil.generateBytes(length);
+ DFSTestUtil.writeFile(fs, file, bytes);
+
+ int dnIndex = findFirstDataNode(file, cellSize * dataBlocks);
+ Assert.assertNotEquals(-1, dnIndex);
+ LocatedStripedBlock slb = (LocatedStripedBlock)fs.getClient()
+ .getLocatedBlocks(file.toString(), 0, cellSize * dataBlocks).get(0);
+ final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(slb,
+ cellSize, dataBlocks, parityBlocks);
+ final Block b = blks[0].getBlock().getLocalBlock();
+
+ DataNode dn = cluster.getDataNodes().get(dnIndex);
+ // disable the heartbeat from DN so that the invalidated block record is kept
+ // in NameNode until heartbeat expires and NN mark the dn as dead
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
+
+ try {
+ // delete the file
+ fs.delete(file, true);
+ // check the block is added to invalidateBlocks
+ final FSNamesystem fsn = cluster.getNamesystem();
+ final BlockManager bm = fsn.getBlockManager();
+ DatanodeDescriptor dnd = NameNodeAdapter.getDatanode(fsn, dn.getDatanodeId());
+ Assert.assertTrue(bm.containsInvalidateBlock(
+ blks[0].getLocations()[0], b) || dnd.containsInvalidateBlock(b));
+ } finally {
+ DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
+ }
+ }
+
+ /**
+ * Test reading a file with some blocks(data blocks or parity blocks or both)
+ * deleted or corrupted.
+ * @param src file path
+ * @param fileLength file length
+ * @param dataBlkDelNum the deleted or corrupted number of data blocks.
+ * @param parityBlkDelNum the deleted or corrupted number of parity blocks.
+ * @param deleteBlockFile whether block file is deleted or corrupted.
+ * true is to delete the block file.
+ * false is to corrupt the content of the block file.
+ * @throws IOException
+ */
+ private void testReadWithBlockCorrupted(String src, int fileLength,
+ int dataBlkDelNum, int parityBlkDelNum, boolean deleteBlockFile)
+ throws IOException {
+ LOG.info("testReadWithBlockCorrupted: file = " + src
+ + ", dataBlkDelNum = " + dataBlkDelNum
+ + ", parityBlkDelNum = " + parityBlkDelNum
+ + ", deleteBlockFile? " + deleteBlockFile);
+ int recoverBlkNum = dataBlkDelNum + parityBlkDelNum;
+ Assert.assertTrue("dataBlkDelNum and parityBlkDelNum should be positive",
+ dataBlkDelNum >= 0 && parityBlkDelNum >= 0);
+ Assert.assertTrue("The sum of dataBlkDelNum and parityBlkDelNum " +
+ "should be between 1 ~ " + parityBlocks, recoverBlkNum <= parityBlocks);
+
+ // write a file with the length of writeLen
+ Path srcPath = new Path(src);
+ final byte[] bytes = StripedFileTestUtil.generateBytes(fileLength);
+ DFSTestUtil.writeFile(fs, srcPath, bytes);
+
+ // delete or corrupt some blocks
+ corruptBlocks(srcPath, dataBlkDelNum, parityBlkDelNum, deleteBlockFile);
+
+ // check the file can be read after some blocks were deleted
+ verifyRead(srcPath, fileLength, bytes);
+ }
+
+ private void corruptBlocks(Path srcPath, int dataBlkDelNum,
+ int parityBlkDelNum, boolean deleteBlockFile) throws IOException {
+ int recoverBlkNum = dataBlkDelNum + parityBlkDelNum;
+
+ LocatedBlocks locatedBlocks = getLocatedBlocks(srcPath);
+ LocatedStripedBlock lastBlock =
+ (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+
+ int[] delDataBlkIndices = StripedFileTestUtil.randomArray(0, dataBlocks,
+ dataBlkDelNum);
+ Assert.assertNotNull(delDataBlkIndices);
+ int[] delParityBlkIndices = StripedFileTestUtil.randomArray(dataBlocks,
+ dataBlocks + parityBlocks, parityBlkDelNum);
+ Assert.assertNotNull(delParityBlkIndices);
+
+ int[] delBlkIndices = new int[recoverBlkNum];
+ System.arraycopy(delDataBlkIndices, 0,
+ delBlkIndices, 0, delDataBlkIndices.length);
+ System.arraycopy(delParityBlkIndices, 0,
+ delBlkIndices, delDataBlkIndices.length, delParityBlkIndices.length);
+
+ ExtendedBlock[] delBlocks = new ExtendedBlock[recoverBlkNum];
+ for (int i = 0; i < recoverBlkNum; i++) {
+ delBlocks[i] = StripedBlockUtil
+ .constructInternalBlock(lastBlock.getBlock(),
+ cellSize, dataBlocks, delBlkIndices[i]);
+ if (deleteBlockFile) {
+ // delete the block file
+ cluster.corruptBlockOnDataNodesByDeletingBlockFile(delBlocks[i]);
+ } else {
+ // corrupt the block file
+ cluster.corruptBlockOnDataNodes(delBlocks[i]);
+ }
+ }
+ }
+
+ private LocatedBlocks getLocatedBlocks(Path filePath) throws IOException {
+ return fs.getClient().getLocatedBlocks(filePath.toString(),
+ 0, Long.MAX_VALUE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
new file mode 100644
index 0000000..4c2438d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.cellSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
+
+/**
+ * Test reading a striped file when some of its blocks are missing (not included
+ * in the block locations returned by the NameNode).
+ */
+public class TestReadStripedFileWithMissingBlocks {
+ public static final Log LOG = LogFactory
+ .getLog(TestReadStripedFileWithMissingBlocks.class);
+ private static MiniDFSCluster cluster;
+ private static FileSystem fs;
+ private static Configuration conf = new HdfsConfiguration();
+ private final int fileLength = blockSize * dataBlocks + 123;
+
+ @Before
+ public void setup() throws IOException {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.getFileSystem().getClient().createErasureCodingZone("/",
+ null, cellSize);
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testReadFileWithMissingBlocks1() throws IOException {
+ readFileWithMissingBlocks(new Path("/foo"), fileLength, 1, 0);
+ }
+
+ @Test
+ public void testReadFileWithMissingBlocks2() throws IOException {
+ readFileWithMissingBlocks(new Path("/foo"), fileLength, 1, 1);
+ }
+
+ @Test
+ public void testReadFileWithMissingBlocks3() throws IOException {
+ readFileWithMissingBlocks(new Path("/foo"), fileLength, 1, 2);
+ }
+
+ @Test
+ public void testReadFileWithMissingBlocks4() throws IOException {
+ readFileWithMissingBlocks(new Path("/foo"), fileLength, 2, 0);
+ }
+
+ @Test
+ public void testReadFileWithMissingBlocks5() throws IOException {
+ readFileWithMissingBlocks(new Path("/foo"), fileLength, 2, 1);
+ }
+
+ @Test
+ public void testReadFileWithMissingBlocks6() throws IOException {
+ readFileWithMissingBlocks(new Path("/foo"), fileLength, 3, 0);
+ }
+
+ private void readFileWithMissingBlocks(Path srcPath, int fileLength,
+ int missingDataNum, int missingParityNum)
+ throws IOException {
+ LOG.info("readFileWithMissingBlocks: (" + missingDataNum + ","
+ + missingParityNum + ")");
+ final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
+ DFSTestUtil.writeFile(fs, srcPath, new String(expected));
+ StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
+ int dataBlocks = (fileLength - 1) / cellSize + 1;
+ BlockLocation[] locs = fs.getFileBlockLocations(srcPath, 0, cellSize);
+
+ int[] missingDataNodes = new int[missingDataNum + missingParityNum];
+ for (int i = 0; i < missingDataNum; i++) {
+ missingDataNodes[i] = i;
+ }
+ for (int i = 0; i < missingParityNum; i++) {
+ missingDataNodes[i + missingDataNum] = i +
+ Math.min(StripedFileTestUtil.dataBlocks, dataBlocks);
+ }
+ stopDataNodes(locs, missingDataNodes);
+
+ // make sure there are missing block locations
+ BlockLocation[] newLocs = fs.getFileBlockLocations(srcPath, 0, cellSize);
+ Assert.assertTrue(newLocs[0].getNames().length < locs[0].getNames().length);
+
+ byte[] smallBuf = new byte[1024];
+ byte[] largeBuf = new byte[fileLength + 100];
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
+ smallBuf);
+ StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
+
+ // delete the file
+ fs.delete(srcPath, true);
+ }
+
+ private void stopDataNodes(BlockLocation[] locs, int[] datanodes)
+ throws IOException {
+ if (locs != null && locs.length > 0) {
+ for (int failedDNIdx : datanodes) {
+ String name = (locs[0].getNames())[failedDNIdx];
+ for (DataNode dn : cluster.getDataNodes()) {
+ int port = dn.getXferPort();
+ if (name.contains(Integer.toString(port))) {
+ dn.shutdown();
+ cluster.setDataNodeDead(dn.getDatanodeId());
+ LOG.info("stop datanode " + failedDNIdx);
+ break;
+ }
+ }
+ }
+ }
+ }
+}
[13/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 3d30a19..4f524ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -44,10 +45,13 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.util.ChunkedArrayList;
@@ -73,7 +77,7 @@ class FSDirWriteFileOp {
Block block) throws IOException {
// modify file-> block and blocksMap
// fileNode should be under construction
- BlockInfoContiguousUnderConstruction uc = fileNode.removeLastBlock(block);
+ BlockInfoUnderConstruction uc = fileNode.removeLastBlock(block);
if (uc == null) {
return false;
}
@@ -87,7 +91,7 @@ class FSDirWriteFileOp {
// update space consumed
fsd.updateCount(iip, 0, -fileNode.getPreferredBlockSize(),
- fileNode.getPreferredBlockReplication(), true);
+ fileNode.getPreferredBlockReplication(), true);
return true;
}
@@ -167,9 +171,10 @@ class FSDirWriteFileOp {
String src, long fileId, String clientName,
ExtendedBlock previous, LocatedBlock[] onRetryBlock) throws IOException {
final long blockSize;
- final int replication;
+ final short numTargets;
final byte storagePolicyID;
String clientMachine;
+ final boolean isStriped;
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsn.dir.resolvePath(pc, src, pathComponents);
@@ -195,18 +200,21 @@ class FSDirWriteFileOp {
blockSize = pendingFile.getPreferredBlockSize();
clientMachine = pendingFile.getFileUnderConstructionFeature()
.getClientMachine();
- replication = pendingFile.getFileReplication();
+ isStriped = pendingFile.isStriped();
+ numTargets = isStriped ?
+ HdfsConstants.NUM_DATA_BLOCKS + HdfsConstants.NUM_PARITY_BLOCKS :
+ pendingFile.getFileReplication();
storagePolicyID = pendingFile.getStoragePolicyID();
- return new ValidateAddBlockResult(blockSize, replication, storagePolicyID,
- clientMachine);
+ return new ValidateAddBlockResult(blockSize, numTargets, storagePolicyID,
+ clientMachine, isStriped);
}
- static LocatedBlock makeLocatedBlock(FSNamesystem fsn, Block blk,
+ static LocatedBlock makeLocatedBlock(FSNamesystem fsn, BlockInfo blk,
DatanodeStorageInfo[] locs, long offset) throws IOException {
LocatedBlock lBlk = BlockManager.newLocatedBlock(fsn.getExtendedBlock(blk),
- locs, offset, false);
- fsn.getFSDirectory().getBlockManager()
- .setBlockToken(lBlk, BlockTokenIdentifier.AccessMode.WRITE);
+ blk, locs, offset);
+ fsn.getBlockManager().setBlockToken(lBlk,
+ BlockTokenIdentifier.AccessMode.WRITE);
return lBlk;
}
@@ -236,8 +244,9 @@ class FSDirWriteFileOp {
} else {
// add new chosen targets to already allocated block and return
BlockInfo lastBlockInFile = pendingFile.getLastBlock();
- ((BlockInfoContiguousUnderConstruction) lastBlockInFile)
- .setExpectedLocations(targets);
+ final BlockInfoUnderConstruction uc
+ = (BlockInfoUnderConstruction)lastBlockInFile;
+ uc.setExpectedLocations(targets);
offset = pendingFile.computeFileSize();
return makeLocatedBlock(fsn, lastBlockInFile, targets, offset);
}
@@ -248,15 +257,17 @@ class FSDirWriteFileOp {
ExtendedBlock.getLocalBlock(previous));
// allocate new block, record block locations in INode.
- Block newBlock = fsn.createNewBlock();
+ final boolean isStriped = pendingFile.isStriped();
+ // allocate new block, record block locations in INode.
+ Block newBlock = fsn.createNewBlock(isStriped);
INodesInPath inodesInPath = INodesInPath.fromINode(pendingFile);
- saveAllocatedBlock(fsn, src, inodesInPath, newBlock, targets);
+ saveAllocatedBlock(fsn, src, inodesInPath, newBlock, targets, isStriped);
persistNewBlock(fsn, src, pendingFile);
offset = pendingFile.computeFileSize();
// Return located block
- return makeLocatedBlock(fsn, newBlock, targets, offset);
+ return makeLocatedBlock(fsn, fsn.getStoredBlock(newBlock), targets, offset);
}
static DatanodeStorageInfo[] chooseTargetForNewBlock(
@@ -277,9 +288,10 @@ class FSDirWriteFileOp {
: Arrays.asList(favoredNodes);
// choose targets for the new block to be allocated.
- return bm.chooseTarget4NewBlock(src, r.replication, clientNode,
+ return bm.chooseTarget4NewBlock(src, r.numTargets, clientNode,
excludedNodesSet, r.blockSize,
- favoredNodesList, r.storagePolicyID);
+ favoredNodesList, r.storagePolicyID,
+ r.isStriped);
}
/**
@@ -468,22 +480,22 @@ class FSDirWriteFileOp {
long preferredBlockSize, boolean underConstruction, String clientName,
String clientMachine, byte storagePolicyId) {
final INodeFile newNode;
+ Preconditions.checkNotNull(existing);
assert fsd.hasWriteLock();
- if (underConstruction) {
- newNode = newINodeFile(id, permissions, modificationTime,
- modificationTime, replication,
- preferredBlockSize,
- storagePolicyId);
- newNode.toUnderConstruction(clientName, clientMachine);
- } else {
- newNode = newINodeFile(id, permissions, modificationTime,
- atime, replication,
- preferredBlockSize,
- storagePolicyId);
- }
-
- newNode.setLocalName(localName);
try {
+ // check if the file is in an EC zone
+ final boolean isStriped = FSDirErasureCodingOp.isInErasureCodingZone(
+ fsd.getFSNamesystem(), existing);
+ if (underConstruction) {
+ newNode = newINodeFile(id, permissions, modificationTime,
+ modificationTime, replication, preferredBlockSize, storagePolicyId,
+ isStriped);
+ newNode.toUnderConstruction(clientName, clientMachine);
+ } else {
+ newNode = newINodeFile(id, permissions, modificationTime, atime,
+ replication, preferredBlockSize, storagePolicyId, isStriped);
+ }
+ newNode.setLocalName(localName);
INodesInPath iip = fsd.addINode(existing, newNode);
if (iip != null) {
if (aclEntries != null) {
@@ -509,23 +521,38 @@ class FSDirWriteFileOp {
*/
private static BlockInfo addBlock(
FSDirectory fsd, String path, INodesInPath inodesInPath, Block block,
- DatanodeStorageInfo[] targets) throws IOException {
+ DatanodeStorageInfo[] targets, boolean isStriped) throws IOException {
fsd.writeLock();
try {
final INodeFile fileINode = inodesInPath.getLastINode().asFile();
Preconditions.checkState(fileINode.isUnderConstruction());
- // check quota limits and updated space consumed
- fsd.updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(),
- fileINode.getPreferredBlockReplication(), true);
-
// associate new last block for the file
- BlockInfoContiguousUnderConstruction blockInfo =
- new BlockInfoContiguousUnderConstruction(
- block,
- fileINode.getFileReplication(),
- HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION,
+ final BlockInfo blockInfo;
+ if (isStriped) {
+ ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsd.getFSNamesystem(), inodesInPath);
+ ECSchema ecSchema = ecZone.getSchema();
+ short numDataUnits = (short) ecSchema.getNumDataUnits();
+ short numParityUnits = (short) ecSchema.getNumParityUnits();
+ short numLocations = (short) (numDataUnits + numParityUnits);
+
+ // check quota limits and updated space consumed
+ fsd.updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(),
+ numLocations, true);
+ blockInfo = new BlockInfoStripedUnderConstruction(block, ecSchema,
+ ecZone.getCellSize(),
+ HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
+ } else {
+ // check quota limits and updated space consumed
+ fsd.updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(),
+ fileINode.getPreferredBlockReplication(), true);
+
+ short numLocations = fileINode.getFileReplication();
+ blockInfo = new BlockInfoContiguousUnderConstruction(block,
+ numLocations, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION,
targets);
+ }
fsd.getBlockManager().addBlockCollection(blockInfo, fileINode);
fileINode.addBlock(blockInfo);
@@ -551,22 +578,24 @@ class FSDirWriteFileOp {
String clientName, String clientMachine)
throws IOException {
+ Preconditions.checkNotNull(existing);
long modTime = now();
- INodeFile newNode = newINodeFile(fsd.allocateNewInodeId(), permissions,
- modTime, modTime, replication, preferredBlockSize);
- newNode.setLocalName(localName.getBytes(Charsets.UTF_8));
- newNode.toUnderConstruction(clientName, clientMachine);
-
INodesInPath newiip;
fsd.writeLock();
try {
+ final boolean isStriped = FSDirErasureCodingOp.isInErasureCodingZone(
+ fsd.getFSNamesystem(), existing);
+ INodeFile newNode = newINodeFile(fsd.allocateNewInodeId(), permissions,
+ modTime, modTime, replication, preferredBlockSize, isStriped);
+ newNode.setLocalName(localName.getBytes(Charsets.UTF_8));
+ newNode.toUnderConstruction(clientName, clientMachine);
newiip = fsd.addINode(existing, newNode);
} finally {
fsd.writeUnlock();
}
if (newiip == null) {
NameNode.stateChangeLog.info("DIR* addFile: failed to add " +
- existing.getPath() + "/" + localName);
+ existing.getPath() + "/" + localName);
return null;
}
@@ -579,7 +608,7 @@ class FSDirWriteFileOp {
private static FileState analyzeFileState(
FSNamesystem fsn, String src, long fileId, String clientName,
ExtendedBlock previous, LocatedBlock[] onRetryBlock)
- throws IOException {
+ throws IOException {
assert fsn.hasReadLock();
checkBlock(fsn, previous);
@@ -662,8 +691,8 @@ class FSDirWriteFileOp {
"allocation of a new block in " + src + ". Returning previously" +
" allocated block " + lastBlockInFile);
long offset = file.computeFileSize();
- BlockInfoContiguousUnderConstruction lastBlockUC =
- (BlockInfoContiguousUnderConstruction) lastBlockInFile;
+ BlockInfoUnderConstruction lastBlockUC =
+ (BlockInfoUnderConstruction) lastBlockInFile;
onRetryBlock[0] = makeLocatedBlock(fsn, lastBlockInFile,
lastBlockUC.getExpectedStorageLocations(), offset);
return new FileState(file, src, iip);
@@ -688,14 +717,8 @@ class FSDirWriteFileOp {
checkBlock(fsn, last);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
src = fsn.dir.resolvePath(pc, src, pathComponents);
- boolean success = completeFileInternal(fsn, src, holder,
- ExtendedBlock.getLocalBlock(last),
- fileId);
- if (success) {
- NameNode.stateChangeLog.info("DIR* completeFile: " + srcArg
- + " is closed by " + holder);
- }
- return success;
+ return completeFileInternal(fsn, src, holder,
+ ExtendedBlock.getLocalBlock(last), fileId);
}
private static boolean completeFileInternal(
@@ -760,16 +783,18 @@ class FSDirWriteFileOp {
private static INodeFile newINodeFile(
long id, PermissionStatus permissions, long mtime, long atime,
- short replication, long preferredBlockSize, byte storagePolicyId) {
+ short replication, long preferredBlockSize, byte storagePolicyId,
+ boolean isStriped) {
return new INodeFile(id, null, permissions, mtime, atime,
BlockInfo.EMPTY_ARRAY, replication, preferredBlockSize,
- storagePolicyId);
+ storagePolicyId, isStriped);
}
private static INodeFile newINodeFile(long id, PermissionStatus permissions,
- long mtime, long atime, short replication, long preferredBlockSize) {
+ long mtime, long atime, short replication, long preferredBlockSize,
+ boolean isStriped) {
return newINodeFile(id, permissions, mtime, atime, replication,
- preferredBlockSize, (byte)0);
+ preferredBlockSize, (byte)0, isStriped);
}
/**
@@ -797,13 +822,12 @@ class FSDirWriteFileOp {
* @param targets target datanodes where replicas of the new block is placed
* @throws QuotaExceededException If addition of block exceeds space quota
*/
- private static void saveAllocatedBlock(
- FSNamesystem fsn, String src, INodesInPath inodesInPath, Block newBlock,
- DatanodeStorageInfo[] targets)
- throws IOException {
+ private static void saveAllocatedBlock(FSNamesystem fsn, String src,
+ INodesInPath inodesInPath, Block newBlock, DatanodeStorageInfo[] targets,
+ boolean isStriped) throws IOException {
assert fsn.hasWriteLock();
- BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock,
- targets);
+ BlockInfo b = addBlock(fsn.dir, src, inodesInPath, newBlock, targets,
+ isStriped);
NameNode.stateChangeLog.info("BLOCK* allocate " + b + " for " + src);
DatanodeStorageInfo.incrementBlocksScheduled(targets);
}
@@ -852,17 +876,19 @@ class FSDirWriteFileOp {
static class ValidateAddBlockResult {
final long blockSize;
- final int replication;
+ final int numTargets;
final byte storagePolicyID;
final String clientMachine;
+ final boolean isStriped;
ValidateAddBlockResult(
- long blockSize, int replication, byte storagePolicyID,
- String clientMachine) {
+ long blockSize, int numTargets, byte storagePolicyID,
+ String clientMachine, boolean isStriped) {
this.blockSize = blockSize;
- this.replication = replication;
+ this.numTargets = numTargets;
this.storagePolicyID = storagePolicyID;
this.clientMachine = clientMachine;
+ this.isStriped = isStriped;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index ccee1ae..ce48595 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -201,6 +201,9 @@ public class FSDirectory implements Closeable {
@VisibleForTesting
public final EncryptionZoneManager ezManager;
+ @VisibleForTesting
+ public final ErasureCodingZoneManager ecZoneManager;
+
/**
* Caches frequently used file names used in {@link INode} to reuse
* byte[] objects and reduce heap usage.
@@ -292,6 +295,7 @@ public class FSDirectory implements Closeable {
namesystem = ns;
this.editLog = ns.getEditLog();
ezManager = new EncryptionZoneManager(this, conf);
+ ecZoneManager = new ErasureCodingZoneManager(this);
}
FSNamesystem getFSNamesystem() {
@@ -513,7 +517,7 @@ public class FSDirectory implements Closeable {
final INodeFile fileINode = iip.getLastINode().asFile();
EnumCounters<StorageType> typeSpaceDeltas =
getStorageTypeDeltas(fileINode.getStoragePolicyID(), ssDelta,
- replication, replication);;
+ replication, replication);
updateCount(iip, iip.length() - 1,
new QuotaCounts.Builder().nameSpace(nsDelta).storageSpace(ssDelta * replication).
typeSpaces(typeSpaceDeltas).build(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 3dd076d..246fcd8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -36,16 +36,20 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
@@ -413,7 +417,9 @@ public class FSEditLogLoader {
// Update the salient file attributes.
newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
newFile.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- updateBlocks(fsDir, addCloseOp, iip, newFile);
+ ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsDir.getFSNamesystem(), iip);
+ updateBlocks(fsDir, addCloseOp, iip, newFile, ecZone);
break;
}
case OP_CLOSE: {
@@ -433,7 +439,9 @@ public class FSEditLogLoader {
// Update the salient file attributes.
file.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
file.setModificationTime(addCloseOp.mtime, Snapshot.CURRENT_STATE_ID);
- updateBlocks(fsDir, addCloseOp, iip, file);
+ ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsDir.getFSNamesystem(), iip);
+ updateBlocks(fsDir, addCloseOp, iip, file, ecZone);
// Now close the file
if (!file.isUnderConstruction() &&
@@ -491,8 +499,10 @@ public class FSEditLogLoader {
INodesInPath iip = fsDir.getINodesInPath(path, true);
INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// Update in-memory data structures
- updateBlocks(fsDir, updateOp, iip, oldFile);
-
+ ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsDir.getFSNamesystem(), iip);
+ updateBlocks(fsDir, updateOp, iip, oldFile, ecZone);
+
if (toAddRetryCache) {
fsNamesys.addCacheEntry(updateOp.rpcClientId, updateOp.rpcCallId);
}
@@ -505,9 +515,12 @@ public class FSEditLogLoader {
FSNamesystem.LOG.debug(op.opCode + ": " + path +
" new block id : " + addBlockOp.getLastBlock().getBlockId());
}
- INodeFile oldFile = INodeFile.valueOf(fsDir.getINode(path), path);
+ INodesInPath iip = fsDir.getINodesInPath(path, true);
+ INodeFile oldFile = INodeFile.valueOf(iip.getLastINode(), path);
// add the new block to the INodeFile
- addNewBlock(fsDir, addBlockOp, oldFile);
+ ErasureCodingZone ecZone = FSDirErasureCodingOp.getErasureCodingZone(
+ fsDir.getFSNamesystem(), iip);
+ addNewBlock(addBlockOp, oldFile, ecZone);
break;
}
case OP_SET_REPLICATION: {
@@ -787,8 +800,15 @@ public class FSEditLogLoader {
}
case OP_ALLOCATE_BLOCK_ID: {
AllocateBlockIdOp allocateBlockIdOp = (AllocateBlockIdOp) op;
- fsNamesys.getBlockIdManager().setLastAllocatedBlockId(
- allocateBlockIdOp.blockId);
+ if (BlockIdManager.isStripedBlockID(allocateBlockIdOp.blockId)) {
+ // ALLOCATE_BLOCK_ID is added for sequential block id, thus if the id
+ // is negative, it must belong to striped blocks
+ fsNamesys.getBlockIdManager().setLastAllocatedStripedBlockId(
+ allocateBlockIdOp.blockId);
+ } else {
+ fsNamesys.getBlockIdManager().setLastAllocatedContiguousBlockId(
+ allocateBlockIdOp.blockId);
+ }
break;
}
case OP_ROLLING_UPGRADE_START: {
@@ -941,16 +961,16 @@ public class FSEditLogLoader {
/**
* Add a new block into the given INodeFile
*/
- private void addNewBlock(FSDirectory fsDir, AddBlockOp op, INodeFile file)
- throws IOException {
+ private void addNewBlock(AddBlockOp op, INodeFile file,
+ ErasureCodingZone ecZone) throws IOException {
BlockInfo[] oldBlocks = file.getBlocks();
Block pBlock = op.getPenultimateBlock();
Block newBlock= op.getLastBlock();
if (pBlock != null) { // the penultimate block is not null
- Preconditions.checkState(oldBlocks != null && oldBlocks.length > 0);
+ assert oldBlocks != null && oldBlocks.length > 0;
// compare pBlock with the last block of oldBlocks
- Block oldLastBlock = oldBlocks[oldBlocks.length - 1];
+ BlockInfo oldLastBlock = oldBlocks[oldBlocks.length - 1];
if (oldLastBlock.getBlockId() != pBlock.getBlockId()
|| oldLastBlock.getGenerationStamp() != pBlock.getGenerationStamp()) {
throw new IOException(
@@ -960,19 +980,25 @@ public class FSEditLogLoader {
}
oldLastBlock.setNumBytes(pBlock.getNumBytes());
- if (oldLastBlock instanceof BlockInfoContiguousUnderConstruction) {
- fsNamesys.getBlockManager().forceCompleteBlock(file,
- (BlockInfoContiguousUnderConstruction) oldLastBlock);
+ if (!oldLastBlock.isComplete()) {
+ fsNamesys.getBlockManager().forceCompleteBlock(file, oldLastBlock);
fsNamesys.getBlockManager().processQueuedMessagesForBlock(pBlock);
}
} else { // the penultimate block is null
Preconditions.checkState(oldBlocks == null || oldBlocks.length == 0);
}
// add the new block
- BlockInfo newBI = new BlockInfoContiguousUnderConstruction(
- newBlock, file.getPreferredBlockReplication());
- fsNamesys.getBlockManager().addBlockCollection(newBI, file);
- file.addBlock(newBI);
+ final BlockInfo newBlockInfo;
+ boolean isStriped = ecZone != null;
+ if (isStriped) {
+ newBlockInfo = new BlockInfoStripedUnderConstruction(newBlock,
+ ecZone.getSchema(), ecZone.getCellSize());
+ } else {
+ newBlockInfo = new BlockInfoContiguousUnderConstruction(newBlock,
+ file.getPreferredBlockReplication());
+ }
+ fsNamesys.getBlockManager().addBlockCollectionWithCheck(newBlockInfo, file);
+ file.addBlock(newBlockInfo);
fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
}
@@ -981,7 +1007,8 @@ public class FSEditLogLoader {
* @throws IOException
*/
private void updateBlocks(FSDirectory fsDir, BlockListUpdatingOp op,
- INodesInPath iip, INodeFile file) throws IOException {
+ INodesInPath iip, INodeFile file, ErasureCodingZone ecZone)
+ throws IOException {
// Update its block list
BlockInfo[] oldBlocks = file.getBlocks();
Block[] newBlocks = op.getBlocks();
@@ -1010,11 +1037,10 @@ public class FSEditLogLoader {
oldBlock.getGenerationStamp() != newBlock.getGenerationStamp();
oldBlock.setGenerationStamp(newBlock.getGenerationStamp());
- if (oldBlock instanceof BlockInfoContiguousUnderConstruction &&
+ if (!oldBlock.isComplete() &&
(!isLastBlock || op.shouldCompleteLastBlock())) {
changeMade = true;
- fsNamesys.getBlockManager().forceCompleteBlock(file,
- (BlockInfoContiguousUnderConstruction) oldBlock);
+ fsNamesys.getBlockManager().forceCompleteBlock(file, oldBlock);
}
if (changeMade) {
// The state or gen-stamp of the block has changed. So, we may be
@@ -1041,25 +1067,38 @@ public class FSEditLogLoader {
throw new IOException("Trying to delete non-existant block " + oldBlock);
}
} else if (newBlocks.length > oldBlocks.length) {
+ final boolean isStriped = ecZone != null;
// We're adding blocks
for (int i = oldBlocks.length; i < newBlocks.length; i++) {
Block newBlock = newBlocks[i];
- BlockInfo newBI;
+ final BlockInfo newBI;
if (!op.shouldCompleteLastBlock()) {
// TODO: shouldn't this only be true for the last block?
// what about an old-version fsync() where fsync isn't called
// until several blocks in?
- newBI = new BlockInfoContiguousUnderConstruction(
- newBlock, file.getPreferredBlockReplication());
+ if (isStriped) {
+ newBI = new BlockInfoStripedUnderConstruction(newBlock,
+ ecZone.getSchema(), ecZone.getCellSize());
+ } else {
+ newBI = new BlockInfoContiguousUnderConstruction(newBlock,
+ file.getPreferredBlockReplication());
+ }
} else {
// OP_CLOSE should add finalized blocks. This code path
// is only executed when loading edits written by prior
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
- newBI = new BlockInfoContiguous(newBlock,
- file.getPreferredBlockReplication());
+ // TODO: ECSchema can be restored from persisted file (HDFS-7859).
+ if (isStriped) {
+ newBI = new BlockInfoStriped(newBlock,
+ ErasureCodingSchemaManager.getSystemDefaultSchema(),
+ ecZone.getCellSize());
+ } else {
+ newBI = new BlockInfoContiguous(newBlock,
+ file.getPreferredBlockReplication());
+ }
}
- fsNamesys.getBlockManager().addBlockCollection(newBI, file);
+ fsNamesys.getBlockManager().addBlockCollectionWithCheck(newBI, file);
file.addBlock(newBI);
fsNamesys.getBlockManager().processQueuedMessagesForBlock(newBlock);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index 30517d0..1a1d6b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutFlags;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
@@ -359,7 +360,8 @@ public class FSImageFormat {
// read the max sequential block ID.
long maxSequentialBlockId = in.readLong();
- namesystem.getBlockIdManager().setLastAllocatedBlockId(maxSequentialBlockId);
+ namesystem.getBlockIdManager().setLastAllocatedContiguousBlockId(
+ maxSequentialBlockId);
} else {
long startingGenStamp = namesystem.getBlockIdManager()
@@ -691,7 +693,7 @@ public class FSImageFormat {
if (blocks != null) {
final BlockManager bm = namesystem.getBlockManager();
for (int i = 0; i < blocks.length; i++) {
- file.setBlock(i, bm.addBlockCollection(blocks[i], file));
+ file.setBlock(i, bm.addBlockCollectionWithCheck(blocks[i], file));
}
}
}
@@ -791,14 +793,14 @@ public class FSImageFormat {
counter.increment();
}
- final INodeFile file = new INodeFile(inodeId, localName, permissions,
+ INodeFile file = new INodeFile(inodeId, localName, permissions,
modificationTime, atime, blocks, replication, blockSize);
if (underConstruction) {
file.toUnderConstruction(clientName, clientMachine);
}
- return fileDiffs == null ? file : new INodeFile(file, fileDiffs);
- } else if (numBlocks == -1) {
- //directory
+ return fileDiffs == null ? file : new INodeFile(file, fileDiffs);
+ } else if (numBlocks == -1) {
+ //directory
//read quotas
final long nsQuota = in.readLong();
@@ -897,7 +899,7 @@ public class FSImageFormat {
final long preferredBlockSize = in.readLong();
return new INodeFileAttributes.SnapshotCopy(name, permissions, null, modificationTime,
- accessTime, replication, preferredBlockSize, (byte) 0, null);
+ accessTime, replication, preferredBlockSize, (byte) 0, null, false);
}
public INodeDirectoryAttributes loadINodeDirectoryAttributes(DataInput in)
@@ -961,8 +963,8 @@ public class FSImageFormat {
if (oldnode.numBlocks() > 0) {
BlockInfo ucBlock = cons.getLastBlock();
// we do not replace the inode, just replace the last block of oldnode
- BlockInfo info = namesystem.getBlockManager().addBlockCollection(
- ucBlock, oldnode);
+ BlockInfo info = namesystem.getBlockManager()
+ .addBlockCollectionWithCheck(ucBlock, oldnode);
oldnode.setBlock(oldnode.numBlocks() - 1, info);
}
@@ -1269,7 +1271,7 @@ public class FSImageFormat {
out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampV1());
out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampV2());
out.writeLong(sourceNamesystem.getBlockIdManager().getGenerationStampAtblockIdSwitch());
- out.writeLong(sourceNamesystem.getBlockIdManager().getLastAllocatedBlockId());
+ out.writeLong(sourceNamesystem.getBlockIdManager().getLastAllocatedContiguousBlockId());
out.writeLong(context.getTxId());
out.writeLong(sourceNamesystem.dir.getLastInodeId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index e8378e5..c4fc1ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -45,6 +45,8 @@ import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStripedUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SaverContext;
@@ -68,6 +70,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.protobuf.ByteString;
+import org.apache.hadoop.io.erasurecode.ECSchema;
@InterfaceAudience.Private
public final class FSImageFormatPBINode {
@@ -218,7 +221,7 @@ public final class FSImageFormatPBINode {
final BlockInfo[] blocks = file.getBlocks();
if (blocks != null) {
for (int i = 0; i < blocks.length; i++) {
- file.setBlock(i, bm.addBlockCollection(blocks[i], file));
+ file.setBlock(i, bm.addBlockCollectionWithCheck(blocks[i], file));
}
}
}
@@ -331,27 +334,39 @@ public final class FSImageFormatPBINode {
INodeSection.INodeFile f = n.getFile();
List<BlockProto> bp = f.getBlocksList();
short replication = (short) f.getReplication();
+ boolean isStriped = f.getIsStriped();
LoaderContext state = parent.getLoaderContext();
+ ECSchema schema = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ if (isStriped) {
+ Preconditions.checkState(f.hasStripingCellSize());
+ }
BlockInfo[] blocks = new BlockInfo[bp.size()];
- for (int i = 0, e = bp.size(); i < e; ++i) {
- blocks[i] =
- new BlockInfoContiguous(PBHelper.convert(bp.get(i)), replication);
+ for (int i = 0; i < bp.size(); ++i) {
+ BlockProto b = bp.get(i);
+ if (isStriped) {
+ blocks[i] = new BlockInfoStriped(PBHelper.convert(b), schema,
+ (int)f.getStripingCellSize());
+ } else {
+ blocks[i] = new BlockInfoContiguous(PBHelper.convert(b),
+ replication);
+ }
}
+
final PermissionStatus permissions = loadPermission(f.getPermission(),
parent.getLoaderContext().getStringTable());
final INodeFile file = new INodeFile(n.getId(),
n.getName().toByteArray(), permissions, f.getModificationTime(),
f.getAccessTime(), blocks, replication, f.getPreferredBlockSize(),
- (byte)f.getStoragePolicyID());
+ (byte)f.getStoragePolicyID(), isStriped);
if (f.hasAcl()) {
int[] entries = AclEntryStatusFormat.toInt(loadAclEntries(
f.getAcl(), state.getStringTable()));
file.addAclFeature(new AclFeature(entries));
}
-
+
if (f.hasXAttrs()) {
file.addXAttrFeature(new XAttrFeature(
loadXAttrs(f.getXAttrs(), state.getStringTable())));
@@ -364,8 +379,16 @@ public final class FSImageFormatPBINode {
if (blocks.length > 0) {
BlockInfo lastBlk = file.getLastBlock();
// replace the last block of file
- file.setBlock(file.numBlocks() - 1, new BlockInfoContiguousUnderConstruction(
- lastBlk, replication));
+ final BlockInfo ucBlk;
+ if (isStriped) {
+ BlockInfoStriped striped = (BlockInfoStriped) lastBlk;
+ ucBlk = new BlockInfoStripedUnderConstruction(striped,
+ schema, (int)f.getStripingCellSize());
+ } else {
+ ucBlk = new BlockInfoContiguousUnderConstruction(lastBlk,
+ replication);
+ }
+ file.setBlock(file.numBlocks() - 1, ucBlk);
}
}
return file;
@@ -479,7 +502,8 @@ public final class FSImageFormatPBINode {
.setPermission(buildPermissionStatus(file, state.getStringMap()))
.setPreferredBlockSize(file.getPreferredBlockSize())
.setReplication(file.getFileReplication())
- .setStoragePolicyID(file.getLocalStoragePolicyID());
+ .setStoragePolicyID(file.getLocalStoragePolicyID())
+ .setIsStriped(file.isStriped());
AclFeature f = file.getAclFeature();
if (f != null) {
@@ -633,13 +657,24 @@ public final class FSImageFormatPBINode {
private void save(OutputStream out, INodeFile n) throws IOException {
INodeSection.INodeFile.Builder b = buildINodeFile(n,
parent.getSaverContext());
+ BlockInfo[] blocks = n.getBlocks();
- if (n.getBlocks() != null) {
+ if (blocks != null) {
for (Block block : n.getBlocks()) {
b.addBlocks(PBHelper.convert(block));
}
}
+ if (n.isStriped()) {
+ if (blocks != null && blocks.length > 0) {
+ BlockInfo firstBlock = blocks[0];
+ Preconditions.checkState(firstBlock.isStriped());
+ b.setStripingCellSize(((BlockInfoStriped)firstBlock).getCellSize());
+ } else {
+ b.setStripingCellSize(HdfsConstants.BLOCK_STRIPED_CELL_SIZE);
+ }
+ }
+
FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature();
if (uc != null) {
INodeSection.FileUnderConstructionFeature f =
@@ -668,7 +703,7 @@ public final class FSImageFormatPBINode {
r.writeDelimitedTo(out);
}
- private final INodeSection.INode.Builder buildINodeCommon(INode n) {
+ private INodeSection.INode.Builder buildINodeCommon(INode n) {
return INodeSection.INode.newBuilder()
.setId(n.getId())
.setName(ByteString.copyFrom(n.getLocalNameBytes()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 7c8a857..ef0cc1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -297,7 +297,11 @@ public final class FSImageFormatProtobuf {
blockIdManager.setGenerationStampV1(s.getGenstampV1());
blockIdManager.setGenerationStampV2(s.getGenstampV2());
blockIdManager.setGenerationStampV1Limit(s.getGenstampV1Limit());
- blockIdManager.setLastAllocatedBlockId(s.getLastAllocatedBlockId());
+ blockIdManager.setLastAllocatedContiguousBlockId(s.getLastAllocatedBlockId());
+ if (s.hasLastAllocatedStripedBlockId()) {
+ blockIdManager.setLastAllocatedStripedBlockId(
+ s.getLastAllocatedStripedBlockId());
+ }
imgTxId = s.getTransactionId();
if (s.hasRollingUpgradeStartTime()
&& fsn.getFSImage().hasRollbackFSImage()) {
@@ -549,7 +553,8 @@ public final class FSImageFormatProtobuf {
.setGenstampV1(blockIdManager.getGenerationStampV1())
.setGenstampV1Limit(blockIdManager.getGenerationStampV1Limit())
.setGenstampV2(blockIdManager.getGenerationStampV2())
- .setLastAllocatedBlockId(blockIdManager.getLastAllocatedBlockId())
+ .setLastAllocatedBlockId(blockIdManager.getLastAllocatedContiguousBlockId())
+ .setLastAllocatedStripedBlockId(blockIdManager.getLastAllocatedStripedBlockId())
.setTransactionId(context.getTxId());
// We use the non-locked version of getNamespaceInfo here since
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index f71cf0b..c204f55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -125,12 +125,12 @@ public class FSImageSerialization {
short blockReplication = in.readShort();
long modificationTime = in.readLong();
long preferredBlockSize = in.readLong();
-
+
int numBlocks = in.readInt();
BlockInfo[] blocks = new BlockInfo[numBlocks];
Block blk = new Block();
int i = 0;
- for (; i < numBlocks-1; i++) {
+ for (; i < numBlocks - 1; i++) {
blk.readFields(in);
blocks[i] = new BlockInfoContiguous(blk, blockReplication);
}
@@ -138,8 +138,9 @@ public class FSImageSerialization {
if(numBlocks > 0) {
blk.readFields(in);
blocks[i] = new BlockInfoContiguousUnderConstruction(
- blk, blockReplication, BlockUCState.UNDER_CONSTRUCTION, null);
+ blk, blockReplication, BlockUCState.UNDER_CONSTRUCTION, null);
}
+
PermissionStatus perm = PermissionStatus.read(in);
String clientName = readString(in);
String clientMachine = readString(in);
@@ -180,9 +181,9 @@ public class FSImageSerialization {
/**
* Serialize a {@link INodeFile} node
- * @param node The node to write
+ * @param file The node to write
* @param out The {@link DataOutputStream} where the fields are written
- * @param writeBlock Whether to write block information
+ * @param writeUnderConstruction Whether to write block information
*/
public static void writeINodeFile(INodeFile file, DataOutput out,
boolean writeUnderConstruction) throws IOException {
@@ -305,7 +306,7 @@ public class FSImageSerialization {
if (!isWithName) {
Preconditions.checkState(ref instanceof INodeReference.DstReference);
// dst snapshot id
- out.writeInt(((INodeReference.DstReference) ref).getDstSnapshotId());
+ out.writeInt(ref.getDstSnapshotId());
} else {
out.writeInt(((INodeReference.WithName) ref).getLastSnapshotId());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4cc3073..c0bfcc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import static org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT;
@@ -58,9 +57,9 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROL
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ENABLE_RETRY_CACHE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC_DEFAULT;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_INODE_ATTRIBUTES_PROVIDER_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY;
@@ -138,6 +137,7 @@ import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
@@ -149,6 +149,7 @@ import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
@@ -157,10 +158,8 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
-import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.ServiceFailedException;
-import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
@@ -168,6 +167,7 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
@@ -177,14 +177,17 @@ import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeException;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@@ -200,6 +203,9 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -251,6 +257,7 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.RetryCache;
import org.apache.hadoop.ipc.Server;
@@ -414,6 +421,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private final BlockManager blockManager;
private final SnapshotManager snapshotManager;
private final CacheManager cacheManager;
+ private final ErasureCodingSchemaManager ecSchemaManager;
private final DatanodeStatistics datanodeStatistics;
private String nameserviceId;
@@ -590,6 +598,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
leaseManager.removeAllLeases();
snapshotManager.clearSnapshottableDirs();
cacheManager.clear();
+ ecSchemaManager.clear();
setImageLoaded(false);
blockManager.clear();
}
@@ -827,6 +836,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
this.dir = new FSDirectory(this, conf);
this.snapshotManager = new SnapshotManager(dir);
this.cacheManager = new CacheManager(this, conf, blockManager);
+ this.ecSchemaManager = new ErasureCodingSchemaManager();
this.safeMode = new SafeModeInfo(conf);
this.topConf = new TopConf(conf);
this.auditLoggers = initAuditLoggers(conf);
@@ -1755,8 +1765,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
LocatedBlocks blocks = res.blocks;
if (blocks != null) {
+ List<LocatedBlock> blkList = blocks.getLocatedBlocks();
+ if (blkList == null || blkList.size() == 0 ||
+ blkList.get(0) instanceof LocatedStripedBlock) {
+ // no need to sort locations for striped blocks
+ return blocks;
+ }
blockManager.getDatanodeManager().sortLocatedBlocks(
- clientMachine, blocks.getLocatedBlocks());
+ clientMachine, blkList);
// lastBlock is not part of getLocatedBlocks(), might need to sort it too
LocatedBlock lastBlock = blocks.getLastLocatedBlock();
@@ -2114,7 +2130,19 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (!DFSUtil.isValidName(src)) {
throw new InvalidPathException(src);
}
- blockManager.verifyReplication(src, replication, clientMachine);
+
+ checkOperation(OperationCategory.READ);
+ readLock();
+ try {
+ checkOperation(OperationCategory.READ);
+ if (!FSDirErasureCodingOp.isInErasureCodingZone(this, src)) {
+ blockManager.verifyReplication(src, replication, clientMachine);
+ }
+ } finally {
+ readUnlock();
+ }
+
+ checkOperation(OperationCategory.WRITE);
if (blockSize < minBlockSize) {
throw new IOException("Specified block size is less than configured" +
" minimum value (" + DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY
@@ -2455,6 +2483,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final long preferredblocksize;
final byte storagePolicyID;
final List<DatanodeStorageInfo> chosen;
+ final boolean isStriped;
checkOperation(OperationCategory.READ);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
FSPermissionChecker pc = getPermissionChecker();
@@ -2481,6 +2510,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
clientnode = blockManager.getDatanodeManager().getDatanodeByHost(clientMachine);
preferredblocksize = file.getPreferredBlockSize();
storagePolicyID = file.getStoragePolicyID();
+ isStriped = file.isStriped();
//find datanode storages
final DatanodeManager dm = blockManager.getDatanodeManager();
@@ -2496,7 +2526,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// choose new datanodes.
final DatanodeStorageInfo[] targets = blockManager.chooseTarget4AdditionalDatanode(
src, numAdditionalNodes, clientnode, chosen,
- excludes, preferredblocksize, storagePolicyID);
+ excludes, preferredblocksize, storagePolicyID, isStriped);
final LocatedBlock lb = BlockManager.newLocatedBlock(
blk, targets, -1, false);
blockManager.setBlockToken(lb, BlockTokenIdentifier.AccessMode.COPY);
@@ -2590,15 +2620,20 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
writeUnlock();
}
getEditLog().logSync();
+ if (success) {
+ NameNode.stateChangeLog.info("DIR* completeFile: " + src
+ + " is closed by " + holder);
+ }
return success;
}
/**
* Create new block with a unique block id and a new generation stamp.
+ * @param isStriped is the file under striping or contiguous layout?
*/
- Block createNewBlock() throws IOException {
+ Block createNewBlock(boolean isStriped) throws IOException {
assert hasWriteLock();
- Block b = new Block(nextBlockId(), 0, 0);
+ Block b = new Block(nextBlockId(isStriped), 0, 0);
// Increment the generation stamp for every new block.
b.setGenerationStamp(nextGenerationStamp(false));
return b;
@@ -2792,7 +2827,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (trackBlockCounts) {
if (b.isComplete()) {
numRemovedComplete++;
- if (blockManager.checkMinReplication(b)) {
+ if (blockManager.hasMinStorage(b, b.numNodes())) {
numRemovedSafe++;
}
}
@@ -3024,7 +3059,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
curBlock = blocks[nrCompleteBlocks];
if(!curBlock.isComplete())
break;
- assert blockManager.checkMinReplication(curBlock) :
+ assert blockManager.hasMinStorage(curBlock) :
"A COMPLETE block is not minimally replicated in " + src;
}
@@ -3059,8 +3094,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
BlockInfo penultimateBlock = pendingFile.getPenultimateBlock();
// If penultimate block doesn't exist then its minReplication is met
- boolean penultimateBlockMinReplication = penultimateBlock == null ? true :
- blockManager.checkMinReplication(penultimateBlock);
+ boolean penultimateBlockMinStorage = penultimateBlock == null ||
+ blockManager.hasMinStorage(penultimateBlock);
switch(lastBlockState) {
case COMPLETE:
@@ -3068,8 +3103,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
break;
case COMMITTED:
// Close file if committed blocks are minimally replicated
- if(penultimateBlockMinReplication &&
- blockManager.checkMinReplication(lastBlock)) {
+ if(penultimateBlockMinStorage &&
+ blockManager.hasMinStorage(lastBlock)) {
finalizeINodeFileUnderConstruction(src, pendingFile,
iip.getLatestSnapshotId());
NameNode.stateChangeLog.warn("BLOCK*"
@@ -3089,16 +3124,19 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throw new AlreadyBeingCreatedException(message);
case UNDER_CONSTRUCTION:
case UNDER_RECOVERY:
- final BlockInfoContiguousUnderConstruction uc = (BlockInfoContiguousUnderConstruction)lastBlock;
+ // TODO support truncate of striped blocks
+ final BlockInfoUnderConstruction uc =
+ (BlockInfoUnderConstruction)lastBlock;
// determine if last block was intended to be truncated
Block recoveryBlock = uc.getTruncateBlock();
boolean truncateRecovery = recoveryBlock != null;
boolean copyOnTruncate = truncateRecovery &&
- recoveryBlock.getBlockId() != uc.getBlockId();
+ recoveryBlock.getBlockId() != uc.toBlock().getBlockId();
assert !copyOnTruncate ||
- recoveryBlock.getBlockId() < uc.getBlockId() &&
- recoveryBlock.getGenerationStamp() < uc.getGenerationStamp() &&
- recoveryBlock.getNumBytes() > uc.getNumBytes() :
+ recoveryBlock.getBlockId() < uc.toBlock().getBlockId() &&
+ recoveryBlock.getGenerationStamp() < uc.toBlock().
+ getGenerationStamp() &&
+ recoveryBlock.getNumBytes() > uc.toBlock().getNumBytes() :
"wrong recoveryBlock";
// setup the last block locations from the blockManager if not known
@@ -3106,7 +3144,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
uc.setExpectedLocations(blockManager.getStorages(lastBlock));
}
- if (uc.getNumExpectedLocations() == 0 && uc.getNumBytes() == 0) {
+ if (uc.getNumExpectedLocations() == 0 &&
+ uc.toBlock().getNumBytes() == 0) {
// There is no datanode reported to this block.
// may be client have crashed before writing data to pipeline.
// This blocks doesn't need any recovery.
@@ -3119,10 +3158,11 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return true;
}
// start recovery of the last block for this file
- long blockRecoveryId = nextGenerationStamp(blockIdManager.isLegacyBlock(uc));
+ long blockRecoveryId =
+ nextGenerationStamp(blockIdManager.isLegacyBlock(uc.toBlock()));
lease = reassignLease(lease, src, recoveryLeaseHolder, pendingFile);
if(copyOnTruncate) {
- uc.setGenerationStamp(blockRecoveryId);
+ uc.toBlock().setGenerationStamp(blockRecoveryId);
} else if(truncateRecovery) {
recoveryBlock.setGenerationStamp(blockRecoveryId);
}
@@ -3167,10 +3207,32 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
// Adjust disk space consumption if required
- final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
+ final long diff;
+ final short replicationFactor;
+ if (fileINode.isStriped()) {
+ final ErasureCodingZone ecZone = FSDirErasureCodingOp
+ .getErasureCodingZone(this, iip);
+ final ECSchema ecSchema = ecZone.getSchema();
+ final short numDataUnits = (short) ecSchema.getNumDataUnits();
+ final short numParityUnits = (short) ecSchema.getNumParityUnits();
+
+ final long numBlocks = numDataUnits + numParityUnits;
+ final long fullBlockGroupSize =
+ fileINode.getPreferredBlockSize() * numBlocks;
+
+ final BlockInfoStriped striped = new BlockInfoStriped(commitBlock,
+ ecSchema, ecZone.getCellSize());
+ final long actualBlockGroupSize = striped.spaceConsumed();
+
+ diff = fullBlockGroupSize - actualBlockGroupSize;
+ replicationFactor = (short) 1;
+ } else {
+ diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
+ replicationFactor = fileINode.getFileReplication();
+ }
if (diff > 0) {
try {
- dir.updateSpaceConsumed(iip, 0, -diff, fileINode.getFileReplication());
+ dir.updateSpaceConsumed(iip, 0, -diff, replicationFactor);
} catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e);
}
@@ -3200,14 +3262,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
@VisibleForTesting
- BlockInfo getStoredBlock(Block block) {
+ public BlockInfo getStoredBlock(Block block) {
return blockManager.getStoredBlock(block);
}
@Override
- public boolean isInSnapshot(BlockInfoContiguousUnderConstruction blockUC) {
+ public boolean isInSnapshot(BlockCollection bc) {
assert hasReadLock();
- final BlockCollection bc = blockUC.getBlockCollection();
if (bc == null || !(bc instanceof INodeFile)
|| !bc.isUnderConstruction()) {
return false;
@@ -3252,7 +3313,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
waitForLoadingFSImage();
writeLock();
boolean copyTruncate = false;
- BlockInfoContiguousUnderConstruction truncatedBlock = null;
+ BlockInfo truncatedBlock = null;
try {
checkOperation(OperationCategory.WRITE);
// If a DN tries to commit to the standby, the recovery will
@@ -3309,9 +3370,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
return;
}
- truncatedBlock = (BlockInfoContiguousUnderConstruction) iFile
- .getLastBlock();
- long recoveryId = truncatedBlock.getBlockRecoveryId();
+ truncatedBlock = iFile.getLastBlock();
+ final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)truncatedBlock;
+ final long recoveryId = uc.getBlockRecoveryId();
copyTruncate = truncatedBlock.getBlockId() != storedBlock.getBlockId();
if(recoveryId != newgenerationstamp) {
throw new IOException("The recovery id " + newgenerationstamp
@@ -3325,8 +3386,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (remove) {
blockManager.removeBlock(storedBlock);
}
- }
- else {
+ } else {
// update last block
if(!copyTruncate) {
storedBlock.setGenerationStamp(newgenerationstamp);
@@ -3360,9 +3420,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
trimmedTargets.get(i).getStorageInfo(trimmedStorages.get(i));
if (storageInfo != null) {
if(copyTruncate) {
- storageInfo.addBlock(truncatedBlock);
+ storageInfo.addBlock(truncatedBlock, truncatedBlock);
} else {
- storageInfo.addBlock(storedBlock);
+ storageInfo.addBlock(storedBlock, storedBlock);
}
}
}
@@ -3374,12 +3434,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
trimmedTargets.toArray(new DatanodeID[trimmedTargets.size()]),
trimmedStorages.toArray(new String[trimmedStorages.size()]));
if(copyTruncate) {
- iFile.setLastBlock(truncatedBlock, trimmedStorageInfos);
+ iFile.convertLastBlockToUC(truncatedBlock, trimmedStorageInfos);
} else {
- iFile.setLastBlock(storedBlock, trimmedStorageInfos);
+ iFile.convertLastBlockToUC(storedBlock, trimmedStorageInfos);
if (closeFile) {
- blockManager.markBlockReplicasAsCorrupt(storedBlock,
- oldGenerationStamp, oldNumBytes, trimmedStorageInfos);
+ blockManager.markBlockReplicasAsCorrupt(oldBlock.getLocalBlock(),
+ storedBlock, oldGenerationStamp, oldNumBytes,
+ trimmedStorageInfos);
}
}
}
@@ -3387,7 +3448,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (closeFile) {
if(copyTruncate) {
src = closeFileCommitBlocks(iFile, truncatedBlock);
- if(!iFile.isBlockInLatestSnapshot(storedBlock)) {
+ if(!iFile.isBlockInLatestSnapshot((BlockInfoContiguous) storedBlock)) {
blockManager.removeBlock(storedBlock);
}
} else {
@@ -3714,7 +3775,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
while (it.hasNext()) {
Block b = it.next();
- BlockInfo blockInfo = blockManager.getStoredBlock(b);
+ BlockInfo blockInfo = getStoredBlock(b);
if (blockInfo.getBlockCollection().getStoragePolicyID()
== lpPolicy.getId()) {
filesToDelete.add(blockInfo.getBlockCollection());
@@ -4353,10 +4414,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Increment number of safe blocks if current block has
* reached minimal replication.
- * @param replication current replication
+ * @param storageNum current number of replicas or number of internal blocks
+ * of a striped block group
+ * @param storedBlock current storedBlock which is either a
+ * BlockInfoContiguous or a BlockInfoStriped
*/
- private synchronized void incrementSafeBlockCount(short replication) {
- if (replication == safeReplication) {
+ private synchronized void incrementSafeBlockCount(short storageNum,
+ BlockInfo storedBlock) {
+ final int safe = storedBlock.isStriped() ?
+ ((BlockInfoStriped) storedBlock).getRealDataBlockNum() : safeReplication;
+ if (storageNum == safe) {
this.blockSafe++;
// Report startup progress only if we haven't completed startup yet.
@@ -4649,12 +4716,12 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
@Override
- public void incrementSafeBlockCount(int replication) {
+ public void incrementSafeBlockCount(int storageNum, BlockInfo storedBlock) {
// safeMode is volatile, and may be set to null at any time
SafeModeInfo safeMode = this.safeMode;
if (safeMode == null)
return;
- safeMode.incrementSafeBlockCount((short)replication);
+ safeMode.incrementSafeBlockCount((short) storageNum, storedBlock);
}
@Override
@@ -5154,11 +5221,13 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Increments, logs and then returns the block ID
+ * @param isStriped is the file under striping or contiguous layout?
*/
- private long nextBlockId() throws IOException {
+ private long nextBlockId(boolean isStriped) throws IOException {
assert hasWriteLock();
checkNameNodeSafeMode("Cannot get next block ID");
- final long blockId = blockIdManager.nextBlockId();
+ final long blockId = isStriped ?
+ blockIdManager.nextStripedBlockId() : blockIdManager.nextContiguousBlockId();
getEditLog().logAllocateBlockId(blockId);
// NB: callers sync the log
return blockId;
@@ -5262,29 +5331,32 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* Get a new generation stamp together with an access token for
* a block under construction
*
- * This method is called for recovering a failed pipeline or setting up
- * a pipeline to append to a block.
+ * This method is called for recovering a failed write or setting up
+ * a block for appended.
*
* @param block a block
* @param clientName the name of a client
* @return a located block with a new generation stamp and an access token
* @throws IOException if any error occurs
*/
- LocatedBlock updateBlockForPipeline(ExtendedBlock block,
+ LocatedBlock bumpBlockGenerationStamp(ExtendedBlock block,
String clientName) throws IOException {
- LocatedBlock locatedBlock;
+ final LocatedBlock locatedBlock;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
// check vadility of parameters
- checkUCBlock(block, clientName);
+ final INodeFile file = checkUCBlock(block, clientName);
// get a new generation stamp and an access token
block.setGenerationStamp(nextGenerationStamp(blockIdManager.isLegacyBlock(block.getLocalBlock())));
- locatedBlock = new LocatedBlock(block, new DatanodeInfo[0]);
- blockManager.setBlockToken(locatedBlock, BlockTokenIdentifier.AccessMode.WRITE);
+
+ locatedBlock = BlockManager.newLocatedBlock(
+ block, file.getLastBlock(), null, -1);
+ blockManager.setBlockToken(locatedBlock,
+ BlockTokenIdentifier.AccessMode.WRITE);
} finally {
writeUnlock();
}
@@ -5336,21 +5408,27 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
assert hasWriteLock();
// check the vadility of the block and lease holder name
final INodeFile pendingFile = checkUCBlock(oldBlock, clientName);
- final BlockInfoContiguousUnderConstruction blockinfo
- = (BlockInfoContiguousUnderConstruction)pendingFile.getLastBlock();
+ final BlockInfo lastBlock = pendingFile.getLastBlock();
+ final BlockInfoUnderConstruction blockinfo = (BlockInfoUnderConstruction)lastBlock;
// check new GS & length: this is not expected
- if (newBlock.getGenerationStamp() <= blockinfo.getGenerationStamp() ||
- newBlock.getNumBytes() < blockinfo.getNumBytes()) {
- String msg = "Update " + oldBlock + " (len = " +
- blockinfo.getNumBytes() + ") to an older state: " + newBlock +
- " (len = " + newBlock.getNumBytes() +")";
+ if (newBlock.getGenerationStamp() <= lastBlock.getGenerationStamp()) {
+ final String msg = "Update " + oldBlock + " but the new block " + newBlock
+ + " does not have a larger generation stamp than the last block "
+ + lastBlock;
+ LOG.warn(msg);
+ throw new IOException(msg);
+ }
+ if (newBlock.getNumBytes() < lastBlock.getNumBytes()) {
+ final String msg = "Update " + oldBlock + " (size="
+ + oldBlock.getNumBytes() + ") to a smaller size block " + newBlock
+ + " (size=" + newBlock.getNumBytes() + ")";
LOG.warn(msg);
throw new IOException(msg);
}
// Update old block with the new generation stamp and new length
- blockinfo.setNumBytes(newBlock.getNumBytes());
+ lastBlock.setNumBytes(newBlock.getNumBytes());
blockinfo.setGenerationStampAndVerifyReplicas(newBlock.getGenerationStamp());
// find the DatanodeDescriptor objects
@@ -6122,17 +6200,29 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
public FSDirectory getFSDirectory() {
return dir;
}
+
/** Set the FSDirectory. */
@VisibleForTesting
public void setFSDirectory(FSDirectory dir) {
this.dir = dir;
}
+
/** @return the cache manager. */
@Override
public CacheManager getCacheManager() {
return cacheManager;
}
+ /** @return the ErasureCodingSchemaManager. */
+ public ErasureCodingSchemaManager getErasureCodingSchemaManager() {
+ return ecSchemaManager;
+ }
+
+ /** @return the ErasureCodingZoneManager. */
+ public ErasureCodingZoneManager getErasureCodingZoneManager() {
+ return dir.ecZoneManager;
+ }
+
@Override // NameNodeMXBean
public String getCorruptFiles() {
List<String> list = new ArrayList<String>();
@@ -7078,6 +7168,85 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
+ /**
+ * Create an erasure coding zone on directory src.
+ * @param srcArg the path of a directory which will be the root of the
+ * erasure coding zone. The directory must be empty.
+ * @param schema ECSchema for the erasure coding zone
+ * @param cellSize Cell size of stripe
+ * @throws AccessControlException if the caller is not the superuser.
+ * @throws UnresolvedLinkException if the path can't be resolved.
+ * @throws SafeModeException if the Namenode is in safe mode.
+ */
+ void createErasureCodingZone(final String srcArg, final ECSchema schema,
+ int cellSize, final boolean logRetryCache) throws IOException,
+ UnresolvedLinkException, SafeModeException, AccessControlException {
+ checkSuperuserPrivilege();
+ checkOperation(OperationCategory.WRITE);
+ HdfsFileStatus resultingStat = null;
+ boolean success = false;
+ writeLock();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot create erasure coding zone on " + srcArg);
+ resultingStat = FSDirErasureCodingOp.createErasureCodingZone(this,
+ srcArg, schema, cellSize, logRetryCache);
+ success = true;
+ } finally {
+ writeUnlock();
+ if (success) {
+ getEditLog().logSync();
+ }
+ logAuditEvent(success, "createErasureCodingZone", srcArg, null,
+ resultingStat);
+ }
+ }
+
+ /**
+ * Get the erasure coding zone information for specified path
+ */
+ ErasureCodingZone getErasureCodingZone(String src)
+ throws AccessControlException, UnresolvedLinkException, IOException {
+ checkOperation(OperationCategory.READ);
+ readLock();
+ try {
+ checkOperation(OperationCategory.READ);
+ return getErasureCodingZoneForPath(src);
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * Get available erasure coding schemas
+ */
+ ECSchema[] getErasureCodingSchemas() throws IOException {
+ checkOperation(OperationCategory.READ);
+ waitForLoadingFSImage();
+ readLock();
+ try {
+ checkOperation(OperationCategory.READ);
+ return FSDirErasureCodingOp.getErasureCodingSchemas(this);
+ } finally {
+ readUnlock();
+ }
+ }
+
+ /**
+ * Get the ECSchema specified by the name
+ */
+ ECSchema getErasureCodingSchema(String schemaName) throws IOException {
+ checkOperation(OperationCategory.READ);
+ waitForLoadingFSImage();
+ readLock();
+ try {
+ checkOperation(OperationCategory.READ);
+ return FSDirErasureCodingOp.getErasureCodingSchema(this, schemaName);
+ } finally {
+ readUnlock();
+ }
+ }
+
void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag,
boolean logRetryCache)
throws IOException {
@@ -7260,5 +7429,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
+ @Override
+ public ErasureCodingZone getErasureCodingZoneForPath(String src)
+ throws IOException {
+ return FSDirErasureCodingOp.getErasureCodingZone(this, src);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
index d07ae1f..900839a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FileUnderConstructionFeature.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
/**
@@ -61,9 +60,9 @@ public class FileUnderConstructionFeature implements INode.Feature {
BlockInfo lastBlock = f.getLastBlock();
assert (lastBlock != null) : "The last block for path "
+ f.getFullPathName() + " is null when updating its length";
- assert (lastBlock instanceof BlockInfoContiguousUnderConstruction)
+ assert !lastBlock.isComplete()
: "The last block for path " + f.getFullPathName()
- + " is not a BlockInfoUnderConstruction when updating its length";
+ + " is not a BlockInfoUnderConstruction when updating its length";
lastBlock.setNumBytes(lastBlockLength);
}
@@ -76,9 +75,8 @@ public class FileUnderConstructionFeature implements INode.Feature {
final BlocksMapUpdateInfo collectedBlocks) {
final BlockInfo[] blocks = f.getBlocks();
if (blocks != null && blocks.length > 0
- && blocks[blocks.length - 1] instanceof BlockInfoContiguousUnderConstruction) {
- BlockInfoContiguousUnderConstruction lastUC =
- (BlockInfoContiguousUnderConstruction) blocks[blocks.length - 1];
+ && !blocks[blocks.length - 1].isComplete()) {
+ BlockInfo lastUC = blocks[blocks.length - 1];
if (lastUC.getNumBytes() == 0) {
// this is a 0-sized block. do not need check its UC state here
collectedBlocks.addDeleteBlock(lastUC);
[18/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
index 23e8f57..4701538 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtilClient;
+import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* Interface that represents the over the wire information
@@ -58,10 +59,11 @@ public class HdfsLocatedFileStatus extends HdfsFileStatus {
int block_replication, long blocksize, long modification_time,
long access_time, FsPermission permission, String owner, String group,
byte[] symlink, byte[] path, long fileId, LocatedBlocks locations,
- int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) {
+ int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy,
+ ECSchema schema, int stripeCellSize) {
super(length, isdir, block_replication, blocksize, modification_time,
access_time, permission, owner, group, symlink, path, fileId,
- childrenNum, feInfo, storagePolicy);
+ childrenNum, feInfo, storagePolicy, schema, stripeCellSize);
this.locations = locations;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 8e81fdc..f988ae3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -201,6 +202,12 @@ import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathR
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECSchemasRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECSchemasResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingZoneRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingZoneResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.CreateErasureCodingZoneRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.CreateErasureCodingZoneResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeIDProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.DatanodeInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
@@ -216,6 +223,7 @@ import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenResponseProto;
import org.apache.hadoop.security.proto.SecurityProtos.GetDelegationTokenRequestProto;
@@ -422,7 +430,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
req.getClientName(), flags);
AppendResponseProto.Builder builder = AppendResponseProto.newBuilder();
if (result.getLastBlock() != null) {
- builder.setBlock(PBHelper.convert(result.getLastBlock()));
+ builder.setBlock(PBHelper.convertLocatedBlock(result.getLastBlock()));
}
if (result.getFileStatus() != null) {
builder.setStat(PBHelper.convert(result.getFileStatus()));
@@ -498,7 +506,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
(favor == null || favor.size() == 0) ? null : favor
.toArray(new String[favor.size()]));
return AddBlockResponseProto.newBuilder()
- .setBlock(PBHelper.convert(result)).build();
+ .setBlock(PBHelper.convertLocatedBlock(result)).build();
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -522,7 +530,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
new DatanodeInfoProto[excludesList.size()])),
req.getNumAdditionalNodes(), req.getClientName());
return GetAdditionalDatanodeResponseProto.newBuilder().setBlock(
- PBHelper.convert(result))
+ PBHelper.convertLocatedBlock(result))
.build();
} catch (IOException e) {
throw new ServiceException(e);
@@ -548,8 +556,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
ReportBadBlocksRequestProto req) throws ServiceException {
try {
List<LocatedBlockProto> bl = req.getBlocksList();
- server.reportBadBlocks(PBHelper.convertLocatedBlock(
- bl.toArray(new LocatedBlockProto[bl.size()])));
+ server.reportBadBlocks(PBHelper.convertLocatedBlocks(
+ bl.toArray(new LocatedBlockProto[bl.size()])));
} catch (IOException e) {
throw new ServiceException(e);
}
@@ -953,8 +961,8 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, UpdateBlockForPipelineRequestProto req)
throws ServiceException {
try {
- LocatedBlockProto result = PBHelper.convert(server
- .updateBlockForPipeline(PBHelper.convert(req.getBlock()),
+ LocatedBlockProto result = PBHelper.convertLocatedBlock(
+ server.updateBlockForPipeline(PBHelper.convert(req.getBlock()),
req.getClientName()));
return UpdateBlockForPipelineResponseProto.newBuilder().setBlock(result)
.build();
@@ -1394,6 +1402,21 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+ public CreateErasureCodingZoneResponseProto createErasureCodingZone(
+ RpcController controller, CreateErasureCodingZoneRequestProto req)
+ throws ServiceException {
+ try {
+ ECSchema schema = req.hasSchema() ? PBHelper.convertECSchema(req
+ .getSchema()) : null;
+ int cellSize = req.hasCellSize() ? req.getCellSize() : 0;
+ server.createErasureCodingZone(req.getSrc(), schema, cellSize);
+ return CreateErasureCodingZoneResponseProto.newBuilder().build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
public SetXAttrResponseProto setXAttr(RpcController controller,
SetXAttrRequestProto req) throws ServiceException {
try {
@@ -1514,4 +1537,35 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
throw new ServiceException(e);
}
}
+
+ @Override
+ public GetECSchemasResponseProto getECSchemas(RpcController controller,
+ GetECSchemasRequestProto request) throws ServiceException {
+ try {
+ ECSchema[] ecSchemas = server.getECSchemas();
+ GetECSchemasResponseProto.Builder resBuilder = GetECSchemasResponseProto
+ .newBuilder();
+ for (ECSchema ecSchema : ecSchemas) {
+ resBuilder.addSchemas(PBHelper.convertECSchema(ecSchema));
+ }
+ return resBuilder.build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public GetErasureCodingZoneResponseProto getErasureCodingZone(RpcController controller,
+ GetErasureCodingZoneRequestProto request) throws ServiceException {
+ try {
+ ErasureCodingZone ecZone = server.getErasureCodingZone(request.getSrc());
+ GetErasureCodingZoneResponseProto.Builder builder = GetErasureCodingZoneResponseProto.newBuilder();
+ if (ecZone != null) {
+ builder.setECZone(PBHelper.convertErasureCodingZone(ecZone));
+ }
+ return builder.build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index d6afa6e..342da0c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -161,10 +162,16 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.Trunca
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdateBlockForPipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.UpdatePipelineRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SetStoragePolicyRequestProto;
-import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos;
+import org.apache.hadoop.hdfs.protocol.proto.*;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncryptionZoneRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECSchemasRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetECSchemasResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingZoneRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingZoneResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.CreateErasureCodingZoneRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ECSchemaProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.RemoveXAttrRequestProto;
@@ -176,6 +183,7 @@ import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
import org.apache.hadoop.ipc.ProtocolTranslator;
@@ -233,6 +241,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
VOID_GET_STORAGE_POLICIES_REQUEST =
GetStoragePoliciesRequestProto.newBuilder().build();
+ private final static GetECSchemasRequestProto
+ VOID_GET_ECSCHEMAS_REQUEST = GetECSchemasRequestProto
+ .newBuilder().build();
+
public ClientNamenodeProtocolTranslatorPB(ClientNamenodeProtocolPB proxy) {
rpcProxy = proxy;
}
@@ -328,7 +340,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
try {
AppendResponseProto res = rpcProxy.append(null, req);
LocatedBlock lastBlock = res.hasBlock() ? PBHelper
- .convert(res.getBlock()) : null;
+ .convertLocatedBlockProto(res.getBlock()) : null;
HdfsFileStatus stat = (res.hasStat()) ? PBHelper.convert(res.getStat())
: null;
return new LastBlockWithStatus(lastBlock, stat);
@@ -416,7 +428,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
req.addAllFavoredNodes(Arrays.asList(favoredNodes));
}
try {
- return PBHelper.convert(rpcProxy.addBlock(null, req.build()).getBlock());
+ return PBHelper.convertLocatedBlockProto(
+ rpcProxy.addBlock(null, req.build()).getBlock());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -441,8 +454,8 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setClientName(clientName)
.build();
try {
- return PBHelper.convert(rpcProxy.getAdditionalDatanode(null, req)
- .getBlock());
+ return PBHelper.convertLocatedBlockProto(
+ rpcProxy.getAdditionalDatanode(null, req).getBlock());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
}
@@ -469,7 +482,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
ReportBadBlocksRequestProto req = ReportBadBlocksRequestProto.newBuilder()
- .addAllBlocks(Arrays.asList(PBHelper.convertLocatedBlock(blocks)))
+ .addAllBlocks(Arrays.asList(PBHelper.convertLocatedBlocks(blocks)))
.build();
try {
rpcProxy.reportBadBlocks(null, req);
@@ -901,7 +914,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setClientName(clientName)
.build();
try {
- return PBHelper.convert(
+ return PBHelper.convertLocatedBlockProto(
rpcProxy.updateBlockForPipeline(null, req).getBlock());
} catch (ServiceException e) {
throw ProtobufHelper.getRemoteException(e);
@@ -1407,6 +1420,26 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
+ public void createErasureCodingZone(String src, ECSchema schema, int cellSize)
+ throws IOException {
+ final CreateErasureCodingZoneRequestProto.Builder builder =
+ CreateErasureCodingZoneRequestProto.newBuilder();
+ builder.setSrc(src);
+ if (schema != null) {
+ builder.setSchema(PBHelper.convertECSchema(schema));
+ }
+ if (cellSize > 0) {
+ builder.setCellSize(cellSize);
+ }
+ CreateErasureCodingZoneRequestProto req = builder.build();
+ try {
+ rpcProxy.createErasureCodingZone(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
throws IOException {
SetXAttrRequestProto req = SetXAttrRequestProto.newBuilder()
@@ -1528,4 +1561,36 @@ public class ClientNamenodeProtocolTranslatorPB implements
throw ProtobufHelper.getRemoteException(e);
}
}
+
+ @Override
+ public ECSchema[] getECSchemas() throws IOException {
+ try {
+ GetECSchemasResponseProto response = rpcProxy.getECSchemas(null,
+ VOID_GET_ECSCHEMAS_REQUEST);
+ ECSchema[] schemas = new ECSchema[response.getSchemasCount()];
+ int i = 0;
+ for (ECSchemaProto schemaProto : response.getSchemasList()) {
+ schemas[i++] = PBHelper.convertECSchema(schemaProto);
+ }
+ return schemas;
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public ErasureCodingZone getErasureCodingZone(String src) throws IOException {
+ GetErasureCodingZoneRequestProto req = GetErasureCodingZoneRequestProto.newBuilder()
+ .setSrc(src).build();
+ try {
+ GetErasureCodingZoneResponseProto response = rpcProxy.getErasureCodingZone(
+ null, req);
+ if (response.hasECZone()) {
+ return PBHelper.convertErasureCodingZone(response.getECZone());
+ }
+ return null;
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
index 94028a2..e71e24c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
@@ -281,7 +281,7 @@ public class DatanodeProtocolClientSideTranslatorPB implements
ReportBadBlocksRequestProto.Builder builder = ReportBadBlocksRequestProto
.newBuilder();
for (int i = 0; i < blocks.length; i++) {
- builder.addBlocks(i, PBHelper.convert(blocks[i]));
+ builder.addBlocks(i, PBHelper.convertLocatedBlock(blocks[i]));
}
ReportBadBlocksRequestProto req = builder.build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
index e133ec7..09ba564 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
@@ -259,7 +259,7 @@ public class DatanodeProtocolServerSideTranslatorPB implements
List<LocatedBlockProto> lbps = request.getBlocksList();
LocatedBlock [] blocks = new LocatedBlock [lbps.size()];
for(int i=0; i<lbps.size(); i++) {
- blocks[i] = PBHelper.convert(lbps.get(i));
+ blocks[i] = PBHelper.convertLocatedBlockProto(lbps.get(i));
}
try {
impl.reportBadBlocks(blocks);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 4ca5b26..a97e2ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -28,8 +28,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
@@ -72,6 +77,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -81,6 +87,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
@@ -93,7 +100,7 @@ import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTyp
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos;
+import org.apache.hadoop.hdfs.protocol.proto.*;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
@@ -114,6 +121,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmI
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto;
+import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockECRecoveryCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockIdCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeCommandProto;
@@ -125,7 +133,10 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDele
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.VolumeFailureSummaryProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockReportContextProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.BlockECRecoveryInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.ErasureCodingZoneProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ECSchemaOptionEntryProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ECSchemaProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
@@ -172,7 +183,6 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageUuidsProto;
-import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
@@ -191,12 +201,15 @@ import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
import org.apache.hadoop.hdfs.server.protocol.BalancerBandwidthCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockIdCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.StripedBlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
@@ -223,6 +236,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
import org.apache.hadoop.hdfs.util.ExactSizeInputStream;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
@@ -429,22 +443,32 @@ public class PBHelper {
}
public static BlockWithLocationsProto convert(BlockWithLocations blk) {
- return BlockWithLocationsProto.newBuilder()
- .setBlock(convert(blk.getBlock()))
+ BlockWithLocationsProto.Builder builder = BlockWithLocationsProto
+ .newBuilder().setBlock(convert(blk.getBlock()))
.addAllDatanodeUuids(Arrays.asList(blk.getDatanodeUuids()))
.addAllStorageUuids(Arrays.asList(blk.getStorageIDs()))
- .addAllStorageTypes(convertStorageTypes(blk.getStorageTypes()))
- .build();
+ .addAllStorageTypes(convertStorageTypes(blk.getStorageTypes()));
+ if (blk instanceof StripedBlockWithLocations) {
+ StripedBlockWithLocations sblk = (StripedBlockWithLocations) blk;
+ builder.setIndices(getByteString(sblk.getIndices()));
+ builder.setDataBlockNum(sblk.getDataBlockNum());
+ }
+ return builder.build();
}
public static BlockWithLocations convert(BlockWithLocationsProto b) {
final List<String> datanodeUuids = b.getDatanodeUuidsList();
final List<String> storageUuids = b.getStorageUuidsList();
final List<StorageTypeProto> storageTypes = b.getStorageTypesList();
- return new BlockWithLocations(convert(b.getBlock()),
+ BlockWithLocations blk = new BlockWithLocations(convert(b.getBlock()),
datanodeUuids.toArray(new String[datanodeUuids.size()]),
storageUuids.toArray(new String[storageUuids.size()]),
convertStorageTypes(storageTypes, storageUuids.size()));
+ if (b.hasIndices()) {
+ blk = new StripedBlockWithLocations(blk, b.getIndices().toByteArray(),
+ (short) b.getDataBlockNum());
+ }
+ return blk;
}
public static BlocksWithLocationsProto convert(BlocksWithLocations blks) {
@@ -609,7 +633,7 @@ public class PBHelper {
if (b == null) {
return null;
}
- LocatedBlockProto lb = PBHelper.convert((LocatedBlock)b);
+ LocatedBlockProto lb = PBHelper.convertLocatedBlock(b);
RecoveringBlockProto.Builder builder = RecoveringBlockProto.newBuilder();
builder.setBlock(lb).setNewGenStamp(b.getNewGenerationStamp());
if(b.getNewBlock() != null)
@@ -759,7 +783,7 @@ public class PBHelper {
}
}
- public static LocatedBlockProto convert(LocatedBlock b) {
+ public static LocatedBlockProto convertLocatedBlock(LocatedBlock b) {
if (b == null) return null;
Builder builder = LocatedBlockProto.newBuilder();
DatanodeInfo[] locs = b.getLocations();
@@ -780,21 +804,30 @@ public class PBHelper {
StorageType[] storageTypes = b.getStorageTypes();
if (storageTypes != null) {
- for (int i = 0; i < storageTypes.length; ++i) {
- builder.addStorageTypes(PBHelper.convertStorageType(storageTypes[i]));
+ for (StorageType storageType : storageTypes) {
+ builder.addStorageTypes(PBHelper.convertStorageType(storageType));
}
}
final String[] storageIDs = b.getStorageIDs();
if (storageIDs != null) {
builder.addAllStorageIDs(Arrays.asList(storageIDs));
}
+ if (b instanceof LocatedStripedBlock) {
+ LocatedStripedBlock sb = (LocatedStripedBlock) b;
+ int[] indices = sb.getBlockIndices();
+ Token<BlockTokenIdentifier>[] blockTokens = sb.getBlockTokens();
+ for (int i = 0; i < indices.length; i++) {
+ builder.addBlockIndex(indices[i]);
+ builder.addBlockTokens(PBHelper.convert(blockTokens[i]));
+ }
+ }
return builder.setB(PBHelper.convert(b.getBlock()))
.setBlockToken(PBHelper.convert(b.getBlockToken()))
.setCorrupt(b.isCorrupt()).setOffset(b.getStartOffset()).build();
}
- public static LocatedBlock convert(LocatedBlockProto proto) {
+ public static LocatedBlock convertLocatedBlockProto(LocatedBlockProto proto) {
if (proto == null) return null;
List<DatanodeInfoProto> locs = proto.getLocsList();
DatanodeInfo[] targets = new DatanodeInfo[locs.size()];
@@ -814,6 +847,15 @@ public class PBHelper {
storageIDs = proto.getStorageIDsList().toArray(new String[storageIDsCount]);
}
+ int[] indices = null;
+ final int indexCount = proto.getBlockIndexCount();
+ if (indexCount > 0) {
+ indices = new int[indexCount];
+ for (int i = 0; i < indexCount; i++) {
+ indices[i] = proto.getBlockIndex(i);
+ }
+ }
+
// Set values from the isCached list, re-using references from loc
List<DatanodeInfo> cachedLocs = new ArrayList<DatanodeInfo>(locs.size());
List<Boolean> isCachedList = proto.getIsCachedList();
@@ -823,9 +865,23 @@ public class PBHelper {
}
}
- LocatedBlock lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets,
- storageIDs, storageTypes, proto.getOffset(), proto.getCorrupt(),
- cachedLocs.toArray(new DatanodeInfo[0]));
+ final LocatedBlock lb;
+ if (indices == null) {
+ lb = new LocatedBlock(PBHelper.convert(proto.getB()), targets, storageIDs,
+ storageTypes, proto.getOffset(), proto.getCorrupt(),
+ cachedLocs.toArray(new DatanodeInfo[cachedLocs.size()]));
+ } else {
+ lb = new LocatedStripedBlock(PBHelper.convert(proto.getB()), targets,
+ storageIDs, storageTypes, indices, proto.getOffset(),
+ proto.getCorrupt(),
+ cachedLocs.toArray(new DatanodeInfo[cachedLocs.size()]));
+ List<TokenProto> tokenProtos = proto.getBlockTokensList();
+ Token<BlockTokenIdentifier>[] blockTokens = new Token[indices.length];
+ for (int i = 0; i < indices.length; i++) {
+ blockTokens[i] = PBHelper.convert(tokenProtos.get(i));
+ }
+ ((LocatedStripedBlock) lb).setBlockTokens(blockTokens);
+ }
lb.setBlockToken(PBHelper.convert(proto.getBlockToken()));
return lb;
@@ -918,6 +974,8 @@ public class PBHelper {
return REG_CMD;
case BlockIdCommand:
return PBHelper.convert(proto.getBlkIdCmd());
+ case BlockECRecoveryCommand:
+ return PBHelper.convert(proto.getBlkECRecoveryCmd());
default:
return null;
}
@@ -1068,6 +1126,11 @@ public class PBHelper {
builder.setCmdType(DatanodeCommandProto.Type.BlockIdCommand).
setBlkIdCmd(PBHelper.convert((BlockIdCommand) datanodeCommand));
break;
+ case DatanodeProtocol.DNA_ERASURE_CODING_RECOVERY:
+ builder.setCmdType(DatanodeCommandProto.Type.BlockECRecoveryCommand)
+ .setBlkECRecoveryCmd(
+ convert((BlockECRecoveryCommand) datanodeCommand));
+ break;
case DatanodeProtocol.DNA_UNKNOWN: //Not expected
default:
builder.setCmdType(DatanodeCommandProto.Type.NullDatanodeCommand);
@@ -1241,36 +1304,36 @@ public class PBHelper {
}
// Located Block Arrays and Lists
- public static LocatedBlockProto[] convertLocatedBlock(LocatedBlock[] lb) {
+ public static LocatedBlockProto[] convertLocatedBlocks(LocatedBlock[] lb) {
if (lb == null) return null;
- return convertLocatedBlock2(Arrays.asList(lb)).toArray(
- new LocatedBlockProto[lb.length]);
+ return convertLocatedBlocks2(Arrays.asList(lb))
+ .toArray(new LocatedBlockProto[lb.length]);
}
- public static LocatedBlock[] convertLocatedBlock(LocatedBlockProto[] lb) {
+ public static LocatedBlock[] convertLocatedBlocks(LocatedBlockProto[] lb) {
if (lb == null) return null;
- return convertLocatedBlock(Arrays.asList(lb)).toArray(
- new LocatedBlock[lb.length]);
+ return convertLocatedBlocks(Arrays.asList(lb))
+ .toArray(new LocatedBlock[lb.length]);
}
- public static List<LocatedBlock> convertLocatedBlock(
+ public static List<LocatedBlock> convertLocatedBlocks(
List<LocatedBlockProto> lb) {
if (lb == null) return null;
final int len = lb.size();
- List<LocatedBlock> result =
- new ArrayList<LocatedBlock>(len);
- for (int i = 0; i < len; ++i) {
- result.add(PBHelper.convert(lb.get(i)));
+ List<LocatedBlock> result = new ArrayList<>(len);
+ for (LocatedBlockProto aLb : lb) {
+ result.add(PBHelper.convertLocatedBlockProto(aLb));
}
return result;
}
- public static List<LocatedBlockProto> convertLocatedBlock2(List<LocatedBlock> lb) {
+ public static List<LocatedBlockProto> convertLocatedBlocks2(
+ List<LocatedBlock> lb) {
if (lb == null) return null;
final int len = lb.size();
- List<LocatedBlockProto> result = new ArrayList<LocatedBlockProto>(len);
- for (int i = 0; i < len; ++i) {
- result.add(PBHelper.convert(lb.get(i)));
+ List<LocatedBlockProto> result = new ArrayList<>(len);
+ for (LocatedBlock aLb : lb) {
+ result.add(PBHelper.convertLocatedBlock(aLb));
}
return result;
}
@@ -1280,11 +1343,13 @@ public class PBHelper {
public static LocatedBlocks convert(LocatedBlocksProto lb) {
return new LocatedBlocks(
lb.getFileLength(), lb.getUnderConstruction(),
- PBHelper.convertLocatedBlock(lb.getBlocksList()),
- lb.hasLastBlock() ? PBHelper.convert(lb.getLastBlock()) : null,
+ PBHelper.convertLocatedBlocks(lb.getBlocksList()),
+ lb.hasLastBlock() ?
+ PBHelper.convertLocatedBlockProto(lb.getLastBlock()) : null,
lb.getIsLastBlockComplete(),
- lb.hasFileEncryptionInfo() ? convert(lb.getFileEncryptionInfo()) :
- null);
+ lb.hasFileEncryptionInfo() ? convert(lb.getFileEncryptionInfo()) : null,
+ lb.hasECSchema() ? convertECSchema(lb.getECSchema()) : null,
+ lb.hasStripeCellSize() ? lb.getStripeCellSize() : 0);
}
public static LocatedBlocksProto convert(LocatedBlocks lb) {
@@ -1294,14 +1359,21 @@ public class PBHelper {
LocatedBlocksProto.Builder builder =
LocatedBlocksProto.newBuilder();
if (lb.getLastLocatedBlock() != null) {
- builder.setLastBlock(PBHelper.convert(lb.getLastLocatedBlock()));
+ builder.setLastBlock(
+ PBHelper.convertLocatedBlock(lb.getLastLocatedBlock()));
}
if (lb.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(lb.getFileEncryptionInfo()));
}
+ if (lb.getECSchema() != null) {
+ builder.setECSchema(convertECSchema(lb.getECSchema()));
+ }
+ if (lb.getStripeCellSize() != 0) {
+ builder.setStripeCellSize(lb.getStripeCellSize());
+ }
return builder.setFileLength(lb.getFileLength())
.setUnderConstruction(lb.isUnderConstruction())
- .addAllBlocks(PBHelper.convertLocatedBlock2(lb.getLocatedBlocks()))
+ .addAllBlocks(PBHelper.convertLocatedBlocks2(lb.getLocatedBlocks()))
.setIsLastBlockComplete(lb.isLastBlockComplete()).build();
}
@@ -1441,7 +1513,9 @@ public class PBHelper {
fs.hasChildrenNum() ? fs.getChildrenNum() : -1,
fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null,
fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy()
- : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED);
+ : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
+ fs.hasEcSchema() ? PBHelper.convertECSchema(fs.getEcSchema()) : null,
+ fs.hasStripeCellSize() ? fs.getStripeCellSize() : 0);
}
public static SnapshottableDirectoryStatus convert(
@@ -1502,6 +1576,10 @@ public class PBHelper {
builder.setLocations(PBHelper.convert(locations));
}
}
+ if(fs.getECSchema() != null) {
+ builder.setEcSchema(PBHelper.convertECSchema(fs.getECSchema()));
+ }
+ builder.setStripeCellSize(fs.getStripeCellSize());
return builder.build();
}
@@ -3070,4 +3148,176 @@ public class PBHelper {
setLeaseId(context.getLeaseId()).
build();
}
+
+ public static ECSchema convertECSchema(ECSchemaProto schema) {
+ List<ECSchemaOptionEntryProto> optionsList = schema.getOptionsList();
+ Map<String, String> options = new HashMap<>(optionsList.size());
+ for (ECSchemaOptionEntryProto option : optionsList) {
+ options.put(option.getKey(), option.getValue());
+ }
+ return new ECSchema(schema.getSchemaName(), schema.getCodecName(),
+ schema.getDataUnits(), schema.getParityUnits(), options);
+ }
+
+ public static ECSchemaProto convertECSchema(ECSchema schema) {
+ ECSchemaProto.Builder builder = ECSchemaProto.newBuilder()
+ .setSchemaName(schema.getSchemaName())
+ .setCodecName(schema.getCodecName())
+ .setDataUnits(schema.getNumDataUnits())
+ .setParityUnits(schema.getNumParityUnits());
+ Set<Entry<String, String>> entrySet = schema.getExtraOptions().entrySet();
+ for (Entry<String, String> entry : entrySet) {
+ builder.addOptions(ECSchemaOptionEntryProto.newBuilder()
+ .setKey(entry.getKey()).setValue(entry.getValue()).build());
+ }
+ return builder.build();
+ }
+
+ public static ErasureCodingZoneProto convertErasureCodingZone(
+ ErasureCodingZone ecZone) {
+ return ErasureCodingZoneProto.newBuilder().setDir(ecZone.getDir())
+ .setSchema(convertECSchema(ecZone.getSchema()))
+ .setCellSize(ecZone.getCellSize()).build();
+ }
+
+ public static ErasureCodingZone convertErasureCodingZone(
+ ErasureCodingZoneProto ecZoneProto) {
+ return new ErasureCodingZone(ecZoneProto.getDir(),
+ convertECSchema(ecZoneProto.getSchema()), ecZoneProto.getCellSize());
+ }
+
+ public static BlockECRecoveryInfo convertBlockECRecoveryInfo(
+ BlockECRecoveryInfoProto blockEcRecoveryInfoProto) {
+ ExtendedBlockProto blockProto = blockEcRecoveryInfoProto.getBlock();
+ ExtendedBlock block = convert(blockProto);
+
+ DatanodeInfosProto sourceDnInfosProto = blockEcRecoveryInfoProto
+ .getSourceDnInfos();
+ DatanodeInfo[] sourceDnInfos = convert(sourceDnInfosProto);
+
+ DatanodeInfosProto targetDnInfosProto = blockEcRecoveryInfoProto
+ .getTargetDnInfos();
+ DatanodeInfo[] targetDnInfos = convert(targetDnInfosProto);
+
+ StorageUuidsProto targetStorageUuidsProto = blockEcRecoveryInfoProto
+ .getTargetStorageUuids();
+ String[] targetStorageUuids = convert(targetStorageUuidsProto);
+
+ StorageTypesProto targetStorageTypesProto = blockEcRecoveryInfoProto
+ .getTargetStorageTypes();
+ StorageType[] convertStorageTypes = convertStorageTypes(
+ targetStorageTypesProto.getStorageTypesList(), targetStorageTypesProto
+ .getStorageTypesList().size());
+
+ List<Integer> liveBlockIndicesList = blockEcRecoveryInfoProto
+ .getLiveBlockIndicesList();
+ short[] liveBlkIndices = new short[liveBlockIndicesList.size()];
+ for (int i = 0; i < liveBlockIndicesList.size(); i++) {
+ liveBlkIndices[i] = liveBlockIndicesList.get(i).shortValue();
+ }
+
+ ECSchema ecSchema = convertECSchema(blockEcRecoveryInfoProto.getEcSchema());
+ int cellSize = blockEcRecoveryInfoProto.getCellSize();
+
+ return new BlockECRecoveryInfo(block, sourceDnInfos, targetDnInfos,
+ targetStorageUuids, convertStorageTypes, liveBlkIndices, ecSchema,
+ cellSize);
+ }
+
+ public static BlockECRecoveryInfoProto convertBlockECRecoveryInfo(
+ BlockECRecoveryInfo blockEcRecoveryInfo) {
+ BlockECRecoveryInfoProto.Builder builder = BlockECRecoveryInfoProto
+ .newBuilder();
+ builder.setBlock(convert(blockEcRecoveryInfo.getExtendedBlock()));
+
+ DatanodeInfo[] sourceDnInfos = blockEcRecoveryInfo.getSourceDnInfos();
+ builder.setSourceDnInfos(convertToDnInfosProto(sourceDnInfos));
+
+ DatanodeInfo[] targetDnInfos = blockEcRecoveryInfo.getTargetDnInfos();
+ builder.setTargetDnInfos(convertToDnInfosProto(targetDnInfos));
+
+ String[] targetStorageIDs = blockEcRecoveryInfo.getTargetStorageIDs();
+ builder.setTargetStorageUuids(convertStorageIDs(targetStorageIDs));
+
+ StorageType[] targetStorageTypes = blockEcRecoveryInfo
+ .getTargetStorageTypes();
+ builder.setTargetStorageTypes(convertStorageTypesProto(targetStorageTypes));
+
+ short[] liveBlockIndices = blockEcRecoveryInfo.getLiveBlockIndices();
+ builder.addAllLiveBlockIndices(convertIntArray(liveBlockIndices));
+
+ builder.setEcSchema(convertECSchema(blockEcRecoveryInfo.getECSchema()));
+ builder.setCellSize(blockEcRecoveryInfo.getCellSize());
+
+ return builder.build();
+ }
+
+ private static List<Integer> convertIntArray(short[] liveBlockIndices) {
+ List<Integer> liveBlockIndicesList = new ArrayList<Integer>();
+ for (short s : liveBlockIndices) {
+ liveBlockIndicesList.add((int) s);
+ }
+ return liveBlockIndicesList;
+ }
+
+ private static StorageTypesProto convertStorageTypesProto(
+ StorageType[] targetStorageTypes) {
+ StorageTypesProto.Builder builder = StorageTypesProto.newBuilder();
+ for (StorageType storageType : targetStorageTypes) {
+ builder.addStorageTypes(convertStorageType(storageType));
+ }
+ return builder.build();
+ }
+
+ private static StorageUuidsProto convertStorageIDs(String[] targetStorageIDs) {
+ StorageUuidsProto.Builder builder = StorageUuidsProto.newBuilder();
+ for (String storageUuid : targetStorageIDs) {
+ builder.addStorageUuids(storageUuid);
+ }
+ return builder.build();
+ }
+
+ private static DatanodeInfosProto convertToDnInfosProto(DatanodeInfo[] dnInfos) {
+ DatanodeInfosProto.Builder builder = DatanodeInfosProto.newBuilder();
+ for (DatanodeInfo datanodeInfo : dnInfos) {
+ builder.addDatanodes(convert(datanodeInfo));
+ }
+ return builder.build();
+ }
+
+ private static String[] convert(StorageUuidsProto targetStorageUuidsProto) {
+ List<String> storageUuidsList = targetStorageUuidsProto
+ .getStorageUuidsList();
+ String[] storageUuids = new String[storageUuidsList.size()];
+ for (int i = 0; i < storageUuidsList.size(); i++) {
+ storageUuids[i] = storageUuidsList.get(i);
+ }
+ return storageUuids;
+ }
+
+ public static BlockECRecoveryCommandProto convert(
+ BlockECRecoveryCommand blkECRecoveryCmd) {
+ BlockECRecoveryCommandProto.Builder builder = BlockECRecoveryCommandProto
+ .newBuilder();
+ Collection<BlockECRecoveryInfo> blockECRecoveryInfos = blkECRecoveryCmd
+ .getECTasks();
+ for (BlockECRecoveryInfo blkECRecoveryInfo : blockECRecoveryInfos) {
+ builder
+ .addBlockECRecoveryinfo(convertBlockECRecoveryInfo(blkECRecoveryInfo));
+ }
+ return builder.build();
+ }
+
+ public static BlockECRecoveryCommand convert(
+ BlockECRecoveryCommandProto blkECRecoveryCmdProto) {
+ Collection<BlockECRecoveryInfo> blkECRecoveryInfos = new ArrayList<BlockECRecoveryInfo>();
+ List<BlockECRecoveryInfoProto> blockECRecoveryinfoList = blkECRecoveryCmdProto
+ .getBlockECRecoveryinfoList();
+ for (BlockECRecoveryInfoProto blockECRecoveryInfoProto : blockECRecoveryinfoList) {
+ blkECRecoveryInfos
+ .add(convertBlockECRecoveryInfo(blockECRecoveryInfoProto));
+ }
+ return new BlockECRecoveryCommand(DatanodeProtocol.DNA_ERASURE_CODING_RECOVERY,
+ blkECRecoveryInfos);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 742a300..33fdc43 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -48,8 +48,8 @@ import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup;
import org.apache.hadoop.hdfs.server.balancer.Dispatcher.Source;
import org.apache.hadoop.hdfs.server.balancer.Dispatcher.Task;
import org.apache.hadoop.hdfs.server.balancer.Dispatcher.Util;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicies;
import org.apache.hadoop.hdfs.server.namenode.UnsupportedActionException;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
@@ -205,7 +205,9 @@ public class Balancer {
*/
private static void checkReplicationPolicyCompatibility(Configuration conf
) throws UnsupportedActionException {
- if (!(BlockPlacementPolicy.getInstance(conf, null, null, null) instanceof
+ BlockPlacementPolicies placementPolicies =
+ new BlockPlacementPolicies(conf, null, null, null);
+ if (!(placementPolicies.getPolicy(false) instanceof
BlockPlacementPolicyDefault)) {
throw new UnsupportedActionException(
"Balancer without BlockPlacementPolicyDefault");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index b4b06ee..a9e4f41 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.balancer;
import static org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed;
+import static org.apache.hadoop.hdfs.util.StripedBlockUtil.getInternalBlockLength;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
@@ -65,6 +67,7 @@ import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DDatanode.StorageGroup;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.StripedBlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
@@ -149,18 +152,17 @@ public class Dispatcher {
private final Map<Block, DBlock> map = new HashMap<Block, DBlock>();
/**
- * Get the block from the map;
- * if the block is not found, create a new block and put it in the map.
+ * Put block in the map if it's not found
+ * @return the block which be put in the map the first time
*/
- private DBlock get(Block b) {
- DBlock block = map.get(b);
- if (block == null) {
- block = new DBlock(b);
- map.put(b, block);
+ private DBlock putIfAbsent(Block blk, DBlock dblk) {
+ if (!map.containsKey(blk)) {
+ map.put(blk, dblk);
+ return dblk;
}
- return block;
+ return map.get(blk);
}
-
+
/** Remove all blocks except for the moved blocks. */
private void removeAllButRetain(MovedBlocks<StorageGroup> movedBlocks) {
for (Iterator<Block> i = map.keySet().iterator(); i.hasNext();) {
@@ -201,9 +203,9 @@ public class Dispatcher {
}
}
- /** This class keeps track of a scheduled block move */
+ /** This class keeps track of a scheduled reportedBlock move */
public class PendingMove {
- private DBlock block;
+ private DBlock reportedBlock;
private Source source;
private DDatanode proxySource;
private StorageGroup target;
@@ -215,7 +217,7 @@ public class Dispatcher {
@Override
public String toString() {
- final Block b = block != null ? block.getBlock() : null;
+ final Block b = reportedBlock != null ? reportedBlock.getBlock() : null;
String bStr = b != null ? (b + " with size=" + b.getNumBytes() + " ")
: " ";
return bStr + "from " + source.getDisplayName() + " to " + target
@@ -224,8 +226,8 @@ public class Dispatcher {
}
/**
- * Choose a block & a proxy source for this pendingMove whose source &
- * target have already been chosen.
+ * Choose a good block/blockGroup from source & Get reportedBlock from
+ * the block & Choose a proxy source for the reportedBlock.
*
* @return true if a block and its proxy are chosen; false otherwise
*/
@@ -249,7 +251,11 @@ public class Dispatcher {
synchronized (block) {
synchronized (movedBlocks) {
if (isGoodBlockCandidate(source, target, targetStorageType, block)) {
- this.block = block;
+ if (block instanceof DBlockStriped) {
+ reportedBlock = ((DBlockStriped) block).getInternalBlock(source);
+ } else {
+ reportedBlock = block;
+ }
if (chooseProxySource()) {
movedBlocks.put(block);
if (LOG.isDebugEnabled()) {
@@ -276,7 +282,7 @@ public class Dispatcher {
}
// if node group is supported, first try add nodes in the same node group
if (cluster.isNodeGroupAware()) {
- for (StorageGroup loc : block.getLocations()) {
+ for (StorageGroup loc : reportedBlock.getLocations()) {
if (cluster.isOnSameNodeGroup(loc.getDatanodeInfo(), targetDN)
&& addTo(loc)) {
return true;
@@ -284,13 +290,13 @@ public class Dispatcher {
}
}
// check if there is replica which is on the same rack with the target
- for (StorageGroup loc : block.getLocations()) {
+ for (StorageGroup loc : reportedBlock.getLocations()) {
if (cluster.isOnSameRack(loc.getDatanodeInfo(), targetDN) && addTo(loc)) {
return true;
}
}
// find out a non-busy replica
- for (StorageGroup loc : block.getLocations()) {
+ for (StorageGroup loc : reportedBlock.getLocations()) {
if (addTo(loc)) {
return true;
}
@@ -298,7 +304,7 @@ public class Dispatcher {
return false;
}
- /** add to a proxy source for specific block movement */
+ /** add to a proxy source for specific reportedBlock movement */
private boolean addTo(StorageGroup g) {
final DDatanode dn = g.getDDatanode();
if (dn.addPendingBlock(this)) {
@@ -311,6 +317,7 @@ public class Dispatcher {
/** Dispatch the move to the proxy source & wait for the response. */
private void dispatch() {
LOG.info("Start moving " + this);
+ assert !(reportedBlock instanceof DBlockStriped);
Socket sock = new Socket();
DataOutputStream out = null;
@@ -325,7 +332,7 @@ public class Dispatcher {
OutputStream unbufOut = sock.getOutputStream();
InputStream unbufIn = sock.getInputStream();
ExtendedBlock eb = new ExtendedBlock(nnc.getBlockpoolID(),
- block.getBlock());
+ reportedBlock.getBlock());
final KeyManager km = nnc.getKeyManager();
Token<BlockTokenIdentifier> accessToken = km.getAccessToken(eb);
IOStreamPair saslStreams = saslClient.socketSend(sock, unbufOut,
@@ -339,7 +346,7 @@ public class Dispatcher {
sendRequest(out, eb, accessToken);
receiveResponse(in);
- nnc.getBytesMoved().addAndGet(block.getNumBytes());
+ nnc.getBytesMoved().addAndGet(reportedBlock.getNumBytes());
target.getDDatanode().setHasSuccess();
LOG.info("Successfully moved " + this);
} catch (IOException e) {
@@ -368,14 +375,14 @@ public class Dispatcher {
}
}
- /** Send a block replace request to the output stream */
+ /** Send a reportedBlock replace request to the output stream */
private void sendRequest(DataOutputStream out, ExtendedBlock eb,
Token<BlockTokenIdentifier> accessToken) throws IOException {
new Sender(out).replaceBlock(eb, target.storageType, accessToken,
source.getDatanodeInfo().getDatanodeUuid(), proxySource.datanode);
}
- /** Receive a block copy response from the input stream */
+ /** Receive a reportedBlock copy response from the input stream */
private void receiveResponse(DataInputStream in) throws IOException {
BlockOpResponseProto response =
BlockOpResponseProto.parseFrom(vintPrefixed(in));
@@ -383,13 +390,13 @@ public class Dispatcher {
// read intermediate responses
response = BlockOpResponseProto.parseFrom(vintPrefixed(in));
}
- String logInfo = "block move is failed";
+ String logInfo = "reportedBlock move is failed";
DataTransferProtoUtil.checkBlockOpStatus(response, logInfo);
}
/** reset the object */
private void reset() {
- block = null;
+ reportedBlock = null;
source = null;
proxySource = null;
target = null;
@@ -401,6 +408,44 @@ public class Dispatcher {
public DBlock(Block block) {
super(block);
}
+
+ public long getNumBytes(StorageGroup storage) {
+ return super.getNumBytes();
+ }
+ }
+
+ public static class DBlockStriped extends DBlock {
+
+ final byte[] indices;
+ final short dataBlockNum;
+
+ public DBlockStriped(Block block, byte[] indices, short dataBlockNum) {
+ super(block);
+ this.indices = indices;
+ this.dataBlockNum = dataBlockNum;
+ }
+
+ public DBlock getInternalBlock(StorageGroup storage) {
+ int idxInLocs = locations.indexOf(storage);
+ if (idxInLocs == -1) {
+ return null;
+ }
+ byte idxInGroup = indices[idxInLocs];
+ long blkId = getBlock().getBlockId() + idxInGroup;
+ long numBytes = getInternalBlockLength(getNumBytes(),
+ HdfsConstants.BLOCK_STRIPED_CELL_SIZE, dataBlockNum, idxInGroup);
+ Block blk = new Block(getBlock());
+ blk.setBlockId(blkId);
+ blk.setNumBytes(numBytes);
+ DBlock dblk = new DBlock(blk);
+ dblk.addLocation(storage);
+ return dblk;
+ }
+
+ @Override
+ public long getNumBytes(StorageGroup storage) {
+ return getInternalBlock(storage).getNumBytes();
+ }
}
/** The class represents a desired move. */
@@ -476,7 +521,7 @@ public class Dispatcher {
private PendingMove addPendingMove(DBlock block, final PendingMove pm) {
if (getDDatanode().addPendingBlock(pm)) {
if (pm.markMovedIfGoodBlock(block, getStorageType())) {
- incScheduledSize(pm.block.getNumBytes());
+ incScheduledSize(pm.reportedBlock.getNumBytes());
return pm;
} else {
getDDatanode().removePendingBlock(pm);
@@ -651,24 +696,39 @@ public class Dispatcher {
*/
private long getBlockList() throws IOException {
final long size = Math.min(MAX_BLOCKS_SIZE_TO_FETCH, blocksToReceive);
- final BlocksWithLocations newBlocks = nnc.getBlocks(getDatanodeInfo(), size);
+ final BlocksWithLocations newBlksLocs =
+ nnc.getBlocks(getDatanodeInfo(), size);
if (LOG.isTraceEnabled()) {
LOG.trace("getBlocks(" + getDatanodeInfo() + ", "
+ StringUtils.TraditionalBinaryPrefix.long2String(size, "B", 2)
- + ") returns " + newBlocks.getBlocks().length + " blocks.");
+ + ") returns " + newBlksLocs.getBlocks().length + " blocks.");
}
long bytesReceived = 0;
- for (BlockWithLocations blk : newBlocks.getBlocks()) {
- bytesReceived += blk.getBlock().getNumBytes();
+ for (BlockWithLocations blkLocs : newBlksLocs.getBlocks()) {
+
+ DBlock block;
+ if (blkLocs instanceof StripedBlockWithLocations) {
+ StripedBlockWithLocations sblkLocs =
+ (StripedBlockWithLocations) blkLocs;
+ // approximate size
+ bytesReceived += sblkLocs.getBlock().getNumBytes() /
+ sblkLocs.getDataBlockNum();
+ block = new DBlockStriped(sblkLocs.getBlock(), sblkLocs.getIndices(),
+ sblkLocs.getDataBlockNum());
+ } else{
+ bytesReceived += blkLocs.getBlock().getNumBytes();
+ block = new DBlock(blkLocs.getBlock());
+ }
+
synchronized (globalBlocks) {
- final DBlock block = globalBlocks.get(blk.getBlock());
+ block = globalBlocks.putIfAbsent(blkLocs.getBlock(), block);
synchronized (block) {
block.clearLocations();
// update locations
- final String[] datanodeUuids = blk.getDatanodeUuids();
- final StorageType[] storageTypes = blk.getStorageTypes();
+ final String[] datanodeUuids = blkLocs.getDatanodeUuids();
+ final StorageType[] storageTypes = blkLocs.getStorageTypes();
for (int i = 0; i < datanodeUuids.length; i++) {
final StorageGroup g = storageGroupMap.get(
datanodeUuids[i], storageTypes[i]);
@@ -707,6 +767,8 @@ public class Dispatcher {
* target throttling has been considered. They are chosen only when they
* have the capacity to support this block move. The block should be
* dispatched immediately after this method is returned.
+ * If the block is a block group. Only the internal block on this source
+ * will be dispatched.
*
* @return a move that's good for the source to dispatch immediately.
*/
@@ -718,7 +780,7 @@ public class Dispatcher {
if (target.addPendingBlock(pendingBlock)) {
// target is not busy, so do a tentative block allocation
if (pendingBlock.chooseBlockAndProxy()) {
- long blockSize = pendingBlock.block.getNumBytes();
+ long blockSize = pendingBlock.reportedBlock.getNumBytes(this);
incScheduledSize(-blockSize);
task.size -= blockSize;
if (task.size == 0) {
@@ -793,7 +855,7 @@ public class Dispatcher {
blocksToReceive -= getBlockList();
continue;
} catch (IOException e) {
- LOG.warn("Exception while getting block list", e);
+ LOG.warn("Exception while getting reportedBlock list", e);
return;
}
} else {
@@ -937,7 +999,7 @@ public class Dispatcher {
public void executePendingMove(final PendingMove p) {
- // move the block
+ // move the reportedBlock
final DDatanode targetDn = p.target.getDDatanode();
ExecutorService moveExecutor = targetDn.getMoveExecutor();
if (moveExecutor == null) {
@@ -995,17 +1057,17 @@ public class Dispatcher {
}
}
- // wait for all block moving to be done
+ // wait for all reportedBlock moving to be done
waitForMoveCompletion(targets);
return getBytesMoved() - bytesLastMoved;
}
- /** The sleeping period before checking if block move is completed again */
+ /** The sleeping period before checking if reportedBlock move is completed again */
static private long blockMoveWaitTime = 30000L;
/**
- * Wait for all block move confirmations.
+ * Wait for all reportedBlock move confirmations.
* @return true if there is failed move execution
*/
public static boolean waitForMoveCompletion(
@@ -1044,10 +1106,10 @@ public class Dispatcher {
}
/**
- * Decide if the block is a good candidate to be moved from source to target.
- * A block is a good candidate if
+ * Decide if the block/blockGroup is a good candidate to be moved from source
+ * to target. A block is a good candidate if
* 1. the block is not in the process of being moved/has not been moved;
- * 2. the block does not have a replica on the target;
+ * 2. the block does not have a replica/internalBlock on the target;
* 3. doing the move does not reduce the number of racks that the block has
*/
private boolean isGoodBlockCandidate(StorageGroup source, StorageGroup target,
@@ -1064,7 +1126,7 @@ public class Dispatcher {
}
final DatanodeInfo targetDatanode = target.getDatanodeInfo();
if (source.getDatanodeInfo().equals(targetDatanode)) {
- // the block is moved inside same DN
+ // the reportedBlock is moved inside same DN
return true;
}
@@ -1152,7 +1214,7 @@ public class Dispatcher {
movedBlocks.cleanup();
}
- /** set the sleeping period for block move completion check */
+ /** set the sleeping period for reportedBlock move completion check */
@VisibleForTesting
public static void setBlockMoveWaitTime(long time) {
blockMoveWaitTime = time;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
index 02a1d05..928424b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockCollection.java
@@ -39,12 +39,12 @@ public interface BlockCollection {
public ContentSummary computeContentSummary(BlockStoragePolicySuite bsps);
/**
- * @return the number of blocks
+ * @return the number of blocks or block groups
*/
public int numBlocks();
/**
- * Get the blocks.
+ * Get the blocks (striped or contiguous).
*/
public BlockInfo[] getBlocks();
@@ -55,8 +55,8 @@ public interface BlockCollection {
public long getPreferredBlockSize();
/**
- * Get block replication for the collection
- * @return block replication value
+ * Get block replication for the collection.
+ * @return block replication value. Return 0 if the file is erasure coded.
*/
public short getPreferredBlockReplication();
@@ -71,7 +71,7 @@ public interface BlockCollection {
public String getName();
/**
- * Set the block at the given index.
+ * Set the block (contiguous or striped) at the given index.
*/
public void setBlock(int index, BlockInfo blk);
@@ -79,11 +79,16 @@ public interface BlockCollection {
* Convert the last block of the collection to an under-construction block
* and set the locations.
*/
- public BlockInfoContiguousUnderConstruction setLastBlock(BlockInfo lastBlock,
+ public void convertLastBlockToUC(BlockInfo lastBlock,
DatanodeStorageInfo[] targets) throws IOException;
/**
* @return whether the block collection is under construction.
*/
public boolean isUnderConstruction();
+
+ /**
+ * @return whether the block collection is in striping format
+ */
+ public boolean isStriped();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
index feebd87..685cfcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockIdManager.java
@@ -54,10 +54,12 @@ public class BlockIdManager {
* The global block ID space for this file system.
*/
private final SequentialBlockIdGenerator blockIdGenerator;
+ private final SequentialBlockGroupIdGenerator blockGroupIdGenerator;
public BlockIdManager(BlockManager blockManager) {
this.generationStampV1Limit = HdfsConstants.GRANDFATHER_GENERATION_STAMP;
this.blockIdGenerator = new SequentialBlockIdGenerator(blockManager);
+ this.blockGroupIdGenerator = new SequentialBlockGroupIdGenerator(blockManager);
}
/**
@@ -102,21 +104,38 @@ public class BlockIdManager {
}
/**
- * Sets the maximum allocated block ID for this filesystem. This is
+ * Sets the maximum allocated contiguous block ID for this filesystem. This is
* the basis for allocating new block IDs.
*/
- public void setLastAllocatedBlockId(long blockId) {
+ public void setLastAllocatedContiguousBlockId(long blockId) {
blockIdGenerator.skipTo(blockId);
}
/**
- * Gets the maximum sequentially allocated block ID for this filesystem
+ * Gets the maximum sequentially allocated contiguous block ID for this
+ * filesystem
*/
- public long getLastAllocatedBlockId() {
+ public long getLastAllocatedContiguousBlockId() {
return blockIdGenerator.getCurrentValue();
}
/**
+ * Sets the maximum allocated striped block ID for this filesystem. This is
+ * the basis for allocating new block IDs.
+ */
+ public void setLastAllocatedStripedBlockId(long blockId) {
+ blockGroupIdGenerator.skipTo(blockId);
+ }
+
+ /**
+ * Gets the maximum sequentially allocated striped block ID for this
+ * filesystem
+ */
+ public long getLastAllocatedStripedBlockId() {
+ return blockGroupIdGenerator.getCurrentValue();
+ }
+
+ /**
* Sets the current generation stamp for legacy blocks
*/
public void setGenerationStampV1(long stamp) {
@@ -187,10 +206,14 @@ public class BlockIdManager {
/**
* Increments, logs and then returns the block ID
*/
- public long nextBlockId() {
+ public long nextContiguousBlockId() {
return blockIdGenerator.nextValue();
}
+ public long nextStripedBlockId() {
+ return blockGroupIdGenerator.nextValue();
+ }
+
public boolean isGenStampInFuture(Block block) {
if (isLegacyBlock(block)) {
return block.getGenerationStamp() > getGenerationStampV1();
@@ -206,4 +229,27 @@ public class BlockIdManager {
.LAST_RESERVED_BLOCK_ID);
generationStampV1Limit = HdfsConstants.GRANDFATHER_GENERATION_STAMP;
}
-}
\ No newline at end of file
+
+ public static boolean isStripedBlockID(long id) {
+ return id < 0;
+ }
+
+ /**
+ * The last 4 bits of HdfsConstants.BLOCK_GROUP_INDEX_MASK(15) is 1111,
+ * so the last 4 bits of (~HdfsConstants.BLOCK_GROUP_INDEX_MASK) is 0000
+ * and the other 60 bits are 1. Group ID is the first 60 bits of any
+ * data/parity block id in the same striped block group.
+ */
+ public static long convertToStripedID(long id) {
+ return id & (~HdfsServerConstants.BLOCK_GROUP_INDEX_MASK);
+ }
+
+ public static int getBlockIndex(Block reportedBlock) {
+ return (int) (reportedBlock.getBlockId() &
+ HdfsServerConstants.BLOCK_GROUP_INDEX_MASK);
+ }
+
+ SequentialBlockGroupIdGenerator getBlockGroupIdGenerator() {
+ return blockGroupIdGenerator;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index dea31c4..a913535 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -55,16 +55,17 @@ public abstract class BlockInfo extends Block
/**
* Construct an entry for blocksmap
- * @param replication the block's replication factor
+ * @param size the block's replication factor, or the total number of blocks
+ * in the block group
*/
- public BlockInfo(short replication) {
- this.triplets = new Object[3*replication];
+ public BlockInfo(short size) {
+ this.triplets = new Object[3*size];
this.bc = null;
}
- public BlockInfo(Block blk, short replication) {
+ public BlockInfo(Block blk, short size) {
super(blk);
- this.triplets = new Object[3*replication];
+ this.triplets = new Object[3 * size];
this.bc = null;
}
@@ -108,7 +109,7 @@ public abstract class BlockInfo extends Block
BlockInfo info = (BlockInfo)triplets[index*3+1];
assert info == null ||
info.getClass().getName().startsWith(BlockInfo.class.getName()) :
- "BlockInfo is expected at " + index*3;
+ "BlockInfo is expected at " + index*3;
return info;
}
@@ -150,7 +151,7 @@ public abstract class BlockInfo extends Block
*
* @param index - the datanode index
* @param to - block to be set to next on the list of blocks
- * * @return current next block on the list of blocks
+ * @return current next block on the list of blocks
*/
BlockInfo setNext(int index, BlockInfo to) {
assert this.triplets != null : "BlockInfo is not initialized";
@@ -167,14 +168,20 @@ public abstract class BlockInfo extends Block
}
/**
- * Count the number of data-nodes the block belongs to.
+ * Count the number of data-nodes the block currently belongs to (i.e., NN
+ * has received block reports from the DN).
*/
public abstract int numNodes();
/**
- * Add a {@link DatanodeStorageInfo} location for a block.
+ * Add a {@link DatanodeStorageInfo} location for a block
+ * @param storage The storage to add
+ * @param reportedBlock The block reported from the datanode. This is only
+ * used by erasure coded blocks, this block's id contains
+ * information indicating the index of the block in the
+ * corresponding block group.
*/
- abstract boolean addStorage(DatanodeStorageInfo storage);
+ abstract boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock);
/**
* Remove {@link DatanodeStorageInfo} location for a block
@@ -188,6 +195,11 @@ public abstract class BlockInfo extends Block
*/
abstract void replaceBlock(BlockInfo newBlock);
+ public abstract boolean isStriped();
+
+ /** @return true if there is no datanode storage associated with the block */
+ abstract boolean hasNoStorage();
+
/**
* Find specified DatanodeStorageInfo.
* @return DatanodeStorageInfo or null if not found.
@@ -196,10 +208,9 @@ public abstract class BlockInfo extends Block
int len = getCapacity();
for(int idx = 0; idx < len; idx++) {
DatanodeStorageInfo cur = getStorageInfo(idx);
- if(cur == null)
- break;
- if(cur.getDatanodeDescriptor() == dn)
+ if(cur != null && cur.getDatanodeDescriptor() == dn) {
return cur;
+ }
}
return null;
}
@@ -215,9 +226,6 @@ public abstract class BlockInfo extends Block
if (cur == storageInfo) {
return idx;
}
- if (cur == null) {
- break;
- }
}
return -1;
}
@@ -261,12 +269,15 @@ public abstract class BlockInfo extends Block
BlockInfo prev = this.getPrevious(dnIndex);
this.setNext(dnIndex, null);
this.setPrevious(dnIndex, null);
- if(prev != null)
+ if (prev != null) {
prev.setNext(prev.findStorageInfo(storage), next);
- if(next != null)
+ }
+ if (next != null) {
next.setPrevious(next.findStorageInfo(storage), prev);
- if(this == head) // removing the head
+ }
+ if (this == head) { // removing the head
head = next;
+ }
return head;
}
@@ -295,7 +306,7 @@ public abstract class BlockInfo extends Block
/**
* BlockInfo represents a block that is not being constructed.
* In order to start modifying the block, the BlockInfo should be converted
- * to {@link BlockInfoContiguousUnderConstruction}.
+ * to {@link BlockInfoUnderConstruction}.
* @return {@link BlockUCState#COMPLETE}
*/
public BlockUCState getBlockUCState() {
@@ -311,28 +322,6 @@ public abstract class BlockInfo extends Block
return getBlockUCState().equals(BlockUCState.COMPLETE);
}
- /**
- * Convert a complete block to an under construction block.
- * @return BlockInfoUnderConstruction - an under construction block.
- */
- public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
- BlockUCState s, DatanodeStorageInfo[] targets) {
- if(isComplete()) {
- BlockInfoContiguousUnderConstruction ucBlock =
- new BlockInfoContiguousUnderConstruction(this,
- getBlockCollection().getPreferredBlockReplication(), s, targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
- // the block is already under construction
- BlockInfoContiguousUnderConstruction ucBlock =
- (BlockInfoContiguousUnderConstruction)this;
- ucBlock.setBlockUCState(s);
- ucBlock.setExpectedLocations(targets);
- ucBlock.setBlockCollection(getBlockCollection());
- return ucBlock;
- }
-
@Override
public int hashCode() {
// Super implementation is sufficient
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
index eff89a8..7e03ca5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
/**
* Subclass of {@link BlockInfo}, used for a block with replication scheme.
@@ -37,11 +38,12 @@ public class BlockInfoContiguous extends BlockInfo {
/**
* Copy construction.
- * This is used to convert BlockReplicationInfoUnderConstruction
- * @param from BlockReplicationInfo to copy from.
+ * This is used to convert BlockInfoContiguous
+ * @param from BlockInfoContiguous to copy from.
*/
protected BlockInfoContiguous(BlockInfoContiguous from) {
- super(from);
+ this(from, (short) (from.triplets.length / 3));
+ this.setBlockCollection(from.getBlockCollection());
}
/**
@@ -63,7 +65,7 @@ public class BlockInfoContiguous extends BlockInfo {
}
@Override
- boolean addStorage(DatanodeStorageInfo storage) {
+ boolean addStorage(DatanodeStorageInfo storage, Block reportedBlock) {
// find the last null node
int lastNode = ensureCapacity(1);
setStorageInfo(lastNode, storage);
@@ -115,9 +117,42 @@ public class BlockInfoContiguous extends BlockInfo {
assert removed : "currentBlock not found.";
final DatanodeStorageInfo.AddBlockResult result = storage.addBlock(
- newBlock);
+ newBlock, newBlock);
assert result == DatanodeStorageInfo.AddBlockResult.ADDED :
"newBlock already exists.";
}
}
+
+ /**
+ * Convert a complete block to an under construction block.
+ *
+ * @return BlockInfoUnderConstruction - an under construction block.
+ */
+ public BlockInfoContiguousUnderConstruction convertToBlockUnderConstruction(
+ BlockUCState s, DatanodeStorageInfo[] targets) {
+ if (isComplete()) {
+ BlockInfoContiguousUnderConstruction ucBlock =
+ new BlockInfoContiguousUnderConstruction(this,
+ getBlockCollection().getPreferredBlockReplication(), s, targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+ // the block is already under construction
+ BlockInfoContiguousUnderConstruction ucBlock =
+ (BlockInfoContiguousUnderConstruction) this;
+ ucBlock.setBlockUCState(s);
+ ucBlock.setExpectedLocations(targets);
+ ucBlock.setBlockCollection(getBlockCollection());
+ return ucBlock;
+ }
+
+ @Override
+ public final boolean isStriped() {
+ return false;
+ }
+
+ @Override
+ final boolean hasNoStorage() {
+ return getStorageInfo(0) == null;
+ }
}
[08/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index 43f2992..26ed1fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
-import java.util.EnumSet;
import java.util.List;
import java.util.Random;
@@ -69,28 +68,32 @@ import org.junit.Test;
public class TestBlockTokenWithDFS {
- private static final int BLOCK_SIZE = 1024;
- private static final int FILE_SIZE = 2 * BLOCK_SIZE;
+ protected static int BLOCK_SIZE = 1024;
+ protected static int FILE_SIZE = 2 * BLOCK_SIZE;
private static final String FILE_TO_READ = "/fileToRead.dat";
private static final String FILE_TO_WRITE = "/fileToWrite.dat";
private static final String FILE_TO_APPEND = "/fileToAppend.dat";
- private final byte[] rawData = new byte[FILE_SIZE];
{
((Log4JLogger) DFSClient.LOG).getLogger().setLevel(Level.ALL);
+ }
+
+ public static byte[] generateBytes(int fileSize){
Random r = new Random();
+ byte[] rawData = new byte[fileSize];
r.nextBytes(rawData);
+ return rawData;
}
- private void createFile(FileSystem fs, Path filename) throws IOException {
+ private void createFile(FileSystem fs, Path filename, byte[] expected) throws IOException {
FSDataOutputStream out = fs.create(filename);
- out.write(rawData);
+ out.write(expected);
out.close();
}
// read a file using blockSeekTo()
- private boolean checkFile1(FSDataInputStream in) {
- byte[] toRead = new byte[FILE_SIZE];
+ private boolean checkFile1(FSDataInputStream in, byte[] expected) {
+ byte[] toRead = new byte[expected.length];
int totalRead = 0;
int nRead = 0;
try {
@@ -101,27 +104,27 @@ public class TestBlockTokenWithDFS {
return false;
}
assertEquals("Cannot read file.", toRead.length, totalRead);
- return checkFile(toRead);
+ return checkFile(toRead, expected);
}
// read a file using fetchBlockByteRange()
- private boolean checkFile2(FSDataInputStream in) {
- byte[] toRead = new byte[FILE_SIZE];
+ private boolean checkFile2(FSDataInputStream in, byte[] expected) {
+ byte[] toRead = new byte[expected.length];
try {
assertEquals("Cannot read file", toRead.length, in.read(0, toRead, 0,
toRead.length));
} catch (IOException e) {
return false;
}
- return checkFile(toRead);
+ return checkFile(toRead, expected);
}
- private boolean checkFile(byte[] fileToCheck) {
- if (fileToCheck.length != rawData.length) {
+ private boolean checkFile(byte[] fileToCheck, byte[] expected) {
+ if (fileToCheck.length != expected.length) {
return false;
}
for (int i = 0; i < fileToCheck.length; i++) {
- if (fileToCheck[i] != rawData[i]) {
+ if (fileToCheck[i] != expected[i]) {
return false;
}
}
@@ -137,7 +140,7 @@ public class TestBlockTokenWithDFS {
}
// try reading a block using a BlockReader directly
- private static void tryRead(final Configuration conf, LocatedBlock lblock,
+ protected void tryRead(final Configuration conf, LocatedBlock lblock,
boolean shouldSucceed) {
InetSocketAddress targetAddr = null;
IOException ioe = null;
@@ -148,7 +151,7 @@ public class TestBlockTokenWithDFS {
targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr());
blockReader = new BlockReaderFactory(new DfsClientConf(conf)).
- setFileName(BlockReaderFactory.getFileName(targetAddr,
+ setFileName(BlockReaderFactory.getFileName(targetAddr,
"test-blockpoolid", block.getBlockId())).
setBlock(block).
setBlockToken(lblock.getBlockToken()).
@@ -205,7 +208,7 @@ public class TestBlockTokenWithDFS {
}
// get a conf for testing
- private static Configuration getConf(int numDataNodes) {
+ protected Configuration getConf(int numDataNodes) {
Configuration conf = new Configuration();
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
@@ -241,16 +244,16 @@ public class TestBlockTokenWithDFS {
SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
Path fileToAppend = new Path(FILE_TO_APPEND);
FileSystem fs = cluster.getFileSystem();
-
+ byte[] expected = generateBytes(FILE_SIZE);
// write a one-byte file
FSDataOutputStream stm = writeFile(fs, fileToAppend,
(short) numDataNodes, BLOCK_SIZE);
- stm.write(rawData, 0, 1);
+ stm.write(expected, 0, 1);
stm.close();
// open the file again for append
stm = fs.append(fileToAppend);
- int mid = rawData.length - 1;
- stm.write(rawData, 1, mid - 1);
+ int mid = expected.length - 1;
+ stm.write(expected, 1, mid - 1);
stm.hflush();
/*
@@ -267,11 +270,11 @@ public class TestBlockTokenWithDFS {
// remove a datanode to force re-establishing pipeline
cluster.stopDataNode(0);
// append the rest of the file
- stm.write(rawData, mid, rawData.length - mid);
+ stm.write(expected, mid, expected.length - mid);
stm.close();
// check if append is successful
FSDataInputStream in5 = fs.open(fileToAppend);
- assertTrue(checkFile1(in5));
+ assertTrue(checkFile1(in5, expected));
} finally {
if (cluster != null) {
cluster.shutdown();
@@ -303,11 +306,12 @@ public class TestBlockTokenWithDFS {
Path fileToWrite = new Path(FILE_TO_WRITE);
FileSystem fs = cluster.getFileSystem();
+ byte[] expected = generateBytes(FILE_SIZE);
FSDataOutputStream stm = writeFile(fs, fileToWrite, (short) numDataNodes,
BLOCK_SIZE);
// write a partial block
- int mid = rawData.length - 1;
- stm.write(rawData, 0, mid);
+ int mid = expected.length - 1;
+ stm.write(expected, 0, mid);
stm.hflush();
/*
@@ -324,11 +328,11 @@ public class TestBlockTokenWithDFS {
// remove a datanode to force re-establishing pipeline
cluster.stopDataNode(0);
// write the rest of the file
- stm.write(rawData, mid, rawData.length - mid);
+ stm.write(expected, mid, expected.length - mid);
stm.close();
// check if write is successful
FSDataInputStream in4 = fs.open(fileToWrite);
- assertTrue(checkFile1(in4));
+ assertTrue(checkFile1(in4, expected));
} finally {
if (cluster != null) {
cluster.shutdown();
@@ -346,125 +350,137 @@ public class TestBlockTokenWithDFS {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
cluster.waitActive();
assertEquals(numDataNodes, cluster.getDataNodes().size());
+ doTestRead(conf, cluster, false);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
- final NameNode nn = cluster.getNameNode();
- final NamenodeProtocols nnProto = nn.getRpcServer();
- final BlockManager bm = nn.getNamesystem().getBlockManager();
- final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
+ protected void doTestRead(Configuration conf, MiniDFSCluster cluster,
+ boolean isStriped) throws Exception {
+ final int numDataNodes = cluster.getDataNodes().size();
+ final NameNode nn = cluster.getNameNode();
+ final NamenodeProtocols nnProto = nn.getRpcServer();
+ final BlockManager bm = nn.getNamesystem().getBlockManager();
+ final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
- // set a short token lifetime (1 second) initially
- SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
+ // set a short token lifetime (1 second) initially
+ SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
- Path fileToRead = new Path(FILE_TO_READ);
- FileSystem fs = cluster.getFileSystem();
- createFile(fs, fileToRead);
+ Path fileToRead = new Path(FILE_TO_READ);
+ FileSystem fs = cluster.getFileSystem();
+ byte[] expected = generateBytes(FILE_SIZE);
+ createFile(fs, fileToRead, expected);
/*
* setup for testing expiration handling of cached tokens
*/
- // read using blockSeekTo(). Acquired tokens are cached in in1
- FSDataInputStream in1 = fs.open(fileToRead);
- assertTrue(checkFile1(in1));
- // read using blockSeekTo(). Acquired tokens are cached in in2
- FSDataInputStream in2 = fs.open(fileToRead);
- assertTrue(checkFile1(in2));
- // read using fetchBlockByteRange(). Acquired tokens are cached in in3
- FSDataInputStream in3 = fs.open(fileToRead);
- assertTrue(checkFile2(in3));
+ // read using blockSeekTo(). Acquired tokens are cached in in1
+ FSDataInputStream in1 = fs.open(fileToRead);
+ assertTrue(checkFile1(in1,expected));
+ // read using blockSeekTo(). Acquired tokens are cached in in2
+ FSDataInputStream in2 = fs.open(fileToRead);
+ assertTrue(checkFile1(in2,expected));
+ // read using fetchBlockByteRange(). Acquired tokens are cached in in3
+ FSDataInputStream in3 = fs.open(fileToRead);
+ assertTrue(checkFile2(in3,expected));
/*
* testing READ interface on DN using a BlockReader
*/
- DFSClient client = null;
- try {
- client = new DFSClient(new InetSocketAddress("localhost",
+ DFSClient client = null;
+ try {
+ client = new DFSClient(new InetSocketAddress("localhost",
cluster.getNameNodePort()), conf);
- } finally {
- if (client != null) client.close();
- }
- List<LocatedBlock> locatedBlocks = nnProto.getBlockLocations(
- FILE_TO_READ, 0, FILE_SIZE).getLocatedBlocks();
- LocatedBlock lblock = locatedBlocks.get(0); // first block
- Token<BlockTokenIdentifier> myToken = lblock.getBlockToken();
- // verify token is not expired
- assertFalse(SecurityTestUtil.isBlockTokenExpired(myToken));
- // read with valid token, should succeed
- tryRead(conf, lblock, true);
+ } finally {
+ if (client != null) client.close();
+ }
+ List<LocatedBlock> locatedBlocks = nnProto.getBlockLocations(
+ FILE_TO_READ, 0, FILE_SIZE).getLocatedBlocks();
+ LocatedBlock lblock = locatedBlocks.get(0); // first block
+ // verify token is not expired
+ assertFalse(isBlockTokenExpired(lblock));
+ // read with valid token, should succeed
+ tryRead(conf, lblock, true);
/*
* wait till myToken and all cached tokens in in1, in2 and in3 expire
*/
- while (!SecurityTestUtil.isBlockTokenExpired(myToken)) {
- try {
- Thread.sleep(10);
- } catch (InterruptedException ignored) {
- }
+ while (!isBlockTokenExpired(lblock)) {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException ignored) {
}
+ }
/*
* continue testing READ interface on DN using a BlockReader
*/
- // verify token is expired
- assertTrue(SecurityTestUtil.isBlockTokenExpired(myToken));
- // read should fail
- tryRead(conf, lblock, false);
- // use a valid new token
- lblock.setBlockToken(sm.generateToken(lblock.getBlock(),
- EnumSet.of(BlockTokenIdentifier.AccessMode.READ)));
- // read should succeed
- tryRead(conf, lblock, true);
- // use a token with wrong blockID
- ExtendedBlock wrongBlock = new ExtendedBlock(lblock.getBlock()
- .getBlockPoolId(), lblock.getBlock().getBlockId() + 1);
- lblock.setBlockToken(sm.generateToken(wrongBlock,
- EnumSet.of(BlockTokenIdentifier.AccessMode.READ)));
- // read should fail
- tryRead(conf, lblock, false);
- // use a token with wrong access modes
- lblock.setBlockToken(sm.generateToken(lblock.getBlock(),
- EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE,
- BlockTokenIdentifier.AccessMode.COPY,
- BlockTokenIdentifier.AccessMode.REPLACE)));
- // read should fail
- tryRead(conf, lblock, false);
-
- // set a long token lifetime for future tokens
- SecurityTestUtil.setBlockTokenLifetime(sm, 600 * 1000L);
+ // verify token is expired
+ assertTrue(isBlockTokenExpired(lblock));
+ // read should fail
+ tryRead(conf, lblock, false);
+ // use a valid new token
+ bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.READ);
+ // read should succeed
+ tryRead(conf, lblock, true);
+ // use a token with wrong blockID
+ long rightId = lblock.getBlock().getBlockId();
+ long wrongId = rightId + 1;
+ lblock.getBlock().setBlockId(wrongId);
+ bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.READ);
+ lblock.getBlock().setBlockId(rightId);
+ // read should fail
+ tryRead(conf, lblock, false);
+ // use a token with wrong access modes
+ bm.setBlockToken(lblock, BlockTokenIdentifier.AccessMode.WRITE);
+ // read should fail
+ tryRead(conf, lblock, false);
+
+ // set a long token lifetime for future tokens
+ SecurityTestUtil.setBlockTokenLifetime(sm, 600 * 1000L);
/*
* testing that when cached tokens are expired, DFSClient will re-fetch
* tokens transparently for READ.
*/
- // confirm all tokens cached in in1 are expired by now
- List<LocatedBlock> lblocks = DFSTestUtil.getAllBlocks(in1);
- for (LocatedBlock blk : lblocks) {
- assertTrue(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
- }
- // verify blockSeekTo() is able to re-fetch token transparently
- in1.seek(0);
- assertTrue(checkFile1(in1));
-
- // confirm all tokens cached in in2 are expired by now
- List<LocatedBlock> lblocks2 = DFSTestUtil.getAllBlocks(in2);
- for (LocatedBlock blk : lblocks2) {
- assertTrue(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
- }
- // verify blockSeekTo() is able to re-fetch token transparently (testing
- // via another interface method)
+ // confirm all tokens cached in in1 are expired by now
+ List<LocatedBlock> lblocks = DFSTestUtil.getAllBlocks(in1);
+ for (LocatedBlock blk : lblocks) {
+ assertTrue(isBlockTokenExpired(blk));
+ }
+ // verify blockSeekTo() is able to re-fetch token transparently
+ in1.seek(0);
+ assertTrue(checkFile1(in1, expected));
+
+ // confirm all tokens cached in in2 are expired by now
+ List<LocatedBlock> lblocks2 = DFSTestUtil.getAllBlocks(in2);
+ for (LocatedBlock blk : lblocks2) {
+ assertTrue(isBlockTokenExpired(blk));
+ }
+ // verify blockSeekTo() is able to re-fetch token transparently (testing
+ // via another interface method)
+ if (isStriped) {
+ // striped block doesn't support seekToNewSource
+ in2.seek(0);
+ } else {
assertTrue(in2.seekToNewSource(0));
- assertTrue(checkFile1(in2));
+ }
+ assertTrue(checkFile1(in2,expected));
- // confirm all tokens cached in in3 are expired by now
- List<LocatedBlock> lblocks3 = DFSTestUtil.getAllBlocks(in3);
- for (LocatedBlock blk : lblocks3) {
- assertTrue(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
- }
- // verify fetchBlockByteRange() is able to re-fetch token transparently
- assertTrue(checkFile2(in3));
+ // confirm all tokens cached in in3 are expired by now
+ List<LocatedBlock> lblocks3 = DFSTestUtil.getAllBlocks(in3);
+ for (LocatedBlock blk : lblocks3) {
+ assertTrue(isBlockTokenExpired(blk));
+ }
+ // verify fetchBlockByteRange() is able to re-fetch token transparently
+ assertTrue(checkFile2(in3,expected));
/*
* testing that after datanodes are restarted on the same ports, cached
@@ -473,37 +489,42 @@ public class TestBlockTokenWithDFS {
* new tokens can be fetched from namenode).
*/
- // restart datanodes on the same ports that they currently use
- assertTrue(cluster.restartDataNodes(true));
- cluster.waitActive();
- assertEquals(numDataNodes, cluster.getDataNodes().size());
- cluster.shutdownNameNode(0);
+ // restart datanodes on the same ports that they currently use
+ assertTrue(cluster.restartDataNodes(true));
+ cluster.waitActive();
+ assertEquals(numDataNodes, cluster.getDataNodes().size());
+ cluster.shutdownNameNode(0);
- // confirm tokens cached in in1 are still valid
- lblocks = DFSTestUtil.getAllBlocks(in1);
- for (LocatedBlock blk : lblocks) {
- assertFalse(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
- }
- // verify blockSeekTo() still works (forced to use cached tokens)
- in1.seek(0);
- assertTrue(checkFile1(in1));
-
- // confirm tokens cached in in2 are still valid
- lblocks2 = DFSTestUtil.getAllBlocks(in2);
- for (LocatedBlock blk : lblocks2) {
- assertFalse(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
- }
- // verify blockSeekTo() still works (forced to use cached tokens)
+ // confirm tokens cached in in1 are still valid
+ lblocks = DFSTestUtil.getAllBlocks(in1);
+ for (LocatedBlock blk : lblocks) {
+ assertFalse(isBlockTokenExpired(blk));
+ }
+ // verify blockSeekTo() still works (forced to use cached tokens)
+ in1.seek(0);
+ assertTrue(checkFile1(in1,expected));
+
+ // confirm tokens cached in in2 are still valid
+ lblocks2 = DFSTestUtil.getAllBlocks(in2);
+ for (LocatedBlock blk : lblocks2) {
+ assertFalse(isBlockTokenExpired(blk));
+ }
+
+ // verify blockSeekTo() still works (forced to use cached tokens)
+ if (isStriped) {
+ in2.seek(0);
+ } else {
in2.seekToNewSource(0);
- assertTrue(checkFile1(in2));
+ }
+ assertTrue(checkFile1(in2,expected));
- // confirm tokens cached in in3 are still valid
- lblocks3 = DFSTestUtil.getAllBlocks(in3);
- for (LocatedBlock blk : lblocks3) {
- assertFalse(SecurityTestUtil.isBlockTokenExpired(blk.getBlockToken()));
- }
- // verify fetchBlockByteRange() still works (forced to use cached tokens)
- assertTrue(checkFile2(in3));
+ // confirm tokens cached in in3 are still valid
+ lblocks3 = DFSTestUtil.getAllBlocks(in3);
+ for (LocatedBlock blk : lblocks3) {
+ assertFalse(isBlockTokenExpired(blk));
+ }
+ // verify fetchBlockByteRange() still works (forced to use cached tokens)
+ assertTrue(checkFile2(in3,expected));
/*
* testing that when namenode is restarted, cached tokens should still
@@ -512,18 +533,23 @@ public class TestBlockTokenWithDFS {
* setup for this test depends on the previous test.
*/
- // restart the namenode and then shut it down for test
- cluster.restartNameNode(0);
- cluster.shutdownNameNode(0);
+ // restart the namenode and then shut it down for test
+ cluster.restartNameNode(0);
+ cluster.shutdownNameNode(0);
- // verify blockSeekTo() still works (forced to use cached tokens)
- in1.seek(0);
- assertTrue(checkFile1(in1));
- // verify again blockSeekTo() still works (forced to use cached tokens)
+ // verify blockSeekTo() still works (forced to use cached tokens)
+ in1.seek(0);
+ assertTrue(checkFile1(in1,expected));
+ // verify again blockSeekTo() still works (forced to use cached tokens)
+ if (isStriped) {
+ in2.seek(0);
+ } else {
in2.seekToNewSource(0);
- assertTrue(checkFile1(in2));
- // verify fetchBlockByteRange() still works (forced to use cached tokens)
- assertTrue(checkFile2(in3));
+ }
+ assertTrue(checkFile1(in2,expected));
+
+ // verify fetchBlockByteRange() still works (forced to use cached tokens)
+ assertTrue(checkFile2(in3,expected));
/*
* testing that after both namenode and datanodes got restarted (namenode
@@ -532,58 +558,60 @@ public class TestBlockTokenWithDFS {
* setup of this test depends on the previous test.
*/
- // restore the cluster and restart the datanodes for test
- cluster.restartNameNode(0);
- assertTrue(cluster.restartDataNodes(true));
- cluster.waitActive();
- assertEquals(numDataNodes, cluster.getDataNodes().size());
-
- // shutdown namenode so that DFSClient can't get new tokens from namenode
- cluster.shutdownNameNode(0);
-
- // verify blockSeekTo() fails (cached tokens become invalid)
- in1.seek(0);
- assertFalse(checkFile1(in1));
- // verify fetchBlockByteRange() fails (cached tokens become invalid)
- assertFalse(checkFile2(in3));
-
- // restart the namenode to allow DFSClient to re-fetch tokens
- cluster.restartNameNode(0);
- // verify blockSeekTo() works again (by transparently re-fetching
- // tokens from namenode)
- in1.seek(0);
- assertTrue(checkFile1(in1));
+ // restore the cluster and restart the datanodes for test
+ cluster.restartNameNode(0);
+ assertTrue(cluster.restartDataNodes(true));
+ cluster.waitActive();
+ assertEquals(numDataNodes, cluster.getDataNodes().size());
+
+ // shutdown namenode so that DFSClient can't get new tokens from namenode
+ cluster.shutdownNameNode(0);
+
+ // verify blockSeekTo() fails (cached tokens become invalid)
+ in1.seek(0);
+ assertFalse(checkFile1(in1,expected));
+ // verify fetchBlockByteRange() fails (cached tokens become invalid)
+ assertFalse(checkFile2(in3,expected));
+
+ // restart the namenode to allow DFSClient to re-fetch tokens
+ cluster.restartNameNode(0);
+ // verify blockSeekTo() works again (by transparently re-fetching
+ // tokens from namenode)
+ in1.seek(0);
+ assertTrue(checkFile1(in1,expected));
+ if (isStriped) {
+ in2.seek(0);
+ } else {
in2.seekToNewSource(0);
- assertTrue(checkFile1(in2));
- // verify fetchBlockByteRange() works again (by transparently
- // re-fetching tokens from namenode)
- assertTrue(checkFile2(in3));
+ }
+ assertTrue(checkFile1(in2,expected));
+ // verify fetchBlockByteRange() works again (by transparently
+ // re-fetching tokens from namenode)
+ assertTrue(checkFile2(in3,expected));
/*
* testing that when datanodes are restarted on different ports, DFSClient
* is able to re-fetch tokens transparently to connect to them
*/
- // restart datanodes on newly assigned ports
- assertTrue(cluster.restartDataNodes(false));
- cluster.waitActive();
- assertEquals(numDataNodes, cluster.getDataNodes().size());
- // verify blockSeekTo() is able to re-fetch token transparently
- in1.seek(0);
- assertTrue(checkFile1(in1));
- // verify blockSeekTo() is able to re-fetch token transparently
+ // restart datanodes on newly assigned ports
+ assertTrue(cluster.restartDataNodes(false));
+ cluster.waitActive();
+ assertEquals(numDataNodes, cluster.getDataNodes().size());
+ // verify blockSeekTo() is able to re-fetch token transparently
+ in1.seek(0);
+ assertTrue(checkFile1(in1,expected));
+ // verify blockSeekTo() is able to re-fetch token transparently
+ if (isStriped) {
+ in2.seek(0);
+ } else {
in2.seekToNewSource(0);
- assertTrue(checkFile1(in2));
- // verify fetchBlockByteRange() is able to re-fetch token transparently
- assertTrue(checkFile2(in3));
-
- } finally {
- if (cluster != null) {
- cluster.shutdown();
- }
}
- }
+ assertTrue(checkFile1(in2,expected));
+ // verify fetchBlockByteRange() is able to re-fetch token transparently
+ assertTrue(checkFile2(in3,expected));
+ }
/**
* Integration testing of access token, involving NN, DN, and Balancer
*/
@@ -593,4 +621,8 @@ public class TestBlockTokenWithDFS {
conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
new TestBalancer().integrationTest(conf);
}
+
+ protected boolean isBlockTokenExpired(LocatedBlock lb) throws IOException {
+ return SecurityTestUtil.isBlockTokenExpired(lb.getBlockToken());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
new file mode 100644
index 0000000..f985f54
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
+
+ private final static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+ private final static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+ private final static int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final static int stripesPerBlock = 4;
+ private final static int numDNs = dataBlocks + parityBlocks + 2;
+ private static MiniDFSCluster cluster;
+ private static Configuration conf;
+
+ {
+ BLOCK_SIZE = cellSize * stripesPerBlock;
+ FILE_SIZE = BLOCK_SIZE * dataBlocks * 3;
+ }
+
+ private Configuration getConf() {
+ Configuration conf = super.getConf(numDNs);
+ conf.setInt("io.bytes.per.checksum", cellSize);
+ return conf;
+ }
+
+ @Test
+ @Override
+ public void testRead() throws Exception {
+ conf = getConf();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.getFileSystem().getClient()
+ .createErasureCodingZone("/", null, cellSize);
+ try {
+ cluster.waitActive();
+ doTestRead(conf, cluster, true);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+
+ /**
+ * tested at {@link org.apache.hadoop.hdfs.TestDFSStripedOutputStreamWithFailure#testBlockTokenExpired()}
+ */
+ @Test
+ @Override
+ public void testWrite(){
+ }
+
+ @Test
+ @Override
+ public void testAppend() throws Exception {
+ //TODO: support Append for striped file
+ }
+
+ @Test
+ @Override
+ public void testEnd2End() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+ new TestBalancer().integrationTestWithStripedFile(conf);
+ }
+
+ @Override
+ protected void tryRead(final Configuration conf, LocatedBlock lblock,
+ boolean shouldSucceed) {
+ LocatedStripedBlock lsb = (LocatedStripedBlock) lblock;
+ LocatedBlock[] internalBlocks = StripedBlockUtil.parseStripedBlockGroup
+ (lsb, cellSize, dataBlocks, parityBlocks);
+ for (LocatedBlock internalBlock : internalBlocks) {
+ super.tryRead(conf, internalBlock, shouldSucceed);
+ }
+ }
+
+ @Override
+ protected boolean isBlockTokenExpired(LocatedBlock lb) throws IOException {
+ LocatedStripedBlock lsb = (LocatedStripedBlock) lb;
+ LocatedBlock[] internalBlocks = StripedBlockUtil.parseStripedBlockGroup
+ (lsb, cellSize, dataBlocks, parityBlocks);
+ for (LocatedBlock internalBlock : internalBlocks) {
+ if(super.isBlockTokenExpired(internalBlock)){
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
index 6fc30ba..c1218a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestHeartbeatHandling.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
-import org.apache.hadoop.util.Time;
import org.junit.Test;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
index cea6865..b11b48a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
@@ -185,9 +185,12 @@ public class TestNameNodePrunesMissingStorages {
String datanodeUuid;
// Find the first storage which this block is in.
try {
+ BlockInfo storedBlock =
+ cluster.getNamesystem().getBlockManager().
+ getStoredBlock(block.getLocalBlock());
Iterator<DatanodeStorageInfo> storageInfoIter =
cluster.getNamesystem().getBlockManager().
- getStorages(block.getLocalBlock()).iterator();
+ blocksMap.getStorages(storedBlock).iterator();
assertTrue(storageInfoIter.hasNext());
DatanodeStorageInfo info = storageInfoIter.next();
storageIdToRemove = info.getStorageID();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
index 1c3f075..c33667d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNodeCount.java
@@ -100,7 +100,7 @@ public class TestNodeCount {
DatanodeDescriptor nonExcessDN = null;
for(DatanodeStorageInfo storage : bm.blocksMap.getStorages(block.getLocalBlock())) {
final DatanodeDescriptor dn = storage.getDatanodeDescriptor();
- Collection<Block> blocks = bm.excessReplicateMap.get(dn.getDatanodeUuid());
+ Collection<BlockInfo> blocks = bm.excessReplicateMap.get(dn.getDatanodeUuid());
if (blocks == null || !blocks.contains(block.getLocalBlock()) ) {
nonExcessDN = dn;
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
index 2d7bb44..83b3aa0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
@@ -34,7 +34,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
@@ -42,7 +41,6 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.util.Time;
import org.junit.Test;
public class TestOverReplicatedBlocks {
@@ -185,7 +183,7 @@ public class TestOverReplicatedBlocks {
// All replicas for deletion should be scheduled on lastDN.
// And should not actually be deleted, because lastDN does not heartbeat.
namesystem.readLock();
- Collection<Block> dnBlocks =
+ Collection<BlockInfo> dnBlocks =
namesystem.getBlockManager().excessReplicateMap.get(lastDNid);
assertEquals("Replicas on node " + lastDNid + " should have been deleted",
SMALL_FILE_LENGTH / SMALL_BLOCK_SIZE, dnBlocks.size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 6553185..f5af898 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -1208,8 +1208,17 @@ public class TestReplicationPolicy {
BlockManager bm = new BlockManager(mockNS, new HdfsConfiguration());
UnderReplicatedBlocks underReplicatedBlocks = bm.neededReplications;
- BlockInfo block1 = genBlockInfo(ThreadLocalRandom.current().nextLong());
- BlockInfo block2 = genBlockInfo(ThreadLocalRandom.current().nextLong());
+ long blkID1 = ThreadLocalRandom.current().nextLong();
+ if (blkID1 < 0) {
+ blkID1 *= -1;
+ }
+ long blkID2 = ThreadLocalRandom.current().nextLong();
+ if (blkID2 < 0) {
+ blkID2 *= -1;
+ }
+
+ BlockInfo block1 = genBlockInfo(blkID1);
+ BlockInfo block2 = genBlockInfo(blkID2);
// Adding QUEUE_UNDER_REPLICATED block
underReplicatedBlocks.add(block1, 0, 1, 1);
@@ -1224,7 +1233,7 @@ public class TestReplicationPolicy {
chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1);
assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
- final BlockInfo info = new BlockInfoContiguous(block1, (short) 1);
+ final BlockInfoContiguous info = new BlockInfoContiguous(block1, (short) 1);
final BlockCollection mbc = mock(BlockCollection.class);
when(mbc.getLastBlock()).thenReturn(info);
when(mbc.getPreferredBlockSize()).thenReturn(block1.getNumBytes() + 1);
@@ -1247,12 +1256,12 @@ public class TestReplicationPolicy {
when(storage.getState()).thenReturn(DatanodeStorage.State.NORMAL);
when(storage.getDatanodeDescriptor()).thenReturn(dn);
when(storage.removeBlock(any(BlockInfo.class))).thenReturn(true);
- when(storage.addBlock(any(BlockInfo.class))).thenReturn
+ when(storage.addBlock(any(BlockInfoContiguous.class))).thenReturn
(DatanodeStorageInfo.AddBlockResult.ADDED);
- ucBlock.addStorage(storage);
+ ucBlock.addStorage(storage, ucBlock);
- when(mbc.setLastBlock((BlockInfo) any(), (DatanodeStorageInfo[]) any()))
- .thenReturn(ucBlock);
+ BlockInfo lastBlk = mbc.getLastBlock();
+ when(mbc.getLastBlock()).thenReturn(lastBlk, ucBlock);
bm.convertLastBlockToUnderConstruction(mbc, 0L);
@@ -1287,7 +1296,7 @@ public class TestReplicationPolicy {
chosenBlocks = underReplicatedBlocks.chooseUnderReplicatedBlocks(1);
assertTheChosenBlocks(chosenBlocks, 1, 0, 0, 0, 0);
- bm.setReplication((short)0, (short)1, "", block1);
+ bm.setReplication((short)0, (short)1, "", (BlockInfoContiguous) block1);
// Choose 1 block from UnderReplicatedBlocks. Then it should pick 1 block
// from QUEUE_VERY_UNDER_REPLICATED.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
new file mode 100644
index 0000000..2f2356f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BLOCK_GROUP_INDEX_MASK;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.MAX_BLOCKS_IN_GROUP;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Tests the sequential blockGroup ID generation mechanism and blockGroup ID
+ * collision handling.
+ */
+public class TestSequentialBlockGroupId {
+ private static final Log LOG = LogFactory
+ .getLog("TestSequentialBlockGroupId");
+
+ private final short REPLICATION = 1;
+ private final long SEED = 0;
+ private final int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+ private final int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+ private final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+
+ private final int stripesPerBlock = 2;
+ private final int blockSize = cellSize * stripesPerBlock;
+ private final int numDNs = dataBlocks + parityBlocks + 2;
+ private final int blockGrpCount = 4;
+ private final int fileLen = blockSize * dataBlocks * blockGrpCount;
+
+ private MiniDFSCluster cluster;
+ private FileSystem fs;
+ private SequentialBlockGroupIdGenerator blockGrpIdGenerator;
+ private Path eczone = new Path("/eczone");
+
+ @Before
+ public void setup() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.waitActive();
+
+ fs = cluster.getFileSystem();
+ blockGrpIdGenerator = cluster.getNamesystem().getBlockIdManager()
+ .getBlockGroupIdGenerator();
+ fs.mkdirs(eczone);
+ cluster.getFileSystem().getClient()
+ .createErasureCodingZone("/eczone", null, cellSize);
+ }
+
+ @After
+ public void teardown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Test that blockGroup IDs are generating unique value.
+ */
+ @Test(timeout = 60000)
+ public void testBlockGroupIdGeneration() throws IOException {
+ long blockGroupIdInitialValue = blockGrpIdGenerator.getCurrentValue();
+
+ // Create a file that is 4 blocks long.
+ Path path = new Path(eczone, "testBlockGrpIdGeneration.dat");
+ DFSTestUtil.createFile(fs, path, cellSize, fileLen, blockSize, REPLICATION,
+ SEED);
+ List<LocatedBlock> blocks = DFSTestUtil.getAllBlocks(fs, path);
+ assertThat("Wrong BlockGrps", blocks.size(), is(blockGrpCount));
+
+ // initialising the block group generator for verifying the block id
+ blockGrpIdGenerator.setCurrentValue(blockGroupIdInitialValue);
+ // Ensure that the block IDs are generating unique value.
+ for (int i = 0; i < blocks.size(); ++i) {
+ blockGrpIdGenerator
+ .skipTo((blockGrpIdGenerator.getCurrentValue() & ~BLOCK_GROUP_INDEX_MASK)
+ + MAX_BLOCKS_IN_GROUP);
+ long nextBlockExpectedId = blockGrpIdGenerator.getCurrentValue();
+ long nextBlockGrpId = blocks.get(i).getBlock().getBlockId();
+ LOG.info("BlockGrp" + i + " id is " + nextBlockGrpId);
+ assertThat("BlockGrpId mismatches!", nextBlockGrpId,
+ is(nextBlockExpectedId));
+ }
+ }
+
+ /**
+ * Test that collisions in the blockGroup ID space are handled gracefully.
+ */
+ @Test(timeout = 60000)
+ public void testTriggerBlockGroupIdCollision() throws IOException {
+ long blockGroupIdInitialValue = blockGrpIdGenerator.getCurrentValue();
+
+ // Create a file with a few blocks to rev up the global block ID
+ // counter.
+ Path path1 = new Path(eczone, "testBlockGrpIdCollisionDetection_file1.dat");
+ DFSTestUtil.createFile(fs, path1, cellSize, fileLen, blockSize,
+ REPLICATION, SEED);
+ List<LocatedBlock> blocks1 = DFSTestUtil.getAllBlocks(fs, path1);
+ assertThat("Wrong BlockGrps", blocks1.size(), is(blockGrpCount));
+
+ // Rewind the block ID counter in the name system object. This will result
+ // in block ID collisions when we try to allocate new blocks.
+ blockGrpIdGenerator.setCurrentValue(blockGroupIdInitialValue);
+
+ // Trigger collisions by creating a new file.
+ Path path2 = new Path(eczone, "testBlockGrpIdCollisionDetection_file2.dat");
+ DFSTestUtil.createFile(fs, path2, cellSize, fileLen, blockSize,
+ REPLICATION, SEED);
+ List<LocatedBlock> blocks2 = DFSTestUtil.getAllBlocks(fs, path2);
+ assertThat("Wrong BlockGrps", blocks2.size(), is(blockGrpCount));
+
+ // Make sure that file1 and file2 block IDs are different
+ for (LocatedBlock locBlock1 : blocks1) {
+ long blockId1 = locBlock1.getBlock().getBlockId();
+ for (LocatedBlock locBlock2 : blocks2) {
+ long blockId2 = locBlock2.getBlock().getBlockId();
+ assertThat("BlockGrpId mismatches!", blockId1, is(not(blockId2)));
+ }
+ }
+ }
+
+ /**
+ * Test that collisions in the blockGroup ID when the id is occupied by legacy
+ * block.
+ */
+ @Test(timeout = 60000)
+ public void testTriggerBlockGroupIdCollisionWithLegacyBlockId()
+ throws Exception {
+ long blockGroupIdInitialValue = blockGrpIdGenerator.getCurrentValue();
+ blockGrpIdGenerator
+ .skipTo((blockGrpIdGenerator.getCurrentValue() & ~BLOCK_GROUP_INDEX_MASK)
+ + MAX_BLOCKS_IN_GROUP);
+ final long curBlockGroupIdValue = blockGrpIdGenerator.getCurrentValue();
+
+ // Creates contiguous block with negative blockId so that it would trigger
+ // collision during blockGroup Id generation
+ FSNamesystem fsn = cluster.getNamesystem();
+ // Replace SequentialBlockIdGenerator with a spy
+ SequentialBlockIdGenerator blockIdGenerator = spy(fsn.getBlockIdManager()
+ .getBlockIdGenerator());
+ Whitebox.setInternalState(fsn.getBlockIdManager(), "blockIdGenerator",
+ blockIdGenerator);
+ SequentialBlockIdGenerator spySequentialBlockIdGenerator = new SequentialBlockIdGenerator(
+ null) {
+ @Override
+ public long nextValue() {
+ return curBlockGroupIdValue;
+ }
+ };
+ final Answer<Object> delegator = new GenericTestUtils.DelegateAnswer(
+ spySequentialBlockIdGenerator);
+ doAnswer(delegator).when(blockIdGenerator).nextValue();
+
+ Path path1 = new Path("/testCollisionWithLegacyBlock_file1.dat");
+ DFSTestUtil.createFile(fs, path1, 1024, REPLICATION, SEED);
+
+ List<LocatedBlock> contiguousBlocks = DFSTestUtil.getAllBlocks(fs, path1);
+ assertThat(contiguousBlocks.size(), is(1));
+ Assert.assertEquals("Unexpected BlockId!", curBlockGroupIdValue,
+ contiguousBlocks.get(0).getBlock().getBlockId());
+
+ // Reset back to the initial value to trigger collision
+ blockGrpIdGenerator.setCurrentValue(blockGroupIdInitialValue);
+ // Trigger collisions by creating a new file.
+ Path path2 = new Path(eczone, "testCollisionWithLegacyBlock_file2.dat");
+ DFSTestUtil.createFile(fs, path2, cellSize, fileLen, blockSize,
+ REPLICATION, SEED);
+ List<LocatedBlock> blocks2 = DFSTestUtil.getAllBlocks(fs, path2);
+ assertThat("Wrong BlockGrps", blocks2.size(), is(blockGrpCount));
+
+ // Make sure that file1 and file2 block IDs are different
+ for (LocatedBlock locBlock1 : contiguousBlocks) {
+ long blockId1 = locBlock1.getBlock().getBlockId();
+ for (LocatedBlock locBlock2 : blocks2) {
+ long blockId2 = locBlock2.getBlock().getBlockId();
+ assertThat("BlockGrpId mismatches!", blockId1, is(not(blockId2)));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java
index de36e07..0f419ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestUnderReplicatedBlockQueues.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -28,10 +31,21 @@ import static org.junit.Assert.fail;
public class TestUnderReplicatedBlockQueues {
+ private final ECSchema ecSchema =
+ ErasureCodingSchemaManager.getSystemDefaultSchema();
+ private final int CELLSIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+
private BlockInfo genBlockInfo(long id) {
return new BlockInfoContiguous(new Block(id), (short) 3);
}
+ private BlockInfo genStripedBlockInfo(long id, long numBytes) {
+ BlockInfoStriped sblk = new BlockInfoStriped(new Block(id), ecSchema,
+ CELLSIZE);
+ sblk.setNumBytes(numBytes);
+ return sblk;
+ }
+
/**
* Test that adding blocks with different replication counts puts them
* into different queues
@@ -85,6 +99,54 @@ public class TestUnderReplicatedBlockQueues {
assertEquals(2, queues.getCorruptReplOneBlockSize());
}
+ @Test
+ public void testStripedBlockPriorities() throws Throwable {
+ int dataBlkNum = ecSchema.getNumDataUnits();
+ int parityBlkNUm = ecSchema.getNumParityUnits();
+ doTestStripedBlockPriorities(1, parityBlkNUm);
+ doTestStripedBlockPriorities(dataBlkNum, parityBlkNUm);
+ }
+
+ private void doTestStripedBlockPriorities(int dataBlkNum, int parityBlkNum)
+ throws Throwable {
+ int groupSize = dataBlkNum + parityBlkNum;
+ long numBytes = CELLSIZE * dataBlkNum;
+ UnderReplicatedBlocks queues = new UnderReplicatedBlocks();
+
+ // add a striped block which been left NUM_DATA_BLOCKS internal blocks
+ BlockInfo block1 = genStripedBlockInfo(-100, numBytes);
+ assertAdded(queues, block1, dataBlkNum, 0, groupSize);
+ assertEquals(1, queues.getUnderReplicatedBlockCount());
+ assertEquals(1, queues.size());
+ assertInLevel(queues, block1, UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY);
+
+ // add a striped block which been left NUM_DATA_BLOCKS+1 internal blocks
+ BlockInfo block2 = genStripedBlockInfo(-200, numBytes);
+ assertAdded(queues, block2, dataBlkNum + 1, 0, groupSize);
+ assertEquals(2, queues.getUnderReplicatedBlockCount());
+ assertEquals(2, queues.size());
+ assertInLevel(queues, block2,
+ UnderReplicatedBlocks.QUEUE_VERY_UNDER_REPLICATED);
+
+ // add a striped block which been left NUM_DATA_BLOCKS+2 internal blocks
+ BlockInfo block3 = genStripedBlockInfo(-300, numBytes);
+ assertAdded(queues, block3, dataBlkNum + 2, 0, groupSize);
+ assertEquals(3, queues.getUnderReplicatedBlockCount());
+ assertEquals(3, queues.size());
+ assertInLevel(queues, block3,
+ UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED);
+
+ // add a corrupted block
+ BlockInfo block_corrupt = genStripedBlockInfo(-400, numBytes);
+ assertEquals(0, queues.getCorruptBlockSize());
+ assertAdded(queues, block_corrupt, dataBlkNum - 1, 0, groupSize);
+ assertEquals(4, queues.size());
+ assertEquals(3, queues.getUnderReplicatedBlockCount());
+ assertEquals(1, queues.getCorruptBlockSize());
+ assertInLevel(queues, block_corrupt,
+ UnderReplicatedBlocks.QUEUE_WITH_CORRUPT_BLOCKS);
+ }
+
private void assertAdded(UnderReplicatedBlocks queues,
BlockInfo block,
int curReplicas,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 6e5f07c..82b77c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -100,7 +100,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
public static byte simulatedByte(Block b, long offsetInBlk) {
byte firstByte = (byte) (b.getBlockId() & BYTE_MASK);
- return (byte) ((firstByte + offsetInBlk) & BYTE_MASK);
+ return (byte) ((firstByte + offsetInBlk % 29) & BYTE_MASK);
}
public static final String CONFIG_PROPERTY_CAPACITY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
index 989e216..d8c651f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestIncrementalBrVariations.java
@@ -227,15 +227,6 @@ public class TestIncrementalBrVariations {
return new Block(10000000L, 100L, 1048576L);
}
- private static StorageReceivedDeletedBlocks[] makeReportForReceivedBlock(
- Block block, DatanodeStorage storage) {
- ReceivedDeletedBlockInfo[] receivedBlocks = new ReceivedDeletedBlockInfo[1];
- receivedBlocks[0] = new ReceivedDeletedBlockInfo(block, BlockStatus.RECEIVED_BLOCK, null);
- StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[1];
- reports[0] = new StorageReceivedDeletedBlocks(storage, receivedBlocks);
- return reports;
- }
-
/**
* Verify that the NameNode can learn about new storages from incremental
* block reports.
@@ -251,8 +242,9 @@ public class TestIncrementalBrVariations {
// Generate a report for a fake block on a fake storage.
final String newStorageUuid = UUID.randomUUID().toString();
final DatanodeStorage newStorage = new DatanodeStorage(newStorageUuid);
- StorageReceivedDeletedBlocks[] reports = makeReportForReceivedBlock(
- getDummyBlock(), newStorage);
+ StorageReceivedDeletedBlocks[] reports = DFSTestUtil.
+ makeReportForReceivedBlock(getDummyBlock(), BlockStatus.RECEIVED_BLOCK,
+ newStorage);
// Send the report to the NN.
cluster.getNameNodeRpc().blockReceivedAndDeleted(dn0Reg, poolId, reports);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index d3d814c..14503b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -34,11 +35,17 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.MiniDFSNNTopology;
+import org.apache.hadoop.hdfs.NameNodeProxies;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.balancer.Dispatcher;
import org.apache.hadoop.hdfs.server.balancer.Dispatcher.DBlock;
import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.mover.Mover.MLocation;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.test.GenericTestUtils;
@@ -99,7 +106,7 @@ public class TestMover {
final LocatedBlock lb = dfs.getClient().getLocatedBlocks(file, 0).get(0);
final List<MLocation> locations = MLocation.toLocations(lb);
final MLocation ml = locations.get(0);
- final DBlock db = mover.newDBlock(lb.getBlock().getLocalBlock(), locations);
+ final DBlock db = mover.newDBlock(lb, locations, null);
final List<StorageType> storageTypes = new ArrayList<StorageType>(
Arrays.asList(StorageType.DEFAULT, StorageType.DEFAULT));
@@ -409,4 +416,120 @@ public class TestMover {
cluster.shutdown();
}
}
+
+ int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+ int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+ private final static int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final static int stripesPerBlock = 4;
+ static int DEFAULT_STRIPE_BLOCK_SIZE = cellSize * stripesPerBlock;
+
+ static void initConfWithStripe(Configuration conf) {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_STRIPE_BLOCK_SIZE);
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ Dispatcher.setBlockMoveWaitTime(3000L);
+ }
+
+ @Test(timeout = 300000)
+ public void testMoverWithStripedFile() throws Exception {
+ final Configuration conf = new HdfsConfiguration();
+ initConfWithStripe(conf);
+
+ // start 10 datanodes
+ int numOfDatanodes =10;
+ int storagesPerDatanode=2;
+ long capacity = 10 * DEFAULT_STRIPE_BLOCK_SIZE;
+ long[][] capacities = new long[numOfDatanodes][storagesPerDatanode];
+ for (int i = 0; i < numOfDatanodes; i++) {
+ for(int j=0;j<storagesPerDatanode;j++){
+ capacities[i][j]=capacity;
+ }
+ }
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numOfDatanodes)
+ .storagesPerDatanode(storagesPerDatanode)
+ .storageTypes(new StorageType[][]{
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.DISK},
+ {StorageType.DISK, StorageType.ARCHIVE},
+ {StorageType.DISK, StorageType.ARCHIVE},
+ {StorageType.DISK, StorageType.ARCHIVE},
+ {StorageType.DISK, StorageType.ARCHIVE},
+ {StorageType.DISK, StorageType.ARCHIVE}})
+ .storageCapacities(capacities)
+ .build();
+
+ try {
+ cluster.waitActive();
+
+ // set "/bar" directory with HOT storage policy.
+ ClientProtocol client = NameNodeProxies.createProxy(conf,
+ cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy();
+ String barDir = "/bar";
+ client.mkdirs(barDir, new FsPermission((short) 777), true);
+ client.setStoragePolicy(barDir,
+ HdfsConstants.HOT_STORAGE_POLICY_NAME);
+ // set "/bar" directory with EC zone.
+ client.createErasureCodingZone(barDir, null, 0);
+
+ // write file to barDir
+ final String fooFile = "/bar/foo";
+ long fileLen = 20 * DEFAULT_STRIPE_BLOCK_SIZE ;
+ DFSTestUtil.createFile(cluster.getFileSystem(), new Path(fooFile),
+ fileLen,(short) 3, 0);
+
+ // verify storage types and locations
+ LocatedBlocks locatedBlocks =
+ client.getBlockLocations(fooFile, 0, fileLen);
+ for(LocatedBlock lb : locatedBlocks.getLocatedBlocks()){
+ for( StorageType type : lb.getStorageTypes()){
+ Assert.assertEquals(StorageType.DISK, type);
+ }
+ }
+ DFSTestUtil.verifyLocatedStripedBlocks(locatedBlocks,
+ dataBlocks + parityBlocks);
+
+ // start 5 more datanodes
+ numOfDatanodes +=5;
+ capacities = new long[5][storagesPerDatanode];
+ for (int i = 0; i < 5; i++) {
+ for(int j=0;j<storagesPerDatanode;j++){
+ capacities[i][j]=capacity;
+ }
+ }
+ cluster.startDataNodes(conf, 5,
+ new StorageType[][]{
+ {StorageType.ARCHIVE, StorageType.ARCHIVE},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE},
+ {StorageType.ARCHIVE, StorageType.ARCHIVE}},
+ true, null, null, null,capacities, null, false, false, false, null);
+ cluster.triggerHeartbeats();
+
+ // move file to ARCHIVE
+ client.setStoragePolicy(barDir, "COLD");
+ // run Mover
+ int rc = ToolRunner.run(conf, new Mover.Cli(),
+ new String[] { "-p", barDir });
+ Assert.assertEquals("Movement to ARCHIVE should be successfull", 0, rc);
+
+ // verify storage types and locations
+ locatedBlocks = client.getBlockLocations(fooFile, 0, fileLen);
+ for(LocatedBlock lb : locatedBlocks.getLocatedBlocks()){
+ for( StorageType type : lb.getStorageTypes()){
+ Assert.assertEquals(StorageType.ARCHIVE, type);
+ }
+ }
+ DFSTestUtil.verifyLocatedStripedBlocks(locatedBlocks,
+ dataBlocks + parityBlocks);
+
+ }finally{
+ cluster.shutdown();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
index b314584..a2cb434 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
@@ -71,8 +71,13 @@ public class NameNodeAdapter {
public static HdfsFileStatus getFileInfo(NameNode namenode, String src,
boolean resolveLink) throws AccessControlException, UnresolvedLinkException,
StandbyException, IOException {
- return FSDirStatAndListingOp.getFileInfo(namenode.getNamesystem()
- .getFSDirectory(), src, resolveLink);
+ namenode.getNamesystem().readLock();
+ try {
+ return FSDirStatAndListingOp.getFileInfo(namenode.getNamesystem()
+ .getFSDirectory(), src, resolveLink);
+ } finally {
+ namenode.getNamesystem().readUnlock();
+ }
}
public static boolean mkdirs(NameNode namenode, String src,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
new file mode 100644
index 0000000..337911d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestAddOverReplicatedStripedBlocks {
+
+ private MiniDFSCluster cluster;
+ private DistributedFileSystem fs;
+ private final Path dirPath = new Path("/striped");
+ private Path filePath = new Path(dirPath, "file");
+ private final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS;
+ private final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS;
+ private final short GROUP_SIZE = DATA_BLK_NUM + PARITY_BLK_NUM;
+ private final int CELLSIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final int NUM_STRIPE_PER_BLOCK = 4;
+ private final int BLOCK_SIZE = NUM_STRIPE_PER_BLOCK * CELLSIZE;
+ private final int numDNs = GROUP_SIZE + 3;
+
+ @Before
+ public void setup() throws IOException {
+ Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ // disable block recovery
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
+ SimulatedFSDataset.setFactory(conf);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ fs.mkdirs(dirPath);
+ fs.getClient().createErasureCodingZone(dirPath.toString(), null, CELLSIZE);
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testProcessOverReplicatedStripedBlock() throws Exception {
+ // create a file which has exact one block group to the first GROUP_SIZE DNs
+ long fileLen = DATA_BLK_NUM * BLOCK_SIZE;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, 1,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
+ long gs = bg.getBlock().getGenerationStamp();
+ String bpid = bg.getBlock().getBlockPoolId();
+ long groupId = bg.getBlock().getBlockId();
+ Block blk = new Block(groupId, BLOCK_SIZE, gs);
+ for (int i = 0; i < GROUP_SIZE; i++) {
+ blk.setBlockId(groupId + i);
+ cluster.injectBlocks(i, Arrays.asList(blk), bpid);
+ }
+ cluster.triggerBlockReports();
+
+ // let a internal block be over replicated with 2 redundant blocks.
+ blk.setBlockId(groupId + 2);
+ cluster.injectBlocks(numDNs - 3, Arrays.asList(blk), bpid);
+ cluster.injectBlocks(numDNs - 2, Arrays.asList(blk), bpid);
+ // let a internal block be over replicated with 1 redundant block.
+ blk.setBlockId(groupId + 6);
+ cluster.injectBlocks(numDNs - 1, Arrays.asList(blk), bpid);
+
+ // update blocksMap
+ cluster.triggerBlockReports();
+ // add to invalidates
+ cluster.triggerHeartbeats();
+ // datanode delete block
+ cluster.triggerHeartbeats();
+ // update blocksMap
+ cluster.triggerBlockReports();
+
+ // verify that all internal blocks exists
+ lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ DFSTestUtil.verifyLocatedStripedBlocks(lbs, GROUP_SIZE);
+ }
+
+ @Test
+ public void testProcessOverReplicatedSBSmallerThanFullBlocks()
+ throws Exception {
+ // Create a EC file which doesn't fill full internal blocks.
+ int fileLen = CELLSIZE * (DATA_BLK_NUM - 1);
+ byte[] content = new byte[fileLen];
+ DFSTestUtil.writeFile(fs, filePath, new String(content));
+ LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
+ long gs = bg.getBlock().getGenerationStamp();
+ String bpid = bg.getBlock().getBlockPoolId();
+ long groupId = bg.getBlock().getBlockId();
+ Block blk = new Block(groupId, BLOCK_SIZE, gs);
+ cluster.triggerBlockReports();
+ List<DatanodeInfo> infos = Arrays.asList(bg.getLocations());
+
+ // let a internal block be over replicated with 2 redundant blocks.
+ // Therefor number of internal blocks is over GROUP_SIZE. (5 data blocks +
+ // 3 parity blocks + 2 redundant blocks > GROUP_SIZE)
+ blk.setBlockId(groupId + 2);
+ List<DataNode> dataNodeList = cluster.getDataNodes();
+ for (int i = 0; i < numDNs; i++) {
+ if (!infos.contains(dataNodeList.get(i).getDatanodeId())) {
+ cluster.injectBlocks(i, Arrays.asList(blk), bpid);
+ System.out.println("XXX: inject block into datanode " + i);
+ }
+ }
+
+ // update blocksMap
+ cluster.triggerBlockReports();
+ // add to invalidates
+ cluster.triggerHeartbeats();
+ // datanode delete block
+ cluster.triggerHeartbeats();
+ // update blocksMap
+ cluster.triggerBlockReports();
+
+ // verify that all internal blocks exists
+ lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ DFSTestUtil.verifyLocatedStripedBlocks(lbs, GROUP_SIZE - 1);
+ }
+
+ @Test
+ public void testProcessOverReplicatedAndCorruptStripedBlock()
+ throws Exception {
+ long fileLen = DATA_BLK_NUM * BLOCK_SIZE;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, 1,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
+ long gs = bg.getBlock().getGenerationStamp();
+ String bpid = bg.getBlock().getBlockPoolId();
+ long groupId = bg.getBlock().getBlockId();
+ Block blk = new Block(groupId, BLOCK_SIZE, gs);
+ BlockInfoStriped blockInfo = new BlockInfoStriped(blk,
+ ErasureCodingSchemaManager.getSystemDefaultSchema(), CELLSIZE);
+ for (int i = 0; i < GROUP_SIZE; i++) {
+ blk.setBlockId(groupId + i);
+ cluster.injectBlocks(i, Arrays.asList(blk), bpid);
+ }
+ cluster.triggerBlockReports();
+
+ // let a internal block be corrupt
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ List<DatanodeInfo> infos = Arrays.asList(bg.getLocations());
+ List<String> storages = Arrays.asList(bg.getStorageIDs());
+ cluster.getNamesystem().writeLock();
+ try {
+ bm.findAndMarkBlockAsCorrupt(lbs.getLastLocatedBlock().getBlock(),
+ infos.get(0), storages.get(0), "TEST");
+ } finally {
+ cluster.getNamesystem().writeUnlock();
+ }
+ assertEquals(1, bm.countNodes(blockInfo).corruptReplicas());
+
+ // let a internal block be over replicated with 2 redundant block.
+ blk.setBlockId(groupId + 2);
+ cluster.injectBlocks(numDNs - 3, Arrays.asList(blk), bpid);
+ cluster.injectBlocks(numDNs - 2, Arrays.asList(blk), bpid);
+
+ // update blocksMap
+ cluster.triggerBlockReports();
+ // add to invalidates
+ cluster.triggerHeartbeats();
+ // datanode delete block
+ cluster.triggerHeartbeats();
+ // update blocksMap
+ cluster.triggerBlockReports();
+
+ // verify that all internal blocks exists
+ lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ DFSTestUtil.verifyLocatedStripedBlocks(lbs, GROUP_SIZE);
+ }
+
+ @Test
+ public void testProcessOverReplicatedAndMissingStripedBlock()
+ throws Exception {
+ long fileLen = CELLSIZE * DATA_BLK_NUM;
+ DFSTestUtil.createStripedFile(cluster, filePath, null, 1,
+ NUM_STRIPE_PER_BLOCK, false);
+ LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
+ long gs = bg.getBlock().getGenerationStamp();
+ String bpid = bg.getBlock().getBlockPoolId();
+ long groupId = bg.getBlock().getBlockId();
+ Block blk = new Block(groupId, BLOCK_SIZE, gs);
+ // only inject GROUP_SIZE - 1 blocks, so there is one block missing
+ for (int i = 0; i < GROUP_SIZE - 1; i++) {
+ blk.setBlockId(groupId + i);
+ cluster.injectBlocks(i, Arrays.asList(blk), bpid);
+ }
+ cluster.triggerBlockReports();
+
+ // let a internal block be over replicated with 2 redundant blocks.
+ // Therefor number of internal blocks is over GROUP_SIZE. (5 data blocks +
+ // 3 parity blocks + 2 redundant blocks > GROUP_SIZE)
+ blk.setBlockId(groupId + 2);
+ cluster.injectBlocks(numDNs - 3, Arrays.asList(blk), bpid);
+ cluster.injectBlocks(numDNs - 2, Arrays.asList(blk), bpid);
+
+ // update blocksMap
+ cluster.triggerBlockReports();
+ // add to invalidates
+ cluster.triggerHeartbeats();
+ // datanode delete block
+ cluster.triggerHeartbeats();
+ // update blocksMap
+ cluster.triggerBlockReports();
+
+ // Since one block is missing, when over-replicated blocks got deleted,
+ // we are left GROUP_SIZE - 1 blocks.
+ lbs = cluster.getNameNodeRpc().getBlockLocations(
+ filePath.toString(), 0, fileLen);
+ DFSTestUtil.verifyLocatedStripedBlocks(lbs, GROUP_SIZE - 1);
+ }
+
+}
[03/23] hadoop git commit: YARN-3999. RM hangs on draing events.
Contributed by Jian He
Posted by zh...@apache.org.
YARN-3999. RM hangs on draing events. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ae716fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ae716fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ae716fa
Branch: refs/heads/HDFS-7285-merge
Commit: 3ae716fa696b87e849dae40225dc59fb5ed114cb
Parents: 7c796fd
Author: Xuan <xg...@apache.org>
Authored: Tue Aug 11 18:25:11 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Tue Aug 11 18:25:11 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/util/JvmPauseMonitor.java | 12 +++++---
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +++
.../hadoop/yarn/event/AsyncDispatcher.java | 8 ++++-
.../src/main/resources/yarn-default.xml | 9 ++++++
.../hadoop/yarn/event/TestAsyncDispatcher.java | 25 ++++++++++++---
.../resourcemanager/RMActiveServiceContext.java | 30 ------------------
.../server/resourcemanager/RMContextImpl.java | 23 +++++++-------
.../server/resourcemanager/ResourceManager.java | 32 ++++++++++----------
.../server/resourcemanager/TestAppManager.java | 5 +--
.../resourcemanager/TestResourceManager.java | 1 +
.../TestRMAppLogAggregationStatus.java | 4 +--
.../rmapp/TestRMAppTransitions.java | 6 ++--
.../attempt/TestRMAppAttemptTransitions.java | 8 ++---
.../scheduler/capacity/TestUtils.java | 9 +++---
.../scheduler/fifo/TestFifoScheduler.java | 17 +++++------
16 files changed, 104 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
index 1fe7796..cd5afc6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
@@ -83,11 +83,13 @@ public class JvmPauseMonitor {
public void stop() {
shouldRun = false;
- monitorThread.interrupt();
- try {
- monitorThread.join();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ if (monitorThread != null) {
+ monitorThread.interrupt();
+ try {
+ monitorThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a1f6861..f31dd4b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -771,6 +771,7 @@ Release 2.7.2 - UNRELEASED
YARN-3978. Configurably turn off the saving of container info in Generic AHS
(Eric Payne via jeagles)
+
OPTIMIZATIONS
BUG FIXES
@@ -801,6 +802,8 @@ Release 2.7.2 - UNRELEASED
YARN-3990. AsyncDispatcher may overloaded with RMAppNodeUpdateEvent when
Node is connected/disconnected (Bibin A Chundatt via jlowe)
+ YARN-3999. RM hangs on draing events. (Jian He via xgong)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index f1baf5c..93f7ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1300,6 +1300,11 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
= 30 * 1000;
+ public static final String DISPATCHER_DRAIN_EVENTS_TIMEOUT =
+ YARN_PREFIX + "dispatcher.drain-events.timeout";
+
+ public static final long DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT = 300000;
+
/**
* CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
* entries
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index f670112..6cdfaad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import com.google.common.annotations.VisibleForTesting;
@@ -138,9 +139,14 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
if (drainEventsOnStop) {
blockNewEvents = true;
LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
+ long endTime = System.currentTimeMillis() + getConfig()
+ .getLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT,
+ YarnConfiguration.DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT);
+
synchronized (waitForDrained) {
while (!drained && eventHandlingThread != null
- && eventHandlingThread.isAlive()) {
+ && eventHandlingThread.isAlive()
+ && System.currentTimeMillis() < endTime) {
waitForDrained.wait(1000);
LOG.info("Waiting for AsyncDispatcher to drain. Thread state is :" +
eventHandlingThread.getState());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 8b3a3af..402377d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -91,6 +91,15 @@
</property>
<property>
+ <description>Timeout in milliseconds when YARN dispatcher tries to drain the
+ events. Typically, this happens when service is stopping. e.g. RM drains
+ the ATS events dispatcher when stopping.
+ </description>
+ <name>yarn.dispatcher.drain-events.timeout</name>
+ <value>300000</value>
+ </property>
+
+ <property>
<description>The expiry interval for application master reporting.</description>
<name>yarn.am.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index b5fd923..ba0deff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -18,18 +18,17 @@
package org.apache.hadoop.yarn.event;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.junit.Assert;
import org.junit.Test;
+import static org.mockito.Mockito.*;
+
public class TestAsyncDispatcher {
/* This test checks whether dispatcher hangs on close if following two things
@@ -58,5 +57,23 @@ public class TestAsyncDispatcher {
eventQueue.isEmpty());
disp.close();
}
+
+ // Test dispatcher should timeout on draining events.
+ @Test(timeout=10000)
+ public void testDispatchStopOnTimeout() throws Exception {
+ BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<Event>();
+ eventQueue = spy(eventQueue);
+ // simulate dispatcher is not drained.
+ when(eventQueue.isEmpty()).thenReturn(false);
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 2000);
+ DrainDispatcher disp = new DrainDispatcher(eventQueue);
+ disp.init(conf);
+ disp.setDrainEventsOnStop();
+ disp.start();
+ disp.waitForEventThreadToWait();
+ disp.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index e4c72d8..1abb14e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -92,8 +92,6 @@ public class RMActiveServiceContext {
private NodesListManager nodesListManager;
private ResourceTrackerService resourceTrackerService;
private ApplicationMasterService applicationMasterService;
- private RMApplicationHistoryWriter rmApplicationHistoryWriter;
- private SystemMetricsPublisher systemMetricsPublisher;
private RMNodeLabelsManager nodeLabelManager;
private long epoch;
private Clock systemClock = new SystemClock();
@@ -117,7 +115,6 @@ public class RMActiveServiceContext {
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter,
ResourceScheduler scheduler) {
this();
this.setContainerAllocationExpirer(containerAllocationExpirer);
@@ -128,7 +125,6 @@ public class RMActiveServiceContext {
this.setContainerTokenSecretManager(containerTokenSecretManager);
this.setNMTokenSecretManager(nmTokenSecretManager);
this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
- this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
this.setScheduler(scheduler);
RMStateStore nullStore = new NullRMStateStore();
@@ -370,32 +366,6 @@ public class RMActiveServiceContext {
@Private
@Unstable
- public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
- return rmApplicationHistoryWriter;
- }
-
- @Private
- @Unstable
- public void setSystemMetricsPublisher(
- SystemMetricsPublisher systemMetricsPublisher) {
- this.systemMetricsPublisher = systemMetricsPublisher;
- }
-
- @Private
- @Unstable
- public SystemMetricsPublisher getSystemMetricsPublisher() {
- return systemMetricsPublisher;
- }
-
- @Private
- @Unstable
- public void setRMApplicationHistoryWriter(
- RMApplicationHistoryWriter rmApplicationHistoryWriter) {
- this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
- }
-
- @Private
- @Unstable
public long getEpoch() {
return this.epoch;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 8cadc3b..d6d573d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -68,6 +68,9 @@ public class RMContextImpl implements RMContext {
private Configuration yarnConfiguration;
+ private RMApplicationHistoryWriter rmApplicationHistoryWriter;
+ private SystemMetricsPublisher systemMetricsPublisher;
+
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -87,7 +90,6 @@ public class RMContextImpl implements RMContext {
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter,
ResourceScheduler scheduler) {
this();
this.setDispatcher(rmDispatcher);
@@ -95,7 +97,7 @@ public class RMContextImpl implements RMContext {
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
delegationTokenRenewer, appTokenSecretManager,
containerTokenSecretManager, nmTokenSecretManager,
- clientToAMTokenSecretManager, rmApplicationHistoryWriter,
+ clientToAMTokenSecretManager,
scheduler));
ConfigurationProvider provider = new LocalConfigurationProvider();
@@ -112,8 +114,7 @@ public class RMContextImpl implements RMContext {
AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
- ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) {
this(
rmDispatcher,
containerAllocationExpirer,
@@ -123,9 +124,7 @@ public class RMContextImpl implements RMContext {
appTokenSecretManager,
containerTokenSecretManager,
nmTokenSecretManager,
- clientToAMTokenSecretManager,
- rmApplicationHistoryWriter,
- null);
+ clientToAMTokenSecretManager, null);
}
@Override
@@ -351,25 +350,25 @@ public class RMContextImpl implements RMContext {
@Override
public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
- return activeServiceContext.getRMApplicationHistoryWriter();
+ return this.rmApplicationHistoryWriter;
}
@Override
public void setSystemMetricsPublisher(
SystemMetricsPublisher systemMetricsPublisher) {
- activeServiceContext.setSystemMetricsPublisher(systemMetricsPublisher);
+ this.systemMetricsPublisher = systemMetricsPublisher;
}
@Override
public SystemMetricsPublisher getSystemMetricsPublisher() {
- return activeServiceContext.getSystemMetricsPublisher();
+ return this.systemMetricsPublisher;
}
@Override
public void setRMApplicationHistoryWriter(
RMApplicationHistoryWriter rmApplicationHistoryWriter) {
- activeServiceContext
- .setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+ this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
+
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 817565b..d6d9629 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -250,7 +250,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
adminService = createAdminService();
addService(adminService);
rmContext.setRMAdminService(adminService);
-
+
rmContext.setYarnConfiguration(conf);
createAndInitActiveServices();
@@ -259,6 +259,15 @@ public class ResourceManager extends CompositeService implements Recoverable {
YarnConfiguration.RM_BIND_HOST,
WebAppUtils.getRMWebAppURLWithoutScheme(this.conf));
+ RMApplicationHistoryWriter rmApplicationHistoryWriter =
+ createRMApplicationHistoryWriter();
+ addService(rmApplicationHistoryWriter);
+ rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+
+ SystemMetricsPublisher systemMetricsPublisher = createSystemMetricsPublisher();
+ addService(systemMetricsPublisher);
+ rmContext.setSystemMetricsPublisher(systemMetricsPublisher);
+
super.serviceInit(this.conf);
}
@@ -411,7 +420,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setActiveServiceContext(activeServiceContext);
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
-
rmSecretManagerService = createRMSecretManagerService();
addService(rmSecretManagerService);
@@ -468,15 +476,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setDelegationTokenRenewer(delegationTokenRenewer);
}
- RMApplicationHistoryWriter rmApplicationHistoryWriter =
- createRMApplicationHistoryWriter();
- addService(rmApplicationHistoryWriter);
- rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
-
- SystemMetricsPublisher systemMetricsPublisher = createSystemMetricsPublisher();
- addService(systemMetricsPublisher);
- rmContext.setSystemMetricsPublisher(systemMetricsPublisher);
-
// Register event handler for NodesListManager
nodesListManager = new NodesListManager(rmContext);
rmDispatcher.register(NodesListManagerEventType.class, nodesListManager);
@@ -596,11 +595,13 @@ public class ResourceManager extends CompositeService implements Recoverable {
@Override
protected void serviceStop() throws Exception {
- DefaultMetricsSystem.shutdown();
+ super.serviceStop();
+
if (pauseMonitor != null) {
pauseMonitor.stop();
}
+ DefaultMetricsSystem.shutdown();
if (rmContext != null) {
RMStateStore store = rmContext.getStateStore();
try {
@@ -610,7 +611,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
}
- super.serviceStop();
}
protected void createPolicyMonitors() {
@@ -1033,12 +1033,12 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
LOG.info("Transitioning to standby state");
- if (rmContext.getHAServiceState() ==
- HAServiceProtocol.HAServiceState.ACTIVE) {
+ HAServiceState state = rmContext.getHAServiceState();
+ rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
+ if (state == HAServiceProtocol.HAServiceState.ACTIVE) {
stopActiveServices();
reinitialize(initialize);
}
- rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
LOG.info("Transitioned to standby state");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index f073763..cbeae5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -120,7 +120,7 @@ public class TestAppManager{
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
RMContext context = new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, null, null, null, null, writer) {
+ null, null, null, null, null) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return map;
@@ -128,7 +128,8 @@ public class TestAppManager{
};
((RMContextImpl)context).setStateStore(mock(RMStateStore.class));
metricsPublisher = mock(SystemMetricsPublisher.class);
- ((RMContextImpl)context).setSystemMetricsPublisher(metricsPublisher);
+ context.setSystemMetricsPublisher(metricsPublisher);
+ context.setRMApplicationHistoryWriter(writer);
return context;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
index fbf54fc..9ceeffb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
@@ -66,6 +66,7 @@ public class TestResourceManager {
@After
public void tearDown() throws Exception {
+ resourceManager.stop();
}
private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
index 9af4290..fccfa19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
@@ -87,9 +87,9 @@ public class TestRMAppLogAggregationStatus {
rmContext =
new RMContextImpl(rmDispatcher, null, null, null,
- null, null, null, null, null,
- new RMApplicationHistoryWriter());
+ null, null, null, null, null);
rmContext.setSystemMetricsPublisher(new SystemMetricsPublisher());
+ rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
scheduler = mock(YarnScheduler.class);
doAnswer(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 72f1dff..2e64d61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -212,11 +212,11 @@ public class TestRMAppTransitions {
renewer, new AMRMTokenSecretManager(conf, this.rmContext),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(),
- writer);
+ new ClientToAMTokenSecretManagerInRM());
((RMContextImpl)realRMContext).setStateStore(store);
publisher = mock(SystemMetricsPublisher.class);
- ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);
+ realRMContext.setSystemMetricsPublisher(publisher);
+ realRMContext.setRMApplicationHistoryWriter(writer);
this.rmContext = spy(realRMContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index c074ad9..9afb5a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -258,14 +258,14 @@ public class TestRMAppAttemptTransitions {
null, amRMTokenManager,
new RMContainerTokenSecretManager(conf),
nmTokenManager,
- clientToAMTokenManager,
- writer);
+ clientToAMTokenManager);
store = mock(RMStateStore.class);
((RMContextImpl) rmContext).setStateStore(store);
publisher = mock(SystemMetricsPublisher.class);
- ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher);
-
+ rmContext.setSystemMetricsPublisher(publisher);
+ rmContext.setRMApplicationHistoryWriter(writer);
+
scheduler = mock(YarnScheduler.class);
masterService = mock(ApplicationMasterService.class);
applicationMasterLauncher = mock(ApplicationMasterLauncher.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index c95b937..5ffcaad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.RMActiveServiceContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
@@ -103,7 +102,7 @@ public class TestUtils {
new AMRMTokenSecretManager(conf, null),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(), writer);
+ new ClientToAMTokenSecretManagerInRM());
RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
when(
nlm.getQueueResource(any(String.class), any(Set.class),
@@ -117,8 +116,8 @@ public class TestUtils {
when(nlm.getResourceByLabel(any(String.class), any(Resource.class)))
.thenAnswer(new Answer<Resource>() {
- @Override
- public Resource answer(InvocationOnMock invocation) throws Throwable {
+ @Override public Resource answer(InvocationOnMock invocation)
+ throws Throwable {
Object[] args = invocation.getArguments();
return (Resource) args[1];
}
@@ -126,7 +125,7 @@ public class TestUtils {
rmContext.setNodeLabelManager(nlm);
rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
-
+ rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
ResourceScheduler mockScheduler = mock(ResourceScheduler.class);
when(mockScheduler.getResourceCalculator()).thenReturn(
new DefaultResourceCalculator());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ae716fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 5f9030f..6607211 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -172,7 +172,7 @@ public class TestFifoScheduler {
FifoScheduler scheduler = new FifoScheduler();
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
RMContext rmContext = new RMContextImpl(dispatcher, null,
- null, null, null, null, null, null, null, writer, scheduler);
+ null, null, null, null, null, null, null, scheduler);
((RMContextImpl) rmContext).setSystemMetricsPublisher(
mock(SystemMetricsPublisher.class));
@@ -218,10 +218,10 @@ public class TestFifoScheduler {
FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
- null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
- scheduler);
- ((RMContextImpl) rmContext).setSystemMetricsPublisher(
- mock(SystemMetricsPublisher.class));
+ null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
+ rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
+ rmContext.setRMApplicationHistoryWriter(
+ mock(RMApplicationHistoryWriter.class));
((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
scheduler.setRMContext(rmContext);
@@ -300,10 +300,9 @@ public class TestFifoScheduler {
}
};
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
- null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
- scheduler);
- ((RMContextImpl) rmContext).setSystemMetricsPublisher(
- mock(SystemMetricsPublisher.class));
+ null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
+ rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
+ rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
NullRMNodeLabelsManager nlm = new NullRMNodeLabelsManager();
nlm.init(new Configuration());
[22/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
new file mode 100644
index 0000000..3ed3e20
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureEncoder.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Reed-Solomon erasure encoder that encodes a block group.
+ *
+ * It implements {@link ErasureCoder}.
+ */
+public class RSErasureEncoder extends AbstractErasureEncoder {
+ private RawErasureEncoder rawEncoder;
+
+ public RSErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public RSErasureEncoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareEncodingStep(final ECBlockGroup blockGroup) {
+
+ RawErasureEncoder rawEncoder = checkCreateRSRawEncoder();
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+ return new ErasureEncodingStep(inputBlocks,
+ getOutputBlocks(blockGroup), rawEncoder);
+ }
+
+ private RawErasureEncoder checkCreateRSRawEncoder() {
+ if (rawEncoder == null) {
+ rawEncoder = CodecUtil.createRSRawEncoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+ }
+ return rawEncoder;
+ }
+
+ @Override
+ public void release() {
+ if (rawEncoder != null) {
+ rawEncoder.release();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
new file mode 100644
index 0000000..a847418
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureDecoder.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+
+/**
+ * Xor erasure decoder that decodes a block group.
+ *
+ * It implements {@link ErasureCoder}.
+ */
+public class XORErasureDecoder extends AbstractErasureDecoder {
+
+ public XORErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public XORErasureDecoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareDecodingStep(
+ final ECBlockGroup blockGroup) {
+ RawErasureDecoder rawDecoder = CodecUtil.createXORRawDecoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+ return new ErasureDecodingStep(inputBlocks,
+ getErasedIndexes(inputBlocks),
+ getOutputBlocks(blockGroup), rawDecoder);
+ }
+
+ /**
+ * Which blocks were erased ? For XOR it's simple we only allow and return one
+ * erased block, either data or parity.
+ * @param blockGroup
+ * @return output blocks to recover
+ */
+ @Override
+ protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
+ /**
+ * If more than one blocks (either data or parity) erased, then it's not
+ * edible to recover. We don't have the check here since it will be done
+ * by upper level: ErasreCoder call can be avoid if not possible to recover
+ * at all.
+ */
+ int erasedNum = getNumErasedBlocks(blockGroup);
+ ECBlock[] outputBlocks = new ECBlock[erasedNum];
+
+ int idx = 0;
+ for (int i = 0; i < getNumParityUnits(); i++) {
+ if (blockGroup.getParityBlocks()[i].isErased()) {
+ outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
+ }
+ }
+
+ for (int i = 0; i < getNumDataUnits(); i++) {
+ if (blockGroup.getDataBlocks()[i].isErased()) {
+ outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
+ }
+ }
+
+ return outputBlocks;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java
new file mode 100644
index 0000000..5c4bcdd
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/XORErasureEncoder.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Xor erasure encoder that encodes a block group.
+ *
+ * It implements {@link ErasureCoder}.
+ */
+public class XORErasureEncoder extends AbstractErasureEncoder {
+
+ public XORErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public XORErasureEncoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareEncodingStep(
+ final ECBlockGroup blockGroup) {
+ RawErasureEncoder rawEncoder = CodecUtil.createXORRawEncoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+
+ return new ErasureEncodingStep(inputBlocks,
+ getOutputBlocks(blockGroup), rawEncoder);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java
new file mode 100644
index 0000000..bdc1624
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/grouper/BlockGrouper.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.grouper;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+/**
+ * As part of a codec, to handle how to form a block group for encoding
+ * and provide instructions on how to recover erased blocks from a block group
+ */
+public class BlockGrouper {
+
+ private ECSchema schema;
+
+ /**
+ * Set EC schema.
+ * @param schema
+ */
+ public void setSchema(ECSchema schema) {
+ this.schema = schema;
+ }
+
+ /**
+ * Get EC schema.
+ * @return
+ */
+ protected ECSchema getSchema() {
+ return schema;
+ }
+
+ /**
+ * Get required data blocks count in a BlockGroup.
+ * @return count of required data blocks
+ */
+ public int getRequiredNumDataBlocks() {
+ return schema.getNumDataUnits();
+ }
+
+ /**
+ * Get required parity blocks count in a BlockGroup.
+ * @return count of required parity blocks
+ */
+ public int getRequiredNumParityBlocks() {
+ return schema.getNumParityUnits();
+ }
+
+ /**
+ * Calculating and organizing BlockGroup, to be called by ECManager
+ * @param dataBlocks Data blocks to compute parity blocks against
+ * @param parityBlocks To be computed parity blocks
+ * @return
+ */
+ public ECBlockGroup makeBlockGroup(ECBlock[] dataBlocks,
+ ECBlock[] parityBlocks) {
+
+ ECBlockGroup blockGroup = new ECBlockGroup(dataBlocks, parityBlocks);
+ return blockGroup;
+ }
+
+ /**
+ * Given a BlockGroup, tell if any of the missing blocks can be recovered,
+ * to be called by ECManager
+ * @param blockGroup a blockGroup that may contain erased blocks but not sure
+ * recoverable or not
+ * @return true if any erased block recoverable, false otherwise
+ */
+ public boolean anyRecoverable(ECBlockGroup blockGroup) {
+ int erasedCount = blockGroup.getErasedCount();
+
+ return erasedCount > 0 && erasedCount <= getRequiredNumParityBlocks();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
new file mode 100644
index 0000000..4b7461e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureCoder.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configured;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * A common class of basic facilities to be shared by encoder and decoder
+ *
+ * It implements the {@link RawErasureCoder} interface.
+ */
+public abstract class AbstractRawErasureCoder
+ extends Configured implements RawErasureCoder {
+
+ private final int numDataUnits;
+ private final int numParityUnits;
+
+ public AbstractRawErasureCoder(int numDataUnits, int numParityUnits) {
+ this.numDataUnits = numDataUnits;
+ this.numParityUnits = numParityUnits;
+ }
+
+ @Override
+ public int getNumDataUnits() {
+ return numDataUnits;
+ }
+
+ @Override
+ public int getNumParityUnits() {
+ return numParityUnits;
+ }
+
+ @Override
+ public boolean preferDirectBuffer() {
+ return false;
+ }
+
+ @Override
+ public void release() {
+ // Nothing to do by default
+ }
+
+ /**
+ * Ensure a buffer filled with ZERO bytes from current readable/writable
+ * position.
+ * @param buffer a buffer ready to read / write certain size bytes
+ * @return the buffer itself, with ZERO bytes written, the position and limit
+ * are not changed after the call
+ */
+ protected ByteBuffer resetBuffer(ByteBuffer buffer) {
+ int pos = buffer.position();
+ for (int i = pos; i < buffer.limit(); ++i) {
+ buffer.put((byte) 0);
+ }
+ buffer.position(pos);
+
+ return buffer;
+ }
+
+ /**
+ * Ensure the buffer (either input or output) ready to read or write with ZERO
+ * bytes fully in specified length of len.
+ * @param buffer bytes array buffer
+ * @return the buffer itself
+ */
+ protected byte[] resetBuffer(byte[] buffer, int offset, int len) {
+ for (int i = offset; i < len; ++i) {
+ buffer[i] = (byte) 0;
+ }
+
+ return buffer;
+ }
+
+ /**
+ * Check and ensure the buffers are of the length specified by dataLen, also
+ * ensure the buffers are direct buffers or not according to isDirectBuffer.
+ * @param buffers the buffers to check
+ * @param allowNull whether to allow any element to be null or not
+ * @param dataLen the length of data available in the buffer to ensure with
+ * @param isDirectBuffer is direct buffer or not to ensure with
+ */
+ protected void ensureLengthAndType(ByteBuffer[] buffers, boolean allowNull,
+ int dataLen, boolean isDirectBuffer) {
+ for (ByteBuffer buffer : buffers) {
+ if (buffer == null && !allowNull) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid buffer found, not allowing null");
+ } else if (buffer != null) {
+ if (buffer.remaining() != dataLen) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid buffer, not of length " + dataLen);
+ }
+ if (buffer.isDirect() != isDirectBuffer) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid buffer, isDirect should be " + isDirectBuffer);
+ }
+ }
+ }
+ }
+
+ /**
+ * Check and ensure the buffers are of the length specified by dataLen.
+ * @param buffers the buffers to check
+ * @param allowNull whether to allow any element to be null or not
+ * @param dataLen the length of data available in the buffer to ensure with
+ */
+ protected void ensureLength(byte[][] buffers,
+ boolean allowNull, int dataLen) {
+ for (byte[] buffer : buffers) {
+ if (buffer == null && !allowNull) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid buffer found, not allowing null");
+ } else if (buffer != null && buffer.length != dataLen) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid buffer not of length " + dataLen);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
new file mode 100644
index 0000000..931cda1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureDecoder.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * An abstract raw erasure decoder that's to be inherited by new decoders.
+ *
+ * It implements the {@link RawErasureDecoder} interface.
+ */
+public abstract class AbstractRawErasureDecoder extends AbstractRawErasureCoder
+ implements RawErasureDecoder {
+
+ public AbstractRawErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ @Override
+ public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
+ ByteBuffer[] outputs) {
+ checkParameters(inputs, erasedIndexes, outputs);
+
+ ByteBuffer validInput = findFirstValidInput(inputs);
+ boolean usingDirectBuffer = validInput.isDirect();
+ int dataLen = validInput.remaining();
+ if (dataLen == 0) {
+ return;
+ }
+ ensureLengthAndType(inputs, true, dataLen, usingDirectBuffer);
+ ensureLengthAndType(outputs, false, dataLen, usingDirectBuffer);
+
+ if (usingDirectBuffer) {
+ doDecode(inputs, erasedIndexes, outputs);
+ return;
+ }
+
+ int[] inputOffsets = new int[inputs.length];
+ int[] outputOffsets = new int[outputs.length];
+ byte[][] newInputs = new byte[inputs.length][];
+ byte[][] newOutputs = new byte[outputs.length][];
+
+ ByteBuffer buffer;
+ for (int i = 0; i < inputs.length; ++i) {
+ buffer = inputs[i];
+ if (buffer != null) {
+ inputOffsets[i] = buffer.arrayOffset() + buffer.position();
+ newInputs[i] = buffer.array();
+ }
+ }
+
+ for (int i = 0; i < outputs.length; ++i) {
+ buffer = outputs[i];
+ outputOffsets[i] = buffer.arrayOffset() + buffer.position();
+ newOutputs[i] = buffer.array();
+ }
+
+ doDecode(newInputs, inputOffsets, dataLen,
+ erasedIndexes, newOutputs, outputOffsets);
+
+ for (int i = 0; i < inputs.length; ++i) {
+ buffer = inputs[i];
+ if (buffer != null) {
+ // dataLen bytes consumed
+ buffer.position(buffer.position() + dataLen);
+ }
+ }
+ }
+
+ /**
+ * Perform the real decoding using Direct ByteBuffer.
+ * @param inputs Direct ByteBuffers expected
+ * @param erasedIndexes indexes of erased units in the inputs array
+ * @param outputs Direct ByteBuffers expected
+ */
+ protected abstract void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
+ ByteBuffer[] outputs);
+
+ @Override
+ public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs) {
+ checkParameters(inputs, erasedIndexes, outputs);
+
+ byte[] validInput = findFirstValidInput(inputs);
+ int dataLen = validInput.length;
+ if (dataLen == 0) {
+ return;
+ }
+ ensureLength(inputs, true, dataLen);
+ ensureLength(outputs, false, dataLen);
+
+ int[] inputOffsets = new int[inputs.length]; // ALL ZERO
+ int[] outputOffsets = new int[outputs.length]; // ALL ZERO
+
+ doDecode(inputs, inputOffsets, dataLen, erasedIndexes, outputs,
+ outputOffsets);
+ }
+
+ /**
+ * Perform the real decoding using bytes array, supporting offsets and
+ * lengths.
+ * @param inputs the input byte arrays to read data from
+ * @param inputOffsets offsets for the input byte arrays to read data from
+ * @param dataLen how much data are to be read from
+ * @param erasedIndexes indexes of erased units in the inputs array
+ * @param outputs the output byte arrays to write resultant data into
+ * @param outputOffsets offsets from which to write resultant data into
+ */
+ protected abstract void doDecode(byte[][] inputs, int[] inputOffsets,
+ int dataLen, int[] erasedIndexes,
+ byte[][] outputs, int[] outputOffsets);
+
+ @Override
+ public void decode(ECChunk[] inputs, int[] erasedIndexes,
+ ECChunk[] outputs) {
+ ByteBuffer[] newInputs = ECChunk.toBuffers(inputs);
+ ByteBuffer[] newOutputs = ECChunk.toBuffers(outputs);
+ decode(newInputs, erasedIndexes, newOutputs);
+ }
+
+ /**
+ * Check and validate decoding parameters, throw exception accordingly. The
+ * checking assumes it's a MDS code. Other code can override this.
+ * @param inputs input buffers to check
+ * @param erasedIndexes indexes of erased units in the inputs array
+ * @param outputs output buffers to check
+ */
+ protected <T> void checkParameters(T[] inputs, int[] erasedIndexes,
+ T[] outputs) {
+ if (inputs.length != getNumParityUnits() + getNumDataUnits()) {
+ throw new IllegalArgumentException("Invalid inputs length");
+ }
+
+ if (erasedIndexes.length != outputs.length) {
+ throw new HadoopIllegalArgumentException(
+ "erasedIndexes and outputs mismatch in length");
+ }
+
+ if (erasedIndexes.length > getNumParityUnits()) {
+ throw new HadoopIllegalArgumentException(
+ "Too many erased, not recoverable");
+ }
+
+ int validInputs = 0;
+ for (T input : inputs) {
+ if (input != null) {
+ validInputs += 1;
+ }
+ }
+
+ if (validInputs < getNumDataUnits()) {
+ throw new HadoopIllegalArgumentException(
+ "No enough valid inputs are provided, not recoverable");
+ }
+ }
+
+ /**
+ * Get indexes into inputs array for items marked as null, either erased or
+ * not to read.
+ * @return indexes into inputs array
+ */
+ protected <T> int[] getErasedOrNotToReadIndexes(T[] inputs) {
+ int[] invalidIndexes = new int[inputs.length];
+ int idx = 0;
+ for (int i = 0; i < inputs.length; i++) {
+ if (inputs[i] == null) {
+ invalidIndexes[idx++] = i;
+ }
+ }
+
+ return Arrays.copyOf(invalidIndexes, idx);
+ }
+
+ /**
+ * Find the valid input from all the inputs.
+ * @param inputs input buffers to look for valid input
+ * @return the first valid input
+ */
+ protected static <T> T findFirstValidInput(T[] inputs) {
+ for (T input : inputs) {
+ if (input != null) {
+ return input;
+ }
+ }
+
+ throw new HadoopIllegalArgumentException(
+ "Invalid inputs are found, all being null");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
new file mode 100644
index 0000000..a0b3cfe
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/AbstractRawErasureEncoder.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+
+/**
+ * An abstract raw erasure encoder that's to be inherited by new encoders.
+ *
+ * It implements the {@link RawErasureEncoder} interface.
+ */
+public abstract class AbstractRawErasureEncoder extends AbstractRawErasureCoder
+ implements RawErasureEncoder {
+
+ public AbstractRawErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ @Override
+ public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ checkParameters(inputs, outputs);
+
+ boolean usingDirectBuffer = inputs[0].isDirect();
+ int dataLen = inputs[0].remaining();
+ if (dataLen == 0) {
+ return;
+ }
+ ensureLengthAndType(inputs, false, dataLen, usingDirectBuffer);
+ ensureLengthAndType(outputs, false, dataLen, usingDirectBuffer);
+
+ if (usingDirectBuffer) {
+ doEncode(inputs, outputs);
+ return;
+ }
+
+ int[] inputOffsets = new int[inputs.length];
+ int[] outputOffsets = new int[outputs.length];
+ byte[][] newInputs = new byte[inputs.length][];
+ byte[][] newOutputs = new byte[outputs.length][];
+
+ ByteBuffer buffer;
+ for (int i = 0; i < inputs.length; ++i) {
+ buffer = inputs[i];
+ inputOffsets[i] = buffer.arrayOffset() + buffer.position();
+ newInputs[i] = buffer.array();
+ }
+
+ for (int i = 0; i < outputs.length; ++i) {
+ buffer = outputs[i];
+ outputOffsets[i] = buffer.arrayOffset() + buffer.position();
+ newOutputs[i] = buffer.array();
+ }
+
+ doEncode(newInputs, inputOffsets, dataLen, newOutputs, outputOffsets);
+
+ for (int i = 0; i < inputs.length; ++i) {
+ buffer = inputs[i];
+ buffer.position(buffer.position() + dataLen); // dataLen bytes consumed
+ }
+ }
+
+ /**
+ * Perform the real encoding work using direct ByteBuffer
+ * @param inputs Direct ByteBuffers expected
+ * @param outputs Direct ByteBuffers expected
+ */
+ protected abstract void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs);
+
+ @Override
+ public void encode(byte[][] inputs, byte[][] outputs) {
+ checkParameters(inputs, outputs);
+ int dataLen = inputs[0].length;
+ if (dataLen == 0) {
+ return;
+ }
+ ensureLength(inputs, false, dataLen);
+ ensureLength(outputs, false, dataLen);
+
+ int[] inputOffsets = new int[inputs.length]; // ALL ZERO
+ int[] outputOffsets = new int[outputs.length]; // ALL ZERO
+
+ doEncode(inputs, inputOffsets, dataLen, outputs, outputOffsets);
+ }
+
+ /**
+ * Perform the real encoding work using bytes array, supporting offsets
+ * and lengths.
+ * @param inputs the input byte arrays to read data from
+ * @param inputOffsets offsets for the input byte arrays to read data from
+ * @param dataLen how much data are to be read from
+ * @param outputs the output byte arrays to write resultant data into
+ * @param outputOffsets offsets from which to write resultant data into
+ */
+ protected abstract void doEncode(byte[][] inputs, int[] inputOffsets,
+ int dataLen, byte[][] outputs,
+ int[] outputOffsets);
+
+ @Override
+ public void encode(ECChunk[] inputs, ECChunk[] outputs) {
+ ByteBuffer[] newInputs = ECChunk.toBuffers(inputs);
+ ByteBuffer[] newOutputs = ECChunk.toBuffers(outputs);
+ encode(newInputs, newOutputs);
+ }
+
+ /**
+ * Check and validate decoding parameters, throw exception accordingly.
+ * @param inputs input buffers to check
+ * @param outputs output buffers to check
+ */
+ protected <T> void checkParameters(T[] inputs, T[] outputs) {
+ if (inputs.length != getNumDataUnits()) {
+ throw new HadoopIllegalArgumentException("Invalid inputs length");
+ }
+ if (outputs.length != getNumParityUnits()) {
+ throw new HadoopIllegalArgumentException("Invalid outputs length");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
new file mode 100644
index 0000000..57e6957
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawDecoder.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A raw erasure decoder in RS code scheme in pure Java in case native one
+ * isn't available in some environment. Please always use native implementations
+ * when possible.
+ *
+ * Currently this implementation will compute and decode not to read units
+ * unnecessarily due to the underlying implementation limit in GF. This will be
+ * addressed in HADOOP-11871.
+ */
+public class RSRawDecoder extends AbstractRawErasureDecoder {
+ // To describe and calculate the needed Vandermonde matrix
+ private int[] errSignature;
+ private int[] primitivePower;
+
+ /**
+ * We need a set of reusable buffers either for the bytes array
+ * decoding version or direct buffer decoding version. Normally not both.
+ *
+ * For output, in addition to the valid buffers from the caller
+ * passed from above, we need to provide extra buffers for the internal
+ * decoding implementation. For output, the caller should provide no more
+ * than numParityUnits but at least one buffers. And the left buffers will be
+ * borrowed from either bytesArrayBuffers, for the bytes array version.
+ *
+ */
+ // Reused buffers for decoding with bytes arrays
+ private byte[][] bytesArrayBuffers = new byte[getNumParityUnits()][];
+ private byte[][] adjustedByteArrayOutputsParameter =
+ new byte[getNumParityUnits()][];
+ private int[] adjustedOutputOffsets = new int[getNumParityUnits()];
+
+ // Reused buffers for decoding with direct ByteBuffers
+ private ByteBuffer[] directBuffers = new ByteBuffer[getNumParityUnits()];
+ private ByteBuffer[] adjustedDirectBufferOutputsParameter =
+ new ByteBuffer[getNumParityUnits()];
+
+ public RSRawDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ if (numDataUnits + numParityUnits >= RSUtil.GF.getFieldSize()) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid numDataUnits and numParityUnits");
+ }
+
+ this.errSignature = new int[numParityUnits];
+ this.primitivePower = RSUtil.getPrimitivePower(numDataUnits,
+ numParityUnits);
+ }
+
+ private void doDecodeImpl(ByteBuffer[] inputs, int[] erasedIndexes,
+ ByteBuffer[] outputs) {
+ ByteBuffer valid = findFirstValidInput(inputs);
+ int dataLen = valid.remaining();
+ for (int i = 0; i < erasedIndexes.length; i++) {
+ errSignature[i] = primitivePower[erasedIndexes[i]];
+ RSUtil.GF.substitute(inputs, dataLen, outputs[i], primitivePower[i]);
+ }
+
+ RSUtil.GF.solveVandermondeSystem(errSignature,
+ outputs, erasedIndexes.length);
+ }
+
+ private void doDecodeImpl(byte[][] inputs, int[] inputOffsets,
+ int dataLen, int[] erasedIndexes,
+ byte[][] outputs, int[] outputOffsets) {
+ for (int i = 0; i < erasedIndexes.length; i++) {
+ errSignature[i] = primitivePower[erasedIndexes[i]];
+ RSUtil.GF.substitute(inputs, inputOffsets, dataLen, outputs[i],
+ outputOffsets[i], primitivePower[i]);
+ }
+
+ RSUtil.GF.solveVandermondeSystem(errSignature, outputs, outputOffsets,
+ erasedIndexes.length, dataLen);
+ }
+
+ @Override
+ protected void doDecode(byte[][] inputs, int[] inputOffsets,
+ int dataLen, int[] erasedIndexes,
+ byte[][] outputs, int[] outputOffsets) {
+ /**
+ * As passed parameters are friendly to callers but not to the underlying
+ * implementations, so we have to adjust them before calling doDecodeImpl.
+ */
+
+ int[] erasedOrNotToReadIndexes = getErasedOrNotToReadIndexes(inputs);
+
+ // Prepare for adjustedOutputsParameter
+
+ // First reset the positions needed this time
+ for (int i = 0; i < erasedOrNotToReadIndexes.length; i++) {
+ adjustedByteArrayOutputsParameter[i] = null;
+ adjustedOutputOffsets[i] = 0;
+ }
+ // Use the caller passed buffers in erasedIndexes positions
+ for (int outputIdx = 0, i = 0; i < erasedIndexes.length; i++) {
+ boolean found = false;
+ for (int j = 0; j < erasedOrNotToReadIndexes.length; j++) {
+ // If this index is one requested by the caller via erasedIndexes, then
+ // we use the passed output buffer to avoid copying data thereafter.
+ if (erasedIndexes[i] == erasedOrNotToReadIndexes[j]) {
+ found = true;
+ adjustedByteArrayOutputsParameter[j] = resetBuffer(
+ outputs[outputIdx], outputOffsets[outputIdx], dataLen);
+ adjustedOutputOffsets[j] = outputOffsets[outputIdx];
+ outputIdx++;
+ }
+ }
+ if (!found) {
+ throw new HadoopIllegalArgumentException(
+ "Inputs not fully corresponding to erasedIndexes in null places");
+ }
+ }
+ // Use shared buffers for other positions (not set yet)
+ for (int bufferIdx = 0, i = 0; i < erasedOrNotToReadIndexes.length; i++) {
+ if (adjustedByteArrayOutputsParameter[i] == null) {
+ adjustedByteArrayOutputsParameter[i] = resetBuffer(
+ checkGetBytesArrayBuffer(bufferIdx, dataLen), 0, dataLen);
+ adjustedOutputOffsets[i] = 0; // Always 0 for such temp output
+ bufferIdx++;
+ }
+ }
+
+ doDecodeImpl(inputs, inputOffsets, dataLen, erasedOrNotToReadIndexes,
+ adjustedByteArrayOutputsParameter, adjustedOutputOffsets);
+ }
+
+ @Override
+ protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
+ ByteBuffer[] outputs) {
+ ByteBuffer validInput = findFirstValidInput(inputs);
+ int dataLen = validInput.remaining();
+
+ /**
+ * As passed parameters are friendly to callers but not to the underlying
+ * implementations, so we have to adjust them before calling doDecodeImpl.
+ */
+
+ int[] erasedOrNotToReadIndexes = getErasedOrNotToReadIndexes(inputs);
+
+ // Prepare for adjustedDirectBufferOutputsParameter
+
+ // First reset the positions needed this time
+ for (int i = 0; i < erasedOrNotToReadIndexes.length; i++) {
+ adjustedDirectBufferOutputsParameter[i] = null;
+ }
+ // Use the caller passed buffers in erasedIndexes positions
+ for (int outputIdx = 0, i = 0; i < erasedIndexes.length; i++) {
+ boolean found = false;
+ for (int j = 0; j < erasedOrNotToReadIndexes.length; j++) {
+ // If this index is one requested by the caller via erasedIndexes, then
+ // we use the passed output buffer to avoid copying data thereafter.
+ if (erasedIndexes[i] == erasedOrNotToReadIndexes[j]) {
+ found = true;
+ adjustedDirectBufferOutputsParameter[j] =
+ resetBuffer(outputs[outputIdx++]);
+ }
+ }
+ if (!found) {
+ throw new HadoopIllegalArgumentException(
+ "Inputs not fully corresponding to erasedIndexes in null places");
+ }
+ }
+ // Use shared buffers for other positions (not set yet)
+ for (int bufferIdx = 0, i = 0; i < erasedOrNotToReadIndexes.length; i++) {
+ if (adjustedDirectBufferOutputsParameter[i] == null) {
+ ByteBuffer buffer = checkGetDirectBuffer(bufferIdx, dataLen);
+ buffer.position(0);
+ buffer.limit(dataLen);
+ adjustedDirectBufferOutputsParameter[i] = resetBuffer(buffer);
+ bufferIdx++;
+ }
+ }
+
+ doDecodeImpl(inputs, erasedOrNotToReadIndexes,
+ adjustedDirectBufferOutputsParameter);
+ }
+
+ private byte[] checkGetBytesArrayBuffer(int idx, int bufferLen) {
+ if (bytesArrayBuffers[idx] == null ||
+ bytesArrayBuffers[idx].length < bufferLen) {
+ bytesArrayBuffers[idx] = new byte[bufferLen];
+ }
+ return bytesArrayBuffers[idx];
+ }
+
+ private ByteBuffer checkGetDirectBuffer(int idx, int bufferLen) {
+ if (directBuffers[idx] == null ||
+ directBuffers[idx].capacity() < bufferLen) {
+ directBuffers[idx] = ByteBuffer.allocateDirect(bufferLen);
+ }
+ return directBuffers[idx];
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
new file mode 100644
index 0000000..efeee90
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawEncoder.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A raw erasure encoder in RS code scheme in pure Java in case native one
+ * isn't available in some environment. Please always use native implementations
+ * when possible.
+ */
+public class RSRawEncoder extends AbstractRawErasureEncoder {
+ private int[] generatingPolynomial;
+
+ public RSRawEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+
+ assert (getNumDataUnits() + getNumParityUnits() < RSUtil.GF.getFieldSize());
+
+ int[] primitivePower = RSUtil.getPrimitivePower(numDataUnits,
+ numParityUnits);
+ // compute generating polynomial
+ int[] gen = {1};
+ int[] poly = new int[2];
+ for (int i = 0; i < numParityUnits; i++) {
+ poly[0] = primitivePower[i];
+ poly[1] = 1;
+ gen = RSUtil.GF.multiply(gen, poly);
+ }
+ // generating polynomial has all generating roots
+ generatingPolynomial = gen;
+ }
+
+ @Override
+ protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ // parity units + data units
+ ByteBuffer[] all = new ByteBuffer[outputs.length + inputs.length];
+ System.arraycopy(outputs, 0, all, 0, outputs.length);
+ System.arraycopy(inputs, 0, all, outputs.length, inputs.length);
+
+ // Compute the remainder
+ RSUtil.GF.remainder(all, generatingPolynomial);
+ }
+
+ @Override
+ protected void doEncode(byte[][] inputs, int[] inputOffsets,
+ int dataLen, byte[][] outputs,
+ int[] outputOffsets) {
+ // parity units + data units
+ byte[][] all = new byte[outputs.length + inputs.length][];
+ System.arraycopy(outputs, 0, all, 0, outputs.length);
+ System.arraycopy(inputs, 0, all, outputs.length, inputs.length);
+
+ int[] offsets = new int[inputOffsets.length + outputOffsets.length];
+ System.arraycopy(outputOffsets, 0, offsets, 0, outputOffsets.length);
+ System.arraycopy(inputOffsets, 0, offsets,
+ outputOffsets.length, inputOffsets.length);
+
+ // Compute the remainder
+ RSUtil.GF.remainder(all, offsets, dataLen, generatingPolynomial);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
new file mode 100644
index 0000000..5db49e3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+/**
+ * A raw coder factory for raw Reed-Solomon coder in Java.
+ */
+public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
+
+ @Override
+ public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) {
+ return new RSRawEncoder(numDataUnits, numParityUnits);
+ }
+
+ @Override
+ public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) {
+ return new RSRawDecoder(numDataUnits, numParityUnits);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
new file mode 100644
index 0000000..0ddb460
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.conf.Configurable;
+
+/**
+ * RawErasureCoder is a common interface for {@link RawErasureEncoder} and
+ * {@link RawErasureDecoder} as both encoder and decoder share some properties.
+ *
+ * RawErasureCoder is part of ErasureCodec framework, where ErasureCoder is
+ * used to encode/decode a group of blocks (BlockGroup) according to the codec
+ * specific BlockGroup layout and logic. An ErasureCoder extracts chunks of
+ * data from the blocks and can employ various low level RawErasureCoders to
+ * perform encoding/decoding against the chunks.
+ *
+ * To distinguish from ErasureCoder, here RawErasureCoder is used to mean the
+ * low level constructs, since it only takes care of the math calculation with
+ * a group of byte buffers.
+ */
+public interface RawErasureCoder extends Configurable {
+
+ /**
+ * The number of data input units for the coding. A unit can be a byte,
+ * chunk or buffer or even a block.
+ * @return count of data input units
+ */
+ public int getNumDataUnits();
+
+ /**
+ * The number of parity output units for the coding. A unit can be a byte,
+ * chunk, buffer or even a block.
+ * @return count of parity output units
+ */
+ public int getNumParityUnits();
+
+ /**
+ * Tell if direct buffer is preferred or not. It's for callers to
+ * decide how to allocate coding chunk buffers, using DirectByteBuffer or
+ * bytes array. It will return false by default.
+ * @return true if native buffer is preferred for performance consideration,
+ * otherwise false.
+ */
+ public boolean preferDirectBuffer();
+
+ /**
+ * Should be called when release this coder. Good chance to release encoding
+ * or decoding buffers
+ */
+ public void release();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
new file mode 100644
index 0000000..280daf3
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+/**
+ * Raw erasure coder factory that can be used to create raw encoder and decoder.
+ * It helps in configuration since only one factory class is needed to be
+ * configured.
+ */
+public interface RawErasureCoderFactory {
+
+ /**
+ * Create raw erasure encoder.
+ * @param numDataUnits number of data units in a coding group
+ * @param numParityUnits number of parity units in a coding group
+ * @return raw erasure encoder
+ */
+ public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits);
+
+ /**
+ * Create raw erasure decoder.
+ * @param numDataUnits number of data units in a coding group
+ * @param numParityUnits number of parity units in a coding group
+ * @return raw erasure decoder
+ */
+ public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
new file mode 100644
index 0000000..e2d01d9
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureDecoder.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+
+/**
+ * RawErasureDecoder performs decoding given chunks of input data and generates
+ * missing data that corresponds to an erasure code scheme, like XOR and
+ * Reed-Solomon.
+ *
+ * It extends the {@link RawErasureCoder} interface.
+ */
+public interface RawErasureDecoder extends RawErasureCoder {
+
+ /**
+ * Decode with inputs and erasedIndexes, generates outputs.
+ * How to prepare for inputs:
+ * 1. Create an array containing parity units + data units. Please note the
+ * parity units should be first or before the data units.
+ * 2. Set null in the array locations specified via erasedIndexes to indicate
+ * they're erased and no data are to read from;
+ * 3. Set null in the array locations for extra redundant items, as they're
+ * not necessary to read when decoding. For example in RS-6-3, if only 1
+ * unit is really erased, then we have 2 extra items as redundant. They can
+ * be set as null to indicate no data will be used from them.
+ *
+ * For an example using RS (6, 3), assuming sources (d0, d1, d2, d3, d4, d5)
+ * and parities (p0, p1, p2), d2 being erased. We can and may want to use only
+ * 6 units like (d1, d3, d4, d5, p0, p2) to recover d2. We will have:
+ * inputs = [p0, null(p1), p2, null(d0), d1, null(d2), d3, d4, d5]
+ * erasedIndexes = [5] // index of d2 into inputs array
+ * outputs = [a-writable-buffer]
+ *
+ * Note, for both inputs and outputs, no mixing of on-heap buffers and direct
+ * buffers are allowed.
+ *
+ * @param inputs inputs to read data from, contents may change after the call
+ * @param erasedIndexes indexes of erased units in the inputs array
+ * @param outputs outputs to write into for data generated according to
+ * erasedIndexes, ready for reading the result data from after
+ * the call
+ */
+ public void decode(ByteBuffer[] inputs, int[] erasedIndexes,
+ ByteBuffer[] outputs);
+
+ /**
+ * Decode with inputs and erasedIndexes, generates outputs. More see above.
+ * @param inputs inputs to read data from, contents may change after the call
+ * @param erasedIndexes indexes of erased units in the inputs array
+ * @param outputs outputs to write into for data generated according to
+ * erasedIndexes, ready for reading the result data from after
+ * the call
+ */
+ public void decode(byte[][] inputs, int[] erasedIndexes, byte[][] outputs);
+
+ /**
+ * Decode with inputs and erasedIndexes, generates outputs. More see above.
+ *
+ * Note, for both input and output ECChunks, no mixing of on-heap buffers and
+ * direct buffers are allowed.
+ *
+ * @param inputs inputs to read data from, contents may change after the call
+ * @param erasedIndexes indexes of erased units in the inputs array
+ * @param outputs outputs to write into for data generated according to
+ * erasedIndexes, ready for reading the result data from after
+ * the call
+ */
+ public void decode(ECChunk[] inputs, int[] erasedIndexes, ECChunk[] outputs);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
new file mode 100644
index 0000000..7571f09
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureEncoder.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+import java.nio.ByteBuffer;
+
+/**
+ * RawErasureEncoder performs encoding given chunks of input data and generates
+ * parity outputs that corresponds to an erasure code scheme, like XOR and
+ * Reed-Solomon.
+ *
+ * It extends the {@link RawErasureCoder} interface.
+ */
+public interface RawErasureEncoder extends RawErasureCoder {
+
+ /**
+ * Encode with inputs and generates outputs.
+ *
+ * Note, for both inputs and outputs, no mixing of on-heap buffers and direct
+ * buffers are allowed.
+ *
+ * @param inputs inputs to read data from, contents may change after the call
+ * @param outputs
+ */
+ public void encode(ByteBuffer[] inputs, ByteBuffer[] outputs);
+
+ /**
+ * Encode with inputs and generates outputs
+ * @param inputs inputs to read data from, contents may change after the call
+ * @param outputs outputs to write into for data generated, ready for reading
+ * the result data from after the call
+ */
+ public void encode(byte[][] inputs, byte[][] outputs);
+
+ /**
+ * Encode with inputs and generates outputs.
+ *
+ * Note, for both input and output ECChunks, no mixing of on-heap buffers and
+ * direct buffers are allowed.
+ *
+ * @param inputs inputs to read data from, contents may change after the call
+ * @param outputs outputs to write into for data generated, ready for reading
+ * the result data from after the call
+ */
+ public void encode(ECChunk[] inputs, ECChunk[] outputs);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
new file mode 100644
index 0000000..e20e543
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawDecoder.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A raw decoder in XOR code scheme in pure Java, adapted from HDFS-RAID.
+ *
+ * XOR code is an important primitive code scheme in erasure coding and often
+ * used in advanced codes, like HitchHiker and LRC, though itself is rarely
+ * deployed independently.
+ */
+public class XORRawDecoder extends AbstractRawErasureDecoder {
+
+ public XORRawDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ @Override
+ protected void doDecode(ByteBuffer[] inputs, int[] erasedIndexes,
+ ByteBuffer[] outputs) {
+ ByteBuffer output = outputs[0];
+ resetBuffer(output);
+
+ int erasedIdx = erasedIndexes[0];
+
+ // Process the inputs.
+ int iIdx, oIdx;
+ for (int i = 0; i < inputs.length; i++) {
+ // Skip the erased location.
+ if (i == erasedIdx) {
+ continue;
+ }
+
+ for (iIdx = inputs[i].position(), oIdx = output.position();
+ iIdx < inputs[i].limit();
+ iIdx++, oIdx++) {
+ output.put(oIdx, (byte) (output.get(oIdx) ^ inputs[i].get(iIdx)));
+ }
+ }
+ }
+
+ @Override
+ protected void doDecode(byte[][] inputs, int[] inputOffsets, int dataLen,
+ int[] erasedIndexes, byte[][] outputs,
+ int[] outputOffsets) {
+ byte[] output = outputs[0];
+ resetBuffer(output, outputOffsets[0], dataLen);
+
+ int erasedIdx = erasedIndexes[0];
+
+ // Process the inputs.
+ int iIdx, oIdx;
+ for (int i = 0; i < inputs.length; i++) {
+ // Skip the erased location.
+ if (i == erasedIdx) {
+ continue;
+ }
+
+ for (iIdx = inputOffsets[i], oIdx = outputOffsets[0];
+ iIdx < inputOffsets[i] + dataLen; iIdx++, oIdx++) {
+ output[oIdx] ^= inputs[i][iIdx];
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
new file mode 100644
index 0000000..f4d242e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawEncoder.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A raw encoder in XOR code scheme in pure Java, adapted from HDFS-RAID.
+ *
+ * XOR code is an important primitive code scheme in erasure coding and often
+ * used in advanced codes, like HitchHiker and LRC, though itself is rarely
+ * deployed independently.
+ */
+public class XORRawEncoder extends AbstractRawErasureEncoder {
+
+ public XORRawEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ protected void doEncode(ByteBuffer[] inputs, ByteBuffer[] outputs) {
+ ByteBuffer output = outputs[0];
+ resetBuffer(output);
+
+ // Get the first buffer's data.
+ int iIdx, oIdx;
+ for (iIdx = inputs[0].position(), oIdx = output.position();
+ iIdx < inputs[0].limit(); iIdx++, oIdx++) {
+ output.put(oIdx, inputs[0].get(iIdx));
+ }
+
+ // XOR with everything else.
+ for (int i = 1; i < inputs.length; i++) {
+ for (iIdx = inputs[i].position(), oIdx = output.position();
+ iIdx < inputs[i].limit();
+ iIdx++, oIdx++) {
+ output.put(oIdx, (byte) (output.get(oIdx) ^ inputs[i].get(iIdx)));
+ }
+ }
+ }
+
+ @Override
+ protected void doEncode(byte[][] inputs, int[] inputOffsets, int dataLen,
+ byte[][] outputs, int[] outputOffsets) {
+ byte[] output = outputs[0];
+ resetBuffer(output, outputOffsets[0], dataLen);
+
+ // Get the first buffer's data.
+ int iIdx, oIdx;
+ for (iIdx = inputOffsets[0], oIdx = outputOffsets[0];
+ iIdx < inputOffsets[0] + dataLen; iIdx++, oIdx++) {
+ output[oIdx] = inputs[0][iIdx];
+ }
+
+ // XOR with everything else.
+ for (int i = 1; i < inputs.length; i++) {
+ for (iIdx = inputOffsets[i], oIdx = outputOffsets[0];
+ iIdx < inputOffsets[i] + dataLen; iIdx++, oIdx++) {
+ output[oIdx] ^= inputs[i][iIdx];
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
new file mode 100644
index 0000000..de20c95
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+/**
+ * A raw coder factory for raw XOR coder.
+ */
+public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
+
+ @Override
+ public RawErasureEncoder createEncoder(int numDataUnits, int numParityUnits) {
+ return new XORRawEncoder(numDataUnits, numParityUnits);
+ }
+
+ @Override
+ public RawErasureDecoder createDecoder(int numDataUnits, int numParityUnits) {
+ return new XORRawDecoder(numDataUnits, numParityUnits);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java
new file mode 100644
index 0000000..c8f133f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/DumpUtil.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder.util;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+/**
+ * A dump utility class for debugging data erasure coding/decoding issues. Don't
+ * suggest they are used in runtime production codes.
+ */
+public final class DumpUtil {
+ private static final String HEX_CHARS_STR = "0123456789ABCDEF";
+ private static final char[] HEX_CHARS = HEX_CHARS_STR.toCharArray();
+
+ private DumpUtil() {
+ // No called
+ }
+
+ /**
+ * Convert bytes into format like 0x02 02 00 80.
+ */
+ public static String bytesToHex(byte[] bytes, int limit) {
+ if (limit > bytes.length) {
+ limit = bytes.length;
+ }
+ int len = limit * 2;
+ len += limit; // for ' ' appended for each char
+ len += 2; // for '0x' prefix
+ char[] hexChars = new char[len];
+ hexChars[0] = '0';
+ hexChars[1] = 'x';
+ for (int j = 0; j < limit; j++) {
+ int v = bytes[j] & 0xFF;
+ hexChars[j * 3 + 2] = HEX_CHARS[v >>> 4];
+ hexChars[j * 3 + 3] = HEX_CHARS[v & 0x0F];
+ hexChars[j * 3 + 4] = ' ';
+ }
+
+ return new String(hexChars);
+ }
+
+ /**
+ * Print data in hex format in an array of chunks.
+ * @param header
+ * @param chunks
+ */
+ public static void dumpChunks(String header, ECChunk[] chunks) {
+ System.out.println();
+ System.out.println(header);
+ for (int i = 0; i < chunks.length; i++) {
+ dumpChunk(chunks[i]);
+ }
+ System.out.println();
+ }
+
+ /**
+ * Print data in hex format in a chunk.
+ * @param chunk
+ */
+ public static void dumpChunk(ECChunk chunk) {
+ String str;
+ if (chunk == null) {
+ str = "<EMPTY>";
+ } else {
+ byte[] bytes = chunk.toBytesArray();
+ str = DumpUtil.bytesToHex(bytes, 16);
+ }
+ System.out.println(str);
+ }
+}
[15/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicies.java
new file mode 100644
index 0000000..622b258
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicies.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.util.ReflectionUtils;
+
+public class BlockPlacementPolicies{
+
+ private final BlockPlacementPolicy replicationPolicy;
+ private final BlockPlacementPolicy ecPolicy;
+
+ public BlockPlacementPolicies(Configuration conf, FSClusterStats stats,
+ NetworkTopology clusterMap,
+ Host2NodesMap host2datanodeMap){
+ final Class<? extends BlockPlacementPolicy> replicatorClass = conf
+ .getClass(DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
+ DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
+ BlockPlacementPolicy.class);
+ replicationPolicy = ReflectionUtils.newInstance(replicatorClass, conf);
+ replicationPolicy.initialize(conf, stats, clusterMap, host2datanodeMap);
+ final Class<? extends BlockPlacementPolicy> blockPlacementECClass =
+ conf.getClass(DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_KEY,
+ DFSConfigKeys.DFS_BLOCK_PLACEMENT_EC_CLASSNAME_DEFAULT,
+ BlockPlacementPolicy.class);
+ ecPolicy = ReflectionUtils.newInstance(blockPlacementECClass, conf);
+ ecPolicy.initialize(conf, stats, clusterMap, host2datanodeMap);
+ }
+
+ public BlockPlacementPolicy getPolicy(boolean isStriped){
+ if (isStriped) {
+ return ecPolicy;
+ } else {
+ return replicationPolicy;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
index 9696179..86aaf79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicy.java
@@ -145,31 +145,7 @@ public abstract class BlockPlacementPolicy {
abstract protected void initialize(Configuration conf, FSClusterStats stats,
NetworkTopology clusterMap,
Host2NodesMap host2datanodeMap);
-
- /**
- * Get an instance of the configured Block Placement Policy based on the
- * the configuration property
- * {@link DFSConfigKeys#DFS_BLOCK_REPLICATOR_CLASSNAME_KEY}.
- *
- * @param conf the configuration to be used
- * @param stats an object that is used to retrieve the load on the cluster
- * @param clusterMap the network topology of the cluster
- * @return an instance of BlockPlacementPolicy
- */
- public static BlockPlacementPolicy getInstance(Configuration conf,
- FSClusterStats stats,
- NetworkTopology clusterMap,
- Host2NodesMap host2datanodeMap) {
- final Class<? extends BlockPlacementPolicy> replicatorClass = conf.getClass(
- DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_KEY,
- DFSConfigKeys.DFS_BLOCK_REPLICATOR_CLASSNAME_DEFAULT,
- BlockPlacementPolicy.class);
- final BlockPlacementPolicy replicator = ReflectionUtils.newInstance(
- replicatorClass, conf);
- replicator.initialize(conf, stats, clusterMap, host2datanodeMap);
- return replicator;
- }
-
+
/**
* Adjust rackmap, moreThanOne, and exactlyOne after removing replica on cur.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolarent.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolarent.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolarent.java
new file mode 100644
index 0000000..4dbf384
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyRackFaultTolarent.java
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.net.Node;
+import org.apache.hadoop.net.NodeBase;
+
+import java.util.*;
+
+/**
+ * The class is responsible for choosing the desired number of targets
+ * for placing block replicas.
+ * The strategy is that it tries its best to place the replicas to most racks.
+ */
+@InterfaceAudience.Private
+public class BlockPlacementPolicyRackFaultTolarent extends BlockPlacementPolicyDefault {
+
+ @Override
+ protected int[] getMaxNodesPerRack(int numOfChosen, int numOfReplicas) {
+ int clusterSize = clusterMap.getNumOfLeaves();
+ int totalNumOfReplicas = numOfChosen + numOfReplicas;
+ if (totalNumOfReplicas > clusterSize) {
+ numOfReplicas -= (totalNumOfReplicas-clusterSize);
+ totalNumOfReplicas = clusterSize;
+ }
+ // No calculation needed when there is only one rack or picking one node.
+ int numOfRacks = clusterMap.getNumOfRacks();
+ if (numOfRacks == 1 || totalNumOfReplicas <= 1) {
+ return new int[] {numOfReplicas, totalNumOfReplicas};
+ }
+ if(totalNumOfReplicas<numOfRacks){
+ return new int[] {numOfReplicas, 1};
+ }
+ int maxNodesPerRack = (totalNumOfReplicas - 1) / numOfRacks + 1;
+ return new int[] {numOfReplicas, maxNodesPerRack};
+ }
+
+ /**
+ * Choose numOfReplicas in order:
+ * 1. If total replica expected is less than numOfRacks in cluster, it choose
+ * randomly.
+ * 2. If total replica expected is bigger than numOfRacks, it choose:
+ * 2a. Fill each rack exactly (maxNodesPerRack-1) replicas.
+ * 2b. For some random racks, place one more replica to each one of them, until
+ * numOfReplicas have been chosen. <br>
+ * In the end, the difference of the numbers of replicas for each two racks
+ * is no more than 1.
+ * Either way it always prefer local storage.
+ * @return local node of writer
+ */
+ @Override
+ protected Node chooseTargetInOrder(int numOfReplicas,
+ Node writer,
+ final Set<Node> excludedNodes,
+ final long blocksize,
+ final int maxNodesPerRack,
+ final List<DatanodeStorageInfo> results,
+ final boolean avoidStaleNodes,
+ final boolean newBlock,
+ EnumMap<StorageType, Integer> storageTypes)
+ throws NotEnoughReplicasException {
+ int totalReplicaExpected = results.size() + numOfReplicas;
+ int numOfRacks = clusterMap.getNumOfRacks();
+ if (totalReplicaExpected < numOfRacks ||
+ totalReplicaExpected % numOfRacks == 0) {
+ writer = chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ return writer;
+ }
+
+ assert totalReplicaExpected > (maxNodesPerRack -1) * numOfRacks;
+
+ // Calculate numOfReplicas for filling each rack exactly (maxNodesPerRack-1)
+ // replicas.
+ HashMap<String, Integer> rackCounts = new HashMap<>();
+ for (DatanodeStorageInfo dsInfo : results) {
+ String rack = dsInfo.getDatanodeDescriptor().getNetworkLocation();
+ Integer count = rackCounts.get(rack);
+ if (count != null) {
+ rackCounts.put(rack, count + 1);
+ } else {
+ rackCounts.put(rack, 1);
+ }
+ }
+ int excess = 0; // Sum of the above (maxNodesPerRack-1) part of nodes in results
+ for (int count : rackCounts.values()) {
+ if (count > maxNodesPerRack -1) {
+ excess += count - (maxNodesPerRack -1);
+ }
+ }
+ numOfReplicas = Math.min(totalReplicaExpected - results.size(),
+ (maxNodesPerRack -1) * numOfRacks - (results.size() - excess));
+
+ // Fill each rack exactly (maxNodesPerRack-1) replicas.
+ writer = chooseOnce(numOfReplicas, writer, new HashSet<>(excludedNodes),
+ blocksize, maxNodesPerRack -1, results, avoidStaleNodes, storageTypes);
+
+ for (DatanodeStorageInfo resultStorage : results) {
+ addToExcludedNodes(resultStorage.getDatanodeDescriptor(), excludedNodes);
+ }
+
+ // For some racks, place one more replica to each one of them.
+ numOfReplicas = totalReplicaExpected - results.size();
+ chooseOnce(numOfReplicas, writer, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+
+ return writer;
+ }
+
+ /**
+ * Randomly choose <i>numOfReplicas</i> targets from the given <i>scope</i>.
+ * Except that 1st replica prefer local storage.
+ * @return local node of writer.
+ */
+ private Node chooseOnce(int numOfReplicas,
+ Node writer,
+ final Set<Node> excludedNodes,
+ final long blocksize,
+ final int maxNodesPerRack,
+ final List<DatanodeStorageInfo> results,
+ final boolean avoidStaleNodes,
+ EnumMap<StorageType, Integer> storageTypes)
+ throws NotEnoughReplicasException {
+ if (numOfReplicas == 0) {
+ return writer;
+ }
+ writer = chooseLocalStorage(writer, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes, true)
+ .getDatanodeDescriptor();
+ if (--numOfReplicas == 0) {
+ return writer;
+ }
+ chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ return writer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index 0dbf485..0e0fd91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -43,8 +43,15 @@ class BlocksMap {
@Override
public boolean hasNext() {
- return blockInfo != null && nextIdx < blockInfo.getCapacity()
- && blockInfo.getDatanode(nextIdx) != null;
+ if (blockInfo == null) {
+ return false;
+ }
+ while (nextIdx < blockInfo.getCapacity() &&
+ blockInfo.getDatanode(nextIdx) == null) {
+ // note that for striped blocks there may be null in the triplets
+ nextIdx++;
+ }
+ return nextIdx < blockInfo.getCapacity();
}
@Override
@@ -123,12 +130,16 @@ class BlocksMap {
return;
blockInfo.setBlockCollection(null);
- for(int idx = blockInfo.numNodes()-1; idx >= 0; idx--) {
+ final int size = blockInfo instanceof BlockInfoContiguous ?
+ blockInfo.numNodes() : blockInfo.getCapacity();
+ for(int idx = size - 1; idx >= 0; idx--) {
DatanodeDescriptor dn = blockInfo.getDatanode(idx);
- dn.removeBlock(blockInfo); // remove from the list and wipe the location
+ if (dn != null) {
+ dn.removeBlock(blockInfo); // remove from the list and wipe the location
+ }
}
}
-
+
/** Returns the block object it it exists in the map. */
BlockInfo getStoredBlock(Block b) {
return blocks.get(b);
@@ -190,8 +201,8 @@ class BlocksMap {
// remove block from the data-node list and the node from the block info
boolean removed = node.removeBlock(info);
- if (info.getDatanode(0) == null // no datanodes left
- && info.isDeleted()) { // does not belong to a file
+ if (info.hasNoStorage() // no datanodes left
+ && info.isDeleted()) { // does not belong to a file
blocks.remove(b); // remove block from the map
}
return removed;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 7e12a99..3cf9db6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -31,8 +31,8 @@ import java.util.Queue;
import java.util.Set;
import com.google.common.annotations.VisibleForTesting;
-
import com.google.common.collect.ImmutableList;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -41,13 +41,16 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.util.IntrusiveCollection;
import org.apache.hadoop.util.Time;
@@ -220,9 +223,12 @@ public class DatanodeDescriptor extends DatanodeInfo {
/** A queue of blocks to be replicated by this datanode */
private final BlockQueue<BlockTargetPair> replicateBlocks =
new BlockQueue<>();
+ /** A queue of blocks to be erasure coded by this datanode */
+ private final BlockQueue<BlockECRecoveryInfo> erasurecodeBlocks =
+ new BlockQueue<>();
/** A queue of blocks to be recovered by this datanode */
- private final BlockQueue<BlockInfoContiguousUnderConstruction> recoverBlocks =
- new BlockQueue<BlockInfoContiguousUnderConstruction>();
+ private final BlockQueue<BlockInfoUnderConstruction> recoverBlocks =
+ new BlockQueue<>();
/** A set of blocks to be invalidated by this datanode */
private final LightWeightHashSet<Block> invalidateBlocks =
new LightWeightHashSet<>();
@@ -280,7 +286,8 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
- DatanodeStorageInfo[] getStorageInfos() {
+ @VisibleForTesting
+ public DatanodeStorageInfo[] getStorageInfos() {
synchronized (storageMap) {
final Collection<DatanodeStorageInfo> storages = storageMap.values();
return storages.toArray(new DatanodeStorageInfo[storages.size()]);
@@ -377,6 +384,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
this.invalidateBlocks.clear();
this.recoverBlocks.clear();
this.replicateBlocks.clear();
+ this.erasurecodeBlocks.clear();
}
// pendingCached, cached, and pendingUncached are protected by the
// FSN lock.
@@ -577,6 +585,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
Iterator<BlockInfo> getBlockIterator() {
return new BlockIterator(getStorageInfos());
}
+
Iterator<BlockInfo> getBlockIterator(final String storageID) {
return new BlockIterator(getStorageInfo(storageID));
}
@@ -598,9 +607,23 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
/**
+ * Store block erasure coding work.
+ */
+ void addBlockToBeErasureCoded(ExtendedBlock block,
+ DatanodeDescriptor[] sources, DatanodeStorageInfo[] targets,
+ short[] liveBlockIndices, ECSchema ecSchema, int cellSize) {
+ assert (block != null && sources != null && sources.length > 0);
+ BlockECRecoveryInfo task = new BlockECRecoveryInfo(block, sources, targets,
+ liveBlockIndices, ecSchema, cellSize);
+ erasurecodeBlocks.offer(task);
+ BlockManager.LOG.debug("Adding block recovery task " + task + "to "
+ + getName() + ", current queue size is " + erasurecodeBlocks.size());
+ }
+
+ /**
* Store block recovery work.
*/
- void addBlockToBeRecovered(BlockInfoContiguousUnderConstruction block) {
+ void addBlockToBeRecovered(BlockInfoUnderConstruction block) {
if(recoverBlocks.contains(block)) {
// this prevents adding the same block twice to the recovery queue
BlockManager.LOG.info(block + " is already in the recovery queue");
@@ -629,6 +652,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
/**
+ * The number of work items that are pending to be replicated
+ */
+ @VisibleForTesting
+ public int getNumberOfBlocksToBeErasureCoded() {
+ return erasurecodeBlocks.size();
+ }
+
+ /**
* The number of block invalidation items that are pending to
* be sent to the datanode
*/
@@ -642,11 +673,15 @@ public class DatanodeDescriptor extends DatanodeInfo {
return replicateBlocks.poll(maxTransfers);
}
- public BlockInfoContiguousUnderConstruction[] getLeaseRecoveryCommand(int maxTransfers) {
- List<BlockInfoContiguousUnderConstruction> blocks = recoverBlocks.poll(maxTransfers);
+ public List<BlockECRecoveryInfo> getErasureCodeCommand(int maxTransfers) {
+ return erasurecodeBlocks.poll(maxTransfers);
+ }
+
+ public BlockInfoUnderConstruction[] getLeaseRecoveryCommand(int maxTransfers) {
+ List<BlockInfoUnderConstruction> blocks = recoverBlocks.poll(maxTransfers);
if(blocks == null)
return null;
- return blocks.toArray(new BlockInfoContiguousUnderConstruction[blocks.size()]);
+ return blocks.toArray(new BlockInfoUnderConstruction[blocks.size()]);
}
/**
@@ -660,6 +695,13 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
}
+ @VisibleForTesting
+ public boolean containsInvalidateBlock(Block block) {
+ synchronized (invalidateBlocks) {
+ return invalidateBlocks.contains(block);
+ }
+ }
+
/**
* @return Approximate number of blocks currently scheduled to be written
*/
@@ -842,6 +884,10 @@ public class DatanodeDescriptor extends DatanodeInfo {
if (repl > 0) {
sb.append(" ").append(repl).append(" blocks to be replicated;");
}
+ int ec = erasurecodeBlocks.size();
+ if(ec > 0) {
+ sb.append(" ").append(ec).append(" blocks to be erasure coded;");
+ }
int inval = invalidateBlocks.size();
if (inval > 0) {
sb.append(" ").append(inval).append(" blocks to be invalidated;");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 3397bbb..dbd07d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.*;
@@ -1380,29 +1381,29 @@ public class DatanodeManager {
}
//check lease recovery
- BlockInfoContiguousUnderConstruction[] blocks = nodeinfo
+ BlockInfoUnderConstruction[] blocks = nodeinfo
.getLeaseRecoveryCommand(Integer.MAX_VALUE);
if (blocks != null) {
BlockRecoveryCommand brCommand = new BlockRecoveryCommand(
blocks.length);
- for (BlockInfoContiguousUnderConstruction b : blocks) {
+ for (BlockInfoUnderConstruction b : blocks) {
final DatanodeStorageInfo[] storages = b.getExpectedStorageLocations();
// Skip stale nodes during recovery - not heart beated for some time (30s by default).
final List<DatanodeStorageInfo> recoveryLocations =
new ArrayList<>(storages.length);
- for (int i = 0; i < storages.length; i++) {
- if (!storages[i].getDatanodeDescriptor().isStale(staleInterval)) {
- recoveryLocations.add(storages[i]);
+ for (DatanodeStorageInfo storage : storages) {
+ if (!storage.getDatanodeDescriptor().isStale(staleInterval)) {
+ recoveryLocations.add(storage);
}
}
// If we are performing a truncate recovery than set recovery fields
// to old block.
boolean truncateRecovery = b.getTruncateBlock() != null;
boolean copyOnTruncateRecovery = truncateRecovery &&
- b.getTruncateBlock().getBlockId() != b.getBlockId();
+ b.getTruncateBlock().getBlockId() != b.toBlock().getBlockId();
ExtendedBlock primaryBlock = (copyOnTruncateRecovery) ?
new ExtendedBlock(blockPoolId, b.getTruncateBlock()) :
- new ExtendedBlock(blockPoolId, b);
+ new ExtendedBlock(blockPoolId, b.toBlock());
// If we only get 1 replica after eliminating stale nodes, then choose all
// replicas for recovery and let the primary data node handle failures.
DatanodeInfo[] recoveryInfos;
@@ -1419,7 +1420,7 @@ public class DatanodeManager {
recoveryInfos = DatanodeStorageInfo.toDatanodeInfos(storages);
}
if(truncateRecovery) {
- Block recoveryBlock = (copyOnTruncateRecovery) ? b :
+ Block recoveryBlock = (copyOnTruncateRecovery) ? b.toBlock() :
b.getTruncateBlock();
brCommand.add(new RecoveringBlock(primaryBlock, recoveryInfos,
recoveryBlock));
@@ -1439,6 +1440,13 @@ public class DatanodeManager {
cmds.add(new BlockCommand(DatanodeProtocol.DNA_TRANSFER, blockPoolId,
pendingList));
}
+ // checking pending erasure coding tasks
+ List<BlockECRecoveryInfo> pendingECList =
+ nodeinfo.getErasureCodeCommand(maxTransfers);
+ if (pendingECList != null) {
+ cmds.add(new BlockECRecoveryCommand(DatanodeProtocol.DNA_ERASURE_CODING_RECOVERY,
+ pendingECList));
+ }
//check block invalidation
Block[] blks = nodeinfo.getInvalidateBlocks(blockInvalidateLimit);
if (blks != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 216d6d2..bdf9f9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -24,6 +24,7 @@ import java.util.List;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
@@ -209,6 +210,7 @@ public class DatanodeStorageInfo {
return getState() == State.FAILED && numBlocks != 0;
}
+ @VisibleForTesting
public String getStorageID() {
return storageID;
}
@@ -233,7 +235,7 @@ public class DatanodeStorageInfo {
return blockPoolUsed;
}
- public AddBlockResult addBlock(BlockInfo b) {
+ public AddBlockResult addBlock(BlockInfo b, Block reportedBlock) {
// First check whether the block belongs to a different storage
// on the same DN.
AddBlockResult result = AddBlockResult.ADDED;
@@ -252,10 +254,18 @@ public class DatanodeStorageInfo {
}
// add to the head of the data-node list
- b.addStorage(this);
+ b.addStorage(this, reportedBlock);
+ insertToList(b);
+ return result;
+ }
+
+ AddBlockResult addBlock(BlockInfo b) {
+ return addBlock(b, b);
+ }
+
+ public void insertToList(BlockInfo b) {
blockList = b.listInsert(blockList, this);
numBlocks++;
- return result;
}
public boolean removeBlock(BlockInfo b) {
@@ -274,7 +284,6 @@ public class DatanodeStorageInfo {
Iterator<BlockInfo> getBlockIterator() {
return new BlockIterator(blockList);
-
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 797d031..5e3cac2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -36,7 +36,6 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.util.CyclicIteration;
@@ -234,14 +233,14 @@ public class DecommissionManager {
}
/**
- * Checks whether a block is sufficiently replicated for decommissioning.
- * Full-strength replication is not always necessary, hence "sufficient".
+ * Checks whether a block is sufficiently replicated/stored for
+ * decommissioning. For replicated blocks or striped blocks, full-strength
+ * replication or storage is not always necessary, hence "sufficient".
* @return true if sufficient, else false.
*/
- private boolean isSufficientlyReplicated(BlockInfo block,
- BlockCollection bc,
+ private boolean isSufficient(BlockInfo block, BlockCollection bc,
NumberReplicas numberReplicas) {
- final int numExpected = bc.getPreferredBlockReplication();
+ final int numExpected = blockManager.getExpectedReplicaNum(bc, block);
final int numLive = numberReplicas.liveReplicas();
if (!blockManager.isNeededReplication(block, numExpected, numLive)) {
// Block doesn't need replication. Skip.
@@ -255,18 +254,19 @@ public class DecommissionManager {
if (numExpected > numLive) {
if (bc.isUnderConstruction() && block.equals(bc.getLastBlock())) {
// Can decom a UC block as long as there will still be minReplicas
- if (numLive >= blockManager.minReplication) {
+ if (blockManager.hasMinStorage(block, numLive)) {
LOG.trace("UC block {} sufficiently-replicated since numLive ({}) "
- + ">= minR ({})", block, numLive, blockManager.minReplication);
+ + ">= minR ({})", block, numLive,
+ blockManager.getMinStorageNum(block));
return true;
} else {
LOG.trace("UC block {} insufficiently-replicated since numLive "
+ "({}) < minR ({})", block, numLive,
- blockManager.minReplication);
+ blockManager.getMinStorageNum(block));
}
} else {
// Can decom a non-UC as long as the default replication is met
- if (numLive >= blockManager.defaultReplication) {
+ if (numLive >= blockManager.getDefaultStorageNum(block)) {
return true;
}
}
@@ -274,11 +274,11 @@ public class DecommissionManager {
return false;
}
- private static void logBlockReplicationInfo(Block block, BlockCollection bc,
+ private void logBlockReplicationInfo(BlockInfo block, BlockCollection bc,
DatanodeDescriptor srcNode, NumberReplicas num,
Iterable<DatanodeStorageInfo> storages) {
int curReplicas = num.liveReplicas();
- int curExpectedReplicas = bc.getPreferredBlockReplication();
+ int curExpectedReplicas = blockManager.getExpectedReplicaNum(bc, block);
StringBuilder nodeList = new StringBuilder();
for (DatanodeStorageInfo storage : storages) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
@@ -407,14 +407,14 @@ public class DecommissionManager {
// that are insufficiently replicated for further tracking
LOG.debug("Newly-added node {}, doing full scan to find " +
"insufficiently-replicated blocks.", dn);
- blocks = handleInsufficientlyReplicated(dn);
+ blocks = handleInsufficientlyStored(dn);
decomNodeBlocks.put(dn, blocks);
fullScan = true;
} else {
// This is a known datanode, check if its # of insufficiently
// replicated blocks has dropped to zero and if it can be decommed
LOG.debug("Processing decommission-in-progress node {}", dn);
- pruneSufficientlyReplicated(dn, blocks);
+ pruneReliableBlocks(dn, blocks);
}
if (blocks.size() == 0) {
if (!fullScan) {
@@ -426,7 +426,7 @@ public class DecommissionManager {
// marking the datanode as decommissioned
LOG.debug("Node {} has finished replicating current set of "
+ "blocks, checking with the full block map.", dn);
- blocks = handleInsufficientlyReplicated(dn);
+ blocks = handleInsufficientlyStored(dn);
decomNodeBlocks.put(dn, blocks);
}
// If the full scan is clean AND the node liveness is okay,
@@ -467,25 +467,23 @@ public class DecommissionManager {
}
/**
- * Removes sufficiently replicated blocks from the block list of a
- * datanode.
+ * Removes reliable blocks from the block list of a datanode.
*/
- private void pruneSufficientlyReplicated(final DatanodeDescriptor datanode,
+ private void pruneReliableBlocks(final DatanodeDescriptor datanode,
AbstractList<BlockInfo> blocks) {
processBlocksForDecomInternal(datanode, blocks.iterator(), null, true);
}
/**
- * Returns a list of blocks on a datanode that are insufficiently
- * replicated, i.e. are under-replicated enough to prevent decommission.
+ * Returns a list of blocks on a datanode that are insufficiently replicated
+ * or require recovery, i.e. requiring recovery and should prevent
+ * decommission.
* <p/>
- * As part of this, it also schedules replication work for
- * any under-replicated blocks.
+ * As part of this, it also schedules replication/recovery work.
*
- * @param datanode
- * @return List of insufficiently replicated blocks
+ * @return List of blocks requiring recovery
*/
- private AbstractList<BlockInfo> handleInsufficientlyReplicated(
+ private AbstractList<BlockInfo> handleInsufficientlyStored(
final DatanodeDescriptor datanode) {
AbstractList<BlockInfo> insufficient = new ChunkedArrayList<>();
processBlocksForDecomInternal(datanode, datanode.getBlockIterator(),
@@ -496,24 +494,22 @@ public class DecommissionManager {
/**
* Used while checking if decommission-in-progress datanodes can be marked
* as decommissioned. Combines shared logic of
- * pruneSufficientlyReplicated and handleInsufficientlyReplicated.
+ * pruneReliableBlocks and handleInsufficientlyStored.
*
* @param datanode Datanode
* @param it Iterator over the blocks on the
* datanode
- * @param insufficientlyReplicated Return parameter. If it's not null,
+ * @param insufficientList Return parameter. If it's not null,
* will contain the insufficiently
* replicated-blocks from the list.
- * @param pruneSufficientlyReplicated whether to remove sufficiently
- * replicated blocks from the iterator
- * @return true if there are under-replicated blocks in the provided block
- * iterator, else false.
+ * @param pruneReliableBlocks whether to remove blocks reliable
+ * enough from the iterator
*/
private void processBlocksForDecomInternal(
final DatanodeDescriptor datanode,
final Iterator<BlockInfo> it,
- final List<BlockInfo> insufficientlyReplicated,
- boolean pruneSufficientlyReplicated) {
+ final List<BlockInfo> insufficientList,
+ boolean pruneReliableBlocks) {
boolean firstReplicationLog = true;
int underReplicatedBlocks = 0;
int decommissionOnlyReplicas = 0;
@@ -528,7 +524,7 @@ public class DecommissionManager {
it.remove();
continue;
}
- BlockCollection bc = blockManager.blocksMap.getBlockCollection(block);
+ BlockCollection bc = blockManager.getBlockCollection(block);
if (bc == null) {
// Orphan block, will be invalidated eventually. Skip.
continue;
@@ -536,35 +532,34 @@ public class DecommissionManager {
final NumberReplicas num = blockManager.countNodes(block);
final int liveReplicas = num.liveReplicas();
- final int curReplicas = liveReplicas;
// Schedule under-replicated blocks for replication if not already
// pending
if (blockManager.isNeededReplication(block,
- bc.getPreferredBlockReplication(), liveReplicas)) {
+ blockManager.getExpectedReplicaNum(bc, block), liveReplicas)) {
if (!blockManager.neededReplications.contains(block) &&
blockManager.pendingReplications.getNumReplicas(block) == 0 &&
namesystem.isPopulatingReplQueues()) {
// Process these blocks only when active NN is out of safe mode.
blockManager.neededReplications.add(block,
- curReplicas,
+ liveReplicas,
num.decommissionedAndDecommissioning(),
- bc.getPreferredBlockReplication());
+ blockManager.getExpectedReplicaNum(bc, block));
}
}
// Even if the block is under-replicated,
- // it doesn't block decommission if it's sufficiently replicated
- if (isSufficientlyReplicated(block, bc, num)) {
- if (pruneSufficientlyReplicated) {
+ // it doesn't block decommission if it's sufficiently replicated
+ if (isSufficient(block, bc, num)) {
+ if (pruneReliableBlocks) {
it.remove();
}
continue;
}
// We've found an insufficiently replicated block.
- if (insufficientlyReplicated != null) {
- insufficientlyReplicated.add(block);
+ if (insufficientList != null) {
+ insufficientList.add(block);
}
// Log if this is our first time through
if (firstReplicationLog) {
@@ -577,7 +572,7 @@ public class DecommissionManager {
if (bc.isUnderConstruction()) {
underReplicatedInOpenFiles++;
}
- if ((curReplicas == 0) && (num.decommissionedAndDecommissioning() > 0)) {
+ if ((liveReplicas == 0) && (num.decommissionedAndDecommissioning() > 0)) {
decommissionOnlyReplicas++;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java
new file mode 100644
index 0000000..f4600cb7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/ReplicaUnderConstruction.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
+
+/**
+ * ReplicaUnderConstruction contains information about replicas (or blocks
+ * belonging to a block group) while they are under construction.
+ *
+ * The GS, the length and the state of the replica is as reported by the
+ * datanode.
+ *
+ * It is not guaranteed, but expected, that datanodes actually have
+ * corresponding replicas.
+ */
+class ReplicaUnderConstruction extends Block {
+ private final DatanodeStorageInfo expectedLocation;
+ private HdfsServerConstants.ReplicaState state;
+ private boolean chosenAsPrimary;
+
+ ReplicaUnderConstruction(Block block,
+ DatanodeStorageInfo target,
+ HdfsServerConstants.ReplicaState state) {
+ super(block);
+ this.expectedLocation = target;
+ this.state = state;
+ this.chosenAsPrimary = false;
+ }
+
+ /**
+ * Expected block replica location as assigned when the block was allocated.
+ * This defines the pipeline order.
+ * It is not guaranteed, but expected, that the data-node actually has
+ * the replica.
+ */
+ DatanodeStorageInfo getExpectedStorageLocation() {
+ return expectedLocation;
+ }
+
+ /**
+ * Get replica state as reported by the data-node.
+ */
+ HdfsServerConstants.ReplicaState getState() {
+ return state;
+ }
+
+ /**
+ * Whether the replica was chosen for recovery.
+ */
+ boolean getChosenAsPrimary() {
+ return chosenAsPrimary;
+ }
+
+ /**
+ * Set replica state.
+ */
+ void setState(HdfsServerConstants.ReplicaState s) {
+ state = s;
+ }
+
+ /**
+ * Set whether this replica was chosen for recovery.
+ */
+ void setChosenAsPrimary(boolean chosenAsPrimary) {
+ this.chosenAsPrimary = chosenAsPrimary;
+ }
+
+ /**
+ * Is data-node the replica belongs to alive.
+ */
+ boolean isAlive() {
+ return expectedLocation.getDatanodeDescriptor().isAlive;
+ }
+
+ @Override // Block
+ public int hashCode() {
+ return super.hashCode();
+ }
+
+ @Override // Block
+ public boolean equals(Object obj) {
+ // Sufficient to rely on super's implementation
+ return (this == obj) || super.equals(obj);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder b = new StringBuilder(50);
+ appendStringTo(b);
+ return b.toString();
+ }
+
+ @Override
+ public void appendStringTo(StringBuilder sb) {
+ sb.append("ReplicaUC[")
+ .append(expectedLocation)
+ .append("|")
+ .append(state)
+ .append("]");
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
new file mode 100644
index 0000000..479ee4c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockGroupIdGenerator.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.util.SequentialNumber;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BLOCK_GROUP_INDEX_MASK;
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.MAX_BLOCKS_IN_GROUP;
+
+/**
+ * Generate the next valid block group ID by incrementing the maximum block
+ * group ID allocated so far, with the first 2^10 block group IDs reserved.
+ * HDFS-EC introduces a hierarchical protocol to name blocks and groups:
+ * Contiguous: {reserved block IDs | flag | block ID}
+ * Striped: {reserved block IDs | flag | block group ID | index in group}
+ *
+ * Following n bits of reserved block IDs, The (n+1)th bit in an ID
+ * distinguishes contiguous (0) and striped (1) blocks. For a striped block,
+ * bits (n+2) to (64-m) represent the ID of its block group, while the last m
+ * bits represent its index of the group. The value m is determined by the
+ * maximum number of blocks in a group (MAX_BLOCKS_IN_GROUP).
+ *
+ * Note that the {@link #nextValue()} methods requires external lock to
+ * guarantee IDs have no conflicts.
+ */
+@InterfaceAudience.Private
+public class SequentialBlockGroupIdGenerator extends SequentialNumber {
+
+ private final BlockManager blockManager;
+
+ SequentialBlockGroupIdGenerator(BlockManager blockManagerRef) {
+ super(Long.MIN_VALUE);
+ this.blockManager = blockManagerRef;
+ }
+
+ @Override // NumberGenerator
+ public long nextValue() {
+ skipTo((getCurrentValue() & ~BLOCK_GROUP_INDEX_MASK) + MAX_BLOCKS_IN_GROUP);
+ // Make sure there's no conflict with existing random block IDs
+ final Block b = new Block(getCurrentValue());
+ while (hasValidBlockInRange(b)) {
+ skipTo(getCurrentValue() + MAX_BLOCKS_IN_GROUP);
+ b.setBlockId(getCurrentValue());
+ }
+ if (b.getBlockId() >= 0) {
+ throw new IllegalStateException("All negative block group IDs are used, "
+ + "growing into positive IDs, "
+ + "which might conflict with non-erasure coded blocks.");
+ }
+ return getCurrentValue();
+ }
+
+ /**
+ * @param b A block object whose id is set to the starting point for check
+ * @return true if any ID in the range
+ * {id, id+HdfsConstants.MAX_BLOCKS_IN_GROUP} is pointed-to by a file
+ */
+ private boolean hasValidBlockInRange(Block b) {
+ final long id = b.getBlockId();
+ for (int i = 0; i < MAX_BLOCKS_IN_GROUP; i++) {
+ b.setBlockId(id + i);
+ if (blockManager.getBlockCollection(b) != null) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
index eef8857..6074784 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/SequentialBlockIdGenerator.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.util.SequentialNumber;
/**
@@ -54,6 +53,11 @@ public class SequentialBlockIdGenerator extends SequentialNumber {
while(isValidBlock(b)) {
b.setBlockId(super.nextValue());
}
+ if (b.getBlockId() < 0) {
+ throw new IllegalStateException("All positive block IDs are used, " +
+ "wrapping to negative IDs, " +
+ "which might conflict with erasure coded block groups.");
+ }
return b.getBlockId();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
index ebc15b8..7e8f479 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
*
* <p/>
* The policy for choosing which priority to give added blocks
- * is implemented in {@link #getPriority(int, int, int)}.
+ * is implemented in {@link #getPriority(BlockInfo, int, int, int)}.
* </p>
* <p>The queue order is as follows:</p>
* <ol>
@@ -145,14 +145,28 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> {
* @param expectedReplicas expected number of replicas of the block
* @return the priority for the blocks, between 0 and ({@link #LEVEL}-1)
*/
- private int getPriority(int curReplicas,
+ private int getPriority(BlockInfo block,
+ int curReplicas,
int decommissionedReplicas,
int expectedReplicas) {
assert curReplicas >= 0 : "Negative replicas!";
if (curReplicas >= expectedReplicas) {
// Block has enough copies, but not enough racks
return QUEUE_REPLICAS_BADLY_DISTRIBUTED;
- } else if (curReplicas == 0) {
+ }
+ if (block.isStriped()) {
+ BlockInfoStriped sblk = (BlockInfoStriped) block;
+ return getPriorityStriped(curReplicas, decommissionedReplicas,
+ sblk.getRealDataBlockNum(), sblk.getParityBlockNum());
+ } else {
+ return getPriorityContiguous(curReplicas, decommissionedReplicas,
+ expectedReplicas);
+ }
+ }
+
+ private int getPriorityContiguous(int curReplicas, int decommissionedReplicas,
+ int expectedReplicas) {
+ if (curReplicas == 0) {
// If there are zero non-decommissioned replicas but there are
// some decommissioned replicas, then assign them highest priority
if (decommissionedReplicas > 0) {
@@ -161,7 +175,7 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> {
//all we have are corrupt blocks
return QUEUE_WITH_CORRUPT_BLOCKS;
} else if (curReplicas == 1) {
- //only on replica -risk of loss
+ // only one replica, highest risk of loss
// highest priority
return QUEUE_HIGHEST_PRIORITY;
} else if ((curReplicas * 3) < expectedReplicas) {
@@ -174,6 +188,27 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> {
}
}
+ private int getPriorityStriped(int curReplicas, int decommissionedReplicas,
+ short dataBlkNum, short parityBlkNum) {
+ if (curReplicas < dataBlkNum) {
+ // There are some replicas on decommissioned nodes so it's not corrupted
+ if (curReplicas + decommissionedReplicas >= dataBlkNum) {
+ return QUEUE_HIGHEST_PRIORITY;
+ }
+ return QUEUE_WITH_CORRUPT_BLOCKS;
+ } else if (curReplicas == dataBlkNum) {
+ // highest risk of loss, highest priority
+ return QUEUE_HIGHEST_PRIORITY;
+ } else if ((curReplicas - dataBlkNum) * 3 < parityBlkNum + 1) {
+ // can only afford one replica loss
+ // this is considered very under-replicated
+ return QUEUE_VERY_UNDER_REPLICATED;
+ } else {
+ // add to the normal queue for under replicated blocks
+ return QUEUE_UNDER_REPLICATED;
+ }
+ }
+
/** add a block to a under replication queue according to its priority
* @param block a under replication block
* @param curReplicas current number of replicas of the block
@@ -186,7 +221,7 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> {
int decomissionedReplicas,
int expectedReplicas) {
assert curReplicas >= 0 : "Negative replicas!";
- int priLevel = getPriority(curReplicas, decomissionedReplicas,
+ int priLevel = getPriority(block, curReplicas, decomissionedReplicas,
expectedReplicas);
if(priorityQueues.get(priLevel).add(block)) {
if (priLevel == QUEUE_WITH_CORRUPT_BLOCKS &&
@@ -209,7 +244,7 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> {
int oldReplicas,
int decommissionedReplicas,
int oldExpectedReplicas) {
- int priLevel = getPriority(oldReplicas,
+ int priLevel = getPriority(block, oldReplicas,
decommissionedReplicas,
oldExpectedReplicas);
boolean removedBlock = remove(block, priLevel);
@@ -283,9 +318,9 @@ class UnderReplicatedBlocks implements Iterable<BlockInfo> {
int curReplicasDelta, int expectedReplicasDelta) {
int oldReplicas = curReplicas-curReplicasDelta;
int oldExpectedReplicas = curExpectedReplicas-expectedReplicasDelta;
- int curPri = getPriority(curReplicas, decommissionedReplicas,
+ int curPri = getPriority(block, curReplicas, decommissionedReplicas,
curExpectedReplicas);
- int oldPri = getPriority(oldReplicas, decommissionedReplicas,
+ int oldPri = getPriority(block, oldReplicas, decommissionedReplicas,
oldExpectedReplicas);
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("UnderReplicationBlocks.update " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 11194dc..eb45aa2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -393,4 +393,9 @@ public interface HdfsServerConstants {
"raw.hdfs.crypto.file.encryption.info";
String SECURITY_XATTR_UNREADABLE_BY_SUPERUSER =
"security.hdfs.unreadable.by.superuser";
+ String XATTR_ERASURECODING_ZONE =
+ "raw.hdfs.erasurecoding.zone";
+
+ long BLOCK_GROUP_INDEX_MASK = 15;
+ byte MAX_BLOCKS_IN_GROUP = 16;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 92323f1..d77b36d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -32,11 +32,13 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeStatus;
import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.*;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
@@ -722,6 +724,11 @@ class BPOfferService {
dxcs.balanceThrottler.setBandwidth(bandwidth);
}
break;
+ case DatanodeProtocol.DNA_ERASURE_CODING_RECOVERY:
+ LOG.info("DatanodeCommand action: DNA_ERASURE_CODING_RECOVERY");
+ Collection<BlockECRecoveryInfo> ecTasks = ((BlockECRecoveryCommand) cmd).getECTasks();
+ dn.getErasureCodingWorker().processErasureCodingTasks(ecTasks);
+ break;
default:
LOG.warn("Unknown DatanodeCommand action: " + cmd.getAction());
}
@@ -751,6 +758,7 @@ class BPOfferService {
case DatanodeProtocol.DNA_BALANCERBANDWIDTHUPDATE:
case DatanodeProtocol.DNA_CACHE:
case DatanodeProtocol.DNA_UNCACHE:
+ case DatanodeProtocol.DNA_ERASURE_CODING_RECOVERY:
LOG.warn("Got a command from standby NN - ignoring command:" + cmd.getAction());
break;
default:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index abc9390..85f194a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -245,6 +245,33 @@ public class DNConf {
}
/**
+ * Returns true if connect to datanode via hostname
+ *
+ * @return boolean true if connect to datanode via hostname
+ */
+ public boolean getConnectToDnViaHostname() {
+ return connectToDnViaHostname;
+ }
+
+ /**
+ * Returns socket timeout
+ *
+ * @return int socket timeout
+ */
+ public int getSocketTimeout() {
+ return socketTimeout;
+ }
+
+ /**
+ * Returns socket write timeout
+ *
+ * @return int socket write timeout
+ */
+ public int getSocketWriteTimeout() {
+ return socketWriteTimeout;
+ }
+
+ /**
* Returns the SaslPropertiesResolver configured for use with
* DataTransferProtocol, or null if not configured.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index e265dad..1b695e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -87,6 +87,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.management.ObjectName;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -151,6 +152,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
+import org.apache.hadoop.hdfs.server.datanode.erasurecode.ErasureCodingWorker;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
@@ -358,6 +360,8 @@ public class DataNode extends ReconfigurableBase
private String dnUserName = null;
private SpanReceiverHost spanReceiverHost;
+
+ private ErasureCodingWorker ecWorker;
private static final int NUM_CORES = Runtime.getRuntime()
.availableProcessors();
private static final double CONGESTION_RATIO = 1.5;
@@ -1159,6 +1163,8 @@ public class DataNode extends ReconfigurableBase
saslClient = new SaslDataTransferClient(dnConf.conf,
dnConf.saslPropsResolver, dnConf.trustedChannelResolver);
saslServer = new SaslDataTransferServer(dnConf, blockPoolTokenSecretManager);
+ // Initialize ErasureCoding worker
+ ecWorker = new ErasureCodingWorker(conf, this);
}
/**
@@ -1223,6 +1229,10 @@ public class DataNode extends ReconfigurableBase
return UUID.randomUUID().toString();
}
+ public SaslDataTransferClient getSaslClient() {
+ return saslClient;
+ }
+
/**
* Verify that the DatanodeUuid has been initialized. If this is a new
* datanode then we generate a new Datanode Uuid and persist it to disk.
@@ -1485,7 +1495,7 @@ public class DataNode extends ReconfigurableBase
/**
* Creates either NIO or regular depending on socketWriteTimeout.
*/
- protected Socket newSocket() throws IOException {
+ public Socket newSocket() throws IOException {
return (dnConf.socketWriteTimeout > 0) ?
SocketChannel.open().socket() : new Socket();
}
@@ -1894,6 +1904,21 @@ public class DataNode extends ReconfigurableBase
int getXmitsInProgress() {
return xmitsInProgress.get();
}
+
+ /**
+ * Increments the xmitsInProgress count. xmitsInProgress count represents the
+ * number of data replication/reconstruction tasks running currently.
+ */
+ public void incrementXmitsInProgress() {
+ xmitsInProgress.getAndIncrement();
+ }
+
+ /**
+ * Decrements the xmitsInProgress count
+ */
+ public void decrementXmitsInProgress() {
+ xmitsInProgress.getAndDecrement();
+ }
private void reportBadBlock(final BPOfferService bpos,
final ExtendedBlock block, final String msg) {
@@ -2113,7 +2138,7 @@ public class DataNode extends ReconfigurableBase
*/
@Override
public void run() {
- xmitsInProgress.getAndIncrement();
+ incrementXmitsInProgress();
Socket sock = null;
DataOutputStream out = null;
DataInputStream in = null;
@@ -2133,11 +2158,8 @@ public class DataNode extends ReconfigurableBase
//
// Header info
//
- Token<BlockTokenIdentifier> accessToken = BlockTokenSecretManager.DUMMY_TOKEN;
- if (isBlockTokenEnabled) {
- accessToken = blockPoolTokenSecretManager.generateToken(b,
- EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE));
- }
+ Token<BlockTokenIdentifier> accessToken = getBlockAccessToken(b,
+ EnumSet.of(BlockTokenIdentifier.AccessMode.WRITE));
long writeTimeout = dnConf.socketWriteTimeout +
HdfsServerConstants.WRITE_TIMEOUT_EXTENSION * (targets.length-1);
@@ -2195,7 +2217,7 @@ public class DataNode extends ReconfigurableBase
// check if there are any disk problem
checkDiskErrorAsync();
} finally {
- xmitsInProgress.getAndDecrement();
+ decrementXmitsInProgress();
IOUtils.closeStream(blockSender);
IOUtils.closeStream(out);
IOUtils.closeStream(in);
@@ -2204,6 +2226,19 @@ public class DataNode extends ReconfigurableBase
}
}
+ /***
+ * Use BlockTokenSecretManager to generate block token for current user.
+ */
+ public Token<BlockTokenIdentifier> getBlockAccessToken(ExtendedBlock b,
+ EnumSet<AccessMode> mode) throws IOException {
+ Token<BlockTokenIdentifier> accessToken =
+ BlockTokenSecretManager.DUMMY_TOKEN;
+ if (isBlockTokenEnabled) {
+ accessToken = blockPoolTokenSecretManager.generateToken(b, mode);
+ }
+ return accessToken;
+ }
+
/**
* Returns a new DataEncryptionKeyFactory that generates a key from the
* BlockPoolTokenSecretManager, using the block pool ID of the given block.
@@ -2211,7 +2246,7 @@ public class DataNode extends ReconfigurableBase
* @param block for which the factory needs to create a key
* @return DataEncryptionKeyFactory for block's block pool ID
*/
- DataEncryptionKeyFactory getDataEncryptionKeyFactoryForBlock(
+ public DataEncryptionKeyFactory getDataEncryptionKeyFactoryForBlock(
final ExtendedBlock block) {
return new DataEncryptionKeyFactory() {
@Override
@@ -3259,4 +3294,9 @@ public class DataNode extends ReconfigurableBase
checkSuperuserPrivilege();
spanReceiverHost.removeSpanReceiver(id);
}
+
+ public ErasureCodingWorker getErasureCodingWorker(){
+ return ecWorker;
+
+ }
}
[11/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
new file mode 100644
index 0000000..4dc94a0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -0,0 +1,947 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.util;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSStripedOutputStream;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.security.token.Token;
+
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.io.IOException;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * When accessing a file in striped layout, operations on logical byte ranges
+ * in the file need to be mapped to physical byte ranges on block files stored
+ * on DataNodes. This utility class facilities this mapping by defining and
+ * exposing a number of striping-related concepts. The most basic ones are
+ * illustrated in the following diagram. Unless otherwise specified, all
+ * range-related calculations are inclusive (the end offset of the previous
+ * range should be 1 byte lower than the start offset of the next one).
+ *
+ * | <---- Block Group ----> | <- Block Group: logical unit composing
+ * | | striped HDFS files.
+ * blk_0 blk_1 blk_2 <- Internal Blocks: each internal block
+ * | | | represents a physically stored local
+ * v v v block file
+ * +------+ +------+ +------+
+ * |cell_0| |cell_1| |cell_2| <- {@link StripingCell} represents the
+ * +------+ +------+ +------+ logical order that a Block Group should
+ * |cell_3| |cell_4| |cell_5| be accessed: cell_0, cell_1, ...
+ * +------+ +------+ +------+
+ * |cell_6| |cell_7| |cell_8|
+ * +------+ +------+ +------+
+ * |cell_9|
+ * +------+ <- A cell contains cellSize bytes of data
+ */
+@InterfaceAudience.Private
+public class StripedBlockUtil {
+
+ /**
+ * This method parses a striped block group into individual blocks.
+ *
+ * @param bg The striped block group
+ * @param cellSize The size of a striping cell
+ * @param dataBlkNum The number of data blocks
+ * @return An array containing the blocks in the group
+ */
+ public static LocatedBlock[] parseStripedBlockGroup(LocatedStripedBlock bg,
+ int cellSize, int dataBlkNum, int parityBlkNum) {
+ int locatedBGSize = bg.getBlockIndices().length;
+ LocatedBlock[] lbs = new LocatedBlock[dataBlkNum + parityBlkNum];
+ for (short i = 0; i < locatedBGSize; i++) {
+ final int idx = bg.getBlockIndices()[i];
+ // for now we do not use redundant replica of an internal block
+ if (idx < (dataBlkNum + parityBlkNum) && lbs[idx] == null) {
+ lbs[idx] = constructInternalBlock(bg, i, cellSize,
+ dataBlkNum, idx);
+ }
+ }
+ return lbs;
+ }
+
+ /**
+ * This method creates an internal block at the given index of a block group
+ *
+ * @param idxInReturnedLocs The index in the stored locations in the
+ * {@link LocatedStripedBlock} object
+ * @param idxInBlockGroup The logical index in the striped block group
+ * @return The constructed internal block
+ */
+ public static LocatedBlock constructInternalBlock(LocatedStripedBlock bg,
+ int idxInReturnedLocs, int cellSize, int dataBlkNum,
+ int idxInBlockGroup) {
+ final ExtendedBlock blk = constructInternalBlock(
+ bg.getBlock(), cellSize, dataBlkNum, idxInBlockGroup);
+ final LocatedBlock locatedBlock;
+ if (idxInReturnedLocs < bg.getLocations().length) {
+ locatedBlock = new LocatedBlock(blk,
+ new DatanodeInfo[]{bg.getLocations()[idxInReturnedLocs]},
+ new String[]{bg.getStorageIDs()[idxInReturnedLocs]},
+ new StorageType[]{bg.getStorageTypes()[idxInReturnedLocs]},
+ bg.getStartOffset(), bg.isCorrupt(), null);
+ } else {
+ locatedBlock = new LocatedBlock(blk, null, null, null,
+ bg.getStartOffset(), bg.isCorrupt(), null);
+ }
+ Token<BlockTokenIdentifier>[] blockTokens = bg.getBlockTokens();
+ if (idxInReturnedLocs < blockTokens.length) {
+ locatedBlock.setBlockToken(blockTokens[idxInReturnedLocs]);
+ }
+ return locatedBlock;
+ }
+
+ /**
+ * This method creates an internal {@link ExtendedBlock} at the given index
+ * of a block group.
+ */
+ public static ExtendedBlock constructInternalBlock(ExtendedBlock blockGroup,
+ int cellSize, int dataBlkNum, int idxInBlockGroup) {
+ ExtendedBlock block = new ExtendedBlock(blockGroup);
+ block.setBlockId(blockGroup.getBlockId() + idxInBlockGroup);
+ block.setNumBytes(getInternalBlockLength(blockGroup.getNumBytes(),
+ cellSize, dataBlkNum, idxInBlockGroup));
+ return block;
+ }
+
+ /**
+ * Get the size of an internal block at the given index of a block group
+ *
+ * @param dataSize Size of the block group only counting data blocks
+ * @param cellSize The size of a striping cell
+ * @param numDataBlocks The number of data blocks
+ * @param i The logical index in the striped block group
+ * @return The size of the internal block at the specified index
+ */
+ public static long getInternalBlockLength(long dataSize,
+ int cellSize, int numDataBlocks, int i) {
+ Preconditions.checkArgument(dataSize >= 0);
+ Preconditions.checkArgument(cellSize > 0);
+ Preconditions.checkArgument(numDataBlocks > 0);
+ Preconditions.checkArgument(i >= 0);
+ // Size of each stripe (only counting data blocks)
+ final int stripeSize = cellSize * numDataBlocks;
+ // If block group ends at stripe boundary, each internal block has an equal
+ // share of the group
+ final int lastStripeDataLen = (int)(dataSize % stripeSize);
+ if (lastStripeDataLen == 0) {
+ return dataSize / numDataBlocks;
+ }
+
+ final int numStripes = (int) ((dataSize - 1) / stripeSize + 1);
+ return (numStripes - 1L)*cellSize
+ + lastCellSize(lastStripeDataLen, cellSize, numDataBlocks, i);
+ }
+
+ private static int lastCellSize(int size, int cellSize, int numDataBlocks,
+ int i) {
+ if (i < numDataBlocks) {
+ // parity block size (i.e. i >= numDataBlocks) is the same as
+ // the first data block size (i.e. i = 0).
+ size -= i*cellSize;
+ if (size < 0) {
+ size = 0;
+ }
+ }
+ return size > cellSize? cellSize: size;
+ }
+
+ /**
+ * Given a byte's offset in an internal block, calculate the offset in
+ * the block group
+ */
+ public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
+ long offsetInBlk, int idxInBlockGroup) {
+ int cellIdxInBlk = (int) (offsetInBlk / cellSize);
+ return cellIdxInBlk * cellSize * dataBlkNum // n full stripes before offset
+ + idxInBlockGroup * cellSize // m full cells before offset
+ + offsetInBlk % cellSize; // partial cell
+ }
+
+ /**
+ * Get the next completed striped read task
+ *
+ * @return {@link StripingChunkReadResult} indicating the status of the read task
+ * succeeded, and the block index of the task. If the method times
+ * out without getting any completed read tasks, -1 is returned as
+ * block index.
+ * @throws InterruptedException
+ */
+ public static StripingChunkReadResult getNextCompletedStripedRead(
+ CompletionService<Void> readService, Map<Future<Void>, Integer> futures,
+ final long threshold) throws InterruptedException {
+ Preconditions.checkArgument(!futures.isEmpty());
+ Future<Void> future = null;
+ try {
+ if (threshold > 0) {
+ future = readService.poll(threshold, TimeUnit.MILLISECONDS);
+ } else {
+ future = readService.take();
+ }
+ if (future != null) {
+ future.get();
+ return new StripingChunkReadResult(futures.remove(future),
+ StripingChunkReadResult.SUCCESSFUL);
+ } else {
+ return new StripingChunkReadResult(StripingChunkReadResult.TIMEOUT);
+ }
+ } catch (ExecutionException e) {
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug("ExecutionException " + e);
+ }
+ return new StripingChunkReadResult(futures.remove(future),
+ StripingChunkReadResult.FAILED);
+ } catch (CancellationException e) {
+ return new StripingChunkReadResult(futures.remove(future),
+ StripingChunkReadResult.CANCELLED);
+ }
+ }
+
+ /**
+ * Get the total usage of the striped blocks, which is the total of data
+ * blocks and parity blocks
+ *
+ * @param numDataBlkBytes
+ * Size of the block group only counting data blocks
+ * @param dataBlkNum
+ * The number of data blocks
+ * @param parityBlkNum
+ * The number of parity blocks
+ * @param cellSize
+ * The size of a striping cell
+ * @return The total usage of data blocks and parity blocks
+ */
+ public static long spaceConsumedByStripedBlock(long numDataBlkBytes,
+ int dataBlkNum, int parityBlkNum, int cellSize) {
+ int parityIndex = dataBlkNum + 1;
+ long numParityBlkBytes = getInternalBlockLength(numDataBlkBytes, cellSize,
+ dataBlkNum, parityIndex) * parityBlkNum;
+ return numDataBlkBytes + numParityBlkBytes;
+ }
+
+ /**
+ * Initialize the decoding input buffers based on the chunk states in an
+ * {@link AlignedStripe}. For each chunk that was not initially requested,
+ * schedule a new fetch request with the decoding input buffer as transfer
+ * destination.
+ */
+ public static byte[][] initDecodeInputs(AlignedStripe alignedStripe,
+ int dataBlkNum, int parityBlkNum) {
+ byte[][] decodeInputs =
+ new byte[dataBlkNum + parityBlkNum][(int) alignedStripe.getSpanInBlock()];
+ // read the full data aligned stripe
+ for (int i = 0; i < dataBlkNum; i++) {
+ if (alignedStripe.chunks[i] == null) {
+ final int decodeIndex = convertIndex4Decode(i, dataBlkNum, parityBlkNum);
+ alignedStripe.chunks[i] = new StripingChunk(decodeInputs[decodeIndex]);
+ alignedStripe.chunks[i].addByteArraySlice(0,
+ (int) alignedStripe.getSpanInBlock());
+ }
+ }
+ return decodeInputs;
+ }
+
+ /**
+ * Some fetched {@link StripingChunk} might be stored in original application
+ * buffer instead of prepared decode input buffers. Some others are beyond
+ * the range of the internal blocks and should correspond to all zero bytes.
+ * When all pending requests have returned, this method should be called to
+ * finalize decode input buffers.
+ */
+ public static void finalizeDecodeInputs(final byte[][] decodeInputs,
+ int dataBlkNum, int parityBlkNum, AlignedStripe alignedStripe) {
+ for (int i = 0; i < alignedStripe.chunks.length; i++) {
+ final StripingChunk chunk = alignedStripe.chunks[i];
+ final int decodeIndex = convertIndex4Decode(i, dataBlkNum, parityBlkNum);
+ if (chunk != null && chunk.state == StripingChunk.FETCHED) {
+ chunk.copyTo(decodeInputs[decodeIndex]);
+ } else if (chunk != null && chunk.state == StripingChunk.ALLZERO) {
+ Arrays.fill(decodeInputs[decodeIndex], (byte) 0);
+ } else {
+ decodeInputs[decodeIndex] = null;
+ }
+ }
+ }
+
+ /**
+ * Currently decoding requires parity chunks are before data chunks.
+ * The indices are opposite to what we store in NN. In future we may
+ * improve the decoding to make the indices order the same as in NN.
+ *
+ * @param index The index to convert
+ * @param dataBlkNum The number of data blocks
+ * @param parityBlkNum The number of parity blocks
+ * @return converted index
+ */
+ public static int convertIndex4Decode(int index, int dataBlkNum,
+ int parityBlkNum) {
+ return index < dataBlkNum ? index + parityBlkNum : index - dataBlkNum;
+ }
+
+ public static int convertDecodeIndexBack(int index, int dataBlkNum,
+ int parityBlkNum) {
+ return index < parityBlkNum ? index + dataBlkNum : index - parityBlkNum;
+ }
+
+ /**
+ * Decode based on the given input buffers and schema.
+ */
+ public static void decodeAndFillBuffer(final byte[][] decodeInputs,
+ AlignedStripe alignedStripe, int dataBlkNum, int parityBlkNum,
+ RawErasureDecoder decoder) {
+ // Step 1: prepare indices and output buffers for missing data units
+ int[] decodeIndices = new int[parityBlkNum];
+ int pos = 0;
+ for (int i = 0; i < dataBlkNum; i++) {
+ if (alignedStripe.chunks[i] != null &&
+ alignedStripe.chunks[i].state == StripingChunk.MISSING){
+ decodeIndices[pos++] = convertIndex4Decode(i, dataBlkNum, parityBlkNum);
+ }
+ }
+ decodeIndices = Arrays.copyOf(decodeIndices, pos);
+ byte[][] decodeOutputs =
+ new byte[decodeIndices.length][(int) alignedStripe.getSpanInBlock()];
+
+ // Step 2: decode into prepared output buffers
+ decoder.decode(decodeInputs, decodeIndices, decodeOutputs);
+
+ // Step 3: fill original application buffer with decoded data
+ for (int i = 0; i < decodeIndices.length; i++) {
+ int missingBlkIdx = convertDecodeIndexBack(decodeIndices[i],
+ dataBlkNum, parityBlkNum);
+ StripingChunk chunk = alignedStripe.chunks[missingBlkIdx];
+ if (chunk.state == StripingChunk.MISSING) {
+ chunk.copyFrom(decodeOutputs[i]);
+ }
+ }
+ }
+
+ /**
+ * Similar functionality with {@link #divideByteRangeIntoStripes}, but is used
+ * by stateful read and uses ByteBuffer as reading target buffer. Besides the
+ * read range is within a single stripe thus the calculation logic is simpler.
+ */
+ public static AlignedStripe[] divideOneStripe(ECSchema ecSchema,
+ int cellSize, LocatedStripedBlock blockGroup, long rangeStartInBlockGroup,
+ long rangeEndInBlockGroup, ByteBuffer buf) {
+ final int dataBlkNum = ecSchema.getNumDataUnits();
+ // Step 1: map the byte range to StripingCells
+ StripingCell[] cells = getStripingCellsOfByteRange(ecSchema, cellSize,
+ blockGroup, rangeStartInBlockGroup, rangeEndInBlockGroup);
+
+ // Step 2: get the unmerged ranges on each internal block
+ VerticalRange[] ranges = getRangesForInternalBlocks(ecSchema, cellSize,
+ cells);
+
+ // Step 3: merge into stripes
+ AlignedStripe[] stripes = mergeRangesForInternalBlocks(ecSchema, ranges);
+
+ // Step 4: calculate each chunk's position in destination buffer. Since the
+ // whole read range is within a single stripe, the logic is simpler here.
+ int bufOffset = (int) (rangeStartInBlockGroup % (cellSize * dataBlkNum));
+ for (StripingCell cell : cells) {
+ long cellStart = cell.idxInInternalBlk * cellSize + cell.offset;
+ long cellEnd = cellStart + cell.size - 1;
+ for (AlignedStripe s : stripes) {
+ long stripeEnd = s.getOffsetInBlock() + s.getSpanInBlock() - 1;
+ long overlapStart = Math.max(cellStart, s.getOffsetInBlock());
+ long overlapEnd = Math.min(cellEnd, stripeEnd);
+ int overLapLen = (int) (overlapEnd - overlapStart + 1);
+ if (overLapLen > 0) {
+ Preconditions.checkState(s.chunks[cell.idxInStripe] == null);
+ final int pos = (int) (bufOffset + overlapStart - cellStart);
+ buf.position(pos);
+ buf.limit(pos + overLapLen);
+ s.chunks[cell.idxInStripe] = new StripingChunk(buf.slice());
+ }
+ }
+ bufOffset += cell.size;
+ }
+
+ // Step 5: prepare ALLZERO blocks
+ prepareAllZeroChunks(blockGroup, stripes, cellSize, dataBlkNum);
+ return stripes;
+ }
+
+ /**
+ * This method divides a requested byte range into an array of inclusive
+ * {@link AlignedStripe}.
+ * @param ecSchema The codec schema for the file, which carries the numbers
+ * of data / parity blocks
+ * @param cellSize Cell size of stripe
+ * @param blockGroup The striped block group
+ * @param rangeStartInBlockGroup The byte range's start offset in block group
+ * @param rangeEndInBlockGroup The byte range's end offset in block group
+ * @param buf Destination buffer of the read operation for the byte range
+ * @param offsetInBuf Start offset into the destination buffer
+ *
+ * At most 5 stripes will be generated from each logical range, as
+ * demonstrated in the header of {@link AlignedStripe}.
+ */
+ public static AlignedStripe[] divideByteRangeIntoStripes(ECSchema ecSchema,
+ int cellSize, LocatedStripedBlock blockGroup,
+ long rangeStartInBlockGroup, long rangeEndInBlockGroup, byte[] buf,
+ int offsetInBuf) {
+
+ // Step 0: analyze range and calculate basic parameters
+ final int dataBlkNum = ecSchema.getNumDataUnits();
+
+ // Step 1: map the byte range to StripingCells
+ StripingCell[] cells = getStripingCellsOfByteRange(ecSchema, cellSize,
+ blockGroup, rangeStartInBlockGroup, rangeEndInBlockGroup);
+
+ // Step 2: get the unmerged ranges on each internal block
+ VerticalRange[] ranges = getRangesForInternalBlocks(ecSchema, cellSize,
+ cells);
+
+ // Step 3: merge into at most 5 stripes
+ AlignedStripe[] stripes = mergeRangesForInternalBlocks(ecSchema, ranges);
+
+ // Step 4: calculate each chunk's position in destination buffer
+ calcualteChunkPositionsInBuf(cellSize, stripes, cells, buf, offsetInBuf);
+
+ // Step 5: prepare ALLZERO blocks
+ prepareAllZeroChunks(blockGroup, stripes, cellSize, dataBlkNum);
+
+ return stripes;
+ }
+
+ /**
+ * Map the logical byte range to a set of inclusive {@link StripingCell}
+ * instances, each representing the overlap of the byte range to a cell
+ * used by {@link DFSStripedOutputStream} in encoding
+ */
+ @VisibleForTesting
+ private static StripingCell[] getStripingCellsOfByteRange(ECSchema ecSchema,
+ int cellSize, LocatedStripedBlock blockGroup,
+ long rangeStartInBlockGroup, long rangeEndInBlockGroup) {
+ Preconditions.checkArgument(
+ rangeStartInBlockGroup <= rangeEndInBlockGroup &&
+ rangeEndInBlockGroup < blockGroup.getBlockSize());
+ long len = rangeEndInBlockGroup - rangeStartInBlockGroup + 1;
+ int firstCellIdxInBG = (int) (rangeStartInBlockGroup / cellSize);
+ int lastCellIdxInBG = (int) (rangeEndInBlockGroup / cellSize);
+ int numCells = lastCellIdxInBG - firstCellIdxInBG + 1;
+ StripingCell[] cells = new StripingCell[numCells];
+
+ final int firstCellOffset = (int) (rangeStartInBlockGroup % cellSize);
+ final int firstCellSize =
+ (int) Math.min(cellSize - (rangeStartInBlockGroup % cellSize), len);
+ cells[0] = new StripingCell(ecSchema, firstCellSize, firstCellIdxInBG,
+ firstCellOffset);
+ if (lastCellIdxInBG != firstCellIdxInBG) {
+ final int lastCellSize = (int) (rangeEndInBlockGroup % cellSize) + 1;
+ cells[numCells - 1] = new StripingCell(ecSchema, lastCellSize,
+ lastCellIdxInBG, 0);
+ }
+
+ for (int i = 1; i < numCells - 1; i++) {
+ cells[i] = new StripingCell(ecSchema, cellSize, i + firstCellIdxInBG, 0);
+ }
+
+ return cells;
+ }
+
+ /**
+ * Given a logical byte range, mapped to each {@link StripingCell}, calculate
+ * the physical byte range (inclusive) on each stored internal block.
+ */
+ @VisibleForTesting
+ private static VerticalRange[] getRangesForInternalBlocks(ECSchema ecSchema,
+ int cellSize, StripingCell[] cells) {
+ int dataBlkNum = ecSchema.getNumDataUnits();
+ int parityBlkNum = ecSchema.getNumParityUnits();
+
+ VerticalRange ranges[] = new VerticalRange[dataBlkNum + parityBlkNum];
+
+ long earliestStart = Long.MAX_VALUE;
+ long latestEnd = -1;
+ for (StripingCell cell : cells) {
+ // iterate through all cells and update the list of StripeRanges
+ if (ranges[cell.idxInStripe] == null) {
+ ranges[cell.idxInStripe] = new VerticalRange(
+ cell.idxInInternalBlk * cellSize + cell.offset, cell.size);
+ } else {
+ ranges[cell.idxInStripe].spanInBlock += cell.size;
+ }
+ VerticalRange range = ranges[cell.idxInStripe];
+ if (range.offsetInBlock < earliestStart) {
+ earliestStart = range.offsetInBlock;
+ }
+ if (range.offsetInBlock + range.spanInBlock - 1 > latestEnd) {
+ latestEnd = range.offsetInBlock + range.spanInBlock - 1;
+ }
+ }
+
+ // Each parity block should be fetched at maximum range of all data blocks
+ for (int i = dataBlkNum; i < dataBlkNum + parityBlkNum; i++) {
+ ranges[i] = new VerticalRange(earliestStart,
+ latestEnd - earliestStart + 1);
+ }
+
+ return ranges;
+ }
+
+ /**
+ * Merge byte ranges on each internal block into a set of inclusive
+ * {@link AlignedStripe} instances.
+ */
+ private static AlignedStripe[] mergeRangesForInternalBlocks(
+ ECSchema ecSchema, VerticalRange[] ranges) {
+ int dataBlkNum = ecSchema.getNumDataUnits();
+ int parityBlkNum = ecSchema.getNumParityUnits();
+ List<AlignedStripe> stripes = new ArrayList<>();
+ SortedSet<Long> stripePoints = new TreeSet<>();
+ for (VerticalRange r : ranges) {
+ if (r != null) {
+ stripePoints.add(r.offsetInBlock);
+ stripePoints.add(r.offsetInBlock + r.spanInBlock);
+ }
+ }
+
+ long prev = -1;
+ for (long point : stripePoints) {
+ if (prev >= 0) {
+ stripes.add(new AlignedStripe(prev, point - prev,
+ dataBlkNum + parityBlkNum));
+ }
+ prev = point;
+ }
+ return stripes.toArray(new AlignedStripe[stripes.size()]);
+ }
+
+ private static void calcualteChunkPositionsInBuf(int cellSize,
+ AlignedStripe[] stripes, StripingCell[] cells, byte[] buf,
+ int offsetInBuf) {
+ /**
+ * | <--------------- AlignedStripe --------------->|
+ *
+ * |<- length_0 ->|<-- length_1 -->|<- length_2 ->|
+ * +------------------+------------------+----------------+
+ * | cell_0_0_0 | cell_3_1_0 | cell_6_2_0 | <- blk_0
+ * +------------------+------------------+----------------+
+ * _/ \_______________________
+ * | |
+ * v offset_0 v offset_1
+ * +----------------------------------------------------------+
+ * | cell_0_0_0 | cell_1_0_1 and cell_2_0_2 |cell_3_1_0 ...| <- buf
+ * | (partial) | (from blk_1 and blk_2) | |
+ * +----------------------------------------------------------+
+ *
+ * Cell indexing convention defined in {@link StripingCell}
+ */
+ int done = 0;
+ for (StripingCell cell : cells) {
+ long cellStart = cell.idxInInternalBlk * cellSize + cell.offset;
+ long cellEnd = cellStart + cell.size - 1;
+ for (AlignedStripe s : stripes) {
+ long stripeEnd = s.getOffsetInBlock() + s.getSpanInBlock() - 1;
+ long overlapStart = Math.max(cellStart, s.getOffsetInBlock());
+ long overlapEnd = Math.min(cellEnd, stripeEnd);
+ int overLapLen = (int) (overlapEnd - overlapStart + 1);
+ if (overLapLen <= 0) {
+ continue;
+ }
+ if (s.chunks[cell.idxInStripe] == null) {
+ s.chunks[cell.idxInStripe] = new StripingChunk(buf);
+ }
+ s.chunks[cell.idxInStripe].addByteArraySlice(
+ (int)(offsetInBuf + done + overlapStart - cellStart), overLapLen);
+ }
+ done += cell.size;
+ }
+ }
+
+ /**
+ * If a {@link StripingChunk} maps to a byte range beyond an internal block's
+ * size, the chunk should be treated as zero bytes in decoding.
+ */
+ private static void prepareAllZeroChunks(LocatedStripedBlock blockGroup,
+ AlignedStripe[] stripes, int cellSize, int dataBlkNum) {
+ for (AlignedStripe s : stripes) {
+ for (int i = 0; i < dataBlkNum; i++) {
+ long internalBlkLen = getInternalBlockLength(blockGroup.getBlockSize(),
+ cellSize, dataBlkNum, i);
+ if (internalBlkLen <= s.getOffsetInBlock()) {
+ Preconditions.checkState(s.chunks[i] == null);
+ s.chunks[i] = new StripingChunk(StripingChunk.ALLZERO);
+ }
+ }
+ }
+ }
+
+ /**
+ * Cell is the unit of encoding used in {@link DFSStripedOutputStream}. This
+ * size impacts how a logical offset in the file or block group translates
+ * to physical byte offset in a stored internal block. The StripingCell util
+ * class facilitates this calculation. Each StripingCell is inclusive with
+ * its start and end offsets -- e.g., the end logical offset of cell_0_0_0
+ * should be 1 byte lower than the start logical offset of cell_1_0_1.
+ *
+ * | <------- Striped Block Group -------> |
+ * blk_0 blk_1 blk_2
+ * | | |
+ * v v v
+ * +----------+ +----------+ +----------+
+ * |cell_0_0_0| |cell_1_0_1| |cell_2_0_2|
+ * +----------+ +----------+ +----------+
+ * |cell_3_1_0| |cell_4_1_1| |cell_5_1_2| <- {@link #idxInBlkGroup} = 5
+ * +----------+ +----------+ +----------+ {@link #idxInInternalBlk} = 1
+ * {@link #idxInStripe} = 2
+ * A StripingCell is a special instance of {@link StripingChunk} whose offset
+ * and size align with the cell used when writing data.
+ * TODO: consider parity cells
+ */
+ @VisibleForTesting
+ static class StripingCell {
+ final ECSchema schema;
+ /** Logical order in a block group, used when doing I/O to a block group */
+ final int idxInBlkGroup;
+ final int idxInInternalBlk;
+ final int idxInStripe;
+ /**
+ * When a logical byte range is mapped to a set of cells, it might
+ * partially overlap with the first and last cells. This field and the
+ * {@link #size} variable represent the start offset and size of the
+ * overlap.
+ */
+ final int offset;
+ final int size;
+
+ StripingCell(ECSchema ecSchema, int cellSize, int idxInBlkGroup,
+ int offset) {
+ this.schema = ecSchema;
+ this.idxInBlkGroup = idxInBlkGroup;
+ this.idxInInternalBlk = idxInBlkGroup / ecSchema.getNumDataUnits();
+ this.idxInStripe = idxInBlkGroup -
+ this.idxInInternalBlk * ecSchema.getNumDataUnits();
+ this.offset = offset;
+ this.size = cellSize;
+ }
+ }
+
+ /**
+ * Given a requested byte range on a striped block group, an AlignedStripe
+ * represents an inclusive {@link VerticalRange} that is aligned with both
+ * the byte range and boundaries of all internal blocks. As illustrated in
+ * the diagram, any given byte range on a block group leads to 1~5
+ * AlignedStripe's.
+ *
+ * |<-------- Striped Block Group -------->|
+ * blk_0 blk_1 blk_2 blk_3 blk_4
+ * +----+ | +----+ +----+
+ * |full| | | | | | <- AlignedStripe0:
+ * +----+ |~~~~| | |~~~~| |~~~~| 1st cell is partial
+ * |part| | | | | | | | <- AlignedStripe1: byte range
+ * +----+ +----+ +----+ | |~~~~| |~~~~| doesn't start at 1st block
+ * |full| |full| |full| | | | | |
+ * |cell| |cell| |cell| | | | | | <- AlignedStripe2 (full stripe)
+ * | | | | | | | | | | |
+ * +----+ +----+ +----+ | |~~~~| |~~~~|
+ * |full| |part| | | | | | <- AlignedStripe3: byte range
+ * |~~~~| +----+ | |~~~~| |~~~~| doesn't end at last block
+ * | | | | | | | <- AlignedStripe4:
+ * +----+ | +----+ +----+ last cell is partial
+ * |
+ * <---- data blocks ----> | <--- parity --->
+ *
+ * An AlignedStripe is the basic unit of reading from a striped block group,
+ * because within the AlignedStripe, all internal blocks can be processed in
+ * a uniform manner.
+ *
+ * The coverage of an AlignedStripe on an internal block is represented as a
+ * {@link StripingChunk}.
+ *
+ * To simplify the logic of reading a logical byte range from a block group,
+ * a StripingChunk is either completely in the requested byte range or
+ * completely outside the requested byte range.
+ */
+ public static class AlignedStripe {
+ public VerticalRange range;
+ /** status of each chunk in the stripe */
+ public final StripingChunk[] chunks;
+ public int fetchedChunksNum = 0;
+ public int missingChunksNum = 0;
+
+ public AlignedStripe(long offsetInBlock, long length, int width) {
+ Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
+ this.range = new VerticalRange(offsetInBlock, length);
+ this.chunks = new StripingChunk[width];
+ }
+
+ public boolean include(long pos) {
+ return range.include(pos);
+ }
+
+ public long getOffsetInBlock() {
+ return range.offsetInBlock;
+ }
+
+ public long getSpanInBlock() {
+ return range.spanInBlock;
+ }
+
+ @Override
+ public String toString() {
+ return "Offset=" + range.offsetInBlock + ", length=" + range.spanInBlock +
+ ", fetchedChunksNum=" + fetchedChunksNum +
+ ", missingChunksNum=" + missingChunksNum;
+ }
+ }
+
+ /**
+ * A simple utility class representing an arbitrary vertical inclusive range
+ * starting at {@link #offsetInBlock} and lasting for {@link #spanInBlock}
+ * bytes in an internal block. Note that VerticalRange doesn't necessarily
+ * align with {@link StripingCell}.
+ *
+ * |<- Striped Block Group ->|
+ * blk_0
+ * |
+ * v
+ * +-----+
+ * |~~~~~| <-- {@link #offsetInBlock}
+ * | | ^
+ * | | |
+ * | | | {@link #spanInBlock}
+ * | | v
+ * |~~~~~| ---
+ * | |
+ * +-----+
+ */
+ public static class VerticalRange {
+ /** start offset in the block group (inclusive) */
+ public long offsetInBlock;
+ /** length of the stripe range */
+ public long spanInBlock;
+
+ public VerticalRange(long offsetInBlock, long length) {
+ Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
+ this.offsetInBlock = offsetInBlock;
+ this.spanInBlock = length;
+ }
+
+ /** whether a position is in the range */
+ public boolean include(long pos) {
+ return pos >= offsetInBlock && pos < offsetInBlock + spanInBlock;
+ }
+ }
+
+ /**
+ * Indicates the coverage of an {@link AlignedStripe} on an internal block,
+ * and the state of the chunk in the context of the read request.
+ *
+ * |<---------------- Striped Block Group --------------->|
+ * blk_0 blk_1 blk_2 blk_3 blk_4
+ * +---------+ | +----+ +----+
+ * null null |REQUESTED| | |null| |null| <- AlignedStripe0
+ * +---------+ |---------| | |----| |----|
+ * null |REQUESTED| |REQUESTED| | |null| |null| <- AlignedStripe1
+ * +---------+ +---------+ +---------+ | +----+ +----+
+ * |REQUESTED| |REQUESTED| ALLZERO | |null| |null| <- AlignedStripe2
+ * +---------+ +---------+ | +----+ +----+
+ * <----------- data blocks ------------> | <--- parity --->
+ */
+ public static class StripingChunk {
+ /** Chunk has been successfully fetched */
+ public static final int FETCHED = 0x01;
+ /** Chunk has encountered failed when being fetched */
+ public static final int MISSING = 0x02;
+ /** Chunk being fetched (fetching task is in-flight) */
+ public static final int PENDING = 0x04;
+ /**
+ * Chunk is requested either by application or for decoding, need to
+ * schedule read task
+ */
+ public static final int REQUESTED = 0X08;
+ /**
+ * Internal block is short and has no overlap with chunk. Chunk considered
+ * all-zero bytes in codec calculations.
+ */
+ public static final int ALLZERO = 0X0f;
+
+ /**
+ * If a chunk is completely in requested range, the state transition is:
+ * REQUESTED (when AlignedStripe created) -> PENDING -> {FETCHED | MISSING}
+ * If a chunk is completely outside requested range (including parity
+ * chunks), state transition is:
+ * null (AlignedStripe created) -> REQUESTED (upon failure) -> PENDING ...
+ */
+ public int state = REQUESTED;
+
+ public final ChunkByteArray byteArray;
+ public final ByteBuffer byteBuffer;
+
+ public StripingChunk(byte[] buf) {
+ this.byteArray = new ChunkByteArray(buf);
+ byteBuffer = null;
+ }
+
+ public StripingChunk(ByteBuffer buf) {
+ this.byteArray = null;
+ this.byteBuffer = buf;
+ }
+
+ public StripingChunk(int state) {
+ this.byteArray = null;
+ this.byteBuffer = null;
+ this.state = state;
+ }
+
+ public void addByteArraySlice(int offset, int length) {
+ assert byteArray != null;
+ byteArray.offsetsInBuf.add(offset);
+ byteArray.lengthsInBuf.add(length);
+ }
+
+ void copyTo(byte[] target) {
+ assert byteArray != null;
+ byteArray.copyTo(target);
+ }
+
+ void copyFrom(byte[] src) {
+ assert byteArray != null;
+ byteArray.copyFrom(src);
+ }
+ }
+
+ public static class ChunkByteArray {
+ private final byte[] buf;
+ private final List<Integer> offsetsInBuf;
+ private final List<Integer> lengthsInBuf;
+
+ ChunkByteArray(byte[] buf) {
+ this.buf = buf;
+ this.offsetsInBuf = new ArrayList<>();
+ this.lengthsInBuf = new ArrayList<>();
+ }
+
+ public int[] getOffsets() {
+ int[] offsets = new int[offsetsInBuf.size()];
+ for (int i = 0; i < offsets.length; i++) {
+ offsets[i] = offsetsInBuf.get(i);
+ }
+ return offsets;
+ }
+
+ public int[] getLengths() {
+ int[] lens = new int[this.lengthsInBuf.size()];
+ for (int i = 0; i < lens.length; i++) {
+ lens[i] = this.lengthsInBuf.get(i);
+ }
+ return lens;
+ }
+
+ public byte[] buf() {
+ return buf;
+ }
+
+ void copyTo(byte[] target) {
+ int posInBuf = 0;
+ for (int i = 0; i < offsetsInBuf.size(); i++) {
+ System.arraycopy(buf, offsetsInBuf.get(i),
+ target, posInBuf, lengthsInBuf.get(i));
+ posInBuf += lengthsInBuf.get(i);
+ }
+ }
+
+ void copyFrom(byte[] src) {
+ int srcPos = 0;
+ for (int j = 0; j < offsetsInBuf.size(); j++) {
+ System.arraycopy(src, srcPos, buf, offsetsInBuf.get(j),
+ lengthsInBuf.get(j));
+ srcPos += lengthsInBuf.get(j);
+ }
+ }
+ }
+
+ /**
+ * This class represents result from a striped read request.
+ * If the task was successful or the internal computation failed,
+ * an index is also returned.
+ */
+ public static class StripingChunkReadResult {
+ public static final int SUCCESSFUL = 0x01;
+ public static final int FAILED = 0x02;
+ public static final int TIMEOUT = 0x04;
+ public static final int CANCELLED = 0x08;
+
+ public final int index;
+ public final int state;
+
+ public StripingChunkReadResult(int state) {
+ Preconditions.checkArgument(state == TIMEOUT,
+ "Only timeout result should return negative index.");
+ this.index = -1;
+ this.state = state;
+ }
+
+ public StripingChunkReadResult(int index, int state) {
+ Preconditions.checkArgument(state != TIMEOUT,
+ "Timeout result should return negative index.");
+ this.index = index;
+ this.state = state;
+ }
+
+ @Override
+ public String toString() {
+ return "(index=" + index + ", state =" + state + ")";
+ }
+ }
+
+ /**
+ * Check if the information such as IDs and generation stamps in block-i
+ * match block-j, where block-i and block-j are in the same group.
+ */
+ public static void checkBlocks(int j, ExtendedBlock blockj,
+ int i, ExtendedBlock blocki) throws IOException {
+
+ if (!blocki.getBlockPoolId().equals(blockj.getBlockPoolId())) {
+ throw new IOException("Block pool IDs mismatched: block" + j + "="
+ + blockj + ", block" + i + "=" + blocki);
+ }
+ if (blocki.getBlockId() - i != blockj.getBlockId() - j) {
+ throw new IOException("Block IDs mismatched: block" + j + "="
+ + blockj + ", block" + i + "=" + blocki);
+ }
+ if (blocki.getGenerationStamp() != blockj.getGenerationStamp()) {
+ throw new IOException("Generation stamps mismatched: block" + j + "="
+ + blockj + ", block" + i + "=" + blocki);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
index b87e753..9d24b91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
@@ -33,6 +33,7 @@ package hadoop.hdfs.datanode;
import "HAServiceProtocol.proto";
import "hdfs.proto";
+import "erasurecoding.proto";
/**
* Information to identify a datanode to a namenode
@@ -58,6 +59,7 @@ message DatanodeCommandProto {
UnusedUpgradeCommand = 6;
NullDatanodeCommand = 7;
BlockIdCommand = 8;
+ BlockECRecoveryCommand = 9;
}
required Type cmdType = 1; // Type of the command
@@ -71,6 +73,7 @@ message DatanodeCommandProto {
optional KeyUpdateCommandProto keyUpdateCmd = 6;
optional RegisterCommandProto registerCmd = 7;
optional BlockIdCommandProto blkIdCmd = 8;
+ optional BlockECRecoveryCommandProto blkECRecoveryCmd = 9;
}
/**
@@ -145,6 +148,13 @@ message RegisterCommandProto {
}
/**
+ * Block Erasure coding recovery command
+ */
+message BlockECRecoveryCommandProto {
+ repeated BlockECRecoveryInfoProto blockECRecoveryinfo = 1;
+}
+
+/**
* registration - Information of the datanode registering with the namenode
*/
message RegisterDatanodeRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 3bd1d91..3233f66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -73,6 +73,7 @@ message NameSystemSection {
optional uint64 lastAllocatedBlockId = 5;
optional uint64 transactionId = 6;
optional uint64 rollingUpgradeStartTime = 7;
+ optional uint64 lastAllocatedStripedBlockId = 8;
}
/**
@@ -139,6 +140,8 @@ message INodeSection {
optional AclFeatureProto acl = 8;
optional XAttrFeatureProto xAttrs = 9;
optional uint32 storagePolicyID = 10;
+ optional bool isStriped = 11;
+ optional uint64 stripingCellSize = 12;
}
message QuotaByStorageTypeEntryProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 8cb7d5f..fcf2bc1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2370,11 +2370,11 @@
</description>
</property>
- <property>
- <name>dfs.datanode.block-pinning.enabled</name>
- <value>false</value>
- <description>Whether pin blocks on favored DataNode.</description>
- </property>
+<property>
+ <name>dfs.datanode.block-pinning.enabled</name>
+ <value>false</value>
+ <description>Whether pin blocks on favored DataNode.</description>
+</property>
<property>
<name>dfs.client.block.write.locateFollowingBlock.initial.delay.ms</name>
@@ -2412,4 +2412,25 @@
</description>
</property>
+<property>
+ <name>dfs.datanode.stripedread.threshold.millis</name>
+ <value>5000</value>
+ <description>datanode striped read threshold in millisecond.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.stripedread.threads</name>
+ <value>20</value>
+ <description>datanode striped read thread pool size.
+ </description>
+</property>
+
+<property>
+ <name>dfs.datanode.stripedread.buffer.size</name>
+ <value>262144</value>
+ <description>datanode striped read buffer size.
+ </description>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
new file mode 100644
index 0000000..6c06a8d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cli;
+
+import org.apache.hadoop.cli.util.CLICommandErasureCodingCli;
+import org.apache.hadoop.cli.util.CLICommandTypes;
+import org.apache.hadoop.cli.util.CLITestCmd;
+import org.apache.hadoop.cli.util.CommandExecutor;
+import org.apache.hadoop.cli.util.ErasureCodingCliCmdExecutor;
+import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
+
+public class CLITestCmdErasureCoding extends CLITestCmd {
+ public CLITestCmdErasureCoding(String str, CLICommandTypes type) {
+ super(str, type);
+ }
+
+ @Override
+ public CommandExecutor getExecutor(String tag) throws IllegalArgumentException {
+ if (getType() instanceof CLICommandErasureCodingCli)
+ return new ErasureCodingCliCmdExecutor(tag, new ECCli());
+ return super.getExecutor(tag);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java
new file mode 100644
index 0000000..5f01ea2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestErasureCodingCLI.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cli;
+
+import org.apache.hadoop.cli.util.CLICommand;
+import org.apache.hadoop.cli.util.CLICommandErasureCodingCli;
+import org.apache.hadoop.cli.util.CommandExecutor.Result;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.xml.sax.SAXException;
+
+public class TestErasureCodingCLI extends CLITestHelper {
+ private final int NUM_OF_DATANODES = 3;
+ private MiniDFSCluster dfsCluster = null;
+ private FileSystem fs = null;
+ private String namenode = null;
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+
+ dfsCluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(NUM_OF_DATANODES).build();
+ dfsCluster.waitClusterUp();
+ namenode = conf.get(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "file:///");
+
+ username = System.getProperty("user.name");
+
+ fs = dfsCluster.getFileSystem();
+ }
+
+ @Override
+ protected String getTestFile() {
+ return "testErasureCodingConf.xml";
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.close();
+ }
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ }
+ Thread.sleep(2000);
+ super.tearDown();
+ }
+
+ @Override
+ protected String expandCommand(final String cmd) {
+ String expCmd = cmd;
+ expCmd = expCmd.replaceAll("NAMENODE", namenode);
+ expCmd = expCmd.replaceAll("#LF#", System.getProperty("line.separator"));
+ expCmd = super.expandCommand(expCmd);
+ return expCmd;
+ }
+
+ @Override
+ protected TestConfigFileParser getConfigParser() {
+ return new TestErasureCodingAdmin();
+ }
+
+ private class TestErasureCodingAdmin extends
+ CLITestHelper.TestConfigFileParser {
+ @Override
+ public void endElement(String uri, String localName, String qName)
+ throws SAXException {
+ if (qName.equals("ec-admin-command")) {
+ if (testCommands != null) {
+ testCommands.add(new CLITestCmdErasureCoding(charString,
+ new CLICommandErasureCodingCli()));
+ } else if (cleanupCommands != null) {
+ cleanupCommands.add(new CLITestCmdErasureCoding(charString,
+ new CLICommandErasureCodingCli()));
+ }
+ } else {
+ super.endElement(uri, localName, qName);
+ }
+ }
+ }
+
+ @Override
+ protected Result execute(CLICommand cmd) throws Exception {
+ return cmd.getExecutor(namenode).executeCommand(cmd.getCmd());
+ }
+
+ @Test
+ @Override
+ public void testAll() {
+ super.testAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java
new file mode 100644
index 0000000..aafcd9f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/CLICommandErasureCodingCli.java
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cli.util;
+
+public class CLICommandErasureCodingCli implements CLICommandTypes {
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
new file mode 100644
index 0000000..e993313
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/util/ErasureCodingCliCmdExecutor.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cli.util;
+
+import org.apache.hadoop.hdfs.tools.erasurecode.ECCli;
+import org.apache.hadoop.util.ToolRunner;
+
+public class ErasureCodingCliCmdExecutor extends CommandExecutor {
+ protected String namenode = null;
+ protected ECCli admin = null;
+
+ public ErasureCodingCliCmdExecutor(String namenode, ECCli admin) {
+ this.namenode = namenode;
+ this.admin = admin;
+ }
+
+ @Override
+ protected void execute(final String cmd) throws Exception {
+ String[] args = getCommandAsArgs(cmd, "NAMENODE", this.namenode);
+ ToolRunner.run(admin, args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java
index 88b7f37..829cf03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java
@@ -165,20 +165,19 @@ public class BlockReaderTestUtil {
*/
public BlockReader getBlockReader(LocatedBlock testBlock, int offset, int lenToRead)
throws IOException {
- return getBlockReader(cluster, testBlock, offset, lenToRead);
+ return getBlockReader(cluster.getFileSystem(), testBlock, offset, lenToRead);
}
/**
* Get a BlockReader for the given block.
*/
- public static BlockReader getBlockReader(MiniDFSCluster cluster,
- LocatedBlock testBlock, int offset, int lenToRead) throws IOException {
+ public static BlockReader getBlockReader(final DistributedFileSystem fs,
+ LocatedBlock testBlock, int offset, long lenToRead) throws IOException {
InetSocketAddress targetAddr = null;
ExtendedBlock block = testBlock.getBlock();
DatanodeInfo[] nodes = testBlock.getLocations();
targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr());
- final DistributedFileSystem fs = cluster.getFileSystem();
return new BlockReaderFactory(fs.getClient().getConf()).
setInetSocketAddress(targetAddr).
setBlock(block).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index a742757..3f0d6df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -66,6 +66,12 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
@@ -105,6 +111,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.Sender;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
@@ -125,14 +132,19 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo.BlockStatus;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -153,12 +165,8 @@ import org.junit.Assume;
import org.mockito.internal.util.reflection.Whitebox;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Charsets;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Supplier;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS;
/** Utilities for HDFS tests */
public class DFSTestUtil {
@@ -808,15 +816,21 @@ public class DFSTestUtil {
return os.toByteArray();
}
- /* Write the given string to the given file */
- public static void writeFile(FileSystem fs, Path p, String s)
+ /* Write the given bytes to the given file */
+ public static void writeFile(FileSystem fs, Path p, byte[] bytes)
throws IOException {
if (fs.exists(p)) {
fs.delete(p, true);
}
- InputStream is = new ByteArrayInputStream(s.getBytes());
+ InputStream is = new ByteArrayInputStream(bytes);
FSDataOutputStream os = fs.create(p);
- IOUtils.copyBytes(is, os, s.length(), true);
+ IOUtils.copyBytes(is, os, bytes.length, true);
+ }
+
+ /* Write the given string to the given file */
+ public static void writeFile(FileSystem fs, Path p, String s)
+ throws IOException {
+ writeFile(fs, p, s.getBytes());
}
/* Append the given string to the given file */
@@ -1835,7 +1849,7 @@ public class DFSTestUtil {
dn.setLastUpdate(Time.now() + offset);
dn.setLastUpdateMonotonic(Time.monotonicNow() + offset);
}
-
+
/**
* This method takes a set of block locations and fills the provided buffer
* with expected bytes based on simulated content from
@@ -1859,4 +1873,150 @@ public class DFSTestUtil {
}
}
+ public static StorageReceivedDeletedBlocks[] makeReportForReceivedBlock(
+ Block block, BlockStatus blockStatus, DatanodeStorage storage) {
+ ReceivedDeletedBlockInfo[] receivedBlocks = new ReceivedDeletedBlockInfo[1];
+ receivedBlocks[0] = new ReceivedDeletedBlockInfo(block, blockStatus, null);
+ StorageReceivedDeletedBlocks[] reports = new StorageReceivedDeletedBlocks[1];
+ reports[0] = new StorageReceivedDeletedBlocks(storage, receivedBlocks);
+ return reports;
+ }
+
+ /**
+ * Creates the metadata of a file in striped layout. This method only
+ * manipulates the NameNode state without injecting data to DataNode.
+ * You should disable periodical heartbeat before use this.
+ * @param file Path of the file to create
+ * @param dir Parent path of the file
+ * @param numBlocks Number of striped block groups to add to the file
+ * @param numStripesPerBlk Number of striped cells in each block
+ * @param toMkdir
+ */
+ public static void createStripedFile(MiniDFSCluster cluster, Path file, Path dir,
+ int numBlocks, int numStripesPerBlk, boolean toMkdir) throws Exception {
+ DistributedFileSystem dfs = cluster.getFileSystem();
+ // If outer test already created EC zone, dir should be left as null
+ if (toMkdir) {
+ assert dir != null;
+ dfs.mkdirs(dir);
+ try {
+ dfs.getClient().createErasureCodingZone(dir.toString(), null, 0);
+ } catch (IOException e) {
+ if (!e.getMessage().contains("non-empty directory")) {
+ throw e;
+ }
+ }
+ }
+
+ FSDataOutputStream out = null;
+ try {
+ out = dfs.create(file, (short) 1); // create an empty file
+
+ FSNamesystem ns = cluster.getNamesystem();
+ FSDirectory fsdir = ns.getFSDirectory();
+ INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+
+ ExtendedBlock previous = null;
+ for (int i = 0; i < numBlocks; i++) {
+ Block newBlock = addStripedBlockToFile(cluster.getDataNodes(), dfs, ns,
+ file.toString(), fileNode, dfs.getClient().getClientName(),
+ previous, numStripesPerBlk);
+ previous = new ExtendedBlock(ns.getBlockPoolId(), newBlock);
+ }
+
+ dfs.getClient().namenode.complete(file.toString(),
+ dfs.getClient().getClientName(), previous, fileNode.getId());
+ } finally {
+ IOUtils.cleanup(null, out);
+ }
+ }
+
+ /**
+ * Adds a striped block group to a file. This method only manipulates NameNode
+ * states of the file and the block without injecting data to DataNode.
+ * It does mimic block reports.
+ * You should disable periodical heartbeat before use this.
+ * @param dataNodes List DataNodes to host the striped block group
+ * @param previous Previous block in the file
+ * @param numStripes Number of stripes in each block group
+ * @return The added block group
+ */
+ public static Block addStripedBlockToFile(List<DataNode> dataNodes,
+ DistributedFileSystem fs, FSNamesystem ns, String file, INodeFile fileNode,
+ String clientName, ExtendedBlock previous, int numStripes)
+ throws Exception {
+ fs.getClient().namenode.addBlock(file, clientName, previous, null,
+ fileNode.getId(), null);
+
+ final BlockInfo lastBlock = fileNode.getLastBlock();
+ final int groupSize = fileNode.getPreferredBlockReplication();
+ assert dataNodes.size() >= groupSize;
+ // 1. RECEIVING_BLOCK IBR
+ for (int i = 0; i < groupSize; i++) {
+ DataNode dn = dataNodes.get(i);
+ final Block block = new Block(lastBlock.getBlockId() + i, 0,
+ lastBlock.getGenerationStamp());
+ DatanodeStorage storage = new DatanodeStorage(UUID.randomUUID().toString());
+ StorageReceivedDeletedBlocks[] reports = DFSTestUtil
+ .makeReportForReceivedBlock(block,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, storage);
+ for (StorageReceivedDeletedBlocks report : reports) {
+ ns.processIncrementalBlockReport(dn.getDatanodeId(), report);
+ }
+ }
+
+ // 2. RECEIVED_BLOCK IBR
+ for (int i = 0; i < groupSize; i++) {
+ DataNode dn = dataNodes.get(i);
+ final Block block = new Block(lastBlock.getBlockId() + i,
+ numStripes * BLOCK_STRIPED_CELL_SIZE, lastBlock.getGenerationStamp());
+ DatanodeStorage storage = new DatanodeStorage(UUID.randomUUID().toString());
+ StorageReceivedDeletedBlocks[] reports = DFSTestUtil
+ .makeReportForReceivedBlock(block,
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
+ for (StorageReceivedDeletedBlocks report : reports) {
+ ns.processIncrementalBlockReport(dn.getDatanodeId(), report);
+ }
+ }
+
+ lastBlock.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS);
+ return lastBlock;
+ }
+
+ /**
+ * Because currently DFSStripedOutputStream does not support hflush/hsync,
+ * tests can use this method to flush all the buffered data to DataNodes.
+ */
+ public static ExtendedBlock flushInternal(DFSStripedOutputStream out)
+ throws IOException {
+ out.flushInternal();
+ return out.getBlock();
+ }
+
+ /**
+ * Verify that blocks in striped block group are on different nodes, and every
+ * internal blocks exists.
+ */
+ public static void verifyLocatedStripedBlocks(LocatedBlocks lbs,
+ int groupSize) {
+ for (LocatedBlock lb : lbs.getLocatedBlocks()) {
+ assert lb instanceof LocatedStripedBlock;
+ HashSet<DatanodeInfo> locs = new HashSet<>();
+ for (DatanodeInfo datanodeInfo : lb.getLocations()) {
+ locs.add(datanodeInfo);
+ }
+ assertEquals(groupSize, lb.getLocations().length);
+ assertEquals(groupSize, locs.size());
+
+ // verify that every internal blocks exists
+ int[] blockIndices = ((LocatedStripedBlock) lb).getBlockIndices();
+ assertEquals(groupSize, blockIndices.length);
+ HashSet<Integer> found = new HashSet<>();
+ for (int index : blockIndices) {
+ assert index >=0;
+ found.add(index);
+ }
+ assertEquals(groupSize, found.size());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 7052321..65e26df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2113,8 +2113,6 @@ public class MiniDFSCluster {
int node = -1;
for (int i = 0; i < dataNodes.size(); i++) {
DataNode dn = dataNodes.get(i).datanode;
- LOG.info("DN name=" + dnName + " found DN=" + dn +
- " with name=" + dn.getDisplayName());
if (dnName.equals(dn.getDatanodeId().getXferAddr())) {
node = i;
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
new file mode 100644
index 0000000..ca4b2aa
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/StripedFileTestUtil.java
@@ -0,0 +1,261 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.web.ByteRangeInputStream;
+import org.junit.Assert;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+public class StripedFileTestUtil {
+ public static final Log LOG = LogFactory.getLog(StripedFileTestUtil.class);
+
+ static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+ static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+
+ static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ static final int stripesPerBlock = 4;
+ static final int blockSize = cellSize * stripesPerBlock;
+ static final int numDNs = dataBlocks + parityBlocks + 2;
+
+ static final Random random = new Random();
+
+ static byte[] generateBytes(int cnt) {
+ byte[] bytes = new byte[cnt];
+ for (int i = 0; i < cnt; i++) {
+ bytes[i] = getByte(i);
+ }
+ return bytes;
+ }
+
+ static int readAll(FSDataInputStream in, byte[] buf) throws IOException {
+ int readLen = 0;
+ int ret;
+ while ((ret = in.read(buf, readLen, buf.length - readLen)) >= 0 &&
+ readLen <= buf.length) {
+ readLen += ret;
+ }
+ return readLen;
+ }
+
+ static byte getByte(long pos) {
+ final int mod = 29;
+ return (byte) (pos % mod + 1);
+ }
+
+ static void verifyLength(FileSystem fs, Path srcPath, int fileLength)
+ throws IOException {
+ FileStatus status = fs.getFileStatus(srcPath);
+ Assert.assertEquals("File length should be the same", fileLength, status.getLen());
+ }
+
+ static void verifyPread(FileSystem fs, Path srcPath, int fileLength,
+ byte[] expected, byte[] buf) throws IOException {
+ try (FSDataInputStream in = fs.open(srcPath)) {
+ int[] startOffsets = {0, 1, cellSize - 102, cellSize, cellSize + 102,
+ cellSize * (dataBlocks - 1), cellSize * (dataBlocks - 1) + 102,
+ cellSize * dataBlocks, fileLength - 102, fileLength - 1};
+ for (int startOffset : startOffsets) {
+ startOffset = Math.max(0, Math.min(startOffset, fileLength - 1));
+ int remaining = fileLength - startOffset;
+ int offset = startOffset;
+ final byte[] result = new byte[remaining];
+ while (remaining > 0) {
+ int target = Math.min(remaining, buf.length);
+ in.readFully(offset, buf, 0, target);
+ System.arraycopy(buf, 0, result, offset - startOffset, target);
+ remaining -= target;
+ offset += target;
+ }
+ for (int i = 0; i < fileLength - startOffset; i++) {
+ Assert.assertEquals("Byte at " + (startOffset + i) + " is different, "
+ + "the startOffset is " + startOffset,
+ expected[startOffset + i], result[i]);
+ }
+ }
+ }
+ }
+
+ static void verifyStatefulRead(FileSystem fs, Path srcPath, int fileLength,
+ byte[] expected, byte[] buf) throws IOException {
+ try (FSDataInputStream in = fs.open(srcPath)) {
+ final byte[] result = new byte[fileLength];
+ int readLen = 0;
+ int ret;
+ while ((ret = in.read(buf, 0, buf.length)) >= 0) {
+ System.arraycopy(buf, 0, result, readLen, ret);
+ readLen += ret;
+ }
+ Assert.assertEquals("The length of file should be the same to write size",
+ fileLength, readLen);
+ Assert.assertArrayEquals(expected, result);
+ }
+ }
+
+ static void verifyStatefulRead(FileSystem fs, Path srcPath, int fileLength,
+ byte[] expected, ByteBuffer buf) throws IOException {
+ try (FSDataInputStream in = fs.open(srcPath)) {
+ ByteBuffer result = ByteBuffer.allocate(fileLength);
+ int readLen = 0;
+ int ret;
+ while ((ret = in.read(buf)) >= 0) {
+ readLen += ret;
+ buf.flip();
+ result.put(buf);
+ buf.clear();
+ }
+ Assert.assertEquals("The length of file should be the same to write size",
+ fileLength, readLen);
+ Assert.assertArrayEquals(expected, result.array());
+ }
+ }
+
+ static void verifySeek(FileSystem fs, Path srcPath, int fileLength)
+ throws IOException {
+ try (FSDataInputStream in = fs.open(srcPath)) {
+ // seek to 1/2 of content
+ int pos = fileLength / 2;
+ assertSeekAndRead(in, pos, fileLength);
+
+ // seek to 1/3 of content
+ pos = fileLength / 3;
+ assertSeekAndRead(in, pos, fileLength);
+
+ // seek to 0 pos
+ pos = 0;
+ assertSeekAndRead(in, pos, fileLength);
+
+ if (fileLength > cellSize) {
+ // seek to cellSize boundary
+ pos = cellSize - 1;
+ assertSeekAndRead(in, pos, fileLength);
+ }
+
+ if (fileLength > cellSize * dataBlocks) {
+ // seek to striped cell group boundary
+ pos = cellSize * dataBlocks - 1;
+ assertSeekAndRead(in, pos, fileLength);
+ }
+
+ if (fileLength > blockSize * dataBlocks) {
+ // seek to striped block group boundary
+ pos = blockSize * dataBlocks - 1;
+ assertSeekAndRead(in, pos, fileLength);
+ }
+
+ if (!(in.getWrappedStream() instanceof ByteRangeInputStream)) {
+ try {
+ in.seek(-1);
+ Assert.fail("Should be failed if seek to negative offset");
+ } catch (EOFException e) {
+ // expected
+ }
+
+ try {
+ in.seek(fileLength + 1);
+ Assert.fail("Should be failed if seek after EOF");
+ } catch (EOFException e) {
+ // expected
+ }
+ }
+ }
+ }
+
+ static void assertSeekAndRead(FSDataInputStream fsdis, int pos,
+ int writeBytes) throws IOException {
+ fsdis.seek(pos);
+ byte[] buf = new byte[writeBytes];
+ int readLen = StripedFileTestUtil.readAll(fsdis, buf);
+ Assert.assertEquals(readLen, writeBytes - pos);
+ for (int i = 0; i < readLen; i++) {
+ Assert.assertEquals("Byte at " + i + " should be the same",
+ StripedFileTestUtil.getByte(pos + i), buf[i]);
+ }
+ }
+
+ static void killDatanode(MiniDFSCluster cluster, DFSStripedOutputStream out,
+ final int dnIndex, final AtomicInteger pos) {
+ final StripedDataStreamer s = out.getStripedDataStreamer(dnIndex);
+ final DatanodeInfo datanode = getDatanodes(s);
+ LOG.info("killDatanode " + dnIndex + ": " + datanode + ", pos=" + pos);
+ cluster.stopDataNode(datanode.getXferAddr());
+ }
+
+ static DatanodeInfo getDatanodes(StripedDataStreamer streamer) {
+ for(;;) {
+ final DatanodeInfo[] datanodes = streamer.getNodes();
+ if (datanodes != null) {
+ Assert.assertEquals(1, datanodes.length);
+ Assert.assertNotNull(datanodes[0]);
+ return datanodes[0];
+ }
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException ignored) {
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Generate n random and different numbers within
+ * specified non-negative integer range
+ * @param min minimum of the range
+ * @param max maximum of the range
+ * @param n number to be generated
+ * @return
+ */
+ public static int[] randomArray(int min, int max, int n){
+ if (n > (max - min + 1) || max < min || min < 0 || max < 0) {
+ return null;
+ }
+ int[] result = new int[n];
+ for (int i = 0; i < n; i++) {
+ result[i] = -1;
+ }
+
+ int count = 0;
+ while(count < n) {
+ int num = (int) (Math.random() * (max - min)) + min;
+ boolean flag = true;
+ for (int j = 0; j < n; j++) {
+ if(num == result[j]){
+ flag = false;
+ break;
+ }
+ }
+ if(flag){
+ result[count] = num;
+ count++;
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java
index d8aceff..1a767c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderFactory.java
@@ -250,8 +250,8 @@ public class TestBlockReaderFactory {
LocatedBlock lblock = locatedBlocks.get(0); // first block
BlockReader blockReader = null;
try {
- blockReader = BlockReaderTestUtil.
- getBlockReader(cluster, lblock, 0, TEST_FILE_LEN);
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
Assert.fail("expected getBlockReader to fail the first time.");
} catch (Throwable t) {
Assert.assertTrue("expected to see 'TCP reads were disabled " +
@@ -265,8 +265,8 @@ public class TestBlockReaderFactory {
// Second time should succeed.
try {
- blockReader = BlockReaderTestUtil.
- getBlockReader(cluster, lblock, 0, TEST_FILE_LEN);
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
} catch (Throwable t) {
LOG.error("error trying to retrieve a block reader " +
"the second time.", t);
@@ -474,8 +474,8 @@ public class TestBlockReaderFactory {
while (true) {
BlockReader blockReader = null;
try {
- blockReader = BlockReaderTestUtil.
- getBlockReader(cluster, lblock, 0, TEST_FILE_LEN);
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
sem.release();
try {
blockReader.readAll(buf, 0, TEST_FILE_LEN);
@@ -514,8 +514,8 @@ public class TestBlockReaderFactory {
// getting a ClosedChannelException.
BlockReader blockReader = null;
try {
- blockReader = BlockReaderTestUtil.
- getBlockReader(cluster, lblock, 0, TEST_FILE_LEN);
+ blockReader = BlockReaderTestUtil.getBlockReader(
+ cluster.getFileSystem(), lblock, 0, TEST_FILE_LEN);
blockReader.readFully(buf, 0, TEST_FILE_LEN);
} finally {
if (blockReader != null) blockReader.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 441ef9c..c68bd28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -257,12 +257,12 @@ public class TestDFSClientRetries {
Mockito.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
- 1010, 0, null, (byte) 0)).when(mockNN).getFileInfo(anyString());
+ 1010, 0, null, (byte) 0, null, 0)).when(mockNN).getFileInfo(anyString());
Mockito.doReturn(
new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission(
(short) 777), "owner", "group", new byte[0], new byte[0],
- 1010, 0, null, (byte) 0))
+ 1010, 0, null, (byte) 0, null, 0))
.when(mockNN)
.create(anyString(), (FsPermission) anyObject(), anyString(),
(EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(),
@@ -550,7 +550,7 @@ public class TestDFSClientRetries {
badBlocks.add(badLocatedBlock);
return new LocatedBlocks(goodBlockList.getFileLength(), false,
badBlocks, null, true,
- null);
+ null, null, 0);
}
}
[19/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
new file mode 100644
index 0000000..3612063
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -0,0 +1,939 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.ChecksumException;
+import org.apache.hadoop.fs.ReadOption;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.io.ByteBufferPool;
+
+import static org.apache.hadoop.hdfs.util.StripedBlockUtil.AlignedStripe;
+import static org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunk;
+import static org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunkReadResult;
+
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.util.DirectBufferPool;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.Set;
+import java.util.Collection;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Future;
+
+/**
+ * DFSStripedInputStream reads from striped block groups
+ */
+public class DFSStripedInputStream extends DFSInputStream {
+
+ private static class ReaderRetryPolicy {
+ private int fetchEncryptionKeyTimes = 1;
+ private int fetchTokenTimes = 1;
+
+ void refetchEncryptionKey() {
+ fetchEncryptionKeyTimes--;
+ }
+
+ void refetchToken() {
+ fetchTokenTimes--;
+ }
+
+ boolean shouldRefetchEncryptionKey() {
+ return fetchEncryptionKeyTimes > 0;
+ }
+
+ boolean shouldRefetchToken() {
+ return fetchTokenTimes > 0;
+ }
+ }
+
+ /** Used to indicate the buffered data's range in the block group */
+ private static class StripeRange {
+ /** start offset in the block group (inclusive) */
+ final long offsetInBlock;
+ /** length of the stripe range */
+ final long length;
+
+ StripeRange(long offsetInBlock, long length) {
+ Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
+ this.offsetInBlock = offsetInBlock;
+ this.length = length;
+ }
+
+ boolean include(long pos) {
+ return pos >= offsetInBlock && pos < offsetInBlock + length;
+ }
+ }
+
+ private static class BlockReaderInfo {
+ final BlockReader reader;
+ final DatanodeInfo datanode;
+ /**
+ * when initializing block readers, their starting offsets are set to the same
+ * number: the smallest internal block offsets among all the readers. This is
+ * because it is possible that for some internal blocks we have to read
+ * "backwards" for decoding purpose. We thus use this offset array to track
+ * offsets for all the block readers so that we can skip data if necessary.
+ */
+ long blockReaderOffset;
+ LocatedBlock targetBlock;
+ /**
+ * We use this field to indicate whether we should use this reader. In case
+ * we hit any issue with this reader, we set this field to true and avoid
+ * using it for the next stripe.
+ */
+ boolean shouldSkip = false;
+
+ BlockReaderInfo(BlockReader reader, LocatedBlock targetBlock,
+ DatanodeInfo dn, long offset) {
+ this.reader = reader;
+ this.targetBlock = targetBlock;
+ this.datanode = dn;
+ this.blockReaderOffset = offset;
+ }
+
+ void setOffset(long offset) {
+ this.blockReaderOffset = offset;
+ }
+
+ void skip() {
+ this.shouldSkip = true;
+ }
+ }
+
+ private static final DirectBufferPool bufferPool = new DirectBufferPool();
+
+ private final BlockReaderInfo[] blockReaders;
+ private final int cellSize;
+ private final short dataBlkNum;
+ private final short parityBlkNum;
+ private final int groupSize;
+ /** the buffer for a complete stripe */
+ private ByteBuffer curStripeBuf;
+ private ByteBuffer parityBuf;
+ private final ECSchema schema;
+ private final RawErasureDecoder decoder;
+
+ /**
+ * indicate the start/end offset of the current buffered stripe in the
+ * block group
+ */
+ private StripeRange curStripeRange;
+ private final CompletionService<Void> readingService;
+
+ DFSStripedInputStream(DFSClient dfsClient, String src,
+ boolean verifyChecksum, ECSchema schema, int cellSize,
+ LocatedBlocks locatedBlocks) throws IOException {
+ super(dfsClient, src, verifyChecksum, locatedBlocks);
+
+ assert schema != null;
+ this.schema = schema;
+ this.cellSize = cellSize;
+ dataBlkNum = (short) schema.getNumDataUnits();
+ parityBlkNum = (short) schema.getNumParityUnits();
+ groupSize = dataBlkNum + parityBlkNum;
+ blockReaders = new BlockReaderInfo[groupSize];
+ curStripeRange = new StripeRange(0, 0);
+ readingService =
+ new ExecutorCompletionService<>(dfsClient.getStripedReadsThreadPool());
+ decoder = CodecUtil.createRSRawDecoder(dfsClient.getConfiguration(),
+ dataBlkNum, parityBlkNum);
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug("Creating an striped input stream for file " + src);
+ }
+ }
+
+ private void resetCurStripeBuffer() {
+ if (curStripeBuf == null) {
+ curStripeBuf = bufferPool.getBuffer(cellSize * dataBlkNum);
+ }
+ curStripeBuf.clear();
+ curStripeRange = new StripeRange(0, 0);
+ }
+
+ private ByteBuffer getParityBuffer() {
+ if (parityBuf == null) {
+ parityBuf = bufferPool.getBuffer(cellSize * parityBlkNum);
+ }
+ parityBuf.clear();
+ return parityBuf;
+ }
+
+ /**
+ * When seeking into a new block group, create blockReader for each internal
+ * block in the group.
+ */
+ private synchronized void blockSeekTo(long target) throws IOException {
+ if (target >= getFileLength()) {
+ throw new IOException("Attempted to read past end of file");
+ }
+
+ // Will be getting a new BlockReader.
+ closeCurrentBlockReaders();
+
+ // Compute desired striped block group
+ LocatedStripedBlock targetBlockGroup = getBlockGroupAt(target);
+ // Update current position
+ this.pos = target;
+ this.blockEnd = targetBlockGroup.getStartOffset() +
+ targetBlockGroup.getBlockSize() - 1;
+ currentLocatedBlock = targetBlockGroup;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ super.close();
+ if (curStripeBuf != null) {
+ bufferPool.returnBuffer(curStripeBuf);
+ curStripeBuf = null;
+ }
+ if (parityBuf != null) {
+ bufferPool.returnBuffer(parityBuf);
+ parityBuf = null;
+ }
+ }
+
+ /**
+ * Extend the super method with the logic of switching between cells.
+ * When reaching the end of a cell, proceed to the next cell and read it
+ * with the next blockReader.
+ */
+ @Override
+ protected void closeCurrentBlockReaders() {
+ resetCurStripeBuffer();
+ if (blockReaders == null || blockReaders.length == 0) {
+ return;
+ }
+ for (int i = 0; i < groupSize; i++) {
+ closeReader(blockReaders[i]);
+ blockReaders[i] = null;
+ }
+ blockEnd = -1;
+ }
+
+ private void closeReader(BlockReaderInfo readerInfo) {
+ if (readerInfo != null) {
+ IOUtils.cleanup(DFSClient.LOG, readerInfo.reader);
+ readerInfo.skip();
+ }
+ }
+
+ private long getOffsetInBlockGroup() {
+ return getOffsetInBlockGroup(pos);
+ }
+
+ private long getOffsetInBlockGroup(long pos) {
+ return pos - currentLocatedBlock.getStartOffset();
+ }
+
+ /**
+ * Read a new stripe covering the current position, and store the data in the
+ * {@link #curStripeBuf}.
+ */
+ private void readOneStripe(
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
+ throws IOException {
+ resetCurStripeBuffer();
+
+ // compute stripe range based on pos
+ final long offsetInBlockGroup = getOffsetInBlockGroup();
+ final long stripeLen = cellSize * dataBlkNum;
+ final int stripeIndex = (int) (offsetInBlockGroup / stripeLen);
+ final int stripeBufOffset = (int) (offsetInBlockGroup % stripeLen);
+ final int stripeLimit = (int) Math.min(currentLocatedBlock.getBlockSize()
+ - (stripeIndex * stripeLen), stripeLen);
+ StripeRange stripeRange = new StripeRange(offsetInBlockGroup,
+ stripeLimit - stripeBufOffset);
+
+ LocatedStripedBlock blockGroup = (LocatedStripedBlock) currentLocatedBlock;
+ AlignedStripe[] stripes = StripedBlockUtil.divideOneStripe(schema, cellSize,
+ blockGroup, offsetInBlockGroup,
+ offsetInBlockGroup + stripeRange.length - 1, curStripeBuf);
+ final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(
+ blockGroup, cellSize, dataBlkNum, parityBlkNum);
+ // read the whole stripe
+ for (AlignedStripe stripe : stripes) {
+ // Parse group to get chosen DN location
+ StripeReader sreader = new StatefulStripeReader(readingService, stripe,
+ blks, blockReaders, corruptedBlockMap);
+ sreader.readStripe();
+ }
+ curStripeBuf.position(stripeBufOffset);
+ curStripeBuf.limit(stripeLimit);
+ curStripeRange = stripeRange;
+ }
+
+ private Callable<Void> readCells(final BlockReader reader,
+ final DatanodeInfo datanode, final long currentReaderOffset,
+ final long targetReaderOffset, final ByteBufferStrategy[] strategies,
+ final ExtendedBlock currentBlock,
+ final Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
+ return new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ // reader can be null if getBlockReaderWithRetry failed or
+ // the reader hit exception before
+ if (reader == null) {
+ throw new IOException("The BlockReader is null. " +
+ "The BlockReader creation failed or the reader hit exception.");
+ }
+ Preconditions.checkState(currentReaderOffset <= targetReaderOffset);
+ if (currentReaderOffset < targetReaderOffset) {
+ long skipped = reader.skip(targetReaderOffset - currentReaderOffset);
+ Preconditions.checkState(
+ skipped == targetReaderOffset - currentReaderOffset);
+ }
+ int result = 0;
+ for (ByteBufferStrategy strategy : strategies) {
+ result += readToBuffer(reader, datanode, strategy, currentBlock,
+ corruptedBlockMap);
+ }
+ return null;
+ }
+ };
+ }
+
+ private int readToBuffer(BlockReader blockReader,
+ DatanodeInfo currentNode, ByteBufferStrategy strategy,
+ ExtendedBlock currentBlock,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
+ throws IOException {
+ final int targetLength = strategy.buf.remaining();
+ int length = 0;
+ try {
+ while (length < targetLength) {
+ int ret = strategy.doRead(blockReader, 0, 0);
+ if (ret < 0) {
+ throw new IOException("Unexpected EOS from the reader");
+ }
+ length += ret;
+ }
+ return length;
+ } catch (ChecksumException ce) {
+ DFSClient.LOG.warn("Found Checksum error for "
+ + currentBlock + " from " + currentNode
+ + " at " + ce.getPos());
+ // we want to remember which block replicas we have tried
+ addIntoCorruptedBlockMap(currentBlock, currentNode,
+ corruptedBlockMap);
+ throw ce;
+ } catch (IOException e) {
+ DFSClient.LOG.warn("Exception while reading from "
+ + currentBlock + " of " + src + " from "
+ + currentNode, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Seek to a new arbitrary location
+ */
+ @Override
+ public synchronized void seek(long targetPos) throws IOException {
+ if (targetPos > getFileLength()) {
+ throw new EOFException("Cannot seek after EOF");
+ }
+ if (targetPos < 0) {
+ throw new EOFException("Cannot seek to negative offset");
+ }
+ if (closed.get()) {
+ throw new IOException("Stream is closed!");
+ }
+ if (targetPos <= blockEnd) {
+ final long targetOffsetInBlk = getOffsetInBlockGroup(targetPos);
+ if (curStripeRange.include(targetOffsetInBlk)) {
+ int bufOffset = getStripedBufOffset(targetOffsetInBlk);
+ curStripeBuf.position(bufOffset);
+ pos = targetPos;
+ return;
+ }
+ }
+ pos = targetPos;
+ blockEnd = -1;
+ }
+
+ private int getStripedBufOffset(long offsetInBlockGroup) {
+ final long stripeLen = cellSize * dataBlkNum;
+ // compute the position in the curStripeBuf based on "pos"
+ return (int) (offsetInBlockGroup % stripeLen);
+ }
+
+ @Override
+ public synchronized boolean seekToNewSource(long targetPos)
+ throws IOException {
+ return false;
+ }
+
+ @Override
+ protected synchronized int readWithStrategy(ReaderStrategy strategy,
+ int off, int len) throws IOException {
+ dfsClient.checkOpen();
+ if (closed.get()) {
+ throw new IOException("Stream closed");
+ }
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap =
+ new ConcurrentHashMap<>();
+ if (pos < getFileLength()) {
+ try {
+ if (pos > blockEnd) {
+ blockSeekTo(pos);
+ }
+ int realLen = (int) Math.min(len, (blockEnd - pos + 1L));
+ synchronized (infoLock) {
+ if (locatedBlocks.isLastBlockComplete()) {
+ realLen = (int) Math.min(realLen,
+ locatedBlocks.getFileLength() - pos);
+ }
+ }
+
+ /** Number of bytes already read into buffer */
+ int result = 0;
+ while (result < realLen) {
+ if (!curStripeRange.include(getOffsetInBlockGroup())) {
+ readOneStripe(corruptedBlockMap);
+ }
+ int ret = copyToTargetBuf(strategy, off + result, realLen - result);
+ result += ret;
+ pos += ret;
+ }
+ if (dfsClient.stats != null) {
+ dfsClient.stats.incrementBytesRead(result);
+ }
+ return result;
+ } finally {
+ // Check if need to report block replicas corruption either read
+ // was successful or ChecksumException occured.
+ reportCheckSumFailure(corruptedBlockMap,
+ currentLocatedBlock.getLocations().length);
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Copy the data from {@link #curStripeBuf} into the given buffer
+ * @param strategy the ReaderStrategy containing the given buffer
+ * @param offset the offset of the given buffer. Used only when strategy is
+ * a ByteArrayStrategy
+ * @param length target length
+ * @return number of bytes copied
+ */
+ private int copyToTargetBuf(ReaderStrategy strategy, int offset, int length) {
+ final long offsetInBlk = getOffsetInBlockGroup();
+ int bufOffset = getStripedBufOffset(offsetInBlk);
+ curStripeBuf.position(bufOffset);
+ return strategy.copyFrom(curStripeBuf, offset,
+ Math.min(length, curStripeBuf.remaining()));
+ }
+
+ /**
+ * The super method {@link DFSInputStream#refreshLocatedBlock} refreshes
+ * cached LocatedBlock by executing {@link DFSInputStream#getBlockAt} again.
+ * This method extends the logic by first remembering the index of the
+ * internal block, and re-parsing the refreshed block group with the same
+ * index.
+ */
+ @Override
+ protected LocatedBlock refreshLocatedBlock(LocatedBlock block)
+ throws IOException {
+ int idx = BlockIdManager.getBlockIndex(block.getBlock().getLocalBlock());
+ LocatedBlock lb = getBlockGroupAt(block.getStartOffset());
+ // If indexing information is returned, iterate through the index array
+ // to find the entry for position idx in the group
+ LocatedStripedBlock lsb = (LocatedStripedBlock) lb;
+ int i = 0;
+ for (; i < lsb.getBlockIndices().length; i++) {
+ if (lsb.getBlockIndices()[i] == idx) {
+ break;
+ }
+ }
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug("refreshLocatedBlock for striped blocks, offset="
+ + block.getStartOffset() + ". Obtained block " + lb + ", idx=" + idx);
+ }
+ return StripedBlockUtil.constructInternalBlock(
+ lsb, i, cellSize, dataBlkNum, idx);
+ }
+
+ private LocatedStripedBlock getBlockGroupAt(long offset) throws IOException {
+ LocatedBlock lb = super.getBlockAt(offset);
+ assert lb instanceof LocatedStripedBlock : "NameNode" +
+ " should return a LocatedStripedBlock for a striped file";
+ return (LocatedStripedBlock)lb;
+ }
+
+ /**
+ * Real implementation of pread.
+ */
+ @Override
+ protected void fetchBlockByteRange(LocatedBlock block, long start,
+ long end, byte[] buf, int offset,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap)
+ throws IOException {
+ // Refresh the striped block group
+ LocatedStripedBlock blockGroup = getBlockGroupAt(block.getStartOffset());
+
+ AlignedStripe[] stripes = StripedBlockUtil.divideByteRangeIntoStripes(
+ schema, cellSize, blockGroup, start, end, buf, offset);
+ CompletionService<Void> readService = new ExecutorCompletionService<>(
+ dfsClient.getStripedReadsThreadPool());
+ final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(
+ blockGroup, cellSize, dataBlkNum, parityBlkNum);
+ final BlockReaderInfo[] preaderInfos = new BlockReaderInfo[groupSize];
+ try {
+ for (AlignedStripe stripe : stripes) {
+ // Parse group to get chosen DN location
+ StripeReader preader = new PositionStripeReader(readService, stripe,
+ blks, preaderInfos, corruptedBlockMap);
+ preader.readStripe();
+ }
+ } finally {
+ for (BlockReaderInfo preaderInfo : preaderInfos) {
+ closeReader(preaderInfo);
+ }
+ }
+ }
+
+ /**
+ * The reader for reading a complete {@link AlignedStripe}. Note that an
+ * {@link AlignedStripe} may cross multiple stripes with cellSize width.
+ */
+ private abstract class StripeReader {
+ final Map<Future<Void>, Integer> futures = new HashMap<>();
+ final AlignedStripe alignedStripe;
+ final CompletionService<Void> service;
+ final LocatedBlock[] targetBlocks;
+ final Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap;
+ final BlockReaderInfo[] readerInfos;
+
+ StripeReader(CompletionService<Void> service, AlignedStripe alignedStripe,
+ LocatedBlock[] targetBlocks, BlockReaderInfo[] readerInfos,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
+ this.service = service;
+ this.alignedStripe = alignedStripe;
+ this.targetBlocks = targetBlocks;
+ this.readerInfos = readerInfos;
+ this.corruptedBlockMap = corruptedBlockMap;
+ }
+
+ /** prepare all the data chunks */
+ abstract void prepareDecodeInputs();
+
+ /** prepare the parity chunk and block reader if necessary */
+ abstract boolean prepareParityChunk(int index) throws IOException;
+
+ abstract void decode();
+
+ void updateState4SuccessRead(StripingChunkReadResult result) {
+ Preconditions.checkArgument(
+ result.state == StripingChunkReadResult.SUCCESSFUL);
+ readerInfos[result.index].setOffset(alignedStripe.getOffsetInBlock()
+ + alignedStripe.getSpanInBlock());
+ }
+
+ private void checkMissingBlocks() throws IOException {
+ if (alignedStripe.missingChunksNum > parityBlkNum) {
+ clearFutures(futures.keySet());
+ throw new IOException(alignedStripe.missingChunksNum
+ + " missing blocks, the stripe is: " + alignedStripe);
+ }
+ }
+
+ /**
+ * We need decoding. Thus go through all the data chunks and make sure we
+ * submit read requests for all of them.
+ */
+ private void readDataForDecoding() throws IOException {
+ prepareDecodeInputs();
+ for (int i = 0; i < dataBlkNum; i++) {
+ Preconditions.checkNotNull(alignedStripe.chunks[i]);
+ if (alignedStripe.chunks[i].state == StripingChunk.REQUESTED) {
+ if (!readChunk(targetBlocks[i], i)) {
+ alignedStripe.missingChunksNum++;
+ }
+ }
+ }
+ checkMissingBlocks();
+ }
+
+ void readParityChunks(int num) throws IOException {
+ for (int i = dataBlkNum, j = 0; i < dataBlkNum + parityBlkNum && j < num;
+ i++) {
+ if (alignedStripe.chunks[i] == null) {
+ if (prepareParityChunk(i) && readChunk(targetBlocks[i], i)) {
+ j++;
+ } else {
+ alignedStripe.missingChunksNum++;
+ }
+ }
+ }
+ checkMissingBlocks();
+ }
+
+ boolean createBlockReader(LocatedBlock block, int chunkIndex)
+ throws IOException {
+ BlockReader reader = null;
+ final ReaderRetryPolicy retry = new ReaderRetryPolicy();
+ DNAddrPair dnInfo = new DNAddrPair(null, null, null);
+
+ while(true) {
+ try {
+ // the cached block location might have been re-fetched, so always
+ // get it from cache.
+ block = refreshLocatedBlock(block);
+ targetBlocks[chunkIndex] = block;
+
+ // internal block has one location, just rule out the deadNodes
+ dnInfo = getBestNodeDNAddrPair(block, null);
+ if (dnInfo == null) {
+ break;
+ }
+ reader = getBlockReader(block, alignedStripe.getOffsetInBlock(),
+ block.getBlockSize() - alignedStripe.getOffsetInBlock(),
+ dnInfo.addr, dnInfo.storageType, dnInfo.info);
+ } catch (IOException e) {
+ if (e instanceof InvalidEncryptionKeyException &&
+ retry.shouldRefetchEncryptionKey()) {
+ DFSClient.LOG.info("Will fetch a new encryption key and retry, "
+ + "encryption key was invalid when connecting to " + dnInfo.addr
+ + " : " + e);
+ dfsClient.clearDataEncryptionKey();
+ retry.refetchEncryptionKey();
+ } else if (retry.shouldRefetchToken() &&
+ tokenRefetchNeeded(e, dnInfo.addr)) {
+ fetchBlockAt(block.getStartOffset());
+ retry.refetchToken();
+ } else {
+ //TODO: handles connection issues
+ DFSClient.LOG.warn("Failed to connect to " + dnInfo.addr + " for " +
+ "block" + block.getBlock(), e);
+ // re-fetch the block in case the block has been moved
+ fetchBlockAt(block.getStartOffset());
+ addToDeadNodes(dnInfo.info);
+ }
+ }
+ if (reader != null) {
+ readerInfos[chunkIndex] = new BlockReaderInfo(reader, block,
+ dnInfo.info, alignedStripe.getOffsetInBlock());
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private ByteBufferStrategy[] getReadStrategies(StripingChunk chunk) {
+ if (chunk.byteBuffer != null) {
+ ByteBufferStrategy strategy = new ByteBufferStrategy(chunk.byteBuffer);
+ return new ByteBufferStrategy[]{strategy};
+ } else {
+ ByteBufferStrategy[] strategies =
+ new ByteBufferStrategy[chunk.byteArray.getOffsets().length];
+ for (int i = 0; i < strategies.length; i++) {
+ ByteBuffer buffer = ByteBuffer.wrap(chunk.byteArray.buf(),
+ chunk.byteArray.getOffsets()[i], chunk.byteArray.getLengths()[i]);
+ strategies[i] = new ByteBufferStrategy(buffer);
+ }
+ return strategies;
+ }
+ }
+
+ boolean readChunk(final LocatedBlock block, int chunkIndex)
+ throws IOException {
+ final StripingChunk chunk = alignedStripe.chunks[chunkIndex];
+ if (block == null) {
+ chunk.state = StripingChunk.MISSING;
+ return false;
+ }
+ if (readerInfos[chunkIndex] == null) {
+ if (!createBlockReader(block, chunkIndex)) {
+ chunk.state = StripingChunk.MISSING;
+ return false;
+ }
+ } else if (readerInfos[chunkIndex].shouldSkip) {
+ chunk.state = StripingChunk.MISSING;
+ return false;
+ }
+
+ chunk.state = StripingChunk.PENDING;
+ Callable<Void> readCallable = readCells(readerInfos[chunkIndex].reader,
+ readerInfos[chunkIndex].datanode,
+ readerInfos[chunkIndex].blockReaderOffset,
+ alignedStripe.getOffsetInBlock(), getReadStrategies(chunk),
+ block.getBlock(), corruptedBlockMap);
+
+ Future<Void> request = service.submit(readCallable);
+ futures.put(request, chunkIndex);
+ return true;
+ }
+
+ /** read the whole stripe. do decoding if necessary */
+ void readStripe() throws IOException {
+ for (int i = 0; i < dataBlkNum; i++) {
+ if (alignedStripe.chunks[i] != null &&
+ alignedStripe.chunks[i].state != StripingChunk.ALLZERO) {
+ if (!readChunk(targetBlocks[i], i)) {
+ alignedStripe.missingChunksNum++;
+ }
+ }
+ }
+ // There are missing block locations at this stage. Thus we need to read
+ // the full stripe and one more parity block.
+ if (alignedStripe.missingChunksNum > 0) {
+ checkMissingBlocks();
+ readDataForDecoding();
+ // read parity chunks
+ readParityChunks(alignedStripe.missingChunksNum);
+ }
+ // TODO: for a full stripe we can start reading (dataBlkNum + 1) chunks
+
+ // Input buffers for potential decode operation, which remains null until
+ // first read failure
+ while (!futures.isEmpty()) {
+ try {
+ StripingChunkReadResult r = StripedBlockUtil
+ .getNextCompletedStripedRead(service, futures, 0);
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug("Read task returned: " + r + ", for stripe "
+ + alignedStripe);
+ }
+ StripingChunk returnedChunk = alignedStripe.chunks[r.index];
+ Preconditions.checkNotNull(returnedChunk);
+ Preconditions.checkState(returnedChunk.state == StripingChunk.PENDING);
+
+ if (r.state == StripingChunkReadResult.SUCCESSFUL) {
+ returnedChunk.state = StripingChunk.FETCHED;
+ alignedStripe.fetchedChunksNum++;
+ updateState4SuccessRead(r);
+ if (alignedStripe.fetchedChunksNum == dataBlkNum) {
+ clearFutures(futures.keySet());
+ break;
+ }
+ } else {
+ returnedChunk.state = StripingChunk.MISSING;
+ // close the corresponding reader
+ closeReader(readerInfos[r.index]);
+
+ final int missing = alignedStripe.missingChunksNum;
+ alignedStripe.missingChunksNum++;
+ checkMissingBlocks();
+
+ readDataForDecoding();
+ readParityChunks(alignedStripe.missingChunksNum - missing);
+ }
+ } catch (InterruptedException ie) {
+ String err = "Read request interrupted";
+ DFSClient.LOG.error(err);
+ clearFutures(futures.keySet());
+ // Don't decode if read interrupted
+ throw new InterruptedIOException(err);
+ }
+ }
+
+ if (alignedStripe.missingChunksNum > 0) {
+ decode();
+ }
+ }
+ }
+
+ class PositionStripeReader extends StripeReader {
+ private byte[][] decodeInputs = null;
+
+ PositionStripeReader(CompletionService<Void> service,
+ AlignedStripe alignedStripe, LocatedBlock[] targetBlocks,
+ BlockReaderInfo[] readerInfos,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
+ super(service, alignedStripe, targetBlocks, readerInfos,
+ corruptedBlockMap);
+ }
+
+ @Override
+ void prepareDecodeInputs() {
+ if (decodeInputs == null) {
+ decodeInputs = StripedBlockUtil.initDecodeInputs(alignedStripe,
+ dataBlkNum, parityBlkNum);
+ }
+ }
+
+ @Override
+ boolean prepareParityChunk(int index) {
+ Preconditions.checkState(index >= dataBlkNum &&
+ alignedStripe.chunks[index] == null);
+ final int decodeIndex = StripedBlockUtil.convertIndex4Decode(index,
+ dataBlkNum, parityBlkNum);
+ alignedStripe.chunks[index] = new StripingChunk(decodeInputs[decodeIndex]);
+ alignedStripe.chunks[index].addByteArraySlice(0,
+ (int) alignedStripe.getSpanInBlock());
+ return true;
+ }
+
+ @Override
+ void decode() {
+ StripedBlockUtil.finalizeDecodeInputs(decodeInputs, dataBlkNum,
+ parityBlkNum, alignedStripe);
+ StripedBlockUtil.decodeAndFillBuffer(decodeInputs, alignedStripe,
+ dataBlkNum, parityBlkNum, decoder);
+ }
+ }
+
+ class StatefulStripeReader extends StripeReader {
+ ByteBuffer[] decodeInputs;
+
+ StatefulStripeReader(CompletionService<Void> service,
+ AlignedStripe alignedStripe, LocatedBlock[] targetBlocks,
+ BlockReaderInfo[] readerInfos,
+ Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap) {
+ super(service, alignedStripe, targetBlocks, readerInfos,
+ corruptedBlockMap);
+ }
+
+ @Override
+ void prepareDecodeInputs() {
+ if (decodeInputs == null) {
+ decodeInputs = new ByteBuffer[dataBlkNum + parityBlkNum];
+ ByteBuffer cur = curStripeBuf.duplicate();
+ StripedBlockUtil.VerticalRange range = alignedStripe.range;
+ for (int i = 0; i < dataBlkNum; i++) {
+ cur.limit(cur.capacity());
+ int pos = (int) (range.offsetInBlock % cellSize + cellSize * i);
+ cur.position(pos);
+ cur.limit((int) (pos + range.spanInBlock));
+ final int decodeIndex = StripedBlockUtil.convertIndex4Decode(i,
+ dataBlkNum, parityBlkNum);
+ decodeInputs[decodeIndex] = cur.slice();
+ if (alignedStripe.chunks[i] == null) {
+ alignedStripe.chunks[i] = new StripingChunk(
+ decodeInputs[decodeIndex]);
+ }
+ }
+ }
+ }
+
+ @Override
+ boolean prepareParityChunk(int index) throws IOException {
+ Preconditions.checkState(index >= dataBlkNum
+ && alignedStripe.chunks[index] == null);
+ if (blockReaders[index] != null && blockReaders[index].shouldSkip) {
+ alignedStripe.chunks[index] = new StripingChunk(StripingChunk.MISSING);
+ // we have failed the block reader before
+ return false;
+ }
+ final int decodeIndex = StripedBlockUtil.convertIndex4Decode(index,
+ dataBlkNum, parityBlkNum);
+ ByteBuffer buf = getParityBuffer().duplicate();
+ buf.position(cellSize * decodeIndex);
+ buf.limit(cellSize * decodeIndex + (int) alignedStripe.range.spanInBlock);
+ decodeInputs[decodeIndex] = buf.slice();
+ alignedStripe.chunks[index] = new StripingChunk(decodeInputs[decodeIndex]);
+ return true;
+ }
+
+ @Override
+ void decode() {
+ // TODO no copy for data chunks. this depends on HADOOP-12047
+ final int span = (int) alignedStripe.getSpanInBlock();
+ for (int i = 0; i < alignedStripe.chunks.length; i++) {
+ final int decodeIndex = StripedBlockUtil.convertIndex4Decode(i,
+ dataBlkNum, parityBlkNum);
+ if (alignedStripe.chunks[i] != null &&
+ alignedStripe.chunks[i].state == StripingChunk.ALLZERO) {
+ for (int j = 0; j < span; j++) {
+ decodeInputs[decodeIndex].put((byte) 0);
+ }
+ decodeInputs[decodeIndex].flip();
+ } else if (alignedStripe.chunks[i] != null &&
+ alignedStripe.chunks[i].state == StripingChunk.FETCHED) {
+ decodeInputs[decodeIndex].position(0);
+ decodeInputs[decodeIndex].limit(span);
+ }
+ }
+ int[] decodeIndices = new int[parityBlkNum];
+ int pos = 0;
+ for (int i = 0; i < alignedStripe.chunks.length; i++) {
+ if (alignedStripe.chunks[i] != null &&
+ alignedStripe.chunks[i].state == StripingChunk.MISSING) {
+ int decodeIndex = StripedBlockUtil.convertIndex4Decode(i,
+ dataBlkNum, parityBlkNum);
+ if (i < dataBlkNum) {
+ decodeIndices[pos++] = decodeIndex;
+ } else {
+ decodeInputs[decodeIndex] = null;
+ }
+ }
+ }
+ decodeIndices = Arrays.copyOf(decodeIndices, pos);
+
+ final int decodeChunkNum = decodeIndices.length;
+ ByteBuffer[] outputs = new ByteBuffer[decodeChunkNum];
+ for (int i = 0; i < decodeChunkNum; i++) {
+ outputs[i] = decodeInputs[decodeIndices[i]];
+ outputs[i].position(0);
+ outputs[i].limit((int) alignedStripe.range.spanInBlock);
+ decodeInputs[decodeIndices[i]] = null;
+ }
+
+ decoder.decode(decodeInputs, decodeIndices, outputs);
+ }
+ }
+
+ /**
+ * May need online read recovery, zero-copy read doesn't make
+ * sense, so don't support it.
+ */
+ @Override
+ public synchronized ByteBuffer read(ByteBufferPool bufferPool,
+ int maxLength, EnumSet<ReadOption> opts)
+ throws IOException, UnsupportedOperationException {
+ throw new UnsupportedOperationException(
+ "Not support enhanced byte buffer access.");
+ }
+
+ @Override
+ public synchronized void releaseBuffer(ByteBuffer buffer) {
+ throw new UnsupportedOperationException(
+ "Not support enhanced byte buffer access.");
+ }
+
+ /** A variation to {@link DFSInputStream#cancelAll} */
+ private void clearFutures(Collection<Future<Void>> futures) {
+ for (Future<Void> future : futures) {
+ future.cancel(false);
+ }
+ futures.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
new file mode 100644
index 0000000..746b791
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -0,0 +1,653 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.CreateFlag;
+import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.io.MultipleIOException;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.util.DataChecksum;
+import org.apache.hadoop.util.Progressable;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
+
+import com.google.common.base.Preconditions;
+
+
+/**
+ * This class supports writing files in striped layout and erasure coded format.
+ * Each stripe contains a sequence of cells.
+ */
+@InterfaceAudience.Private
+public class DFSStripedOutputStream extends DFSOutputStream {
+ static class MultipleBlockingQueue<T> {
+ private final List<BlockingQueue<T>> queues;
+
+ MultipleBlockingQueue(int numQueue, int queueSize) {
+ queues = new ArrayList<>(numQueue);
+ for (int i = 0; i < numQueue; i++) {
+ queues.add(new LinkedBlockingQueue<T>(queueSize));
+ }
+ }
+
+ boolean isEmpty() {
+ for(int i = 0; i < queues.size(); i++) {
+ if (!queues.get(i).isEmpty()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ int numQueues() {
+ return queues.size();
+ }
+
+ void offer(int i, T object) {
+ final boolean b = queues.get(i).offer(object);
+ Preconditions.checkState(b, "Failed to offer " + object
+ + " to queue, i=" + i);
+ }
+
+ T take(int i) throws InterruptedIOException {
+ try {
+ return queues.get(i).take();
+ } catch(InterruptedException ie) {
+ throw DFSUtil.toInterruptedIOException("take interrupted, i=" + i, ie);
+ }
+ }
+
+ T poll(int i) {
+ return queues.get(i).poll();
+ }
+
+ T peek(int i) {
+ return queues.get(i).peek();
+ }
+ }
+
+ /** Coordinate the communication between the streamers. */
+ class Coordinator {
+ private final MultipleBlockingQueue<LocatedBlock> followingBlocks;
+ private final MultipleBlockingQueue<ExtendedBlock> endBlocks;
+
+ private final MultipleBlockingQueue<LocatedBlock> newBlocks;
+ private final MultipleBlockingQueue<ExtendedBlock> updateBlocks;
+
+ Coordinator(final DfsClientConf conf, final int numDataBlocks,
+ final int numAllBlocks) {
+ followingBlocks = new MultipleBlockingQueue<>(numAllBlocks, 1);
+ endBlocks = new MultipleBlockingQueue<>(numDataBlocks, 1);
+
+ newBlocks = new MultipleBlockingQueue<>(numAllBlocks, 1);
+ updateBlocks = new MultipleBlockingQueue<>(numAllBlocks, 1);
+ }
+
+ MultipleBlockingQueue<LocatedBlock> getFollowingBlocks() {
+ return followingBlocks;
+ }
+
+ MultipleBlockingQueue<LocatedBlock> getNewBlocks() {
+ return newBlocks;
+ }
+
+ MultipleBlockingQueue<ExtendedBlock> getUpdateBlocks() {
+ return updateBlocks;
+ }
+
+ StripedDataStreamer getStripedDataStreamer(int i) {
+ return DFSStripedOutputStream.this.getStripedDataStreamer(i);
+ }
+
+ void offerEndBlock(int i, ExtendedBlock block) {
+ endBlocks.offer(i, block);
+ }
+
+ ExtendedBlock takeEndBlock(int i) throws InterruptedIOException {
+ return endBlocks.take(i);
+ }
+
+ boolean hasAllEndBlocks() {
+ for(int i = 0; i < endBlocks.numQueues(); i++) {
+ if (endBlocks.peek(i) == null) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void setBytesEndBlock(int i, long newBytes, ExtendedBlock block) {
+ ExtendedBlock b = endBlocks.peek(i);
+ if (b == null) {
+ // streamer just has failed, put end block and continue
+ b = block;
+ offerEndBlock(i, b);
+ }
+ b.setNumBytes(newBytes);
+ }
+
+ /** @return a block representing the entire block group. */
+ ExtendedBlock getBlockGroup() {
+ final StripedDataStreamer s0 = getStripedDataStreamer(0);
+ final ExtendedBlock b0 = s0.getBlock();
+ if (b0 == null) {
+ return null;
+ }
+
+ final boolean atBlockGroupBoundary = s0.getBytesCurBlock() == 0 && b0.getNumBytes() > 0;
+ final ExtendedBlock block = new ExtendedBlock(b0);
+ long numBytes = b0.getNumBytes();
+ for (int i = 1; i < numDataBlocks; i++) {
+ final StripedDataStreamer si = getStripedDataStreamer(i);
+ final ExtendedBlock bi = si.getBlock();
+ if (bi != null && bi.getGenerationStamp() > block.getGenerationStamp()) {
+ block.setGenerationStamp(bi.getGenerationStamp());
+ }
+ numBytes += atBlockGroupBoundary? bi.getNumBytes(): si.getBytesCurBlock();
+ }
+ block.setNumBytes(numBytes);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getBlockGroup: " + block + ", numBytes=" + block.getNumBytes());
+ }
+ return block;
+ }
+ }
+
+ /** Buffers for writing the data and parity cells of a stripe. */
+ class CellBuffers {
+ private final ByteBuffer[] buffers;
+ private final byte[][] checksumArrays;
+
+ CellBuffers(int numParityBlocks) throws InterruptedException{
+ if (cellSize % bytesPerChecksum != 0) {
+ throw new HadoopIllegalArgumentException("Invalid values: "
+ + DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY + " (="
+ + bytesPerChecksum + ") must divide cell size (=" + cellSize + ").");
+ }
+
+ checksumArrays = new byte[numParityBlocks][];
+ final int size = getChecksumSize() * (cellSize / bytesPerChecksum);
+ for (int i = 0; i < checksumArrays.length; i++) {
+ checksumArrays[i] = new byte[size];
+ }
+
+ buffers = new ByteBuffer[numAllBlocks];
+ for (int i = 0; i < buffers.length; i++) {
+ buffers[i] = ByteBuffer.wrap(byteArrayManager.newByteArray(cellSize));
+ }
+ }
+
+ private ByteBuffer[] getBuffers() {
+ return buffers;
+ }
+
+ byte[] getChecksumArray(int i) {
+ return checksumArrays[i - numDataBlocks];
+ }
+
+ private int addTo(int i, byte[] b, int off, int len) {
+ final ByteBuffer buf = buffers[i];
+ final int pos = buf.position() + len;
+ Preconditions.checkState(pos <= cellSize);
+ buf.put(b, off, len);
+ return pos;
+ }
+
+ private void clear() {
+ for (int i = 0; i< numAllBlocks; i++) {
+ buffers[i].clear();
+ if (i >= numDataBlocks) {
+ Arrays.fill(buffers[i].array(), (byte) 0);
+ }
+ }
+ }
+
+ private void release() {
+ for (int i = 0; i < numAllBlocks; i++) {
+ byteArrayManager.release(buffers[i].array());
+ }
+ }
+
+ private void flipDataBuffers() {
+ for (int i = 0; i < numDataBlocks; i++) {
+ buffers[i].flip();
+ }
+ }
+ }
+
+ private final Coordinator coordinator;
+ private final CellBuffers cellBuffers;
+ private final RawErasureEncoder encoder;
+ private final List<StripedDataStreamer> streamers;
+ private final DFSPacket[] currentPackets; // current Packet of each streamer
+
+ /** Size of each striping cell, must be a multiple of bytesPerChecksum */
+ private final int cellSize;
+ private final int numAllBlocks;
+ private final int numDataBlocks;
+
+ @Override
+ ExtendedBlock getBlock() {
+ return coordinator.getBlockGroup();
+ }
+
+ /** Construct a new output stream for creating a file. */
+ DFSStripedOutputStream(DFSClient dfsClient, String src, HdfsFileStatus stat,
+ EnumSet<CreateFlag> flag, Progressable progress,
+ DataChecksum checksum, String[] favoredNodes)
+ throws IOException {
+ super(dfsClient, src, stat, flag, progress, checksum, favoredNodes);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Creating DFSStripedOutputStream for " + src);
+ }
+
+ final ECSchema schema = stat.getECSchema();
+ final int numParityBlocks = schema.getNumParityUnits();
+ cellSize = stat.getStripeCellSize();
+ numDataBlocks = schema.getNumDataUnits();
+ numAllBlocks = numDataBlocks + numParityBlocks;
+
+ encoder = CodecUtil.createRSRawEncoder(dfsClient.getConfiguration(),
+ numDataBlocks, numParityBlocks);
+
+ coordinator = new Coordinator(dfsClient.getConf(),
+ numDataBlocks, numAllBlocks);
+ try {
+ cellBuffers = new CellBuffers(numParityBlocks);
+ } catch (InterruptedException ie) {
+ throw DFSUtil.toInterruptedIOException(
+ "Failed to create cell buffers", ie);
+ }
+
+ List<StripedDataStreamer> s = new ArrayList<>(numAllBlocks);
+ for (short i = 0; i < numAllBlocks; i++) {
+ StripedDataStreamer streamer = new StripedDataStreamer(stat,
+ dfsClient, src, progress, checksum, cachingStrategy, byteArrayManager,
+ favoredNodes, i, coordinator);
+ s.add(streamer);
+ }
+ streamers = Collections.unmodifiableList(s);
+ currentPackets = new DFSPacket[streamers.size()];
+ setCurrentStreamer(0);
+ }
+
+ StripedDataStreamer getStripedDataStreamer(int i) {
+ return streamers.get(i);
+ }
+
+ int getCurrentIndex() {
+ return getCurrentStreamer().getIndex();
+ }
+
+ private synchronized StripedDataStreamer getCurrentStreamer() {
+ return (StripedDataStreamer)streamer;
+ }
+
+ private synchronized StripedDataStreamer setCurrentStreamer(int newIdx)
+ throws IOException {
+ // backup currentPacket for current streamer
+ int oldIdx = streamers.indexOf(streamer);
+ if (oldIdx >= 0) {
+ currentPackets[oldIdx] = currentPacket;
+ }
+
+ streamer = streamers.get(newIdx);
+ currentPacket = currentPackets[newIdx];
+ adjustChunkBoundary();
+
+ return getCurrentStreamer();
+ }
+
+ /**
+ * Encode the buffers, i.e. compute parities.
+ *
+ * @param buffers data buffers + parity buffers
+ */
+ private static void encode(RawErasureEncoder encoder, int numData,
+ ByteBuffer[] buffers) {
+ final ByteBuffer[] dataBuffers = new ByteBuffer[numData];
+ final ByteBuffer[] parityBuffers = new ByteBuffer[buffers.length - numData];
+ System.arraycopy(buffers, 0, dataBuffers, 0, dataBuffers.length);
+ System.arraycopy(buffers, numData, parityBuffers, 0, parityBuffers.length);
+
+ encoder.encode(dataBuffers, parityBuffers);
+ }
+
+
+ private void checkStreamers() throws IOException {
+ int count = 0;
+ for(StripedDataStreamer s : streamers) {
+ if (!s.isFailed()) {
+ if (s.getBlock() != null) {
+ s.getErrorState().initExternalError();
+ }
+ count++;
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("checkStreamers: " + streamers);
+ LOG.debug("count=" + count);
+ }
+ if (count < numDataBlocks) {
+ throw new IOException("Failed: the number of remaining blocks = "
+ + count + " < the number of data blocks = " + numDataBlocks);
+ }
+ }
+
+ private void handleStreamerFailure(String err,
+ Exception e) throws IOException {
+ LOG.warn("Failed: " + err + ", " + this, e);
+ getCurrentStreamer().setFailed(true);
+ checkStreamers();
+ currentPacket = null;
+ }
+
+ @Override
+ protected synchronized void writeChunk(byte[] bytes, int offset, int len,
+ byte[] checksum, int ckoff, int cklen) throws IOException {
+ final int index = getCurrentIndex();
+ final StripedDataStreamer current = getCurrentStreamer();
+ final int pos = cellBuffers.addTo(index, bytes, offset, len);
+ final boolean cellFull = pos == cellSize;
+
+ final long oldBytes = current.getBytesCurBlock();
+ if (!current.isFailed()) {
+ try {
+ super.writeChunk(bytes, offset, len, checksum, ckoff, cklen);
+ } catch(Exception e) {
+ handleStreamerFailure("offset=" + offset + ", length=" + len, e);
+ }
+ }
+
+ if (current.isFailed()) {
+ final long newBytes = oldBytes + len;
+ coordinator.setBytesEndBlock(index, newBytes, current.getBlock());
+ current.setBytesCurBlock(newBytes);
+ }
+
+ // Two extra steps are needed when a striping cell is full:
+ // 1. Forward the current index pointer
+ // 2. Generate parity packets if a full stripe of data cells are present
+ if (cellFull) {
+ int next = index + 1;
+ //When all data cells in a stripe are ready, we need to encode
+ //them and generate some parity cells. These cells will be
+ //converted to packets and put to their DataStreamer's queue.
+ if (next == numDataBlocks) {
+ cellBuffers.flipDataBuffers();
+ writeParityCells();
+ next = 0;
+ }
+ setCurrentStreamer(next);
+ }
+ }
+
+ private int stripeDataSize() {
+ return numDataBlocks * cellSize;
+ }
+
+ @Override
+ public void hflush() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void hsync() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected synchronized void start() {
+ for (StripedDataStreamer streamer : streamers) {
+ streamer.start();
+ }
+ }
+
+ @Override
+ synchronized void abort() throws IOException {
+ if (isClosed()) {
+ return;
+ }
+ for (StripedDataStreamer streamer : streamers) {
+ streamer.getLastException().set(new IOException("Lease timeout of "
+ + (dfsClient.getConf().getHdfsTimeout()/1000) +
+ " seconds expired."));
+ }
+ closeThreads(true);
+ dfsClient.endFileLease(fileId);
+ }
+
+ @Override
+ boolean isClosed() {
+ if (closed) {
+ return true;
+ }
+ for(StripedDataStreamer s : streamers) {
+ if (!s.streamerClosed()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ protected void closeThreads(boolean force) throws IOException {
+ final MultipleIOException.Builder b = new MultipleIOException.Builder();
+ try {
+ for (StripedDataStreamer streamer : streamers) {
+ try {
+ streamer.close(force);
+ streamer.join();
+ streamer.closeSocket();
+ } catch (Exception e) {
+ try {
+ handleStreamerFailure("force=" + force, e);
+ } catch (IOException ioe) {
+ b.add(ioe);
+ }
+ } finally {
+ streamer.setSocketToNull();
+ }
+ }
+ } finally {
+ setClosed();
+ }
+ final IOException ioe = b.build();
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+
+ /**
+ * Simply add bytesCurBlock together. Note that this result is not accurately
+ * the size of the block group.
+ */
+ private long getCurrentSumBytes() {
+ long sum = 0;
+ for (int i = 0; i < numDataBlocks; i++) {
+ sum += streamers.get(i).getBytesCurBlock();
+ }
+ return sum;
+ }
+
+ private void writeParityCellsForLastStripe() throws IOException {
+ final long currentBlockGroupBytes = getCurrentSumBytes();
+ if (currentBlockGroupBytes % stripeDataSize() == 0) {
+ return;
+ }
+
+ final int firstCellSize =
+ (int)(getStripedDataStreamer(0).getBytesCurBlock() % cellSize);
+ final int parityCellSize = firstCellSize > 0 && firstCellSize < cellSize?
+ firstCellSize : cellSize;
+ final ByteBuffer[] buffers = cellBuffers.getBuffers();
+
+ for (int i = 0; i < numAllBlocks; i++) {
+ // Pad zero bytes to make all cells exactly the size of parityCellSize
+ // If internal block is smaller than parity block, pad zero bytes.
+ // Also pad zero bytes to all parity cells
+ final int position = buffers[i].position();
+ assert position <= parityCellSize : "If an internal block is smaller" +
+ " than parity block, then its last cell should be small than last" +
+ " parity cell";
+ for (int j = 0; j < parityCellSize - position; j++) {
+ buffers[i].put((byte) 0);
+ }
+ buffers[i].flip();
+ }
+
+ writeParityCells();
+ }
+
+ void writeParityCells() throws IOException {
+ final ByteBuffer[] buffers = cellBuffers.getBuffers();
+ //encode the data cells
+ encode(encoder, numDataBlocks, buffers);
+ for (int i = numDataBlocks; i < numAllBlocks; i++) {
+ writeParity(i, buffers[i], cellBuffers.getChecksumArray(i));
+ }
+ cellBuffers.clear();
+ }
+
+ void writeParity(int index, ByteBuffer buffer, byte[] checksumBuf
+ ) throws IOException {
+ final StripedDataStreamer current = setCurrentStreamer(index);
+ final int len = buffer.limit();
+
+ final long oldBytes = current.getBytesCurBlock();
+ if (!current.isFailed()) {
+ try {
+ DataChecksum sum = getDataChecksum();
+ sum.calculateChunkedSums(buffer.array(), 0, len, checksumBuf, 0);
+ for (int i = 0; i < len; i += sum.getBytesPerChecksum()) {
+ int chunkLen = Math.min(sum.getBytesPerChecksum(), len - i);
+ int ckOffset = i / sum.getBytesPerChecksum() * getChecksumSize();
+ super.writeChunk(buffer.array(), i, chunkLen, checksumBuf, ckOffset,
+ getChecksumSize());
+ }
+ } catch(Exception e) {
+ handleStreamerFailure("oldBytes=" + oldBytes + ", len=" + len, e);
+ }
+ }
+
+ if (current.isFailed()) {
+ final long newBytes = oldBytes + len;
+ current.setBytesCurBlock(newBytes);
+ }
+ }
+
+ @Override
+ void setClosed() {
+ super.setClosed();
+ for (int i = 0; i < numAllBlocks; i++) {
+ streamers.get(i).release();
+ }
+ cellBuffers.release();
+ }
+
+ @Override
+ protected synchronized void closeImpl() throws IOException {
+ if (isClosed()) {
+ final MultipleIOException.Builder b = new MultipleIOException.Builder();
+ for(int i = 0; i < streamers.size(); i++) {
+ final StripedDataStreamer si = getStripedDataStreamer(i);
+ try {
+ si.getLastException().check(true);
+ } catch (IOException e) {
+ b.add(e);
+ }
+ }
+ final IOException ioe = b.build();
+ if (ioe != null) {
+ throw ioe;
+ }
+ return;
+ }
+
+ try {
+ // flush from all upper layers
+ try {
+ flushBuffer();
+ // if the last stripe is incomplete, generate and write parity cells
+ writeParityCellsForLastStripe();
+ enqueueAllCurrentPackets();
+ } catch(Exception e) {
+ handleStreamerFailure("closeImpl", e);
+ }
+
+ for (int i = 0; i < numAllBlocks; i++) {
+ final StripedDataStreamer s = setCurrentStreamer(i);
+ if (!s.isFailed()) {
+ try {
+ if (s.getBytesCurBlock() > 0) {
+ setCurrentPacketToEmpty();
+ }
+ // flush all data to Datanode
+ flushInternal();
+ } catch(Exception e) {
+ handleStreamerFailure("closeImpl", e);
+ }
+ }
+ }
+
+ closeThreads(false);
+ final ExtendedBlock lastBlock = coordinator.getBlockGroup();
+ TraceScope scope = Trace.startSpan("completeFile", Sampler.NEVER);
+ try {
+ completeFile(lastBlock);
+ } finally {
+ scope.close();
+ }
+ dfsClient.endFileLease(fileId);
+ } catch (ClosedChannelException ignored) {
+ } finally {
+ setClosed();
+ }
+ }
+
+ private void enqueueAllCurrentPackets() throws IOException {
+ int idx = streamers.indexOf(getCurrentStreamer());
+ for(int i = 0; i < streamers.size(); i++) {
+ setCurrentStreamer(i);
+ if (currentPacket != null) {
+ enqueueCurrentPacket();
+ }
+ }
+ setCurrentStreamer(idx);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index cae56c0..c06a435 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -36,6 +36,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PAS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.InetAddress;
@@ -55,7 +56,6 @@ import java.util.concurrent.ThreadLocalRandom;
import javax.net.SocketFactory;
-import com.google.common.collect.Sets;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.Option;
@@ -96,6 +96,7 @@ import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
import com.google.protobuf.BlockingService;
@InterfaceAudience.Private
@@ -1527,4 +1528,10 @@ public class DFSUtil {
DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "").isEmpty();
}
+ public static InterruptedIOException toInterruptedIOException(String message,
+ InterruptedException e) {
+ final InterruptedIOException iioe = new InterruptedIOException(message);
+ iioe.initCause(e);
+ return iioe;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 8dd85b7..c78199e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -209,6 +209,7 @@ class DataStreamer extends Daemon {
static class ErrorState {
private boolean error = false;
+ private boolean externalError = false;
private int badNodeIndex = -1;
private int restartingNodeIndex = -1;
private long restartingNodeDeadline = 0;
@@ -220,6 +221,7 @@ class DataStreamer extends Daemon {
synchronized void reset() {
error = false;
+ externalError = false;
badNodeIndex = -1;
restartingNodeIndex = -1;
restartingNodeDeadline = 0;
@@ -229,14 +231,24 @@ class DataStreamer extends Daemon {
return error;
}
+ synchronized boolean hasExternalErrorOnly() {
+ return error && externalError && !isNodeMarked();
+ }
+
synchronized boolean hasDatanodeError() {
- return error && isNodeMarked();
+ return error && (isNodeMarked() || externalError);
}
synchronized void setError(boolean err) {
this.error = err;
}
+ synchronized void initExternalError() {
+ setError(true);
+ this.externalError = true;
+ }
+
+
synchronized void setBadNodeIndex(int index) {
this.badNodeIndex = index;
}
@@ -335,7 +347,7 @@ class DataStreamer extends Daemon {
}
private volatile boolean streamerClosed = false;
- private ExtendedBlock block; // its length is number of bytes acked
+ protected ExtendedBlock block; // its length is number of bytes acked
private Token<BlockTokenIdentifier> accessToken;
private DataOutputStream blockStream;
private DataInputStream blockReplyStream;
@@ -366,12 +378,12 @@ class DataStreamer extends Daemon {
private final LastExceptionInStreamer lastException = new LastExceptionInStreamer();
private Socket s;
- private final DFSClient dfsClient;
- private final String src;
+ protected final DFSClient dfsClient;
+ protected final String src;
/** Only for DataTransferProtocol.writeBlock(..) */
private final DataChecksum checksum4WriteBlock;
private final Progressable progress;
- private final HdfsFileStatus stat;
+ protected final HdfsFileStatus stat;
// appending to existing partial block
private volatile boolean appendChunk = false;
// both dataQueue and ackQueue are protected by dataQueue lock
@@ -397,11 +409,13 @@ class DataStreamer extends Daemon {
private final LoadingCache<DatanodeInfo, DatanodeInfo> excludedNodes;
private final String[] favoredNodes;
- private DataStreamer(HdfsFileStatus stat, DFSClient dfsClient, String src,
+ private DataStreamer(HdfsFileStatus stat, ExtendedBlock block,
+ DFSClient dfsClient, String src,
Progressable progress, DataChecksum checksum,
AtomicReference<CachingStrategy> cachingStrategy,
ByteArrayManager byteArrayManage,
boolean isAppend, String[] favoredNodes) {
+ this.block = block;
this.dfsClient = dfsClient;
this.src = src;
this.progress = progress;
@@ -426,9 +440,8 @@ class DataStreamer extends Daemon {
String src, Progressable progress, DataChecksum checksum,
AtomicReference<CachingStrategy> cachingStrategy,
ByteArrayManager byteArrayManage, String[] favoredNodes) {
- this(stat, dfsClient, src, progress, checksum, cachingStrategy,
+ this(stat, block, dfsClient, src, progress, checksum, cachingStrategy,
byteArrayManage, false, favoredNodes);
- this.block = block;
stage = BlockConstructionStage.PIPELINE_SETUP_CREATE;
}
@@ -442,10 +455,9 @@ class DataStreamer extends Daemon {
String src, Progressable progress, DataChecksum checksum,
AtomicReference<CachingStrategy> cachingStrategy,
ByteArrayManager byteArrayManage) throws IOException {
- this(stat, dfsClient, src, progress, checksum, cachingStrategy,
+ this(stat, lastBlock.getBlock(), dfsClient, src, progress, checksum, cachingStrategy,
byteArrayManage, true, null);
stage = BlockConstructionStage.PIPELINE_SETUP_APPEND;
- block = lastBlock.getBlock();
bytesSent = block.getNumBytes();
accessToken = lastBlock.getBlockToken();
}
@@ -488,7 +500,7 @@ class DataStreamer extends Daemon {
stage = BlockConstructionStage.DATA_STREAMING;
}
- private void endBlock() {
+ protected void endBlock() {
if(LOG.isDebugEnabled()) {
LOG.debug("Closing old block " + block);
}
@@ -574,7 +586,7 @@ class DataStreamer extends Daemon {
// get new block from namenode.
if (stage == BlockConstructionStage.PIPELINE_SETUP_CREATE) {
if(LOG.isDebugEnabled()) {
- LOG.debug("Allocating new block");
+ LOG.debug("Allocating new block " + this);
}
setPipeline(nextBlockOutputStream());
initDataStreaming();
@@ -592,10 +604,7 @@ class DataStreamer extends Daemon {
long lastByteOffsetInBlock = one.getLastByteOffsetBlock();
if (lastByteOffsetInBlock > stat.getBlockSize()) {
throw new IOException("BlockSize " + stat.getBlockSize() +
- " is smaller than data size. " +
- " Offset of packet in block " +
- lastByteOffsetInBlock +
- " Aborting file " + src);
+ " < lastByteOffsetInBlock, " + this + ", " + one);
}
if (one.isLastPacketInBlock()) {
@@ -1069,6 +1078,10 @@ class DataStreamer extends Daemon {
if (!errorState.hasDatanodeError()) {
return false;
}
+ if (errorState.hasExternalErrorOnly() && block == null) {
+ // block is not yet initialized, handle external error later.
+ return false;
+ }
if (response != null) {
LOG.info("Error Recovery for " + block +
" waiting for responder to exit. ");
@@ -1397,15 +1410,28 @@ class DataStreamer extends Daemon {
}
LocatedBlock updateBlockForPipeline() throws IOException {
+ return callUpdateBlockForPipeline(block);
+ }
+
+ LocatedBlock callUpdateBlockForPipeline(ExtendedBlock newBlock) throws IOException {
return dfsClient.namenode.updateBlockForPipeline(
- block, dfsClient.clientName);
+ newBlock, dfsClient.clientName);
+ }
+
+ static ExtendedBlock newBlock(ExtendedBlock b, final long newGS) {
+ return new ExtendedBlock(b.getBlockPoolId(), b.getBlockId(),
+ b.getNumBytes(), newGS);
}
/** update pipeline at the namenode */
ExtendedBlock updatePipeline(long newGS) throws IOException {
- final ExtendedBlock newBlock = new ExtendedBlock(
- block.getBlockPoolId(), block.getBlockId(), block.getNumBytes(), newGS);
- dfsClient.namenode.updatePipeline(dfsClient.clientName, block, newBlock,
+ final ExtendedBlock newBlock = newBlock(block, newGS);
+ return callUpdatePipeline(block, newBlock);
+ }
+
+ ExtendedBlock callUpdatePipeline(ExtendedBlock oldBlock, ExtendedBlock newBlock)
+ throws IOException {
+ dfsClient.namenode.updatePipeline(dfsClient.clientName, oldBlock, newBlock,
nodes, storageIDs);
return newBlock;
}
@@ -1738,6 +1764,10 @@ class DataStreamer extends Daemon {
return accessToken;
}
+ ErrorState getErrorState() {
+ return errorState;
+ }
+
/**
* Put a packet to the data queue
*
@@ -1750,7 +1780,7 @@ class DataStreamer extends Daemon {
dataQueue.addLast(packet);
lastQueuedSeqno = packet.getSeqno();
if (LOG.isDebugEnabled()) {
- LOG.debug("Queued packet " + packet.getSeqno());
+ LOG.debug("Queued " + packet + ", " + this);
}
dataQueue.notifyAll();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 0197cfb..4c9f9cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
@@ -89,6 +90,7 @@ import org.apache.hadoop.hdfs.security.token.block.InvalidBlockTokenException;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Credentials;
@@ -2298,4 +2300,68 @@ public class DistributedFileSystem extends FileSystem {
throws IOException {
return dfs.getInotifyEventStream(lastReadTxid);
}
+
+ /**
+ * Create the erasurecoding zone
+ *
+ * @param path Directory to create the ec zone
+ * @param schema ECSchema for the zone. If not specified default will be used.
+ * @param cellSize Cellsize for the striped erasure coding
+ * @throws IOException
+ */
+ public void createErasureCodingZone(final Path path, final ECSchema schema,
+ final int cellSize) throws IOException {
+ Path absF = fixRelativePart(path);
+ new FileSystemLinkResolver<Void>() {
+ @Override
+ public Void doCall(final Path p) throws IOException,
+ UnresolvedLinkException {
+ dfs.createErasureCodingZone(getPathName(p), schema, cellSize);
+ return null;
+ }
+
+ @Override
+ public Void next(final FileSystem fs, final Path p) throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem myDfs = (DistributedFileSystem) fs;
+ myDfs.createErasureCodingZone(p, schema, cellSize);
+ return null;
+ }
+ throw new UnsupportedOperationException(
+ "Cannot createErasureCodingZone through a symlink to a "
+ + "non-DistributedFileSystem: " + path + " -> " + p);
+ }
+ }.resolve(this, absF);
+ }
+
+ /**
+ * Get ErasureCoding zone information for the specified path
+ *
+ * @param path
+ * @return Returns the zone information if path is in EC zone, null otherwise
+ * @throws IOException
+ */
+ public ErasureCodingZone getErasureCodingZone(final Path path)
+ throws IOException {
+ Path absF = fixRelativePart(path);
+ return new FileSystemLinkResolver<ErasureCodingZone>() {
+ @Override
+ public ErasureCodingZone doCall(final Path p) throws IOException,
+ UnresolvedLinkException {
+ return dfs.getErasureCodingZone(getPathName(p));
+ }
+
+ @Override
+ public ErasureCodingZone next(final FileSystem fs, final Path p)
+ throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem myDfs = (DistributedFileSystem) fs;
+ return myDfs.getErasureCodingZone(p);
+ }
+ throw new UnsupportedOperationException(
+ "Cannot getErasureCodingZone through a symlink to a "
+ + "non-DistributedFileSystem: " + path + " -> " + p);
+ }
+ }.resolve(this, absF);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
index d70f419..70cce7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader.java
@@ -505,4 +505,9 @@ public class RemoteBlockReader extends FSInputChecker implements BlockReader {
public ClientMmap getClientMmap(EnumSet<ReadOption> opts) {
return null;
}
+
+ @Override
+ public DataChecksum getDataChecksum() {
+ return checksum;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
index c368d65..cce44b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/RemoteBlockReader2.java
@@ -474,4 +474,9 @@ public class RemoteBlockReader2 implements BlockReader {
public ClientMmap getClientMmap(EnumSet<ReadOption> opts) {
return null;
}
+
+ @Override
+ public DataChecksum getDataChecksum() {
+ return checksum;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
new file mode 100644
index 0000000..2d51dc4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/StripedDataStreamer.java
@@ -0,0 +1,240 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_PARITY_BLOCKS;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.hadoop.hdfs.DFSStripedOutputStream.Coordinator;
+import org.apache.hadoop.hdfs.DFSStripedOutputStream.MultipleBlockingQueue;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.datanode.CachingStrategy;
+import org.apache.hadoop.hdfs.util.ByteArrayManager;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.util.DataChecksum;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * This class extends {@link DataStreamer} to support writing striped blocks
+ * to datanodes.
+ * A {@link DFSStripedOutputStream} has multiple {@link StripedDataStreamer}s.
+ * Whenever the streamers need to talk the namenode, only the fastest streamer
+ * sends an rpc call to the namenode and then populates the result for the
+ * other streamers.
+ */
+public class StripedDataStreamer extends DataStreamer {
+ /**
+ * This class is designed for multiple threads to share a
+ * {@link MultipleBlockingQueue}. Initially, the queue is empty. The earliest
+ * thread calling poll populates entries to the queue and the other threads
+ * will wait for it. Once the entries are populated, all the threads can poll
+ * their entries.
+ *
+ * @param <T> the queue entry type.
+ */
+ static abstract class ConcurrentPoll<T> {
+ private final MultipleBlockingQueue<T> queue;
+
+ ConcurrentPoll(MultipleBlockingQueue<T> queue) {
+ this.queue = queue;
+ }
+
+ T poll(final int i) throws IOException {
+ for(;;) {
+ synchronized(queue) {
+ final T polled = queue.poll(i);
+ if (polled != null) { // already populated; return polled item.
+ return polled;
+ }
+ if (isReady2Populate()) {
+ populate();
+ return queue.poll(i);
+ }
+ }
+
+ // sleep and then retry.
+ try {
+ Thread.sleep(100);
+ } catch(InterruptedException ie) {
+ throw DFSUtil.toInterruptedIOException(
+ "Sleep interrupted during poll", ie);
+ }
+ }
+ }
+
+ boolean isReady2Populate() {
+ return queue.isEmpty();
+ }
+
+ abstract void populate() throws IOException;
+ }
+
+ private final Coordinator coordinator;
+ private final int index;
+ private volatile boolean failed;
+
+ StripedDataStreamer(HdfsFileStatus stat,
+ DFSClient dfsClient, String src,
+ Progressable progress, DataChecksum checksum,
+ AtomicReference<CachingStrategy> cachingStrategy,
+ ByteArrayManager byteArrayManage, String[] favoredNodes,
+ short index, Coordinator coordinator) {
+ super(stat, null, dfsClient, src, progress, checksum, cachingStrategy,
+ byteArrayManage, favoredNodes);
+ this.index = index;
+ this.coordinator = coordinator;
+ }
+
+ int getIndex() {
+ return index;
+ }
+
+ void setFailed(boolean failed) {
+ this.failed = failed;
+ }
+
+ boolean isFailed() {
+ return failed;
+ }
+
+ private boolean isParityStreamer() {
+ return index >= NUM_DATA_BLOCKS;
+ }
+
+ @Override
+ protected void endBlock() {
+ if (!isParityStreamer()) {
+ coordinator.offerEndBlock(index, block);
+ }
+ super.endBlock();
+ }
+
+ @Override
+ protected LocatedBlock locateFollowingBlock(final DatanodeInfo[] excludedNodes)
+ throws IOException {
+ final MultipleBlockingQueue<LocatedBlock> followingBlocks
+ = coordinator.getFollowingBlocks();
+ return new ConcurrentPoll<LocatedBlock>(followingBlocks) {
+ @Override
+ boolean isReady2Populate() {
+ return super.isReady2Populate()
+ && (block == null || coordinator.hasAllEndBlocks());
+ }
+
+ @Override
+ void populate() throws IOException {
+ getLastException().check(false);
+
+ if (block != null) {
+ // set numByte for the previous block group
+ long bytes = 0;
+ for (int i = 0; i < NUM_DATA_BLOCKS; i++) {
+ final ExtendedBlock b = coordinator.takeEndBlock(i);
+ StripedBlockUtil.checkBlocks(index, block, i, b);
+ bytes += b.getNumBytes();
+ }
+ block.setNumBytes(bytes);
+ block.setBlockId(block.getBlockId() - index);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("locateFollowingBlock: index=" + index + ", block=" + block);
+ }
+
+ final LocatedBlock lb = StripedDataStreamer.super.locateFollowingBlock(
+ excludedNodes);
+ final LocatedBlock[] blocks = StripedBlockUtil.parseStripedBlockGroup(
+ (LocatedStripedBlock)lb,
+ BLOCK_STRIPED_CELL_SIZE, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+
+ for (int i = 0; i < blocks.length; i++) {
+ if (!coordinator.getStripedDataStreamer(i).isFailed()) {
+ if (blocks[i] == null) {
+ getLastException().set(
+ new IOException("Failed to get following block, i=" + i));
+ } else {
+ followingBlocks.offer(i, blocks[i]);
+ }
+ }
+ }
+ }
+ }.poll(index);
+ }
+
+ @Override
+ LocatedBlock updateBlockForPipeline() throws IOException {
+ final MultipleBlockingQueue<LocatedBlock> newBlocks
+ = coordinator.getNewBlocks();
+ return new ConcurrentPoll<LocatedBlock>(newBlocks) {
+ @Override
+ void populate() throws IOException {
+ final ExtendedBlock bg = coordinator.getBlockGroup();
+ final LocatedBlock updated = callUpdateBlockForPipeline(bg);
+ final long newGS = updated.getBlock().getGenerationStamp();
+ final LocatedBlock[] updatedBlks = StripedBlockUtil
+ .parseStripedBlockGroup((LocatedStripedBlock) updated,
+ BLOCK_STRIPED_CELL_SIZE, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+ for (int i = 0; i < NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; i++) {
+ final ExtendedBlock bi = coordinator.getStripedDataStreamer(i).getBlock();
+ if (bi != null) {
+ final LocatedBlock lb = new LocatedBlock(newBlock(bi, newGS),
+ null, null, null, -1, updated.isCorrupt(), null);
+ lb.setBlockToken(updatedBlks[i].getBlockToken());
+ newBlocks.offer(i, lb);
+ } else {
+ final LocatedBlock lb = coordinator.getFollowingBlocks().peek(i);
+ lb.getBlock().setGenerationStamp(newGS);
+ }
+ }
+ }
+ }.poll(index);
+ }
+
+ @Override
+ ExtendedBlock updatePipeline(final long newGS) throws IOException {
+ final MultipleBlockingQueue<ExtendedBlock> updateBlocks
+ = coordinator.getUpdateBlocks();
+ return new ConcurrentPoll<ExtendedBlock>(updateBlocks) {
+ @Override
+ void populate() throws IOException {
+ final ExtendedBlock bg = coordinator.getBlockGroup();
+ final ExtendedBlock newBG = newBlock(bg, newGS);
+ final ExtendedBlock updated = callUpdatePipeline(bg, newBG);
+ for (int i = 0; i < NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS; i++) {
+ final ExtendedBlock bi = coordinator.getStripedDataStreamer(i).getBlock();
+ updateBlocks.offer(i, newBlock(bi, updated.getGenerationStamp()));
+ }
+ }
+ }.poll(index);
+ }
+
+ @Override
+ public String toString() {
+ return "#" + index + ": failed? " + Boolean.toString(failed).charAt(0)
+ + ", " + super.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index 84499bb..5a3c885 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -37,9 +37,11 @@ import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
+import org.apache.hadoop.io.erasurecode.ECSchema;
/**
* The public API for performing administrative functions on HDFS. Those writing
@@ -363,4 +365,42 @@ public class HdfsAdmin {
throws IOException {
dfs.setStoragePolicy(src, policyName);
}
+
+ /**
+ * Create the ErasureCoding zone
+ *
+ * @param path
+ * Directory to create the ErasureCoding zone
+ * @param schema
+ * ECSchema for the zone. If not specified default will be used.
+ * @param cellSize
+ * Cellsize for the striped ErasureCoding
+ * @throws IOException
+ */
+ public void createErasureCodingZone(final Path path, final ECSchema schema,
+ final int cellSize) throws IOException {
+ dfs.createErasureCodingZone(path, schema, cellSize);
+ }
+
+ /**
+ * Get the ErasureCoding zone information for the specified path
+ *
+ * @param path
+ * @return Returns the zone information if path is in EC zone, null otherwise
+ * @throws IOException
+ */
+ public ErasureCodingZone getErasureCodingZone(final Path path)
+ throws IOException {
+ return dfs.getErasureCodingZone(path);
+ }
+
+ /**
+ * Get the ErasureCoding schemas supported.
+ *
+ * @return ECSchemas
+ * @throws IOException
+ */
+ public ECSchema[] getECSchemas() throws IOException {
+ return dfs.getClient().getECSchemas();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index a257e32..9aef436 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -38,6 +38,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_SOCKET_WRITE_TIM
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -101,6 +102,9 @@ public class DfsClientConf {
private final long hedgedReadThresholdMillis;
private final int hedgedReadThreadpoolSize;
+ private final int stripedReadThreadpoolSize;
+
+
public DfsClientConf(Configuration conf) {
// The hdfsTimeout is currently the same as the ipc timeout
hdfsTimeout = Client.getTimeout(conf);
@@ -191,7 +195,7 @@ public class DfsClientConf {
connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
hdfsBlocksMetadataEnabled = conf.getBoolean(
- DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
+ DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
fileBlockStorageLocationsNumThreads = conf.getInt(
DFSConfigKeys.DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_NUM_THREADS,
@@ -215,6 +219,13 @@ public class DfsClientConf {
hedgedReadThreadpoolSize = conf.getInt(
HdfsClientConfigKeys.HedgedRead.THREADPOOL_SIZE_KEY,
HdfsClientConfigKeys.HedgedRead.THREADPOOL_SIZE_DEFAULT);
+
+ stripedReadThreadpoolSize = conf.getInt(
+ HdfsClientConfigKeys.StripedRead.THREADPOOL_SIZE_KEY,
+ HdfsClientConfigKeys.StripedRead.THREADPOOL_SIZE_DEFAULT);
+ Preconditions.checkArgument(stripedReadThreadpoolSize > 0, "The value of " +
+ HdfsClientConfigKeys.StripedRead.THREADPOOL_SIZE_KEY +
+ " must be greater than 0.");
}
private DataChecksum.Type getChecksumType(Configuration conf) {
@@ -492,6 +503,13 @@ public class DfsClientConf {
}
/**
+ * @return the stripedReadThreadpoolSize
+ */
+ public int getStripedReadThreadpoolSize() {
+ return stripedReadThreadpoolSize;
+ }
+
+ /**
* @return the shortCircuitConf
*/
public ShortCircuitConf getShortCircuitConf() {
@@ -744,4 +762,4 @@ public class DfsClientConf {
return builder.toString();
}
}
-}
\ No newline at end of file
+}
[06/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
new file mode 100644
index 0000000..f4efbcf
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestOfflineImageViewerWithStripedBlocks {
+ private static int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+ private static int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+
+ private static MiniDFSCluster cluster;
+ private static DistributedFileSystem fs;
+ private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private static final int stripesPerBlock = 3;
+ private static final int blockSize = cellSize * stripesPerBlock;
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ int numDNs = dataBlocks + parityBlocks + 2;
+ Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.waitActive();
+ cluster.getFileSystem().getClient().createErasureCodingZone("/", null, 0);
+ fs = cluster.getFileSystem();
+ Path eczone = new Path("/eczone");
+ fs.mkdirs(eczone);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testFileEqualToOneStripe() throws Exception {
+ int numBytes = cellSize;
+ testFileSize(numBytes);
+ }
+
+ @Test(timeout = 60000)
+ public void testFileLessThanOneStripe() throws Exception {
+ int numBytes = cellSize - 100;
+ testFileSize(numBytes);
+ }
+
+ @Test(timeout = 60000)
+ public void testFileHavingMultipleBlocks() throws Exception {
+ int numBytes = blockSize * 3;
+ testFileSize(numBytes);
+ }
+
+ @Test(timeout = 60000)
+ public void testFileLargerThanABlockGroup1() throws IOException {
+ testFileSize(blockSize * dataBlocks + cellSize + 123);
+ }
+
+ @Test(timeout = 60000)
+ public void testFileLargerThanABlockGroup2() throws IOException {
+ testFileSize(blockSize * dataBlocks * 3 + cellSize * dataBlocks + cellSize
+ + 123);
+ }
+
+ @Test(timeout = 60000)
+ public void testFileFullBlockGroup() throws IOException {
+ testFileSize(blockSize * dataBlocks);
+ }
+
+ @Test(timeout = 60000)
+ public void testFileMoreThanOneStripe() throws Exception {
+ int numBytes = blockSize + blockSize / 2;
+ testFileSize(numBytes);
+ }
+
+ private void testFileSize(int numBytes) throws IOException,
+ UnresolvedLinkException, SnapshotAccessControlException {
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ File orgFsimage = null;
+ Path file = new Path("/eczone/striped");
+ FSDataOutputStream out = fs.create(file, true);
+ byte[] bytes = DFSTestUtil.generateSequentialBytes(0, numBytes);
+ out.write(bytes);
+ out.close();
+
+ // Write results to the fsimage file
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER, false);
+ fs.saveNamespace();
+
+ // Determine location of fsimage file
+ orgFsimage = FSImageTestUtil.findLatestImageFile(FSImageTestUtil
+ .getFSImage(cluster.getNameNode()).getStorage().getStorageDir(0));
+ if (orgFsimage == null) {
+ throw new RuntimeException("Didn't generate or can't find fsimage");
+ }
+ FSImageLoader loader = FSImageLoader.load(orgFsimage.getAbsolutePath());
+ String fileStatus = loader.getFileStatus("/eczone/striped");
+ long expectedFileSize = bytes.length;
+
+ // Verify space consumed present in BlockInfoStriped
+ FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
+ INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ assertTrue("Invalid block size", fileNode.getBlocks().length > 0);
+ long actualFileSize = 0;
+ for (BlockInfo blockInfo : fileNode.getBlocks()) {
+ assertTrue("Didn't find block striped information",
+ blockInfo instanceof BlockInfoStriped);
+ actualFileSize += blockInfo.getNumBytes();
+ }
+
+ assertEquals("Wrongly computed file size contains striped blocks",
+ expectedFileSize, actualFileSize);
+
+ // Verify space consumed present in filestatus
+ String EXPECTED_FILE_SIZE = "\"length\":"
+ + String.valueOf(expectedFileSize);
+ assertTrue(
+ "Wrongly computed file size contains striped blocks, file status:"
+ + fileStatus + ". Expected file size is : " + EXPECTED_FILE_SIZE,
+ fileStatus.contains(EXPECTED_FILE_SIZE));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
new file mode 100644
index 0000000..5d85073
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
@@ -0,0 +1,279 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.util;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
+import static org.apache.hadoop.hdfs.util.StripedBlockUtil.*;
+
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+/**
+ * Need to cover the following combinations:
+ * 1. Block group size:
+ * 1.1 One byte
+ * 1.2 Smaller than cell
+ * 1.3 One full cell
+ * 1.4 x full cells, where x is smaller than number of data blocks
+ * 1.5 x full cells plus a partial cell
+ * 1.6 One full stripe
+ * 1.7 One full stripe plus a partial cell
+ * 1.8 One full stripe plus x full cells
+ * 1.9 One full stripe plus x full cells plus a partial cell
+ * 1.10 y full stripes, but smaller than full block group size
+ * 1.11 Full block group size
+ *
+ * 2. Byte range start
+ * 2.1 Zero
+ * 2.2 Within first cell
+ * 2.3 End of first cell
+ * 2.4 Start of a middle* cell in the first stripe (* neither first or last)
+ * 2.5 End of middle cell in the first stripe
+ * 2.6 Within a middle cell in the first stripe
+ * 2.7 Start of the last cell in the first stripe
+ * 2.8 Within the last cell in the first stripe
+ * 2.9 End of the last cell in the first stripe
+ * 2.10 Start of a middle stripe
+ * 2.11 Within a middle stripe
+ * 2.12 End of a middle stripe
+ * 2.13 Start of the last stripe
+ * 2.14 Within the last stripe
+ * 2.15 End of the last stripe (last byte)
+ *
+ * 3. Byte range length: same settings as block group size
+ *
+ * We should test in total 11 x 15 x 11 = 1815 combinations
+ * TODO: test parity block logic
+ */
+public class TestStripedBlockUtil {
+ private final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS;
+ private final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS;
+ private final short BLK_GROUP_WIDTH = DATA_BLK_NUM + PARITY_BLK_NUM;
+ private final int CELLSIZE = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final int FULL_STRIPE_SIZE = DATA_BLK_NUM * CELLSIZE;
+ /** number of full stripes in a full block group */
+ private final int BLK_GROUP_STRIPE_NUM = 16;
+ private final ECSchema SCEHMA = ErasureCodingSchemaManager.
+ getSystemDefaultSchema();
+ private final Random random = new Random();
+
+ private int[] blockGroupSizes;
+ private int[] byteRangeStartOffsets;
+ private int[] byteRangeSizes;
+
+ @Before
+ public void setup(){
+ blockGroupSizes = new int[]{1, getDelta(CELLSIZE), CELLSIZE,
+ getDelta(DATA_BLK_NUM) * CELLSIZE,
+ getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
+ FULL_STRIPE_SIZE, FULL_STRIPE_SIZE + getDelta(CELLSIZE),
+ FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE,
+ FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
+ getDelta(BLK_GROUP_STRIPE_NUM) * FULL_STRIPE_SIZE,
+ BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE};
+ byteRangeStartOffsets = new int[] {0, getDelta(CELLSIZE), CELLSIZE - 1};
+ byteRangeSizes = new int[]{1, getDelta(CELLSIZE), CELLSIZE,
+ getDelta(DATA_BLK_NUM) * CELLSIZE,
+ getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
+ FULL_STRIPE_SIZE, FULL_STRIPE_SIZE + getDelta(CELLSIZE),
+ FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE,
+ FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
+ getDelta(BLK_GROUP_STRIPE_NUM) * FULL_STRIPE_SIZE,
+ BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE};
+ }
+
+ private int getDelta(int size) {
+ return 1 + random.nextInt(size - 2);
+ }
+ private byte hashIntToByte(int i) {
+ int BYTE_MASK = 0xff;
+ return (byte) (((i + 13) * 29) & BYTE_MASK);
+ }
+
+ private LocatedStripedBlock createDummyLocatedBlock(int bgSize) {
+ final long blockGroupID = -1048576;
+ DatanodeInfo[] locs = new DatanodeInfo[BLK_GROUP_WIDTH];
+ String[] storageIDs = new String[BLK_GROUP_WIDTH];
+ StorageType[] storageTypes = new StorageType[BLK_GROUP_WIDTH];
+ int[] indices = new int[BLK_GROUP_WIDTH];
+ for (int i = 0; i < BLK_GROUP_WIDTH; i++) {
+ indices[i] = (i + 2) % DATA_BLK_NUM;
+ // Location port always equal to logical index of a block,
+ // for easier verification
+ locs[i] = DFSTestUtil.getLocalDatanodeInfo(indices[i]);
+ storageIDs[i] = locs[i].getDatanodeUuid();
+ storageTypes[i] = StorageType.DISK;
+ }
+ return new LocatedStripedBlock(new ExtendedBlock("pool", blockGroupID,
+ bgSize, 1001), locs, storageIDs, storageTypes, indices, 0, false,
+ null);
+ }
+
+ private byte[][] createInternalBlkBuffers(int bgSize) {
+ byte[][] bufs = new byte[DATA_BLK_NUM + PARITY_BLK_NUM][];
+ int[] pos = new int[DATA_BLK_NUM + PARITY_BLK_NUM];
+ for (int i = 0; i < DATA_BLK_NUM + PARITY_BLK_NUM; i++) {
+ int bufSize = (int) getInternalBlockLength(
+ bgSize, CELLSIZE, DATA_BLK_NUM, i);
+ bufs[i] = new byte[bufSize];
+ pos[i] = 0;
+ }
+ int done = 0;
+ while (done < bgSize) {
+ Preconditions.checkState(done % CELLSIZE == 0);
+ StripingCell cell = new StripingCell(SCEHMA, CELLSIZE, done / CELLSIZE, 0);
+ int idxInStripe = cell.idxInStripe;
+ int size = Math.min(CELLSIZE, bgSize - done);
+ for (int i = 0; i < size; i++) {
+ bufs[idxInStripe][pos[idxInStripe] + i] = hashIntToByte(done + i);
+ }
+ done += size;
+ pos[idxInStripe] += size;
+ }
+
+ return bufs;
+ }
+
+ @Test
+ public void testParseDummyStripedBlock() {
+ LocatedStripedBlock lsb = createDummyLocatedBlock(
+ BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE);
+ LocatedBlock[] blocks = parseStripedBlockGroup(
+ lsb, CELLSIZE, DATA_BLK_NUM, PARITY_BLK_NUM);
+ assertEquals(DATA_BLK_NUM + PARITY_BLK_NUM, blocks.length);
+ for (int i = 0; i < DATA_BLK_NUM; i++) {
+ assertFalse(blocks[i].isStriped());
+ assertEquals(i,
+ BlockIdManager.getBlockIndex(blocks[i].getBlock().getLocalBlock()));
+ assertEquals(0, blocks[i].getStartOffset());
+ assertEquals(1, blocks[i].getLocations().length);
+ assertEquals(i, blocks[i].getLocations()[0].getIpcPort());
+ assertEquals(i, blocks[i].getLocations()[0].getXferPort());
+ }
+ }
+
+ private void verifyInternalBlocks (int numBytesInGroup, int[] expected) {
+ for (int i = 1; i < BLK_GROUP_WIDTH; i++) {
+ assertEquals(expected[i],
+ getInternalBlockLength(numBytesInGroup, CELLSIZE, DATA_BLK_NUM, i));
+ }
+ }
+
+ @Test
+ public void testGetInternalBlockLength () {
+ // A small delta that is smaller than a cell
+ final int delta = 10;
+
+ // Block group is smaller than a cell
+ verifyInternalBlocks(CELLSIZE - delta,
+ new int[] {CELLSIZE - delta, 0, 0, 0, 0, 0,
+ CELLSIZE - delta, CELLSIZE - delta, CELLSIZE - delta});
+
+ // Block group is exactly as large as a cell
+ verifyInternalBlocks(CELLSIZE,
+ new int[] {CELLSIZE, 0, 0, 0, 0, 0,
+ CELLSIZE, CELLSIZE, CELLSIZE});
+
+ // Block group is a little larger than a cell
+ verifyInternalBlocks(CELLSIZE + delta,
+ new int[] {CELLSIZE, delta, 0, 0, 0, 0,
+ CELLSIZE, CELLSIZE, CELLSIZE});
+
+ // Block group contains multiple stripes and ends at stripe boundary
+ verifyInternalBlocks(2 * DATA_BLK_NUM * CELLSIZE,
+ new int[] {2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
+ 2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
+ 2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE});
+
+ // Block group contains multiple stripes and ends at cell boundary
+ // (not ending at stripe boundary)
+ verifyInternalBlocks(2 * DATA_BLK_NUM * CELLSIZE + CELLSIZE,
+ new int[] {3 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
+ 2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
+ 3 * CELLSIZE, 3 * CELLSIZE, 3 * CELLSIZE});
+
+ // Block group contains multiple stripes and doesn't end at cell boundary
+ verifyInternalBlocks(2 * DATA_BLK_NUM * CELLSIZE - delta,
+ new int[] {2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
+ 2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE - delta,
+ 2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE});
+ }
+
+ /**
+ * Test dividing a byte range into aligned stripes and verify the aligned
+ * ranges can be translated back to the byte range.
+ */
+ @Test
+ public void testDivideByteRangeIntoStripes() {
+ byte[] assembled = new byte[BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE];
+ for (int bgSize : blockGroupSizes) {
+ LocatedStripedBlock blockGroup = createDummyLocatedBlock(bgSize);
+ byte[][] internalBlkBufs = createInternalBlkBuffers(bgSize);
+ for (int brStart : byteRangeStartOffsets) {
+ for (int brSize : byteRangeSizes) {
+ if (brStart + brSize > bgSize) {
+ continue;
+ }
+ AlignedStripe[] stripes = divideByteRangeIntoStripes(SCEHMA,
+ CELLSIZE, blockGroup, brStart, brStart + brSize - 1, assembled, 0);
+
+ for (AlignedStripe stripe : stripes) {
+ for (int i = 0; i < DATA_BLK_NUM; i++) {
+ StripingChunk chunk = stripe.chunks[i];
+ if (chunk == null || chunk.state != StripingChunk.REQUESTED) {
+ continue;
+ }
+ int done = 0;
+ for (int j = 0; j < chunk.byteArray.getLengths().length; j++) {
+ System.arraycopy(internalBlkBufs[i],
+ (int) stripe.getOffsetInBlock() + done, assembled,
+ chunk.byteArray.getOffsets()[j],
+ chunk.byteArray.getLengths()[j]);
+ done += chunk.byteArray.getLengths()[j];
+ }
+ }
+ }
+ for (int i = 0; i < brSize; i++) {
+ if (hashIntToByte(brStart + i) != assembled[i]) {
+ System.out.println("Oops");
+ }
+ assertEquals("Byte at " + (brStart + i) + " should be the same",
+ hashIntToByte(brStart + i), assembled[i]);
+ }
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
index 391f190..303d063 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java
@@ -65,7 +65,7 @@ public class TestJsonUtil {
final HdfsFileStatus status = new HdfsFileStatus(1001L, false, 3, 1L << 26,
now, now + 10, new FsPermission((short) 0644), "user", "group",
DFSUtil.string2Bytes("bar"), DFSUtil.string2Bytes("foo"),
- HdfsConstants.GRANDFATHER_INODE_ID, 0, null, (byte) 0);
+ HdfsConstants.GRANDFATHER_INODE_ID, 0, null, (byte) 0, null, 0);
final FileStatus fstatus = toFileStatus(status, parent);
System.out.println("status = " + status);
System.out.println("fstatus = " + fstatus);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
new file mode 100644
index 0000000..70020d5
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -0,0 +1,377 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="testConf.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Normal mode is test. To run just the commands and dump the output
+ to the log, set it to nocompare -->
+ <mode>test</mode>
+
+ <!-- Comparator types:
+ ExactComparator
+ SubstringComparator
+ RegexpComparator
+ TokenComparator
+ -->
+ <tests>
+
+ <!-- Test help options -->
+ <test>
+ <description>help: help for erasure coding command</description>
+ <test-commands>
+ <ec-admin-command>-help</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Usage: hdfs erasurecode [generic options]</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>help: createZone command</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -help createZone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^[ \t]*Create a zone to encode files using a specified schema( )*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-createZone \[-s <schemaName>\] \[-c <cellSize>\] <path>(.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>help: getZone command</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -help getZone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Get information about the EC zone at specified path</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-getZone <path>(.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>help: listSchemas command</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -help listSchemas</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Get the list of ECSchemas supported</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-listSchemas (.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <!-- Test erasure code commands -->
+ <test>
+ <description>createZone : create a zone to encode files</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone -s RS-6-3 /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>EC Zone created successfully at NAMENODE/eczone</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>createZone : create a zone twice</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone /eczone</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -createZone /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Directory /eczone is already in an erasure coding zone</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>createZone : default schema</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone /eczone</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getZone /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Dir: /eczone, Schema: ECSchema=[Name=RS-6-3</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>getZone : get information about the EC zone at specified path not in zone</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /noec</command>
+ <ec-admin-command>-fs NAMENODE -getZone /noec</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /noec</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Path NAMENODE/noec is not in EC zone</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>getZone : get information about the EC zone at specified path</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone -s RS-6-3 /eczone</ec-admin-command>
+ <ec-admin-command>-fs NAMENODE -getZone /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Dir: /eczone, Schema: ECSchema=[Name=RS-6-3</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>getZone : get EC zone at specified file path</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone -s RS-6-3 /eczone</ec-admin-command>
+ <command>-fs NAMENODE -touchz /eczone/ecfile</command>
+ <ec-admin-command>-fs NAMENODE -getZone /eczone/ecfile</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /eczone/ecfile</command>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Dir: /eczone, Schema: ECSchema=[Name=RS-6-3</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>listSchemas : get the list of ECSchemas supported</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -listSchemas</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>RS-6-3</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+<!-- Test illegal parameters -->
+ <test>
+ <description>createZone : illegal parameters - path is missing</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-createZone: <path> is missing(.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>createZone : illegal parameters - schema name is missing</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone -s</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-createZone: option -s requires 1 argument(.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>createZone : illegal parameters - too many arguments</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone /eczone1 /eczone2</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-createZone: Too many arguments</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>createZone : illegal parameters - invalidschema</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /eczone</command>
+ <ec-admin-command>-fs NAMENODE -createZone -s invalidschema /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Schema 'invalidschema' does not match any of the supported schemas. Please select any one of [RS-6-3]</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>createZone : illegal parameters - no such file</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -createZone /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^createZone: `/eczone': No such file or directory(.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>getZone : illegal parameters - path is missing</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -getZone </ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^-getZone: <path> is missing(.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>getZone : illegal parameters - too many arguments</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -getZone /eczone /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /eczone</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-getZone: Too many arguments</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>getZone : illegal parameters - no such file</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -getZone /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^getZone: `/eczone': No such file or directory(.)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>listSchemas : illegal parameters - too many parameters</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -listSchemas /eczone</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-listSchemas: Too many parameters</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ </tests>
+</configuration>
[16/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 508da85..a08bd2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -17,12 +17,12 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX;
import static org.apache.hadoop.util.ExitUtil.terminate;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.BitSet;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
@@ -44,6 +44,7 @@ import javax.management.ObjectName;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -55,10 +56,11 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
@@ -79,6 +81,7 @@ import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.StripedBlockWithLocations;
import org.apache.hadoop.hdfs.server.protocol.DatanodeCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -88,8 +91,14 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
import org.apache.hadoop.metrics2.util.MBeans;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+import static org.apache.hadoop.hdfs.util.StripedBlockUtil.getInternalBlockLength;
+
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.LightWeightGSet;
import org.apache.hadoop.util.Time;
@@ -187,7 +196,11 @@ public class BlockManager implements BlockStatsMXBean {
/** Store blocks -> datanodedescriptor(s) map of corrupt replicas */
final CorruptReplicasMap corruptReplicas = new CorruptReplicasMap();
- /** Blocks to be invalidated. */
+ /**
+ * Blocks to be invalidated.
+ * For a striped block to invalidate, we should track its individual internal
+ * blocks.
+ */
private final InvalidateBlocks invalidateBlocks;
/**
@@ -203,8 +216,8 @@ public class BlockManager implements BlockStatsMXBean {
* Maps a StorageID to the set of blocks that are "extra" for this
* DataNode. We'll eventually remove these extras.
*/
- public final Map<String, LightWeightLinkedSet<Block>> excessReplicateMap =
- new TreeMap<String, LightWeightLinkedSet<Block>>();
+ public final Map<String, LightWeightLinkedSet<BlockInfo>> excessReplicateMap =
+ new TreeMap<>();
/**
* Store set of Blocks that need to be replicated 1 or more times.
@@ -271,12 +284,15 @@ public class BlockManager implements BlockStatsMXBean {
private double replicationQueuesInitProgress = 0.0;
/** for block replicas placement */
- private BlockPlacementPolicy blockplacement;
+ private BlockPlacementPolicies placementPolicies;
private final BlockStoragePolicySuite storagePolicySuite;
/** Check whether name system is running before terminating */
private boolean checkNSRunning = true;
+ /** Check whether there are any non-EC blocks using StripedID */
+ private boolean hasNonEcBlockUsingStripedID = false;
+
public BlockManager(final Namesystem namesystem, final Configuration conf)
throws IOException {
this.namesystem = namesystem;
@@ -292,7 +308,7 @@ public class BlockManager implements BlockStatsMXBean {
// Compute the map capacity by allocating 2% of total memory
blocksMap = new BlocksMap(
LightWeightGSet.computeCapacity(2.0, "BlocksMap"));
- blockplacement = BlockPlacementPolicy.getInstance(
+ placementPolicies = new BlockPlacementPolicies(
conf, datanodeManager.getFSClusterStats(),
datanodeManager.getNetworkTopology(),
datanodeManager.getHost2DatanodeMap());
@@ -494,15 +510,7 @@ public class BlockManager implements BlockStatsMXBean {
@VisibleForTesting
public BlockPlacementPolicy getBlockPlacementPolicy() {
- return blockplacement;
- }
-
- /** Set BlockPlacementPolicy */
- public void setBlockPlacementPolicy(BlockPlacementPolicy newpolicy) {
- if (newpolicy == null) {
- throw new HadoopIllegalArgumentException("newpolicy == null");
- }
- this.blockplacement = newpolicy;
+ return placementPolicies.getPolicy(false);
}
/** Dump meta data to out. */
@@ -552,9 +560,9 @@ public class BlockManager implements BlockStatsMXBean {
NumberReplicas numReplicas = new NumberReplicas();
// source node returned is not used
- chooseSourceDatanode(block, containingNodes,
+ chooseSourceDatanodes(getStoredBlock(block), containingNodes,
containingLiveReplicasNodes, numReplicas,
- UnderReplicatedBlocks.LEVEL);
+ new LinkedList<Short>(), UnderReplicatedBlocks.LEVEL);
// containingLiveReplicasNodes can include READ_ONLY_SHARED replicas which are
// not included in the numReplicas.liveReplicas() count
@@ -601,11 +609,28 @@ public class BlockManager implements BlockStatsMXBean {
return maxReplicationStreams;
}
- /**
- * @return true if the block has minimum replicas
- */
- public boolean checkMinReplication(BlockInfo block) {
- return (countNodes(block).liveReplicas() >= minReplication);
+ public int getDefaultStorageNum(BlockInfo block) {
+ if (block.isStriped()) {
+ return ((BlockInfoStriped) block).getRealTotalBlockNum();
+ } else {
+ return defaultReplication;
+ }
+ }
+
+ public short getMinStorageNum(BlockInfo block) {
+ if (block.isStriped()) {
+ return ((BlockInfoStriped) block).getRealDataBlockNum();
+ } else {
+ return minReplication;
+ }
+ }
+
+ public boolean hasMinStorage(BlockInfo block) {
+ return countNodes(block).liveReplicas() >= getMinStorageNum(block);
+ }
+
+ public boolean hasMinStorage(BlockInfo block, int liveNum) {
+ return liveNum >= getMinStorageNum(block);
}
/**
@@ -617,16 +642,21 @@ public class BlockManager implements BlockStatsMXBean {
* @throws IOException if the block does not have at least a minimal number
* of replicas reported from data-nodes.
*/
- private static boolean commitBlock(
- final BlockInfoContiguousUnderConstruction block, final Block commitBlock)
- throws IOException {
- if (block.getBlockUCState() == BlockUCState.COMMITTED)
- return false;
- assert block.getNumBytes() <= commitBlock.getNumBytes() :
- "commitBlock length is less than the stored one "
- + commitBlock.getNumBytes() + " vs. " + block.getNumBytes();
- block.commitBlock(commitBlock);
- return true;
+ private static boolean commitBlock(final BlockInfo block,
+ final Block commitBlock) throws IOException {
+ if (block instanceof BlockInfoUnderConstruction
+ && block.getBlockUCState() != BlockUCState.COMMITTED) {
+ final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)block;
+
+ assert block.getNumBytes() <= commitBlock.getNumBytes() :
+ "commitBlock length is less than the stored one "
+ + commitBlock.getNumBytes() + " vs. " + block.getNumBytes();
+
+ uc.commitBlock(commitBlock);
+ return true;
+ }
+
+ return false;
}
/**
@@ -649,10 +679,10 @@ public class BlockManager implements BlockStatsMXBean {
if(lastBlock.isComplete())
return false; // already completed (e.g. by syncBlock)
- final boolean b = commitBlock(
- (BlockInfoContiguousUnderConstruction) lastBlock, commitBlock);
- if(countNodes(lastBlock).liveReplicas() >= minReplication)
- completeBlock(bc, bc.numBlocks()-1, false);
+ final boolean b = commitBlock(lastBlock, commitBlock);
+ if (hasMinStorage(lastBlock)) {
+ completeBlock(bc, bc.numBlocks() - 1, false);
+ }
return b;
}
@@ -665,21 +695,28 @@ public class BlockManager implements BlockStatsMXBean {
*/
private BlockInfo completeBlock(final BlockCollection bc,
final int blkIndex, boolean force) throws IOException {
- if(blkIndex < 0)
+ if (blkIndex < 0) {
return null;
+ }
BlockInfo curBlock = bc.getBlocks()[blkIndex];
- if(curBlock.isComplete())
+ if (curBlock.isComplete()) {
return curBlock;
- BlockInfoContiguousUnderConstruction ucBlock =
- (BlockInfoContiguousUnderConstruction) curBlock;
- int numNodes = ucBlock.numNodes();
- if (!force && numNodes < minReplication)
+ }
+
+ int numNodes = curBlock.numNodes();
+ if (!force && !hasMinStorage(curBlock, numNodes)) {
throw new IOException("Cannot complete block: " +
"block does not satisfy minimal replication requirement.");
- if(!force && ucBlock.getBlockUCState() != BlockUCState.COMMITTED)
+ }
+ if (!force && curBlock.getBlockUCState() != BlockUCState.COMMITTED) {
throw new IOException(
"Cannot complete block: block has not been COMMITTED by the client");
- BlockInfo completeBlock = ucBlock.convertToCompleteBlock();
+ }
+
+ final BlockInfo completeBlock
+ = !(curBlock instanceof BlockInfoUnderConstruction)? curBlock
+ : ((BlockInfoUnderConstruction)curBlock).convertToCompleteBlock();
+
// replace penultimate block in file
bc.setBlock(blkIndex, completeBlock);
@@ -690,8 +727,10 @@ public class BlockManager implements BlockStatsMXBean {
// a "forced" completion when a file is getting closed by an
// OP_CLOSE edit on the standby).
namesystem.adjustSafeModeBlockTotals(0, 1);
+ final int minStorage = curBlock.isStriped() ?
+ ((BlockInfoStriped) curBlock).getRealDataBlockNum() : minReplication;
namesystem.incrementSafeBlockCount(
- Math.min(numNodes, minReplication));
+ Math.min(numNodes, minStorage), curBlock);
// replace block in the blocksMap
return blocksMap.replaceBlock(completeBlock);
@@ -700,10 +739,11 @@ public class BlockManager implements BlockStatsMXBean {
private BlockInfo completeBlock(final BlockCollection bc,
final BlockInfo block, boolean force) throws IOException {
BlockInfo[] fileBlocks = bc.getBlocks();
- for(int idx = 0; idx < fileBlocks.length; idx++)
- if(fileBlocks[idx] == block) {
+ for (int idx = 0; idx < fileBlocks.length; idx++) {
+ if (fileBlocks[idx] == block) {
return completeBlock(bc, idx, force);
}
+ }
return block;
}
@@ -713,8 +753,10 @@ public class BlockManager implements BlockStatsMXBean {
* when tailing edit logs as a Standby.
*/
public BlockInfo forceCompleteBlock(final BlockCollection bc,
- final BlockInfoContiguousUnderConstruction block) throws IOException {
- block.commitBlock(block);
+ final BlockInfo block) throws IOException {
+ if (block instanceof BlockInfoUnderConstruction) {
+ ((BlockInfoUnderConstruction)block).commitBlock(block);
+ }
return completeBlock(bc, block, true);
}
@@ -744,8 +786,10 @@ public class BlockManager implements BlockStatsMXBean {
DatanodeStorageInfo[] targets = getStorages(oldBlock);
- BlockInfoContiguousUnderConstruction ucBlock =
- bc.setLastBlock(oldBlock, targets);
+ // convert the last block to UC
+ bc.convertLastBlockToUC(oldBlock, targets);
+ // get the new created uc block
+ BlockInfo ucBlock = bc.getLastBlock();
blocksMap.replaceBlock(ucBlock);
// Remove block from replication queue.
@@ -756,14 +800,17 @@ public class BlockManager implements BlockStatsMXBean {
// remove this block from the list of pending blocks to be deleted.
for (DatanodeStorageInfo storage : targets) {
- invalidateBlocks.remove(storage.getDatanodeDescriptor(), oldBlock);
+ final Block b = getBlockOnStorage(oldBlock, storage);
+ if (b != null) {
+ invalidateBlocks.remove(storage.getDatanodeDescriptor(), b);
+ }
}
// Adjust safe-mode totals, since under-construction blocks don't
// count in safe-mode.
namesystem.adjustSafeModeBlockTotals(
// decrement safe if we had enough
- targets.length >= minReplication ? -1 : 0,
+ hasMinStorage(oldBlock, targets.length) ? -1 : 0,
// always decrement total blocks
-1);
@@ -775,23 +822,25 @@ public class BlockManager implements BlockStatsMXBean {
/**
* Get all valid locations of the block
*/
- private List<DatanodeStorageInfo> getValidLocations(Block block) {
+ private List<DatanodeStorageInfo> getValidLocations(BlockInfo block) {
final List<DatanodeStorageInfo> locations
= new ArrayList<DatanodeStorageInfo>(blocksMap.numNodes(block));
for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
// filter invalidate replicas
- if(!invalidateBlocks.contains(storage.getDatanodeDescriptor(), block)) {
+ Block b = getBlockOnStorage(block, storage);
+ if(b != null &&
+ !invalidateBlocks.contains(storage.getDatanodeDescriptor(), b)) {
locations.add(storage);
}
}
return locations;
}
-
+
private List<LocatedBlock> createLocatedBlockList(
final BlockInfo[] blocks,
final long offset, final long length, final int nrBlocksToReturn,
final AccessMode mode) throws IOException {
- int curBlk = 0;
+ int curBlk;
long curPos = 0, blkSize = 0;
int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length;
for (curBlk = 0; curBlk < nrBlocks; curBlk++) {
@@ -804,10 +853,10 @@ public class BlockManager implements BlockStatsMXBean {
}
if (nrBlocks > 0 && curBlk == nrBlocks) // offset >= end of file
- return Collections.<LocatedBlock>emptyList();
+ return Collections.emptyList();
long endOff = offset + length;
- List<LocatedBlock> results = new ArrayList<LocatedBlock>(blocks.length);
+ List<LocatedBlock> results = new ArrayList<>(blocks.length);
do {
results.add(createLocatedBlock(blocks[curBlk], curPos, mode));
curPos += blocks[curBlk].getNumBytes();
@@ -820,7 +869,7 @@ public class BlockManager implements BlockStatsMXBean {
private LocatedBlock createLocatedBlock(final BlockInfo[] blocks,
final long endPos, final AccessMode mode) throws IOException {
- int curBlk = 0;
+ int curBlk;
long curPos = 0;
int nrBlocks = (blocks[0].getNumBytes() == 0) ? 0 : blocks.length;
for (curBlk = 0; curBlk < nrBlocks; curBlk++) {
@@ -833,7 +882,7 @@ public class BlockManager implements BlockStatsMXBean {
return createLocatedBlock(blocks[curBlk], curPos, mode);
}
-
+
private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos,
final AccessMode mode) throws IOException {
final LocatedBlock lb = createLocatedBlock(blk, pos);
@@ -844,19 +893,25 @@ public class BlockManager implements BlockStatsMXBean {
}
/** @return a LocatedBlock for the given block */
- private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos
- ) throws IOException {
- if (blk instanceof BlockInfoContiguousUnderConstruction) {
- if (blk.isComplete()) {
- throw new IOException(
- "blk instanceof BlockInfoUnderConstruction && blk.isComplete()"
- + ", blk=" + blk);
+ private LocatedBlock createLocatedBlock(final BlockInfo blk, final long pos) {
+ if (!blk.isComplete()) {
+ if (blk.isStriped()) {
+ final BlockInfoStripedUnderConstruction uc =
+ (BlockInfoStripedUnderConstruction) blk;
+ final DatanodeStorageInfo[] storages = uc.getExpectedStorageLocations();
+ final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(),
+ blk);
+ return newLocatedStripedBlock(eb, storages, uc.getBlockIndices(), pos,
+ false);
+ } else {
+ assert blk instanceof BlockInfoContiguousUnderConstruction;
+ final BlockInfoContiguousUnderConstruction uc =
+ (BlockInfoContiguousUnderConstruction) blk;
+ final DatanodeStorageInfo[] storages = uc.getExpectedStorageLocations();
+ final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(),
+ blk);
+ return newLocatedBlock(eb, storages, pos, false);
}
- final BlockInfoContiguousUnderConstruction uc =
- (BlockInfoContiguousUnderConstruction) blk;
- final DatanodeStorageInfo[] storages = uc.getExpectedStorageLocations();
- final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(), blk);
- return newLocatedBlock(eb, storages, pos, false);
}
// get block locations
@@ -873,13 +928,21 @@ public class BlockManager implements BlockStatsMXBean {
numCorruptNodes == numNodes;
final int numMachines = isCorrupt ? numNodes: numNodes - numCorruptNodes;
final DatanodeStorageInfo[] machines = new DatanodeStorageInfo[numMachines];
- int j = 0;
+ final int[] blockIndices = blk.isStriped() ? new int[numMachines] : null;
+ int j = 0, i = 0;
if (numMachines > 0) {
for(DatanodeStorageInfo storage : blocksMap.getStorages(blk)) {
final DatanodeDescriptor d = storage.getDatanodeDescriptor();
final boolean replicaCorrupt = corruptReplicas.isReplicaCorrupt(blk, d);
- if (isCorrupt || (!replicaCorrupt))
+ if (isCorrupt || (!replicaCorrupt)) {
machines[j++] = storage;
+ // TODO this can be more efficient
+ if (blockIndices != null) {
+ int index = ((BlockInfoStriped) blk).getStorageBlockIndex(storage);
+ assert index >= 0;
+ blockIndices[i++] = index;
+ }
+ }
}
}
assert j == machines.length :
@@ -889,7 +952,9 @@ public class BlockManager implements BlockStatsMXBean {
" numCorrupt: " + numCorruptNodes +
" numCorruptRepls: " + numCorruptReplicas;
final ExtendedBlock eb = new ExtendedBlock(namesystem.getBlockPoolId(), blk);
- return newLocatedBlock(eb, machines, pos, isCorrupt);
+ return blockIndices == null ?
+ newLocatedBlock(eb, machines, pos, isCorrupt) :
+ newLocatedStripedBlock(eb, machines, blockIndices, pos, isCorrupt);
}
/** Create a LocatedBlocks. */
@@ -897,14 +962,18 @@ public class BlockManager implements BlockStatsMXBean {
final long fileSizeExcludeBlocksUnderConstruction,
final boolean isFileUnderConstruction, final long offset,
final long length, final boolean needBlockToken,
- final boolean inSnapshot, FileEncryptionInfo feInfo)
+ final boolean inSnapshot, FileEncryptionInfo feInfo,
+ ErasureCodingZone ecZone)
throws IOException {
assert namesystem.hasReadLock();
+ final ECSchema schema = ecZone != null ? ecZone.getSchema() : null;
+ final int cellSize = ecZone != null ? ecZone.getCellSize() : 0;
if (blocks == null) {
return null;
} else if (blocks.length == 0) {
return new LocatedBlocks(0, isFileUnderConstruction,
- Collections.<LocatedBlock>emptyList(), null, false, feInfo);
+ Collections.<LocatedBlock> emptyList(), null, false, feInfo, schema,
+ cellSize);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("blocks = " + java.util.Arrays.asList(blocks));
@@ -927,9 +996,9 @@ public class BlockManager implements BlockStatsMXBean {
fileSizeExcludeBlocksUnderConstruction, mode);
isComplete = true;
}
- return new LocatedBlocks(
- fileSizeExcludeBlocksUnderConstruction, isFileUnderConstruction,
- locatedblocks, lastlb, isComplete, feInfo);
+ return new LocatedBlocks(fileSizeExcludeBlocksUnderConstruction,
+ isFileUnderConstruction, locatedblocks, lastlb, isComplete, feInfo,
+ schema, cellSize);
}
}
@@ -944,9 +1013,23 @@ public class BlockManager implements BlockStatsMXBean {
final AccessMode mode) throws IOException {
if (isBlockTokenEnabled()) {
// Use cached UGI if serving RPC calls.
- b.setBlockToken(blockTokenSecretManager.generateToken(
- NameNode.getRemoteUser().getShortUserName(),
- b.getBlock(), EnumSet.of(mode)));
+ if (b.isStriped()) {
+ LocatedStripedBlock sb = (LocatedStripedBlock) b;
+ int[] indices = sb.getBlockIndices();
+ Token<BlockTokenIdentifier>[] blockTokens = new Token[indices.length];
+ ExtendedBlock internalBlock = new ExtendedBlock(b.getBlock());
+ for (int i = 0; i < indices.length; i++) {
+ internalBlock.setBlockId(b.getBlock().getBlockId() + indices[i]);
+ blockTokens[i] = blockTokenSecretManager.generateToken(
+ NameNode.getRemoteUser().getShortUserName(),
+ internalBlock, EnumSet.of(mode));
+ }
+ sb.setBlockTokens(blockTokens);
+ } else {
+ b.setBlockToken(blockTokenSecretManager.generateToken(
+ NameNode.getRemoteUser().getShortUserName(),
+ b.getBlock(), EnumSet.of(mode)));
+ }
}
}
@@ -1077,7 +1160,7 @@ public class BlockManager implements BlockStatsMXBean {
/** Remove the blocks associated to the given datanode. */
void removeBlocksAssociatedTo(final DatanodeDescriptor node) {
- final Iterator<? extends Block> it = node.getBlockIterator();
+ final Iterator<BlockInfo> it = node.getBlockIterator();
while(it.hasNext()) {
removeStoredBlock(it.next(), node);
}
@@ -1091,12 +1174,15 @@ public class BlockManager implements BlockStatsMXBean {
/** Remove the blocks associated to the given DatanodeStorageInfo. */
void removeBlocksAssociatedTo(final DatanodeStorageInfo storageInfo) {
assert namesystem.hasWriteLock();
- final Iterator<? extends Block> it = storageInfo.getBlockIterator();
+ final Iterator<BlockInfo> it = storageInfo.getBlockIterator();
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
while(it.hasNext()) {
- Block block = it.next();
+ BlockInfo block = it.next();
removeStoredBlock(block, node);
- invalidateBlocks.remove(node, block);
+ final Block b = getBlockOnStorage(block, storageInfo);
+ if (b != null) {
+ invalidateBlocks.remove(node, b);
+ }
}
namesystem.checkSafeMode();
}
@@ -1116,22 +1202,32 @@ public class BlockManager implements BlockStatsMXBean {
* Adds block to list of blocks which will be invalidated on all its
* datanodes.
*/
- private void addToInvalidates(Block b) {
+ private void addToInvalidates(BlockInfo storedBlock) {
if (!namesystem.isPopulatingReplQueues()) {
return;
}
StringBuilder datanodes = new StringBuilder();
- for(DatanodeStorageInfo storage : blocksMap.getStorages(b, State.NORMAL)) {
+ for(DatanodeStorageInfo storage : blocksMap.getStorages(storedBlock,
+ State.NORMAL)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
- invalidateBlocks.add(b, node, false);
- datanodes.append(node).append(" ");
+ final Block b = getBlockOnStorage(storedBlock, storage);
+ if (b != null) {
+ invalidateBlocks.add(b, node, false);
+ datanodes.append(node).append(" ");
+ }
}
if (datanodes.length() != 0) {
- blockLog.debug("BLOCK* addToInvalidates: {} {}", b,
+ blockLog.info("BLOCK* addToInvalidates: {} {}", storedBlock,
datanodes.toString());
}
}
+ private Block getBlockOnStorage(BlockInfo storedBlock,
+ DatanodeStorageInfo storage) {
+ return storedBlock.isStriped() ?
+ ((BlockInfoStriped) storedBlock).getBlockOnStorage(storage) : storedBlock;
+ }
+
/**
* Remove all block invalidation tasks under this datanode UUID;
* used when a datanode registers with a new UUID and the old one
@@ -1155,7 +1251,8 @@ public class BlockManager implements BlockStatsMXBean {
public void findAndMarkBlockAsCorrupt(final ExtendedBlock blk,
final DatanodeInfo dn, String storageID, String reason) throws IOException {
assert namesystem.hasWriteLock();
- final BlockInfo storedBlock = getStoredBlock(blk.getLocalBlock());
+ final Block reportedBlock = blk.getLocalBlock();
+ final BlockInfo storedBlock = getStoredBlock(reportedBlock);
if (storedBlock == null) {
// Check if the replica is in the blockMap, if not
// ignore the request for now. This could happen when BlockScanner
@@ -1172,45 +1269,53 @@ public class BlockManager implements BlockStatsMXBean {
+ ") does not exist");
}
- markBlockAsCorrupt(new BlockToMarkCorrupt(storedBlock,
+ markBlockAsCorrupt(new BlockToMarkCorrupt(reportedBlock, storedBlock,
blk.getGenerationStamp(), reason, Reason.CORRUPTION_REPORTED),
storageID == null ? null : node.getStorageInfo(storageID),
node);
}
/**
- *
- * @param b
+ * Mark a replica (of a contiguous block) or an internal block (of a striped
+ * block group) as corrupt.
+ * @param b Indicating the reported bad block and the corresponding BlockInfo
+ * stored in blocksMap.
* @param storageInfo storage that contains the block, if known. null otherwise.
- * @throws IOException
*/
private void markBlockAsCorrupt(BlockToMarkCorrupt b,
DatanodeStorageInfo storageInfo,
DatanodeDescriptor node) throws IOException {
- if (b.corrupted.isDeleted()) {
- blockLog.debug("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
+ if (b.stored.isDeleted()) {
+ blockLog.info("BLOCK markBlockAsCorrupt: {} cannot be marked as" +
" corrupt as it does not belong to any file", b);
addToInvalidates(b.corrupted, node);
return;
}
short expectedReplicas =
- b.corrupted.getBlockCollection().getPreferredBlockReplication();
+ getExpectedReplicaNum(b.stored.getBlockCollection(), b.stored);
// Add replica to the data-node if it is not already there
if (storageInfo != null) {
- storageInfo.addBlock(b.stored);
+ storageInfo.addBlock(b.stored, b.corrupted);
}
- // Add this replica to corruptReplicas Map
- corruptReplicas.addToCorruptReplicasMap(b.corrupted, node, b.reason,
+ // Add this replica to corruptReplicas Map. For striped blocks, we always
+ // use the id of whole striped block group when adding to corruptReplicas
+ Block corrupted = new Block(b.corrupted);
+ if (b.stored.isStriped()) {
+ corrupted.setBlockId(b.stored.getBlockId());
+ }
+ corruptReplicas.addToCorruptReplicasMap(corrupted, node, b.reason,
b.reasonCode);
NumberReplicas numberOfReplicas = countNodes(b.stored);
boolean hasEnoughLiveReplicas = numberOfReplicas.liveReplicas() >=
expectedReplicas;
- boolean minReplicationSatisfied =
- numberOfReplicas.liveReplicas() >= minReplication;
+
+ boolean minReplicationSatisfied = hasMinStorage(b.stored,
+ numberOfReplicas.liveReplicas());
+
boolean hasMoreCorruptReplicas = minReplicationSatisfied &&
(numberOfReplicas.liveReplicas() + numberOfReplicas.corruptReplicas()) >
expectedReplicas;
@@ -1225,7 +1330,7 @@ public class BlockManager implements BlockStatsMXBean {
if (hasEnoughLiveReplicas || hasMoreCorruptReplicas
|| corruptedDuringWrite) {
// the block is over-replicated so invalidate the replicas immediately
- invalidateBlock(b, node);
+ invalidateBlock(b, node, numberOfReplicas);
} else if (namesystem.isPopulatingReplQueues()) {
// add the block to neededReplication
updateNeededReplications(b.stored, -1, 0);
@@ -1237,8 +1342,8 @@ public class BlockManager implements BlockStatsMXBean {
* @return true if the block was successfully invalidated and no longer
* present in the BlocksMap
*/
- private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn
- ) throws IOException {
+ private boolean invalidateBlock(BlockToMarkCorrupt b, DatanodeInfo dn,
+ NumberReplicas nr) throws IOException {
blockLog.debug("BLOCK* invalidateBlock: {} on {}", b, dn);
DatanodeDescriptor node = getDatanodeManager().getDatanode(dn);
if (node == null) {
@@ -1247,7 +1352,6 @@ public class BlockManager implements BlockStatsMXBean {
}
// Check how many copies we have of the block
- NumberReplicas nr = countNodes(b.stored);
if (nr.replicasOnStaleNodes() > 0) {
blockLog.debug("BLOCK* invalidateBlocks: postponing " +
"invalidation of {} on {} because {} replica(s) are located on " +
@@ -1318,15 +1422,15 @@ public class BlockManager implements BlockStatsMXBean {
}
/**
- * Scan blocks in {@link #neededReplications} and assign replication
- * work to data-nodes they belong to.
+ * Scan blocks in {@link #neededReplications} and assign recovery
+ * (replication or erasure coding) work to data-nodes they belong to.
*
* The number of process blocks equals either twice the number of live
* data-nodes or the number of under-replicated blocks whichever is less.
*
* @return number of blocks scheduled for replication during this iteration.
*/
- int computeReplicationWork(int blocksToProcess) {
+ int computeBlockRecoveryWork(int blocksToProcess) {
List<List<BlockInfo>> blocksToReplicate = null;
namesystem.writeLock();
try {
@@ -1336,30 +1440,32 @@ public class BlockManager implements BlockStatsMXBean {
} finally {
namesystem.writeUnlock();
}
- return computeReplicationWorkForBlocks(blocksToReplicate);
+ return computeRecoveryWorkForBlocks(blocksToReplicate);
}
- /** Replicate a set of blocks
+ /**
+ * Recover a set of blocks to full strength through replication or
+ * erasure coding
*
- * @param blocksToReplicate blocks to be replicated, for each priority
+ * @param blocksToRecover blocks to be recovered, for each priority
* @return the number of blocks scheduled for replication
*/
@VisibleForTesting
- int computeReplicationWorkForBlocks(List<List<BlockInfo>> blocksToReplicate) {
+ int computeRecoveryWorkForBlocks(List<List<BlockInfo>> blocksToRecover) {
int requiredReplication, numEffectiveReplicas;
List<DatanodeDescriptor> containingNodes;
- DatanodeDescriptor srcNode;
- BlockCollection bc = null;
+ BlockCollection bc;
int additionalReplRequired;
int scheduledWork = 0;
- List<ReplicationWork> work = new LinkedList<ReplicationWork>();
+ List<BlockRecoveryWork> recovWork = new LinkedList<>();
+ // Step 1: categorize at-risk blocks into replication and EC tasks
namesystem.writeLock();
try {
synchronized (neededReplications) {
- for (int priority = 0; priority < blocksToReplicate.size(); priority++) {
- for (BlockInfo block : blocksToReplicate.get(priority)) {
+ for (int priority = 0; priority < blocksToRecover.size(); priority++) {
+ for (BlockInfo block : blocksToRecover.get(priority)) {
// block should belong to a file
bc = blocksMap.getBlockCollection(block);
// abandoned block or block reopened for append
@@ -1370,31 +1476,34 @@ public class BlockManager implements BlockStatsMXBean {
continue;
}
- requiredReplication = bc.getPreferredBlockReplication();
+ requiredReplication = getExpectedReplicaNum(bc, block);
// get a source data-node
- containingNodes = new ArrayList<DatanodeDescriptor>();
- List<DatanodeStorageInfo> liveReplicaNodes = new ArrayList<DatanodeStorageInfo>();
+ containingNodes = new ArrayList<>();
+ List<DatanodeStorageInfo> liveReplicaNodes = new ArrayList<>();
NumberReplicas numReplicas = new NumberReplicas();
- srcNode = chooseSourceDatanode(
- block, containingNodes, liveReplicaNodes, numReplicas,
- priority);
- if(srcNode == null) { // block can not be replicated from any node
- LOG.debug("Block " + block + " cannot be repl from any node");
+ List<Short> liveBlockIndices = new ArrayList<>();
+ final DatanodeDescriptor[] srcNodes = chooseSourceDatanodes(block,
+ containingNodes, liveReplicaNodes, numReplicas,
+ liveBlockIndices, priority);
+ if(srcNodes == null || srcNodes.length == 0) {
+ // block can not be replicated from any node
+ LOG.debug("Block " + block + " cannot be recovered " +
+ "from any node");
continue;
}
- // liveReplicaNodes can include READ_ONLY_SHARED replicas which are
+ // liveReplicaNodes can include READ_ONLY_SHARED replicas which are
// not included in the numReplicas.liveReplicas() count
assert liveReplicaNodes.size() >= numReplicas.liveReplicas();
// do not schedule more if enough replicas is already pending
numEffectiveReplicas = numReplicas.liveReplicas() +
pendingReplications.getNumReplicas(block);
-
+
if (numEffectiveReplicas >= requiredReplication) {
if ( (pendingReplications.getNumReplicas(block) > 0) ||
- (blockHasEnoughRacks(block)) ) {
+ (blockHasEnoughRacks(block, requiredReplication)) ) {
neededReplications.remove(block, priority); // remove from neededReplications
blockLog.debug("BLOCK* Removing {} from neededReplications as" +
" it has enough replicas", block);
@@ -1408,9 +1517,20 @@ public class BlockManager implements BlockStatsMXBean {
} else {
additionalReplRequired = 1; // Needed on a new rack
}
- work.add(new ReplicationWork(block, bc, srcNode,
- containingNodes, liveReplicaNodes, additionalReplRequired,
- priority));
+ if (block.isStriped()) {
+ short[] indices = new short[liveBlockIndices.size()];
+ for (int i = 0 ; i < liveBlockIndices.size(); i++) {
+ indices[i] = liveBlockIndices.get(i);
+ }
+ ErasureCodingWork ecw = new ErasureCodingWork(block, bc, srcNodes,
+ containingNodes, liveReplicaNodes, additionalReplRequired,
+ priority, indices);
+ recovWork.add(ecw);
+ } else {
+ recovWork.add(new ReplicationWork(block, bc, srcNodes,
+ containingNodes, liveReplicaNodes, additionalReplRequired,
+ priority));
+ }
}
}
}
@@ -1418,8 +1538,9 @@ public class BlockManager implements BlockStatsMXBean {
namesystem.writeUnlock();
}
+ // Step 2: choose target nodes for each recovery task
final Set<Node> excludedNodes = new HashSet<Node>();
- for(ReplicationWork rw : work){
+ for(BlockRecoveryWork rw : recovWork){
// Exclude all of the containing nodes from being targets.
// This list includes decommissioning or corrupt nodes.
excludedNodes.clear();
@@ -1430,12 +1551,15 @@ public class BlockManager implements BlockStatsMXBean {
// choose replication targets: NOT HOLDING THE GLOBAL LOCK
// It is costly to extract the filename for which chooseTargets is called,
// so for now we pass in the block collection itself.
- rw.chooseTargets(blockplacement, storagePolicySuite, excludedNodes);
+ final BlockPlacementPolicy placementPolicy =
+ placementPolicies.getPolicy(rw.block.isStriped());
+ rw.chooseTargets(placementPolicy, storagePolicySuite, excludedNodes);
}
+ // Step 3: add tasks to the DN
namesystem.writeLock();
try {
- for(ReplicationWork rw : work){
+ for(BlockRecoveryWork rw : recovWork){
final DatanodeStorageInfo[] targets = rw.targets;
if(targets == null || targets.length == 0){
rw.targets = null;
@@ -1454,7 +1578,7 @@ public class BlockManager implements BlockStatsMXBean {
rw.targets = null;
continue;
}
- requiredReplication = bc.getPreferredBlockReplication();
+ requiredReplication = getExpectedReplicaNum(bc, block);
// do not schedule more if enough replicas is already pending
NumberReplicas numReplicas = countNodes(block);
@@ -1463,7 +1587,7 @@ public class BlockManager implements BlockStatsMXBean {
if (numEffectiveReplicas >= requiredReplication) {
if ( (pendingReplications.getNumReplicas(block) > 0) ||
- (blockHasEnoughRacks(block)) ) {
+ (blockHasEnoughRacks(block, requiredReplication)) ) {
neededReplications.remove(block, priority); // remove from neededReplications
rw.targets = null;
blockLog.debug("BLOCK* Removing {} from neededReplications as" +
@@ -1473,8 +1597,8 @@ public class BlockManager implements BlockStatsMXBean {
}
if ( (numReplicas.liveReplicas() >= requiredReplication) &&
- (!blockHasEnoughRacks(block)) ) {
- if (rw.srcNode.getNetworkLocation().equals(
+ (!blockHasEnoughRacks(block, requiredReplication)) ) {
+ if (rw.srcNodes[0].getNetworkLocation().equals(
targets[0].getDatanodeDescriptor().getNetworkLocation())) {
//No use continuing, unless a new rack in this case
continue;
@@ -1482,7 +1606,32 @@ public class BlockManager implements BlockStatsMXBean {
}
// Add block to the to be replicated list
- rw.srcNode.addBlockToBeReplicated(block, targets);
+ if (block.isStriped()) {
+ assert rw instanceof ErasureCodingWork;
+ assert rw.targets.length > 0;
+ String src = block.getBlockCollection().getName();
+ ErasureCodingZone ecZone = null;
+ try {
+ ecZone = namesystem.getErasureCodingZoneForPath(src);
+ } catch (IOException e) {
+ blockLog
+ .warn("Failed to get the EC zone for the file {} ", src);
+ }
+ if (ecZone == null) {
+ blockLog.warn("No EC schema found for the file {}. "
+ + "So cannot proceed for recovery", src);
+ // TODO: we may have to revisit later for what we can do better to
+ // handle this case.
+ continue;
+ }
+ rw.targets[0].getDatanodeDescriptor().addBlockToBeErasureCoded(
+ new ExtendedBlock(namesystem.getBlockPoolId(), block),
+ rw.srcNodes, rw.targets,
+ ((ErasureCodingWork) rw).liveBlockIndicies,
+ ecZone.getSchema(), ecZone.getCellSize());
+ } else {
+ rw.srcNodes[0].addBlockToBeReplicated(block, targets);
+ }
scheduledWork++;
DatanodeStorageInfo.incrementBlocksScheduled(targets);
@@ -1506,15 +1655,15 @@ public class BlockManager implements BlockStatsMXBean {
if (blockLog.isInfoEnabled()) {
// log which blocks have been scheduled for replication
- for(ReplicationWork rw : work){
+ for(BlockRecoveryWork rw : recovWork){
DatanodeStorageInfo[] targets = rw.targets;
if (targets != null && targets.length != 0) {
StringBuilder targetList = new StringBuilder("datanode(s)");
- for (int k = 0; k < targets.length; k++) {
+ for (DatanodeStorageInfo target : targets) {
targetList.append(' ');
- targetList.append(targets[k].getDatanodeDescriptor());
+ targetList.append(target.getDatanodeDescriptor());
}
- blockLog.debug("BLOCK* ask {} to replicate {} to {}", rw.srcNode,
+ blockLog.debug("BLOCK* ask {} to replicate {} to {}", rw.srcNodes,
rw.block, targetList);
}
}
@@ -1530,7 +1679,7 @@ public class BlockManager implements BlockStatsMXBean {
/** Choose target for WebHDFS redirection. */
public DatanodeStorageInfo[] chooseTarget4WebHDFS(String src,
DatanodeDescriptor clientnode, Set<Node> excludes, long blocksize) {
- return blockplacement.chooseTarget(src, 1, clientnode,
+ return placementPolicies.getPolicy(false).chooseTarget(src, 1, clientnode,
Collections.<DatanodeStorageInfo>emptyList(), false, excludes,
blocksize, storagePolicySuite.getDefaultPolicy());
}
@@ -1542,9 +1691,10 @@ public class BlockManager implements BlockStatsMXBean {
List<DatanodeStorageInfo> chosen,
Set<Node> excludes,
long blocksize,
- byte storagePolicyID) {
-
+ byte storagePolicyID,
+ boolean isStriped) {
final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID);
+ final BlockPlacementPolicy blockplacement = placementPolicies.getPolicy(isStriped);
return blockplacement.chooseTarget(src, numAdditionalNodes, clientnode,
chosen, true, excludes, blocksize, storagePolicy);
}
@@ -1562,10 +1712,12 @@ public class BlockManager implements BlockStatsMXBean {
final Set<Node> excludedNodes,
final long blocksize,
final List<String> favoredNodes,
- final byte storagePolicyID) throws IOException {
+ final byte storagePolicyID,
+ final boolean isStriped) throws IOException {
List<DatanodeDescriptor> favoredDatanodeDescriptors =
getDatanodeDescriptors(favoredNodes);
final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(storagePolicyID);
+ final BlockPlacementPolicy blockplacement = placementPolicies.getPolicy(isStriped);
final DatanodeStorageInfo[] targets = blockplacement.chooseTarget(src,
numOfReplicas, client, excludedNodes, blocksize,
favoredDatanodeDescriptors, storagePolicy);
@@ -1600,55 +1752,59 @@ public class BlockManager implements BlockStatsMXBean {
}
/**
- * Parse the data-nodes the block belongs to and choose one,
- * which will be the replication source.
+ * Parse the data-nodes the block belongs to and choose a certain number
+ * from them to be the recovery sources.
*
* We prefer nodes that are in DECOMMISSION_INPROGRESS state to other nodes
* since the former do not have write traffic and hence are less busy.
* We do not use already decommissioned nodes as a source.
- * Otherwise we choose a random node among those that did not reach their
- * replication limits. However, if the replication is of the highest priority
- * and all nodes have reached their replication limits, we will choose a
- * random node despite the replication limit.
+ * Otherwise we randomly choose nodes among those that did not reach their
+ * replication limits. However, if the recovery work is of the highest
+ * priority and all nodes have reached their replication limits, we will
+ * randomly choose the desired number of nodes despite the replication limit.
*
* In addition form a list of all nodes containing the block
* and calculate its replication numbers.
*
* @param block Block for which a replication source is needed
- * @param containingNodes List to be populated with nodes found to contain the
- * given block
- * @param nodesContainingLiveReplicas List to be populated with nodes found to
- * contain live replicas of the given block
- * @param numReplicas NumberReplicas instance to be initialized with the
- * counts of live, corrupt, excess, and
- * decommissioned replicas of the given
- * block.
+ * @param containingNodes List to be populated with nodes found to contain
+ * the given block
+ * @param nodesContainingLiveReplicas List to be populated with nodes found
+ * to contain live replicas of the given
+ * block
+ * @param numReplicas NumberReplicas instance to be initialized with the
+ * counts of live, corrupt, excess, and decommissioned
+ * replicas of the given block.
+ * @param liveBlockIndices List to be populated with indices of healthy
+ * blocks in a striped block group
* @param priority integer representing replication priority of the given
* block
- * @return the DatanodeDescriptor of the chosen node from which to replicate
- * the given block
- */
- @VisibleForTesting
- DatanodeDescriptor chooseSourceDatanode(Block block,
- List<DatanodeDescriptor> containingNodes,
- List<DatanodeStorageInfo> nodesContainingLiveReplicas,
- NumberReplicas numReplicas,
- int priority) {
+ * @return the array of DatanodeDescriptor of the chosen nodes from which to
+ * recover the given block
+ */
+ @VisibleForTesting
+ DatanodeDescriptor[] chooseSourceDatanodes(BlockInfo block,
+ List<DatanodeDescriptor> containingNodes,
+ List<DatanodeStorageInfo> nodesContainingLiveReplicas,
+ NumberReplicas numReplicas,
+ List<Short> liveBlockIndices, int priority) {
containingNodes.clear();
nodesContainingLiveReplicas.clear();
- DatanodeDescriptor srcNode = null;
+ List<DatanodeDescriptor> srcNodes = new ArrayList<>();
int live = 0;
int decommissioned = 0;
int decommissioning = 0;
int corrupt = 0;
int excess = 0;
-
+ liveBlockIndices.clear();
+ final boolean isStriped = block.isStriped();
+
Collection<DatanodeDescriptor> nodesCorrupt = corruptReplicas.getNodes(block);
- for(DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
+ for (DatanodeStorageInfo storage : blocksMap.getStorages(block)) {
final DatanodeDescriptor node = storage.getDatanodeDescriptor();
- LightWeightLinkedSet<Block> excessBlocks =
+ LightWeightLinkedSet<BlockInfo> excessBlocks =
excessReplicateMap.get(node.getDatanodeUuid());
- int countableReplica = storage.getState() == State.NORMAL ? 1 : 0;
+ int countableReplica = storage.getState() == State.NORMAL ? 1 : 0;
if ((nodesCorrupt != null) && (nodesCorrupt.contains(node)))
corrupt += countableReplica;
else if (node.isDecommissionInProgress()) {
@@ -1683,21 +1839,25 @@ public class BlockManager implements BlockStatsMXBean {
if(node.isDecommissioned())
continue;
- // We got this far, current node is a reasonable choice
- if (srcNode == null) {
- srcNode = node;
+ if(isStriped || srcNodes.isEmpty()) {
+ srcNodes.add(node);
+ if (isStriped) {
+ liveBlockIndices.add((short) ((BlockInfoStriped) block).
+ getStorageBlockIndex(storage));
+ }
continue;
}
- // switch to a different node randomly
+ // for replicated block, switch to a different node randomly
// this to prevent from deterministically selecting the same node even
// if the node failed to replicate the block on previous iterations
- if(ThreadLocalRandom.current().nextBoolean())
- srcNode = node;
+ if (!isStriped && ThreadLocalRandom.current().nextBoolean()) {
+ srcNodes.set(0, node);
+ }
}
if(numReplicas != null)
numReplicas.initialize(live, decommissioned, decommissioning, corrupt,
excess, 0);
- return srcNode;
+ return srcNodes.toArray(new DatanodeDescriptor[srcNodes.size()]);
}
/**
@@ -1761,25 +1921,41 @@ public class BlockManager implements BlockStatsMXBean {
* reported by the datanode in the block report.
*/
static class StatefulBlockInfo {
- final BlockInfoContiguousUnderConstruction storedBlock;
+ final BlockInfo storedBlock; // should be UC block
final Block reportedBlock;
final ReplicaState reportedState;
- StatefulBlockInfo(BlockInfoContiguousUnderConstruction storedBlock,
+ StatefulBlockInfo(BlockInfo storedBlock,
Block reportedBlock, ReplicaState reportedState) {
+ Preconditions.checkArgument(
+ storedBlock instanceof BlockInfoContiguousUnderConstruction ||
+ storedBlock instanceof BlockInfoStripedUnderConstruction);
this.storedBlock = storedBlock;
this.reportedBlock = reportedBlock;
this.reportedState = reportedState;
}
}
-
+
+ private static class BlockInfoToAdd {
+ final BlockInfo stored;
+ final Block reported;
+
+ BlockInfoToAdd(BlockInfo stored, Block reported) {
+ this.stored = stored;
+ this.reported = reported;
+ }
+ }
+
/**
* BlockToMarkCorrupt is used to build the "toCorrupt" list, which is a
* list of blocks that should be considered corrupt due to a block report.
*/
private static class BlockToMarkCorrupt {
- /** The corrupted block in a datanode. */
- final BlockInfo corrupted;
+ /**
+ * The corrupted block in a datanode. This is the one reported by the
+ * datanode.
+ */
+ final Block corrupted;
/** The corresponding block stored in the BlockManager. */
final BlockInfo stored;
/** The reason to mark corrupt. */
@@ -1787,8 +1963,7 @@ public class BlockManager implements BlockStatsMXBean {
/** The reason code to be stored */
final Reason reasonCode;
- BlockToMarkCorrupt(BlockInfo corrupted,
- BlockInfo stored, String reason,
+ BlockToMarkCorrupt(Block corrupted, BlockInfo stored, String reason,
Reason reasonCode) {
Preconditions.checkNotNull(corrupted, "corrupted is null");
Preconditions.checkNotNull(stored, "stored is null");
@@ -1799,15 +1974,9 @@ public class BlockManager implements BlockStatsMXBean {
this.reasonCode = reasonCode;
}
- BlockToMarkCorrupt(BlockInfo stored, String reason,
- Reason reasonCode) {
- this(stored, stored, reason, reasonCode);
- }
-
- BlockToMarkCorrupt(BlockInfo stored, long gs, String reason,
- Reason reasonCode) {
- this(new BlockInfoContiguous((BlockInfoContiguous)stored), stored,
- reason, reasonCode);
+ BlockToMarkCorrupt(Block corrupted, BlockInfo stored, long gs,
+ String reason, Reason reasonCode) {
+ this(corrupted, stored, reason, reasonCode);
//the corrupted block in datanode has a different generation stamp
corrupted.setGenerationStamp(gs);
}
@@ -1921,8 +2090,8 @@ public class BlockManager implements BlockStatsMXBean {
metrics.addBlockReport((int) (endTime - startTime));
}
blockLog.info("BLOCK* processReport: from storage {} node {}, " +
- "blocks: {}, hasStaleStorage: {}, processing time: {} msecs", storage
- .getStorageID(), nodeID, newReport.getNumberOfBlocks(),
+ "blocks: {}, hasStaleStorage: {}, processing time: {} msecs", storage
+ .getStorageID(), nodeID, newReport.getNumberOfBlocks(),
node.hasStaleStorages(), (endTime - startTime));
return !node.hasStaleStorages();
}
@@ -1944,13 +2113,16 @@ public class BlockManager implements BlockStatsMXBean {
// more than one storage on a datanode (and because it's a difficult
// assumption to really enforce)
removeStoredBlock(block, zombie.getDatanodeDescriptor());
- invalidateBlocks.remove(zombie.getDatanodeDescriptor(), block);
+ Block b = getBlockOnStorage(block, zombie);
+ if (b != null) {
+ invalidateBlocks.remove(zombie.getDatanodeDescriptor(), b);
+ }
}
assert(zombie.numBlocks() == 0);
LOG.warn("processReport 0x{}: removed {} replicas from storage {}, " +
"which no longer exists on the DataNode.",
- Long.toHexString(context.getReportId()), prevBlocks,
- zombie.getStorageID());
+ Long.toHexString(context.getReportId()), prevBlocks,
+ zombie.getStorageID());
}
/**
@@ -1993,8 +2165,7 @@ public class BlockManager implements BlockStatsMXBean {
if (i >= blocksPerRescan) {
break;
}
-
- BlockInfo bi = blocksMap.getStoredBlock(b);
+ BlockInfo bi = getStoredBlock(b);
if (bi == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("BLOCK* rescanPostponedMisreplicatedBlocks: " +
@@ -2034,11 +2205,11 @@ public class BlockManager implements BlockStatsMXBean {
// Modify the (block-->datanode) map, according to the difference
// between the old and new block report.
//
- Collection<BlockInfo> toAdd = new LinkedList<BlockInfo>();
- Collection<Block> toRemove = new TreeSet<Block>();
- Collection<Block> toInvalidate = new LinkedList<Block>();
- Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<BlockToMarkCorrupt>();
- Collection<StatefulBlockInfo> toUC = new LinkedList<StatefulBlockInfo>();
+ Collection<BlockInfoToAdd> toAdd = new LinkedList<>();
+ Collection<BlockInfo> toRemove = new TreeSet<>();
+ Collection<Block> toInvalidate = new LinkedList<>();
+ Collection<BlockToMarkCorrupt> toCorrupt = new LinkedList<>();
+ Collection<StatefulBlockInfo> toUC = new LinkedList<>();
reportDiff(storageInfo, report,
toAdd, toRemove, toInvalidate, toCorrupt, toUC);
@@ -2047,12 +2218,13 @@ public class BlockManager implements BlockStatsMXBean {
for (StatefulBlockInfo b : toUC) {
addStoredBlockUnderConstruction(b, storageInfo);
}
- for (Block b : toRemove) {
+ for (BlockInfo b : toRemove) {
removeStoredBlock(b, node);
}
int numBlocksLogged = 0;
- for (BlockInfo b : toAdd) {
- addStoredBlock(b, storageInfo, null, numBlocksLogged < maxNumBlocksToLog);
+ for (BlockInfoToAdd b : toAdd) {
+ addStoredBlock(b.stored, b.reported, storageInfo, null,
+ numBlocksLogged < maxNumBlocksToLog);
numBlocksLogged++;
}
if (numBlocksLogged > maxNumBlocksToLog) {
@@ -2073,17 +2245,18 @@ public class BlockManager implements BlockStatsMXBean {
* Mark block replicas as corrupt except those on the storages in
* newStorages list.
*/
- public void markBlockReplicasAsCorrupt(BlockInfo block,
+ public void markBlockReplicasAsCorrupt(Block oldBlock,
+ BlockInfo block,
long oldGenerationStamp, long oldNumBytes,
DatanodeStorageInfo[] newStorages) throws IOException {
assert namesystem.hasWriteLock();
BlockToMarkCorrupt b = null;
if (block.getGenerationStamp() != oldGenerationStamp) {
- b = new BlockToMarkCorrupt(block, oldGenerationStamp,
+ b = new BlockToMarkCorrupt(oldBlock, block, oldGenerationStamp,
"genstamp does not match " + oldGenerationStamp
+ " : " + block.getGenerationStamp(), Reason.GENSTAMP_MISMATCH);
} else if (block.getNumBytes() != oldNumBytes) {
- b = new BlockToMarkCorrupt(block,
+ b = new BlockToMarkCorrupt(oldBlock, block,
"length does not match " + oldNumBytes
+ " : " + block.getNumBytes(), Reason.SIZE_MISMATCH);
} else {
@@ -2141,8 +2314,8 @@ public class BlockManager implements BlockStatsMXBean {
QUEUE_REASON_FUTURE_GENSTAMP);
continue;
}
-
- BlockInfo storedBlock = blocksMap.getStoredBlock(iblk);
+
+ BlockInfo storedBlock = getStoredBlock(iblk);
// If block does not belong to any file, we are done.
if (storedBlock == null) continue;
@@ -2165,38 +2338,38 @@ public class BlockManager implements BlockStatsMXBean {
// If block is under construction, add this replica to its list
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
- ((BlockInfoContiguousUnderConstruction)storedBlock)
- .addReplicaIfNotPresent(storageInfo, iblk, reportedState);
+ final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)storedBlock;
+ uc.addReplicaIfNotPresent(storageInfo, iblk, reportedState);
// OpenFileBlocks only inside snapshots also will be added to safemode
// threshold. So we need to update such blocks to safemode
// refer HDFS-5283
- BlockInfoContiguousUnderConstruction blockUC =
- (BlockInfoContiguousUnderConstruction) storedBlock;
- if (namesystem.isInSnapshot(blockUC)) {
- int numOfReplicas = blockUC.getNumExpectedLocations();
- namesystem.incrementSafeBlockCount(numOfReplicas);
+ if (namesystem.isInSnapshot(storedBlock.getBlockCollection())) {
+ int numOfReplicas = uc.getNumExpectedLocations();
+ namesystem.incrementSafeBlockCount(numOfReplicas, storedBlock);
}
//and fall through to next clause
}
//add replica if appropriate
if (reportedState == ReplicaState.FINALIZED) {
- addStoredBlockImmediate(storedBlock, storageInfo);
+ addStoredBlockImmediate(storedBlock, iblk, storageInfo);
}
}
}
private void reportDiff(DatanodeStorageInfo storageInfo,
- BlockListAsLongs newReport,
- Collection<BlockInfo> toAdd, // add to DatanodeDescriptor
- Collection<Block> toRemove, // remove from DatanodeDescriptor
+ BlockListAsLongs newReport,
+ Collection<BlockInfoToAdd> toAdd, // add to DatanodeDescriptor
+ Collection<BlockInfo> toRemove, // remove from DatanodeDescriptor
Collection<Block> toInvalidate, // should be removed from DN
Collection<BlockToMarkCorrupt> toCorrupt, // add to corrupt replicas list
Collection<StatefulBlockInfo> toUC) { // add to under-construction list
// place a delimiter in the list which separates blocks
// that have been reported from those that have not
- BlockInfo delimiter = new BlockInfoContiguous(new Block(), (short) 1);
- AddBlockResult result = storageInfo.addBlock(delimiter);
+ Block delimiterBlock = new Block();
+ BlockInfo delimiter = new BlockInfoContiguous(delimiterBlock,
+ (short) 1);
+ AddBlockResult result = storageInfo.addBlock(delimiter, delimiterBlock);
assert result == AddBlockResult.ADDED
: "Delimiting block cannot be present in the node";
int headIndex = 0; //currently the delimiter is in the head of the list
@@ -2222,8 +2395,9 @@ public class BlockManager implements BlockStatsMXBean {
// all of them are next to the delimiter
Iterator<BlockInfo> it =
storageInfo.new BlockIterator(delimiter.getNext(0));
- while(it.hasNext())
+ while(it.hasNext()) {
toRemove.add(it.next());
+ }
storageInfo.removeBlock(delimiter);
}
@@ -2260,9 +2434,9 @@ public class BlockManager implements BlockStatsMXBean {
*/
private BlockInfo processReportedBlock(
final DatanodeStorageInfo storageInfo,
- final Block block, final ReplicaState reportedState,
- final Collection<BlockInfo> toAdd,
- final Collection<Block> toInvalidate,
+ final Block block, final ReplicaState reportedState,
+ final Collection<BlockInfoToAdd> toAdd,
+ final Collection<Block> toInvalidate,
final Collection<BlockToMarkCorrupt> toCorrupt,
final Collection<StatefulBlockInfo> toUC) {
@@ -2282,7 +2456,7 @@ public class BlockManager implements BlockStatsMXBean {
}
// find block by blockId
- BlockInfo storedBlock = blocksMap.getStoredBlock(block);
+ BlockInfo storedBlock = getStoredBlock(block);
if(storedBlock == null) {
// If blocksMap does not contain reported block id,
// the replica should be removed from the data-node.
@@ -2325,9 +2499,8 @@ public class BlockManager implements BlockStatsMXBean {
}
if (isBlockUnderConstruction(storedBlock, ucState, reportedState)) {
- toUC.add(new StatefulBlockInfo(
- (BlockInfoContiguousUnderConstruction) storedBlock,
- new Block(block), reportedState));
+ toUC.add(new StatefulBlockInfo(storedBlock, new Block(block),
+ reportedState));
return storedBlock;
}
@@ -2336,7 +2509,7 @@ public class BlockManager implements BlockStatsMXBean {
if (reportedState == ReplicaState.FINALIZED
&& (storedBlock.findStorageInfo(storageInfo) == -1 ||
corruptReplicas.isReplicaCorrupt(storedBlock, dn))) {
- toAdd.add(storedBlock);
+ toAdd.add(new BlockInfoToAdd(storedBlock, block));
}
return storedBlock;
}
@@ -2382,7 +2555,7 @@ public class BlockManager implements BlockStatsMXBean {
if (rbi.getReportedState() == null) {
// This is a DELETE_BLOCK request
DatanodeStorageInfo storageInfo = rbi.getStorageInfo();
- removeStoredBlock(rbi.getBlock(),
+ removeStoredBlock(getStoredBlock(rbi.getBlock()),
storageInfo.getDatanodeDescriptor());
} else {
processAndHandleReportedBlock(rbi.getStorageInfo(),
@@ -2430,12 +2603,27 @@ public class BlockManager implements BlockStatsMXBean {
case COMMITTED:
if (storedBlock.getGenerationStamp() != reported.getGenerationStamp()) {
final long reportedGS = reported.getGenerationStamp();
- return new BlockToMarkCorrupt(storedBlock, reportedGS,
+ return new BlockToMarkCorrupt(new Block(reported), storedBlock, reportedGS,
"block is " + ucState + " and reported genstamp " + reportedGS
+ " does not match genstamp in block map "
+ storedBlock.getGenerationStamp(), Reason.GENSTAMP_MISMATCH);
- } else if (storedBlock.getNumBytes() != reported.getNumBytes()) {
- return new BlockToMarkCorrupt(storedBlock,
+ }
+ boolean wrongSize;
+ if (storedBlock.isStriped()) {
+ assert BlockIdManager.isStripedBlockID(reported.getBlockId());
+ assert storedBlock.getBlockId() ==
+ BlockIdManager.convertToStripedID(reported.getBlockId());
+ BlockInfoStriped stripedBlock = (BlockInfoStriped) storedBlock;
+ int reportedBlkIdx = BlockIdManager.getBlockIndex(reported);
+ wrongSize = reported.getNumBytes() !=
+ getInternalBlockLength(stripedBlock.getNumBytes(),
+ BLOCK_STRIPED_CELL_SIZE,
+ stripedBlock.getDataBlockNum(), reportedBlkIdx);
+ } else {
+ wrongSize = storedBlock.getNumBytes() != reported.getNumBytes();
+ }
+ if (wrongSize) {
+ return new BlockToMarkCorrupt(new Block(reported), storedBlock,
"block is " + ucState + " and reported length " +
reported.getNumBytes() + " does not match " +
"length in block map " + storedBlock.getNumBytes(),
@@ -2446,8 +2634,8 @@ public class BlockManager implements BlockStatsMXBean {
case UNDER_CONSTRUCTION:
if (storedBlock.getGenerationStamp() > reported.getGenerationStamp()) {
final long reportedGS = reported.getGenerationStamp();
- return new BlockToMarkCorrupt(storedBlock, reportedGS, "block is "
- + ucState + " and reported state " + reportedState
+ return new BlockToMarkCorrupt(new Block(reported), storedBlock, reportedGS,
+ "block is " + ucState + " and reported state " + reportedState
+ ", But reported genstamp " + reportedGS
+ " does not match genstamp in block map "
+ storedBlock.getGenerationStamp(), Reason.GENSTAMP_MISMATCH);
@@ -2462,7 +2650,7 @@ public class BlockManager implements BlockStatsMXBean {
return null; // not corrupt
} else if (storedBlock.getGenerationStamp() != reported.getGenerationStamp()) {
final long reportedGS = reported.getGenerationStamp();
- return new BlockToMarkCorrupt(storedBlock, reportedGS,
+ return new BlockToMarkCorrupt(new Block(reported), storedBlock, reportedGS,
"reported " + reportedState + " replica with genstamp " + reportedGS
+ " does not match COMPLETE block's genstamp in block map "
+ storedBlock.getGenerationStamp(), Reason.GENSTAMP_MISMATCH);
@@ -2477,7 +2665,7 @@ public class BlockManager implements BlockStatsMXBean {
"complete with the same genstamp");
return null;
} else {
- return new BlockToMarkCorrupt(storedBlock,
+ return new BlockToMarkCorrupt(new Block(reported), storedBlock,
"reported replica has invalid state " + reportedState,
Reason.INVALID_STATE);
}
@@ -2490,7 +2678,8 @@ public class BlockManager implements BlockStatsMXBean {
" on " + dn + " size " + storedBlock.getNumBytes();
// log here at WARN level since this is really a broken HDFS invariant
LOG.warn(msg);
- return new BlockToMarkCorrupt(storedBlock, msg, Reason.INVALID_STATE);
+ return new BlockToMarkCorrupt(new Block(reported), storedBlock, msg,
+ Reason.INVALID_STATE);
}
}
@@ -2517,13 +2706,14 @@ public class BlockManager implements BlockStatsMXBean {
void addStoredBlockUnderConstruction(StatefulBlockInfo ucBlock,
DatanodeStorageInfo storageInfo) throws IOException {
- BlockInfoContiguousUnderConstruction block = ucBlock.storedBlock;
- block.addReplicaIfNotPresent(
- storageInfo, ucBlock.reportedBlock, ucBlock.reportedState);
+ BlockInfo block = ucBlock.storedBlock;
+ final BlockInfoUnderConstruction uc = (BlockInfoUnderConstruction)block;
+ uc.addReplicaIfNotPresent(storageInfo, ucBlock.reportedBlock,
+ ucBlock.reportedState);
if (ucBlock.reportedState == ReplicaState.FINALIZED &&
(block.findStorageInfo(storageInfo) < 0)) {
- addStoredBlock(block, storageInfo, null, true);
+ addStoredBlock(block, ucBlock.reportedBlock, storageInfo, null, true);
}
}
@@ -2538,39 +2728,40 @@ public class BlockManager implements BlockStatsMXBean {
*
* @throws IOException
*/
- private void addStoredBlockImmediate(BlockInfo storedBlock,
+ private void addStoredBlockImmediate(BlockInfo storedBlock, Block reported,
DatanodeStorageInfo storageInfo)
throws IOException {
assert (storedBlock != null && namesystem.hasWriteLock());
if (!namesystem.isInStartupSafeMode()
|| namesystem.isPopulatingReplQueues()) {
- addStoredBlock(storedBlock, storageInfo, null, false);
+ addStoredBlock(storedBlock, reported, storageInfo, null, false);
return;
}
// just add it
- AddBlockResult result = storageInfo.addBlock(storedBlock);
+ AddBlockResult result = storageInfo.addBlock(storedBlock, reported);
// Now check for completion of blocks and safe block count
int numCurrentReplica = countLiveNodes(storedBlock);
if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED
- && numCurrentReplica >= minReplication) {
+ && hasMinStorage(storedBlock, numCurrentReplica)) {
completeBlock(storedBlock.getBlockCollection(), storedBlock, false);
} else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) {
// check whether safe replication is reached for the block
// only complete blocks are counted towards that.
// In the case that the block just became complete above, completeBlock()
// handles the safe block count maintenance.
- namesystem.incrementSafeBlockCount(numCurrentReplica);
+ namesystem.incrementSafeBlockCount(numCurrentReplica, storedBlock);
}
}
/**
* Modify (block-->datanode) map. Remove block from set of
* needed replications if this takes care of the problem.
- * @return the block that is stored in blockMap.
+ * @return the block that is stored in blocksMap.
*/
private Block addStoredBlock(final BlockInfo block,
+ final Block reportedBlock,
DatanodeStorageInfo storageInfo,
DatanodeDescriptor delNodeHint,
boolean logEveryBlock)
@@ -2578,9 +2769,10 @@ public class BlockManager implements BlockStatsMXBean {
assert block != null && namesystem.hasWriteLock();
BlockInfo storedBlock;
DatanodeDescriptor node = storageInfo.getDatanodeDescriptor();
- if (block instanceof BlockInfoContiguousUnderConstruction) {
+ if (block instanceof BlockInfoContiguousUnderConstruction ||
+ block instanceof BlockInfoStripedUnderConstruction) {
//refresh our copy in case the block got completed in another thread
- storedBlock = blocksMap.getStoredBlock(block);
+ storedBlock = getStoredBlock(block);
} else {
storedBlock = block;
}
@@ -2594,10 +2786,9 @@ public class BlockManager implements BlockStatsMXBean {
return block;
}
BlockCollection bc = storedBlock.getBlockCollection();
- assert bc != null : "Block must belong to a file";
// add block to the datanode
- AddBlockResult result = storageInfo.addBlock(storedBlock);
+ AddBlockResult result = storageInfo.addBlock(storedBlock, reportedBlock);
int curReplicaDelta;
if (result == AddBlockResult.ADDED) {
@@ -2628,7 +2819,7 @@ public class BlockManager implements BlockStatsMXBean {
+ pendingReplications.getNumReplicas(storedBlock);
if(storedBlock.getBlockUCState() == BlockUCState.COMMITTED &&
- numLiveReplicas >= minReplication) {
+ hasMinStorage(storedBlock, numLiveReplicas)) {
storedBlock = completeBlock(bc, storedBlock, false);
} else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) {
// check whether safe replication is reached for the block
@@ -2636,7 +2827,7 @@ public class BlockManager implements BlockStatsMXBean {
// Is no-op if not in safe mode.
// In the case that the block just became complete above, completeBlock()
// handles the safe block count maintenance.
- namesystem.incrementSafeBlockCount(numCurrentReplica);
+ namesystem.incrementSafeBlockCount(numCurrentReplica, storedBlock);
}
// if file is under construction, then done for now
@@ -2650,7 +2841,7 @@ public class BlockManager implements BlockStatsMXBean {
}
// handle underReplication/overReplication
- short fileReplication = bc.getPreferredBlockReplication();
+ short fileReplication = getExpectedReplicaNum(bc, storedBlock);
if (!isNeededReplication(storedBlock, fileReplication, numCurrentReplica)) {
neededReplications.remove(storedBlock, numCurrentReplica,
num.decommissionedAndDecommissioning(), fileReplication);
@@ -2666,11 +2857,12 @@ public class BlockManager implements BlockStatsMXBean {
int numCorruptNodes = num.corruptReplicas();
if (numCorruptNodes != corruptReplicasCount) {
LOG.warn("Inconsistent number of corrupt replicas for " +
- storedBlock + "blockMap has " + numCorruptNodes +
+ storedBlock + ". blockMap has " + numCorruptNodes +
" but corrupt replicas map has " + corruptReplicasCount);
}
- if ((corruptReplicasCount > 0) && (numLiveReplicas >= fileReplication))
- invalidateCorruptReplicas(storedBlock);
+ if ((corruptReplicasCount > 0) && (numLiveReplicas >= fileReplication)) {
+ invalidateCorruptReplicas(storedBlock, reportedBlock, num);
+ }
return storedBlock;
}
@@ -2702,18 +2894,20 @@ public class BlockManager implements BlockStatsMXBean {
*
* @param blk Block whose corrupt replicas need to be invalidated
*/
- private void invalidateCorruptReplicas(BlockInfo blk) {
+ private void invalidateCorruptReplicas(BlockInfo blk, Block reported,
+ NumberReplicas numberReplicas) {
Collection<DatanodeDescriptor> nodes = corruptReplicas.getNodes(blk);
boolean removedFromBlocksMap = true;
if (nodes == null)
return;
// make a copy of the array of nodes in order to avoid
// ConcurrentModificationException, when the block is removed from the node
- DatanodeDescriptor[] nodesCopy = nodes.toArray(new DatanodeDescriptor[0]);
+ DatanodeDescriptor[] nodesCopy =
+ nodes.toArray(new DatanodeDescriptor[nodes.size()]);
for (DatanodeDescriptor node : nodesCopy) {
try {
- if (!invalidateBlock(new BlockToMarkCorrupt(blk, null,
- Reason.ANY), node)) {
+ if (!invalidateBlock(new BlockToMarkCorrupt(reported, blk, null,
+ Reason.ANY), node, numberReplicas)) {
removedFromBlocksMap = false;
}
} catch (IOException e) {
@@ -2864,6 +3058,15 @@ public class BlockManager implements BlockStatsMXBean {
}
/**
+ * Get the value of whether there are any non-EC blocks using StripedID.
+ *
+ * @return Returns the value of whether there are any non-EC blocks using StripedID.
+ */
+ public boolean hasNonEcBlockUsingStripedID(){
+ return hasNonEcBlockUsingStripedID;
+ }
+
+ /**
* Process a single possibly misreplicated block. This adds it to the
* appropriate queues if necessary, and returns a result code indicating
* what happened with it.
@@ -2881,7 +3084,7 @@ public class BlockManager implements BlockStatsMXBean {
}
// calculate current replication
short expectedReplication =
- block.getBlockCollection().getPreferredBlockReplication();
+ getExpectedReplicaNum(block.getBlockCollection(), block);
NumberReplicas num = countNodes(block);
int numCurrentReplica = num.liveReplicas();
// add to under-replicated queue if need to be
@@ -2940,14 +3143,14 @@ public class BlockManager implements BlockStatsMXBean {
* If there are any extras, call chooseExcessReplicates() to
* mark them in the excessReplicateMap.
*/
- private void processOverReplicatedBlock(final Block block,
+ private void processOverReplicatedBlock(final BlockInfo block,
final short replication, final DatanodeDescriptor addedNode,
DatanodeDescriptor delNodeHint) {
assert namesystem.hasWriteLock();
if (addedNode == delNodeHint) {
delNodeHint = null;
}
- Collection<DatanodeStorageInfo> nonExcess = new ArrayList<DatanodeStorageInfo>();
+ Collection<DatanodeStorageInfo> nonExcess = new ArrayList<>();
Collection<DatanodeDescriptor> corruptNodes = corruptReplicas
.getNodes(block);
for(DatanodeStorageInfo storage : blocksMap.getStorages(block, State.NORMAL)) {
@@ -2961,8 +3164,8 @@ public class BlockManager implements BlockStatsMXBean {
postponeBlock(block);
return;
}
- LightWeightLinkedSet<Block> excessBlocks = excessReplicateMap.get(cur
- .getDatanodeUuid());
+ LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get(
+ cur.getDatanodeUuid());
if (excessBlocks == null || !excessBlocks.contains(block)) {
if (!cur.isDecommissionInProgress() && !cur.isDecommissioned()) {
// exclude corrupt replicas
@@ -2972,10 +3175,29 @@ public class BlockManager implements BlockStatsMXBean {
}
}
}
- chooseExcessReplicates(nonExcess, block, replication,
- addedNode, delNodeHint, blockplacement);
+ chooseExcessReplicates(nonExcess, block, replication, addedNode,
+ delNodeHint);
}
+ private void chooseExcessReplicates(
+ final Collection<DatanodeStorageInfo> nonExcess,
+ BlockInfo storedBlock, short replication,
+ DatanodeDescriptor addedNode,
+ DatanodeDescriptor delNodeHint) {
+ assert namesystem.hasWriteLock();
+ // first form a rack to datanodes map and
+ BlockCollection bc = getBlockCollection(storedBlock);
+ if (storedBlock.isStriped()) {
+ chooseExcessReplicasStriped(bc, nonExcess, storedBlock, delNodeHint);
+ } else {
+ final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(
+ bc.getStoragePolicyID());
+ final List<StorageType> excessTypes = storagePolicy.chooseExcess(
+ replication, DatanodeStorageInfo.toStorageTypes(nonExcess));
+ chooseExcessReplicasContiguous(bc, nonExcess, storedBlock,
+ replication, addedNode, delNodeHint, excessTypes);
+ }
+ }
/**
* We want "replication" replicates for the block, but we now have too many.
@@ -2991,23 +3213,16 @@ public class BlockManager implements BlockStatsMXBean {
* If no such a node is available,
* then pick a node with least free space
*/
- private void chooseExcessReplicates(final Collection<DatanodeStorageInfo> nonExcess,
- Block b, short replication,
- DatanodeDescriptor addedNode,
- DatanodeDescriptor delNodeHint,
- BlockPlacementPolicy replicator) {
- assert namesystem.hasWriteLock();
- // first form a rack to datanodes map and
- BlockCollection bc = getBlockCollection(b);
- final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(bc.getStoragePolicyID());
- final List<StorageType> excessTypes = storagePolicy.chooseExcess(
- replication, DatanodeStorageInfo.toStorageTypes(nonExcess));
-
-
- final Map<String, List<DatanodeStorageInfo>> rackMap
- = new HashMap<String, List<DatanodeStorageInfo>>();
- final List<DatanodeStorageInfo> moreThanOne = new ArrayList<DatanodeStorageInfo>();
- final List<DatanodeStorageInfo> exactlyOne = new ArrayList<DatanodeStorageInfo>();
+ private void chooseExcessReplicasContiguous(BlockCollection bc,
+ final Collection<DatanodeStorageInfo> nonExcess,
+ BlockInfo storedBlock, short replication,
+ DatanodeDescriptor addedNode,
+ DatanodeDescriptor delNodeHint,
+ List<StorageType> excessTypes) {
+ BlockPlacementPolicy replicator = placementPolicies.getPolicy(false);
+ final Map<String, List<DatanodeStorageInfo>> rackMap = new HashMap<>();
+ final List<DatanodeStorageInfo> moreThanOne = new ArrayList<>();
+ final List<DatanodeStorageInfo> exactlyOne = new ArrayList<>();
// split nodes into two sets
// moreThanOne contains nodes on rack with more than one replica
@@ -3028,33 +3243,111 @@ public class BlockManager implements BlockStatsMXBean {
moreThanOne, excessTypes)) {
cur = delNodeHintStorage;
} else { // regular excessive replica removal
- cur = replicator.chooseReplicaToDelete(bc, b, replication,
+ cur = replicator.chooseReplicaToDelete(bc, storedBlock, replication,
moreThanOne, exactlyOne, excessTypes);
}
firstOne = false;
-
// adjust rackmap, moreThanOne, and exactlyOne
replicator.adjustSetsWithChosenReplica(rackMap, moreThanOne,
exactlyOne, cur);
- nonExcess.remove(cur);
- addToExcessReplicate(cur.getDatanodeDescriptor(), b);
+ processChosenExcessReplica(nonExcess, cur, storedBlock);
+ }
+ }
- //
- // The 'excessblocks' tracks blocks until we get confirmation
- // that the datanode has deleted them; the only way we remove them
- // is when we get a "removeBlock" message.
- //
- // The 'invalidate' list is used to inform the datanode the block
- // should be deleted. Items are removed from the invalidate list
- // upon giving instructions to the namenode.
- //
- addToInvalidates(b, cur.getDatanodeDescriptor());
- blockLog.debug("BLOCK* chooseExcessReplicates: "
- +"({}, {}) is added to invalidated blocks set", cur, b);
+ /**
+ * We want block group has every internal block, but we have redundant
+ * internal blocks (which have the same index).
+ * In this method, we delete the redundant internal blocks until only one
+ * left for each index.
+ *
+ * The block placement policy will make sure that the left internal blocks are
+ * spread across racks and also try hard to pick one with least free space.
+ */
+ private void chooseExcessReplicasStriped(BlockCollection bc,
+ final Collection<DatanodeStorageInfo> nonExcess,
+ BlockInfo storedBlock,
+ DatanodeDescriptor delNodeHint) {
+ assert storedBlock instanceof BlockInfoStriped;
+ BlockInfoStriped sblk = (BlockInfoStriped) storedBlock;
+ short groupSize = sblk.getTotalBlockNum();
+ BlockPlacementPolicy placementPolicy = placementPolicies.getPolicy(true);
+ List<DatanodeStorageInfo> empty = new ArrayList<>(0);
+
+ // find all duplicated indices
+ BitSet found = new BitSet(groupSize); //indices found
+ BitSet duplicated = new BitSet(groupSize); //indices found more than once
+ HashMap<DatanodeStorageInfo, Integer> storage2index = new HashMap<>();
+ for (DatanodeStorageInfo storage : nonExcess) {
+ int index = sblk.getStorageBlockIndex(storage);
+ assert index >= 0;
+ if (found.get(index)) {
+ duplicated.set(index);
+ }
+ found.set(index);
+ storage2index.put(storage, index);
+ }
+ // the number of target left replicas equals to the of number of the found
+ // indices.
+ int numOfTarget = found.cardinality();
+
+ final BlockStoragePolicy storagePolicy = storagePolicySuite.getPolicy(
+ bc.getStoragePolicyID());
+ final List<StorageType> excessTypes = storagePolicy.chooseExcess(
+ (short)numOfTarget, DatanodeStorageInfo.toStorageTypes(nonExcess));
+
+ // use delHint only if delHint is duplicated
+ final DatanodeStorageInfo delStorageHint =
+ DatanodeStorageInfo.getDatanodeStorageInfo(nonExcess, delNodeHint);
+ if (delStorageHint != null) {
+ Integer index = storage2index.get(delStorageHint);
+ if (index != null && duplicated.get(index)) {
+ processChosenExcessReplica(nonExcess, delStorageHint, storedBlock);
+ }
+ }
+
+ // for each duplicated index, delete some replicas until only one left
+ for (int targetIndex = duplicated.nextSetBit(0); targetIndex >= 0;
+ targetIndex = duplicated.nextSetBit(targetIndex + 1)) {
+ List<DatanodeStorageInfo> candidates = new ArrayList<>();
+ for (DatanodeStorageInfo storage : nonExcess) {
+ int index = storage2index.get(storage);
+ if (index == targetIndex) {
+ candidates.add(storage);
+ }
+ }
+ Block internalBlock = new Block(storedBlock);
+ internalBlock.setBlockId(storedBlock.getBlockId() + targetIndex);
+ while (candidates.size() > 1) {
+ DatanodeStorageInfo target = placementPolicy.chooseReplicaToDelete(bc,
+ internalBlock, (short)1, candidates, empty, excessTypes);
+ processChosenExcessReplica(nonExcess, target, storedBlock);
+ candidates.remove(target);
+ }
+ duplicated.clear(targetIndex);
}
}
+ private void processChosenExcessReplica(
+ final Collection<DatanodeStorageInfo> nonExcess,
+ final DatanodeStorageInfo chosen, BlockInfo storedBlock) {
+ nonExcess.remove(chosen);
+ addToExcessReplicate(chosen.getDatanodeDescriptor(), storedBlock);
+ //
+ // The 'excessblocks' tracks blocks until we get confirmation
+ // that the datanode has deleted them; the only way we remove them
+ // is when we get a "removeBlock" message.
+ //
+ // The 'invalidate' list is used to inform the datanode the block
+ // should be deleted. Items are removed from the invalidate list
+ // upon giving instructions to the datanodes.
+ //
+ final Block blockToInvalidate = getBlockOnStorage(storedBlock, chosen);
+ addToInvalidates(blockToInvalidate, chosen.getDatanodeDescriptor());
+ blockLog.debug("BLOCK* chooseExcessReplicates: "
+ + "({}, {}) is added to invalidated blocks set", chosen, storedBlock);
+ }
+
/** Check if we can use delHint */
static boolean useDelHint(boolean isFirst, DatanodeStorageInfo delHint,
DatanodeStorageInfo added, List<DatanodeStorageInfo> moreThan1Racks,
@@ -3076,17 +3369,18 @@ public class BlockManager implements BlockStatsMXBean {
}
}
- private void addToExcessReplicate(DatanodeInfo dn, Block block) {
+ private void addToExcessReplicate(DatanodeInfo dn, BlockInfo storedBlock) {
assert namesystem.hasWriteLock();
- LightWeightLinkedSet<Block> excessBlocks = excessReplicateMap.get(dn.getDatanodeUuid());
+ LightWeightLinkedSet<BlockInfo> excessBlocks = excessReplicateMap.get(
+ dn.getDatanodeUuid());
if (excessBlocks == null) {
- excessBlocks = new LightWeightLinkedSet<Block>();
+ excessBlocks = new LightWeightLinkedSet<>();
excessReplicateMap.put(dn.getDatanodeUuid(), excessBlocks);
}
- if (excessBlocks.add(block)) {
+ if (excessBlocks.add(storedBlock)) {
excessBlocksCount.incrementAndGet();
blockLog.debug("BLOCK* addToExcessReplicate: ({}, {}) is added to"
- + " excessReplicateMap", dn, block);
+ + " excessReplicateMap", dn, storedBlock);
}
}
@@ -3098,26 +3392,25 @@ public class BlockManager implements BlockStatsMXBean {
QUEUE_REASON_FUTURE_GENSTAMP);
return;
}
- removeStoredBlock(block, node);
+ removeStoredBlock(getStoredBlock(block), node);
}
/**
* Modify (block-->datanode) map. Possibly generate replication tasks, if the
* removed block is still valid.
*/
- public void removeStoredBlock(Block block, DatanodeDescriptor node) {
- blockLog.debug("BLOCK* removeStoredBlock: {} from {}", block, node);
+ public void removeStoredBlock(BlockInfo storedBlock, DatanodeDescriptor node) {
+ blockLog.debug("BLOCK* removeStoredBlock: {} from {}", storedBlock, node);
assert (namesystem.hasWriteLock());
{
- BlockInfo storedBlock = getStoredBlock(block);
if (storedBlock == null || !blocksMap.removeNode(storedBlock, node)) {
blockLog.debug("BLOCK* removeStoredBlock: {} has already been" +
- " removed from node {}", block, node);
+ " removed from node {}", storedBlock, node);
return;
}
CachedBlock cblock = namesystem.getCacheManager().getCachedBlocks()
- .get(new CachedBlock(block.getBlockId(), (short) 0, false));
+ .get(new CachedBlock(storedBlock.getBlockId(), (short) 0, false));
if (cblock != null) {
boolean removed = false;
removed |= node.getPendingCached().remove(cblock);
@@ -3125,7 +3418,7 @@ public class BlockManager implements BlockStatsMXBean {
removed |= node.getPendingUncached().remove(cblock);
if (removed) {
blockLog.debug("BLOCK* removeStoredBlock: {} removed from caching "
- + "related lists on node {}", block, node);
+ + "related lists on node {}", storedBlock, node);
}
}
@@ -3135,7 +3428,7 @@ public class BlockManager implements BlockStatsMXBean {
// necessary. In that case, put block on a possibly-will-
// be-replicated list.
//
- BlockCollection bc = blocksMap.getBlockCollection(block);
+ BlockCollection bc = storedBlock.getBlockCollection();
if (bc != null) {
namesystem.decrementS
<TRUNCATED>
[23/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
HDFS-7285. Erasure Coding Support inside HDFS.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc2564af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc2564af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc2564af
Branch: refs/heads/HDFS-7285-merge
Commit: bc2564afe511eab0a5cfc3cc3d9b68fe5caa7c77
Parents: 1c12adb
Author: Zhe Zhang <zh...@cloudera.com>
Authored: Thu Aug 6 23:24:03 2015 -0700
Committer: Zhe Zhang <zh...@cloudera.com>
Committed: Wed Aug 12 11:17:53 2015 -0700
----------------------------------------------------------------------
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 74 +
.../hadoop/fs/CommonConfigurationKeys.java | 15 +
.../org/apache/hadoop/fs/FSOutputSummer.java | 4 +
.../main/java/org/apache/hadoop/fs/FsShell.java | 8 +-
.../apache/hadoop/io/erasurecode/CodecUtil.java | 144 ++
.../apache/hadoop/io/erasurecode/ECBlock.java | 80 ++
.../hadoop/io/erasurecode/ECBlockGroup.java | 100 ++
.../apache/hadoop/io/erasurecode/ECChunk.java | 87 ++
.../apache/hadoop/io/erasurecode/ECSchema.java | 257 ++++
.../hadoop/io/erasurecode/SchemaLoader.java | 152 +++
.../erasurecode/codec/AbstractErasureCodec.java | 51 +
.../io/erasurecode/codec/ErasureCodec.java | 49 +
.../io/erasurecode/codec/RSErasureCodec.java | 43 +
.../io/erasurecode/codec/XORErasureCodec.java | 44 +
.../erasurecode/coder/AbstractErasureCoder.java | 62 +
.../coder/AbstractErasureCodingStep.java | 59 +
.../coder/AbstractErasureDecoder.java | 167 +++
.../coder/AbstractErasureEncoder.java | 60 +
.../io/erasurecode/coder/ErasureCoder.java | 77 ++
.../io/erasurecode/coder/ErasureCodingStep.java | 55 +
.../erasurecode/coder/ErasureDecodingStep.java | 52 +
.../erasurecode/coder/ErasureEncodingStep.java | 49 +
.../io/erasurecode/coder/RSErasureDecoder.java | 67 +
.../io/erasurecode/coder/RSErasureEncoder.java | 67 +
.../io/erasurecode/coder/XORErasureDecoder.java | 86 ++
.../io/erasurecode/coder/XORErasureEncoder.java | 53 +
.../io/erasurecode/grouper/BlockGrouper.java | 90 ++
.../rawcoder/AbstractRawErasureCoder.java | 138 ++
.../rawcoder/AbstractRawErasureDecoder.java | 207 +++
.../rawcoder/AbstractRawErasureEncoder.java | 136 ++
.../io/erasurecode/rawcoder/RSRawDecoder.java | 216 +++
.../io/erasurecode/rawcoder/RSRawEncoder.java | 79 ++
.../rawcoder/RSRawErasureCoderFactory.java | 34 +
.../erasurecode/rawcoder/RawErasureCoder.java | 66 +
.../rawcoder/RawErasureCoderFactory.java | 42 +
.../erasurecode/rawcoder/RawErasureDecoder.java | 88 ++
.../erasurecode/rawcoder/RawErasureEncoder.java | 64 +
.../io/erasurecode/rawcoder/XORRawDecoder.java | 83 ++
.../io/erasurecode/rawcoder/XORRawEncoder.java | 77 ++
.../rawcoder/XORRawErasureCoderFactory.java | 34 +
.../io/erasurecode/rawcoder/util/DumpUtil.java | 85 ++
.../erasurecode/rawcoder/util/GaloisField.java | 561 ++++++++
.../io/erasurecode/rawcoder/util/RSUtil.java | 39 +
.../hadoop/io/erasurecode/BufferAllocator.java | 91 ++
.../hadoop/io/erasurecode/TestCoderBase.java | 500 +++++++
.../hadoop/io/erasurecode/TestECSchema.java | 51 +
.../hadoop/io/erasurecode/TestSchemaLoader.java | 74 +
.../erasurecode/coder/TestErasureCoderBase.java | 297 ++++
.../erasurecode/coder/TestRSErasureCoder.java | 126 ++
.../io/erasurecode/coder/TestXORCoder.java | 64 +
.../io/erasurecode/rawcoder/TestRSRawCoder.java | 118 ++
.../rawcoder/TestRSRawCoderBase.java | 58 +
.../erasurecode/rawcoder/TestRawCoderBase.java | 232 ++++
.../erasurecode/rawcoder/TestXORRawCoder.java | 66 +
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 +
.../hdfs/client/HdfsClientConfigKeys.java | 12 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 27 +
.../hadoop/hdfs/protocol/ErasureCodingZone.java | 66 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 11 +
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 16 +-
.../hadoop/hdfs/protocol/LocatedBlock.java | 8 +-
.../hadoop/hdfs/protocol/LocatedBlocks.java | 26 +-
.../hdfs/protocol/LocatedStripedBlock.java | 86 ++
.../protocol/SnapshottableDirectoryStatus.java | 2 +-
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 4 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 7 +
.../src/main/proto/erasurecoding.proto | 71 +
.../src/main/proto/hdfs.proto | 36 +-
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 396 ++++++
.../hadoop-hdfs/src/main/bin/hdfs | 6 +
.../org/apache/hadoop/hdfs/BlockReader.java | 10 +-
.../apache/hadoop/hdfs/BlockReaderLocal.java | 5 +
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 5 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 109 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 66 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 23 +-
.../java/org/apache/hadoop/hdfs/DFSPacket.java | 34 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 939 +++++++++++++
.../hadoop/hdfs/DFSStripedOutputStream.java | 653 +++++++++
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 9 +-
.../org/apache/hadoop/hdfs/DataStreamer.java | 72 +-
.../hadoop/hdfs/DistributedFileSystem.java | 66 +
.../apache/hadoop/hdfs/RemoteBlockReader.java | 5 +
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 5 +
.../apache/hadoop/hdfs/StripedDataStreamer.java | 240 ++++
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 40 +
.../hadoop/hdfs/client/impl/DfsClientConf.java | 22 +-
.../hdfs/protocol/HdfsLocatedFileStatus.java | 6 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 68 +-
.../ClientNamenodeProtocolTranslatorPB.java | 79 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 2 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 2 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 326 ++++-
.../hadoop/hdfs/server/balancer/Balancer.java | 6 +-
.../hadoop/hdfs/server/balancer/Dispatcher.java | 150 +-
.../server/blockmanagement/BlockCollection.java | 17 +-
.../server/blockmanagement/BlockIdManager.java | 58 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 73 +-
.../blockmanagement/BlockInfoContiguous.java | 45 +-
.../BlockInfoContiguousUnderConstruction.java | 189 +--
.../blockmanagement/BlockInfoStriped.java | 286 ++++
.../BlockInfoStripedUnderConstruction.java | 298 ++++
.../BlockInfoUnderConstruction.java | 84 ++
.../server/blockmanagement/BlockManager.java | 1281 ++++++++++++------
.../blockmanagement/BlockPlacementPolicies.java | 54 +
.../blockmanagement/BlockPlacementPolicy.java | 26 +-
.../BlockPlacementPolicyRackFaultTolarent.java | 154 +++
.../hdfs/server/blockmanagement/BlocksMap.java | 25 +-
.../blockmanagement/DatanodeDescriptor.java | 62 +-
.../server/blockmanagement/DatanodeManager.java | 24 +-
.../blockmanagement/DatanodeStorageInfo.java | 17 +-
.../blockmanagement/DecommissionManager.java | 83 +-
.../ReplicaUnderConstruction.java | 119 ++
.../SequentialBlockGroupIdGenerator.java | 85 ++
.../SequentialBlockIdGenerator.java | 6 +-
.../blockmanagement/UnderReplicatedBlocks.java | 51 +-
.../hdfs/server/common/HdfsServerConstants.java | 5 +
.../hdfs/server/datanode/BPOfferService.java | 8 +
.../hadoop/hdfs/server/datanode/DNConf.java | 27 +
.../hadoop/hdfs/server/datanode/DataNode.java | 58 +-
.../erasurecode/ErasureCodingWorker.java | 988 ++++++++++++++
.../apache/hadoop/hdfs/server/mover/Mover.java | 36 +-
.../namenode/ErasureCodingSchemaManager.java | 127 ++
.../namenode/ErasureCodingZoneManager.java | 170 +++
.../hdfs/server/namenode/FSDirAppendOp.java | 5 +
.../hdfs/server/namenode/FSDirAttrOp.java | 7 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 6 +
.../server/namenode/FSDirErasureCodingOp.java | 217 +++
.../hdfs/server/namenode/FSDirRenameOp.java | 2 +
.../server/namenode/FSDirStatAndListingOp.java | 27 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 16 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 170 ++-
.../hdfs/server/namenode/FSDirectory.java | 6 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 101 +-
.../hdfs/server/namenode/FSImageFormat.java | 22 +-
.../server/namenode/FSImageFormatPBINode.java | 57 +-
.../server/namenode/FSImageFormatProtobuf.java | 9 +-
.../server/namenode/FSImageSerialization.java | 13 +-
.../hdfs/server/namenode/FSNamesystem.java | 304 ++++-
.../namenode/FileUnderConstructionFeature.java | 10 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 152 ++-
.../server/namenode/INodeFileAttributes.java | 12 +-
.../hdfs/server/namenode/LeaseManager.java | 2 +-
.../server/namenode/NameNodeLayoutVersion.java | 3 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 34 +-
.../hdfs/server/namenode/NamenodeFsck.java | 227 +++-
.../hadoop/hdfs/server/namenode/Namesystem.java | 15 +-
.../hadoop/hdfs/server/namenode/SafeMode.java | 5 +-
.../snapshot/FSImageFormatPBSnapshot.java | 14 +-
.../server/namenode/snapshot/FileDiffList.java | 5 +-
.../server/protocol/BlockECRecoveryCommand.java | 153 +++
.../server/protocol/BlocksWithLocations.java | 25 +
.../hdfs/server/protocol/DatanodeProtocol.java | 1 +
.../hadoop/hdfs/tools/erasurecode/ECCli.java | 48 +
.../hdfs/tools/erasurecode/ECCommand.java | 226 +++
.../hadoop/hdfs/util/StripedBlockUtil.java | 947 +++++++++++++
.../src/main/proto/DatanodeProtocol.proto | 10 +
.../hadoop-hdfs/src/main/proto/fsimage.proto | 3 +
.../src/main/resources/hdfs-default.xml | 31 +-
.../hadoop/cli/CLITestCmdErasureCoding.java | 38 +
.../apache/hadoop/cli/TestErasureCodingCLI.java | 114 ++
.../cli/util/CLICommandErasureCodingCli.java | 21 +
.../cli/util/ErasureCodingCliCmdExecutor.java | 37 +
.../apache/hadoop/hdfs/BlockReaderTestUtil.java | 7 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 182 ++-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 2 -
.../apache/hadoop/hdfs/StripedFileTestUtil.java | 261 ++++
.../hadoop/hdfs/TestBlockReaderFactory.java | 16 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 6 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 335 +++++
.../hadoop/hdfs/TestDFSStripedOutputStream.java | 293 ++++
.../TestDFSStripedOutputStreamWithFailure.java | 329 +++++
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 2 +-
.../org/apache/hadoop/hdfs/TestECSchemas.java | 54 +
.../apache/hadoop/hdfs/TestEncryptionZones.java | 2 +-
.../hadoop/hdfs/TestErasureCodingZones.java | 223 +++
.../hadoop/hdfs/TestFileStatusWithECschema.java | 65 +
.../java/org/apache/hadoop/hdfs/TestLease.java | 4 +-
.../hdfs/TestReadStripedFileWithDecoding.java | 358 +++++
.../TestReadStripedFileWithMissingBlocks.java | 150 ++
.../hadoop/hdfs/TestRecoverStripedFile.java | 400 ++++++
.../org/apache/hadoop/hdfs/TestSafeMode.java | 4 +-
.../hdfs/TestSafeModeWithStripedFile.java | 150 ++
.../hadoop/hdfs/TestWriteReadStripedFile.java | 251 ++++
.../hdfs/TestWriteStripedFileWithFailure.java | 162 +++
.../hadoop/hdfs/protocol/TestLayoutVersion.java | 3 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 170 ++-
.../hdfs/server/balancer/TestBalancer.java | 87 ++
.../blockmanagement/BlockManagerTestUtil.java | 10 +-
.../server/blockmanagement/TestBlockInfo.java | 2 +-
.../blockmanagement/TestBlockInfoStriped.java | 250 ++++
.../TestBlockInfoUnderConstruction.java | 3 +-
.../blockmanagement/TestBlockManager.java | 63 +-
.../blockmanagement/TestBlockTokenWithDFS.java | 422 +++---
.../TestBlockTokenWithDFSStriped.java | 115 ++
.../blockmanagement/TestHeartbeatHandling.java | 1 -
.../TestNameNodePrunesMissingStorages.java | 5 +-
.../server/blockmanagement/TestNodeCount.java | 2 +-
.../TestOverReplicatedBlocks.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 25 +-
.../TestSequentialBlockGroupId.java | 222 +++
.../TestUnderReplicatedBlockQueues.java | 62 +
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../datanode/TestIncrementalBrVariations.java | 14 +-
.../hadoop/hdfs/server/mover/TestMover.java | 125 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 9 +-
.../TestAddOverReplicatedStripedBlocks.java | 266 ++++
.../server/namenode/TestAddStripedBlocks.java | 433 ++++++
.../hdfs/server/namenode/TestDeadDatanode.java | 3 +-
.../server/namenode/TestFSEditLogLoader.java | 273 ++++
.../hdfs/server/namenode/TestFSImage.java | 345 ++++-
.../hdfs/server/namenode/TestFileTruncate.java | 4 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 83 +-
.../hdfs/server/namenode/TestINodeFile.java | 2 +-
.../namenode/TestQuotaWithStripedBlocks.java | 125 ++
.../namenode/TestRecoverStripedBlocks.java | 169 +++
.../server/namenode/TestStripedINodeFile.java | 284 ++++
.../namenode/snapshot/SnapshotTestHelper.java | 2 +-
...TestOfflineImageViewerWithStripedBlocks.java | 162 +++
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 279 ++++
.../apache/hadoop/hdfs/web/TestJsonUtil.java | 2 +-
.../test/resources/testErasureCodingConf.xml | 377 ++++++
223 files changed, 23039 insertions(+), 1587 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
new file mode 100644
index 0000000..1f3006e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -0,0 +1,74 @@
+ BREAKDOWN OF HADOOP-11264 SUBTASKS AND RELATED JIRAS (Common part of HDFS-7285)
+
+ HADOOP-11514. Raw Erasure Coder API for concrete encoding and decoding
+ (Kai Zheng via umamahesh)
+
+ HADOOP-11534. Minor improvements for raw erasure coders
+ ( Kai Zheng via vinayakumarb )
+
+ HADOOP-11541. Raw XOR coder
+ ( Kai Zheng )
+
+ HADOOP-11542. Raw Reed-Solomon coder in pure Java. Contributed by Kai Zheng
+ ( Kai Zheng )
+
+ HADOOP-11643. Define EC schema API for ErasureCodec. Contributed by Kai Zheng
+ ( Kai Zheng )
+
+ HADOOP-11646. Erasure Coder API for encoding and decoding of block group
+ ( Kai Zheng via vinayakumarb )
+
+ HADOOP-11705. Make erasure coder configurable. Contributed by Kai Zheng
+ ( Kai Zheng )
+
+ HADOOP-11706. Refine a little bit erasure coder API. Contributed by Kai Zheng
+ ( Kai Zheng )
+
+ HADOOP-11707. Add factory to create raw erasure coder. Contributed by Kai Zheng
+ ( Kai Zheng )
+
+ HADOOP-11647. Reed-Solomon ErasureCoder. Contributed by Kai Zheng
+ ( Kai Zheng )
+
+ HADOOP-11782 Correct two thrown messages in ECSchema class. Contributed by Xinwei Qin
+ ( Xinwei Qin via Kai Zheng )
+
+ HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang)
+
+ HADOOP-11805 Better to rename some raw erasure coders. Contributed by Kai Zheng
+ ( Kai Zheng )
+
+ HADOOP-11645. Erasure Codec API covering the essential aspects for an erasure code
+ ( Kai Zheng via vinayakumarb )
+
+ HADOOP-11818. Minor improvements for erasurecode classes. (Rakesh R via Kai Zheng)
+
+ HADOOP-11841. Remove unused ecschema-def.xml files. (szetszwo)
+
+ HADOOP-11921. Enhance tests for erasure coders. (Kai Zheng via Zhe Zhang)
+
+ HADOOP-11920. Refactor some codes for erasure coders. (Kai Zheng via Zhe Zhang)
+
+ HADOOP-11566. Add tests and fix for erasure coders to recover erased parity
+ units. (Kai Zheng via Zhe Zhang)
+
+ HADOOP-11938. Enhance ByteBuffer version encode/decode API of raw erasure
+ coder. (Kai Zheng via Zhe Zhang)
+
+ HADOOP-12013. Generate fixed data to perform erasure coder test. (Kai Zheng)
+
+ HADOOP-12029. Remove chunkSize from ECSchema as its not required for coders
+ (vinayakumarb)
+
+ HADOOP-11847. Enhance raw coder allowing to read least required inputs in decoding.
+ (Kai Zheng)
+
+ HADOOP-12011. Allow to dump verbose information to ease debugging in raw erasure coders
+ (Kai Zheng)
+
+ HADOOP-12065. Using more meaningful keys in EC schema. (Kai Zheng)
+
+ HDFS-8557. Allow to configure RS and XOR raw coders (Kai Zheng)
+
+ HADOOP-12060. Fix ByteBuffer usage for raw erasure coders. (Kai Zheng via
+ jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index 2721466..9588254 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -137,6 +137,21 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
false;
/**
+ * Erasure Coding configuration family
+ */
+
+ /** Supported erasure codec classes */
+ public static final String IO_ERASURECODE_CODECS_KEY = "io.erasurecode.codecs";
+
+ /** Raw coder factory for the RS codec. */
+ public static final String IO_ERASURECODE_CODEC_RS_RAWCODER_KEY =
+ "io.erasurecode.codec.rs.rawcoder";
+
+ /** Raw coder factory for the XOR codec. */
+ public static final String IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY =
+ "io.erasurecode.codec.xor.rawcoder";
+
+ /**
* Service Authorization
*/
public static final String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index bdc5585..a8a7494 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@ -196,6 +196,10 @@ abstract public class FSOutputSummer extends OutputStream {
return sum.getChecksumSize();
}
+ protected DataChecksum getDataChecksum() {
+ return sum;
+ }
+
protected TraceScope createWriteTraceScope() {
return NullScope.INSTANCE;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index a0510be..bee7c8c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -124,6 +124,10 @@ public class FsShell extends Configured implements Tool {
return getTrash().getCurrentTrashDir();
}
+ protected String getUsagePrefix() {
+ return usagePrefix;
+ }
+
// NOTE: Usage/Help are inner classes to allow access to outer methods
// that access commandFactory
@@ -207,7 +211,7 @@ public class FsShell extends Configured implements Tool {
}
} else {
// display help or usage for all commands
- out.println(usagePrefix);
+ out.println(getUsagePrefix());
// display list of short usages
ArrayList<Command> instances = new ArrayList<Command>();
@@ -231,7 +235,7 @@ public class FsShell extends Configured implements Tool {
}
private void printInstanceUsage(PrintStream out, Command instance) {
- out.println(usagePrefix + " " + instance.getUsage());
+ out.println(getUsagePrefix() + " " + instance.getUsage());
}
private void printInstanceHelp(PrintStream out, Command instance) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
new file mode 100644
index 0000000..027d58b
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.rawcoder.*;
+
+/**
+ * A codec & coder utility to help create raw coders conveniently.
+ */
+public final class CodecUtil {
+
+ private CodecUtil() { }
+
+ /**
+ * Create RS raw encoder according to configuration.
+ * @param conf configuration possibly with some items to configure the coder
+ * @param numDataUnits number of data units in a coding group
+ * @param numParityUnits number of parity units in a coding group
+ * @return raw encoder
+ */
+ public static RawErasureEncoder createRSRawEncoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ true, numDataUnits, numParityUnits);
+ if (rawCoder == null) {
+ rawCoder = new RSRawEncoder(numDataUnits, numParityUnits);
+ }
+
+ return (RawErasureEncoder) rawCoder;
+ }
+
+ /**
+ * Create RS raw decoder according to configuration.
+ * @param conf configuration possibly with some items to configure the coder
+ * @param numDataUnits number of data units in a coding group
+ * @param numParityUnits number of parity units in a coding group
+ * @return raw decoder
+ */
+ public static RawErasureDecoder createRSRawDecoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ false, numDataUnits, numParityUnits);
+ if (rawCoder == null) {
+ rawCoder = new RSRawDecoder(numDataUnits, numParityUnits);
+ }
+
+ return (RawErasureDecoder) rawCoder;
+ }
+
+ /**
+ * Create XOR raw encoder according to configuration.
+ * @param conf configuration possibly with some items to configure the coder
+ * @param numDataUnits number of data units in a coding group
+ * @param numParityUnits number of parity units in a coding group
+ * @return raw encoder
+ */
+ public static RawErasureEncoder createXORRawEncoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ true, numDataUnits, numParityUnits);
+ if (rawCoder == null) {
+ rawCoder = new XORRawEncoder(numDataUnits, numParityUnits);
+ }
+
+ return (RawErasureEncoder) rawCoder;
+ }
+
+ /**
+ * Create XOR raw decoder according to configuration.
+ * @param conf configuration possibly with some items to configure the coder
+ * @param numDataUnits number of data units in a coding group
+ * @param numParityUnits number of parity units in a coding group
+ * @return raw decoder
+ */
+ public static RawErasureDecoder createXORRawDecoder(
+ Configuration conf, int numDataUnits, int numParityUnits) {
+ RawErasureCoder rawCoder = createRawCoder(conf,
+ CommonConfigurationKeys.IO_ERASURECODE_CODEC_XOR_RAWCODER_KEY,
+ false, numDataUnits, numParityUnits);
+ if (rawCoder == null) {
+ rawCoder = new XORRawDecoder(numDataUnits, numParityUnits);
+ }
+
+ return (RawErasureDecoder) rawCoder;
+ }
+
+ /**
+ * Create raw coder using specified conf and raw coder factory key.
+ * @param conf configuration possibly with some items to configure the coder
+ * @param rawCoderFactoryKey configuration key to find the raw coder factory
+ * @param isEncoder is encoder or not we're going to create
+ * @param numDataUnits number of data units in a coding group
+ * @param numParityUnits number of parity units in a coding group
+ * @return raw coder
+ */
+ public static RawErasureCoder createRawCoder(Configuration conf,
+ String rawCoderFactoryKey, boolean isEncoder, int numDataUnits,
+ int numParityUnits) {
+
+ if (conf == null) {
+ return null;
+ }
+
+ Class<? extends RawErasureCoderFactory> factClass = null;
+ factClass = conf.getClass(rawCoderFactoryKey,
+ factClass, RawErasureCoderFactory.class);
+
+ if (factClass == null) {
+ return null;
+ }
+
+ RawErasureCoderFactory fact;
+ try {
+ fact = factClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new RuntimeException("Failed to create raw coder", e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException("Failed to create raw coder", e);
+ }
+
+ return isEncoder ? fact.createEncoder(numDataUnits, numParityUnits) :
+ fact.createDecoder(numDataUnits, numParityUnits);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java
new file mode 100644
index 0000000..5c0a160
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+/**
+ * A wrapper of block level data source/output that {@link ECChunk}s can be
+ * extracted from. For HDFS, it can be an HDFS block (250MB). Note it only cares
+ * about erasure coding specific logic thus avoids coupling with any HDFS block
+ * details. We can have something like HdfsBlock extend it.
+ */
+public class ECBlock {
+
+ private boolean isParity;
+ private boolean isErased;
+
+ /**
+ * A default constructor. isParity and isErased are false by default.
+ */
+ public ECBlock() {
+ this(false, false);
+ }
+
+ /**
+ * A constructor specifying isParity and isErased.
+ * @param isParity is a parity block
+ * @param isErased is erased or not
+ */
+ public ECBlock(boolean isParity, boolean isErased) {
+ this.isParity = isParity;
+ this.isErased = isErased;
+ }
+
+ /**
+ * Set true if it's for a parity block.
+ * @param isParity is parity or not
+ */
+ public void setParity(boolean isParity) {
+ this.isParity = isParity;
+ }
+
+ /**
+ * Set true if the block is missing.
+ * @param isErased is erased or not
+ */
+ public void setErased(boolean isErased) {
+ this.isErased = isErased;
+ }
+
+ /**
+ *
+ * @return true if it's parity block, otherwise false
+ */
+ public boolean isParity() {
+ return isParity;
+ }
+
+ /**
+ *
+ * @return true if it's erased due to erasure, otherwise false
+ */
+ public boolean isErased() {
+ return isErased;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
new file mode 100644
index 0000000..91e4fb8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+/**
+ * A group of blocks or {@link ECBlock} incurred in an erasure coding task.
+ */
+public class ECBlockGroup {
+
+ private ECBlock[] dataBlocks;
+ private ECBlock[] parityBlocks;
+
+ /**
+ * A constructor specifying data blocks and parity blocks.
+ * @param dataBlocks data blocks in the group
+ * @param parityBlocks parity blocks in the group
+ */
+ public ECBlockGroup(ECBlock[] dataBlocks, ECBlock[] parityBlocks) {
+ this.dataBlocks = dataBlocks;
+ this.parityBlocks = parityBlocks;
+ }
+
+ /**
+ * Get data blocks
+ * @return data blocks
+ */
+ public ECBlock[] getDataBlocks() {
+ return dataBlocks;
+ }
+
+ /**
+ * Get parity blocks
+ * @return parity blocks
+ */
+ public ECBlock[] getParityBlocks() {
+ return parityBlocks;
+ }
+
+ /**
+ * Any erased data block?
+ * @return true if any erased data block, false otherwise
+ */
+ public boolean anyErasedDataBlock() {
+ for (int i = 0; i < dataBlocks.length; ++i) {
+ if (dataBlocks[i].isErased()) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Any erased parity block?
+ * @return true if any erased parity block, false otherwise
+ */
+ public boolean anyErasedParityBlock() {
+ for (int i = 0; i < parityBlocks.length; ++i) {
+ if (parityBlocks[i].isErased()) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * Get erased blocks count
+ * @return erased count of blocks
+ */
+ public int getErasedCount() {
+ int erasedCount = 0;
+
+ for (ECBlock dataBlock : dataBlocks) {
+ if (dataBlock.isErased()) erasedCount++;
+ }
+
+ for (ECBlock parityBlock : parityBlocks) {
+ if (parityBlock.isErased()) erasedCount++;
+ }
+
+ return erasedCount;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
new file mode 100644
index 0000000..d0120d8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A wrapper for ByteBuffer or bytes array for an erasure code chunk.
+ */
+public class ECChunk {
+
+ private ByteBuffer chunkBuffer;
+
+ /**
+ * Wrapping a ByteBuffer
+ * @param buffer buffer to be wrapped by the chunk
+ */
+ public ECChunk(ByteBuffer buffer) {
+ this.chunkBuffer = buffer;
+ }
+
+ /**
+ * Wrapping a bytes array
+ * @param buffer buffer to be wrapped by the chunk
+ */
+ public ECChunk(byte[] buffer) {
+ this.chunkBuffer = ByteBuffer.wrap(buffer);
+ }
+
+ /**
+ * Convert to ByteBuffer
+ * @return ByteBuffer
+ */
+ public ByteBuffer getBuffer() {
+ return chunkBuffer;
+ }
+
+ /**
+ * Convert an array of this chunks to an array of ByteBuffers
+ * @param chunks chunks to convert into buffers
+ * @return an array of ByteBuffers
+ */
+ public static ByteBuffer[] toBuffers(ECChunk[] chunks) {
+ ByteBuffer[] buffers = new ByteBuffer[chunks.length];
+
+ ECChunk chunk;
+ for (int i = 0; i < chunks.length; i++) {
+ chunk = chunks[i];
+ if (chunk == null) {
+ buffers[i] = null;
+ } else {
+ buffers[i] = chunk.getBuffer();
+ }
+ }
+
+ return buffers;
+ }
+
+ /**
+ * Convert to a bytes array, just for test usage.
+ * @return bytes array
+ */
+ public byte[] toBytesArray() {
+ byte[] bytesArr = new byte[chunkBuffer.remaining()];
+ // Avoid affecting the original one
+ chunkBuffer.mark();
+ chunkBuffer.get(bytesArr);
+ chunkBuffer.reset();
+
+ return bytesArr;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
new file mode 100644
index 0000000..fb02476
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECSchema.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Erasure coding schema to housekeeper relevant information.
+ */
+public final class ECSchema {
+ public static final String NUM_DATA_UNITS_KEY = "numDataUnits";
+ public static final String NUM_PARITY_UNITS_KEY = "numParityUnits";
+ public static final String CODEC_NAME_KEY = "codec";
+
+ /**
+ * A friendly and understandable name that can mean what's it, also serves as
+ * the identifier that distinguish it from other schemas.
+ */
+ private final String schemaName;
+
+ /**
+ * The erasure codec name associated.
+ */
+ private final String codecName;
+
+ /**
+ * Number of source data units coded
+ */
+ private final int numDataUnits;
+
+ /**
+ * Number of parity units generated in a coding
+ */
+ private final int numParityUnits;
+
+ /*
+ * An erasure code can have its own specific advanced parameters, subject to
+ * itself to interpret these key-value settings.
+ */
+ private final Map<String, String> extraOptions;
+
+ /**
+ * Constructor with schema name and provided all options. Note the options may
+ * contain additional information for the erasure codec to interpret further.
+ * @param schemaName schema name
+ * @param allOptions all schema options
+ */
+ public ECSchema(String schemaName, Map<String, String> allOptions) {
+ assert (schemaName != null && ! schemaName.isEmpty());
+
+ this.schemaName = schemaName;
+
+ if (allOptions == null || allOptions.isEmpty()) {
+ throw new IllegalArgumentException("No schema options are provided");
+ }
+
+ this.codecName = allOptions.get(CODEC_NAME_KEY);
+ if (codecName == null || codecName.isEmpty()) {
+ throw new IllegalArgumentException("No codec option is provided");
+ }
+
+ int tmpNumDataUnits = extractIntOption(NUM_DATA_UNITS_KEY, allOptions);
+ int tmpNumParityUnits = extractIntOption(NUM_PARITY_UNITS_KEY, allOptions);
+ if (tmpNumDataUnits < 0 || tmpNumParityUnits < 0) {
+ throw new IllegalArgumentException(
+ "No good option for numDataUnits or numParityUnits found ");
+ }
+ this.numDataUnits = tmpNumDataUnits;
+ this.numParityUnits = tmpNumParityUnits;
+
+ allOptions.remove(CODEC_NAME_KEY);
+ allOptions.remove(NUM_DATA_UNITS_KEY);
+ allOptions.remove(NUM_PARITY_UNITS_KEY);
+ // After some cleanup
+ this.extraOptions = Collections.unmodifiableMap(allOptions);
+ }
+
+ /**
+ * Constructor with key parameters provided.
+ * @param schemaName schema name
+ * @param codecName codec name
+ * @param numDataUnits number of data units used in the schema
+ * @param numParityUnits number os parity units used in the schema
+ */
+ public ECSchema(String schemaName, String codecName,
+ int numDataUnits, int numParityUnits) {
+ this(schemaName, codecName, numDataUnits, numParityUnits, null);
+ }
+
+ /**
+ * Constructor with key parameters provided. Note the extraOptions may contain
+ * additional information for the erasure codec to interpret further.
+ * @param schemaName schema name
+ * @param codecName codec name
+ * @param numDataUnits number of data units used in the schema
+ * @param numParityUnits number os parity units used in the schema
+ * @param extraOptions extra options to configure the codec
+ */
+ public ECSchema(String schemaName, String codecName, int numDataUnits,
+ int numParityUnits, Map<String, String> extraOptions) {
+
+ assert (schemaName != null && ! schemaName.isEmpty());
+ assert (codecName != null && ! codecName.isEmpty());
+ assert (numDataUnits > 0 && numParityUnits > 0);
+
+ this.schemaName = schemaName;
+ this.codecName = codecName;
+ this.numDataUnits = numDataUnits;
+ this.numParityUnits = numParityUnits;
+
+ if (extraOptions == null) {
+ extraOptions = new HashMap<>();
+ }
+
+ // After some cleanup
+ this.extraOptions = Collections.unmodifiableMap(extraOptions);
+ }
+
+ private int extractIntOption(String optionKey, Map<String, String> options) {
+ int result = -1;
+
+ try {
+ if (options.containsKey(optionKey)) {
+ result = Integer.parseInt(options.get(optionKey));
+ if (result <= 0) {
+ throw new IllegalArgumentException("Bad option value " + result +
+ " found for " + optionKey);
+ }
+ }
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException("Option value " +
+ options.get(optionKey) + " for " + optionKey +
+ " is found. It should be an integer");
+ }
+
+ return result;
+ }
+
+ /**
+ * Get the schema name
+ * @return schema name
+ */
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ /**
+ * Get the codec name
+ * @return codec name
+ */
+ public String getCodecName() {
+ return codecName;
+ }
+
+ /**
+ * Get extra options specific to a erasure code.
+ * @return extra options
+ */
+ public Map<String, String> getExtraOptions() {
+ return extraOptions;
+ }
+
+ /**
+ * Get required data units count in a coding group
+ * @return count of data units
+ */
+ public int getNumDataUnits() {
+ return numDataUnits;
+ }
+
+ /**
+ * Get required parity units count in a coding group
+ * @return count of parity units
+ */
+ public int getNumParityUnits() {
+ return numParityUnits;
+ }
+
+ /**
+ * Make a meaningful string representation for log output.
+ * @return string representation
+ */
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("ECSchema=[");
+
+ sb.append("Name=" + schemaName + ", ");
+ sb.append("Codec=" + codecName + ", ");
+ sb.append(NUM_DATA_UNITS_KEY + "=" + numDataUnits + ", ");
+ sb.append(NUM_PARITY_UNITS_KEY + "=" + numParityUnits);
+ sb.append((extraOptions.isEmpty() ? "" : ", "));
+
+ int i = 0;
+ for (String opt : extraOptions.keySet()) {
+ sb.append(opt + "=" + extraOptions.get(opt) +
+ (++i < extraOptions.size() ? ", " : ""));
+ }
+
+ sb.append("]");
+
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ ECSchema ecSchema = (ECSchema) o;
+
+ if (numDataUnits != ecSchema.numDataUnits) {
+ return false;
+ }
+ if (numParityUnits != ecSchema.numParityUnits) {
+ return false;
+ }
+ if (!schemaName.equals(ecSchema.schemaName)) {
+ return false;
+ }
+ if (!codecName.equals(ecSchema.codecName)) {
+ return false;
+ }
+ return extraOptions.equals(ecSchema.extraOptions);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = schemaName.hashCode();
+ result = 31 * result + codecName.hashCode();
+ result = 31 * result + extraOptions.hashCode();
+ result = 31 * result + numDataUnits;
+ result = 31 * result + numParityUnits;
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java
new file mode 100644
index 0000000..fce46f8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/SchemaLoader.java
@@ -0,0 +1,152 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.w3c.dom.Text;
+import org.xml.sax.SAXException;
+
+/**
+ * A EC schema loading utility that loads predefined EC schemas from XML file
+ */
+public class SchemaLoader {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ SchemaLoader.class.getName());
+
+ /**
+ * Load predefined ec schemas from configuration file. This file is
+ * expected to be in the XML format.
+ */
+ public List<ECSchema> loadSchema(String schemaFilePath) {
+ File confFile = getSchemaFile(schemaFilePath);
+ if (confFile == null) {
+ LOG.warn("Not found any predefined EC schema file");
+ return Collections.emptyList();
+ }
+
+ try {
+ return loadSchema(confFile);
+ } catch (ParserConfigurationException e) {
+ throw new RuntimeException("Failed to load schema file: " + confFile);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to load schema file: " + confFile);
+ } catch (SAXException e) {
+ throw new RuntimeException("Failed to load schema file: " + confFile);
+ }
+ }
+
+ private List<ECSchema> loadSchema(File schemaFile)
+ throws ParserConfigurationException, IOException, SAXException {
+
+ LOG.info("Loading predefined EC schema file {}", schemaFile);
+
+ // Read and parse the schema file.
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ dbf.setIgnoringComments(true);
+ DocumentBuilder builder = dbf.newDocumentBuilder();
+ Document doc = builder.parse(schemaFile);
+ Element root = doc.getDocumentElement();
+
+ if (!"schemas".equals(root.getTagName())) {
+ throw new RuntimeException("Bad EC schema config file: " +
+ "top-level element not <schemas>");
+ }
+
+ NodeList elements = root.getChildNodes();
+ List<ECSchema> schemas = new ArrayList<ECSchema>();
+ for (int i = 0; i < elements.getLength(); i++) {
+ Node node = elements.item(i);
+ if (node instanceof Element) {
+ Element element = (Element) node;
+ if ("schema".equals(element.getTagName())) {
+ ECSchema schema = loadSchema(element);
+ schemas.add(schema);
+ } else {
+ LOG.warn("Bad element in EC schema configuration file: {}",
+ element.getTagName());
+ }
+ }
+ }
+
+ return schemas;
+ }
+
+ /**
+ * Path to the XML file containing predefined ec schemas. If the path is
+ * relative, it is searched for in the classpath.
+ */
+ private File getSchemaFile(String schemaFilePath) {
+ File schemaFile = new File(schemaFilePath);
+ if (! schemaFile.isAbsolute()) {
+ URL url = Thread.currentThread().getContextClassLoader()
+ .getResource(schemaFilePath);
+ if (url == null) {
+ LOG.warn("{} not found on the classpath.", schemaFilePath);
+ schemaFile = null;
+ } else if (! url.getProtocol().equalsIgnoreCase("file")) {
+ throw new RuntimeException(
+ "EC predefined schema file " + url +
+ " found on the classpath is not on the local filesystem.");
+ } else {
+ schemaFile = new File(url.getPath());
+ }
+ }
+
+ return schemaFile;
+ }
+
+ /**
+ * Loads a schema from a schema element in the configuration file
+ */
+ private ECSchema loadSchema(Element element) {
+ String schemaName = element.getAttribute("name");
+ Map<String, String> ecOptions = new HashMap<String, String>();
+ NodeList fields = element.getChildNodes();
+
+ for (int i = 0; i < fields.getLength(); i++) {
+ Node fieldNode = fields.item(i);
+ if (fieldNode instanceof Element) {
+ Element field = (Element) fieldNode;
+ String tagName = field.getTagName();
+ String value = ((Text) field.getFirstChild()).getData().trim();
+ ecOptions.put(tagName, value);
+ }
+ }
+
+ ECSchema schema = new ECSchema(schemaName, ecOptions);
+ return schema;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
new file mode 100644
index 0000000..0cacfbc
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
+
+/**
+ * Abstract Erasure Codec that implements {@link ErasureCodec}.
+ */
+public abstract class AbstractErasureCodec extends Configured
+ implements ErasureCodec {
+
+ private final ECSchema schema;
+
+ public AbstractErasureCodec(ECSchema schema) {
+ this.schema = schema;
+ }
+
+ public String getName() {
+ return schema.getCodecName();
+ }
+
+ public ECSchema getSchema() {
+ return schema;
+ }
+
+ @Override
+ public BlockGrouper createBlockGrouper() {
+ BlockGrouper blockGrouper = new BlockGrouper();
+ blockGrouper.setSchema(getSchema());
+
+ return blockGrouper;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
new file mode 100644
index 0000000..9aa3db2
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/ErasureCodec.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.io.erasurecode.grouper.BlockGrouper;
+
+/**
+ * Erasure Codec API that's to cover the essential specific aspects of a code.
+ * Currently it cares only block grouper and erasure coder. In future we may
+ * add more aspects here to make the behaviors customizable.
+ */
+public interface ErasureCodec extends Configurable {
+
+ /**
+ * Create block grouper
+ * @return block grouper
+ */
+ public BlockGrouper createBlockGrouper();
+
+ /**
+ * Create Erasure Encoder
+ * @return erasure encoder
+ */
+ public ErasureCoder createEncoder();
+
+ /**
+ * Create Erasure Decoder
+ * @return erasure decoder
+ */
+ public ErasureCoder createDecoder();
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
new file mode 100644
index 0000000..6edd638
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/RSErasureCodec.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.io.erasurecode.coder.RSErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.RSErasureEncoder;
+
+/**
+ * A Reed-Solomon erasure codec.
+ */
+public class RSErasureCodec extends AbstractErasureCodec {
+
+ public RSErasureCodec(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ public ErasureCoder createEncoder() {
+ return new RSErasureEncoder(getSchema());
+ }
+
+ @Override
+ public ErasureCoder createDecoder() {
+ return new RSErasureDecoder(getSchema());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
new file mode 100644
index 0000000..e2dcfa7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/XORErasureCodec.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.coder.ErasureCoder;
+import org.apache.hadoop.io.erasurecode.coder.XORErasureDecoder;
+import org.apache.hadoop.io.erasurecode.coder.XORErasureEncoder;
+
+/**
+ * A XOR erasure codec.
+ */
+public class XORErasureCodec extends AbstractErasureCodec {
+
+ public XORErasureCodec(ECSchema schema) {
+ super(schema);
+ assert(schema.getNumParityUnits() == 1);
+ }
+
+ @Override
+ public ErasureCoder createEncoder() {
+ return new XORErasureEncoder(getSchema());
+ }
+
+ @Override
+ public ErasureCoder createDecoder() {
+ return new XORErasureDecoder(getSchema());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
new file mode 100644
index 0000000..5cd0ee8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCoder.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+/**
+ * A common class of basic facilities to be shared by encoder and decoder
+ *
+ * It implements the {@link ErasureCoder} interface.
+ */
+public abstract class AbstractErasureCoder
+ extends Configured implements ErasureCoder {
+
+ private final int numDataUnits;
+ private final int numParityUnits;
+
+ public AbstractErasureCoder(int numDataUnits, int numParityUnits) {
+ this.numDataUnits = numDataUnits;
+ this.numParityUnits = numParityUnits;
+ }
+
+ public AbstractErasureCoder(ECSchema schema) {
+ this(schema.getNumDataUnits(), schema.getNumParityUnits());
+ }
+
+ @Override
+ public int getNumDataUnits() {
+ return numDataUnits;
+ }
+
+ @Override
+ public int getNumParityUnits() {
+ return numParityUnits;
+ }
+
+ @Override
+ public boolean preferDirectBuffer() {
+ return false;
+ }
+
+ @Override
+ public void release() {
+ // Nothing to do by default
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java
new file mode 100644
index 0000000..c429d49
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureCodingStep.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+
+/**
+ * Abstract class for common facilities shared by {@link ErasureEncodingStep}
+ * and {@link ErasureDecodingStep}.
+ *
+ * It implements {@link ErasureEncodingStep}.
+ */
+public abstract class AbstractErasureCodingStep implements ErasureCodingStep {
+
+ private ECBlock[] inputBlocks;
+ private ECBlock[] outputBlocks;
+
+ /**
+ * Constructor given input blocks and output blocks.
+ * @param inputBlocks
+ * @param outputBlocks
+ */
+ public AbstractErasureCodingStep(ECBlock[] inputBlocks,
+ ECBlock[] outputBlocks) {
+ this.inputBlocks = inputBlocks;
+ this.outputBlocks = outputBlocks;
+ }
+
+ @Override
+ public ECBlock[] getInputBlocks() {
+ return inputBlocks;
+ }
+
+ @Override
+ public ECBlock[] getOutputBlocks() {
+ return outputBlocks;
+ }
+
+ @Override
+ public void finish() {
+ // NOOP by default
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
new file mode 100644
index 0000000..3ea9311
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+/**
+ * An abstract erasure decoder that's to be inherited by new decoders.
+ *
+ * It implements the {@link ErasureCoder} interface.
+ */
+public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
+
+ public AbstractErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public AbstractErasureDecoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
+ // We may have more than this when considering complicate cases. HADOOP-11550
+ return prepareDecodingStep(blockGroup);
+ }
+
+ /**
+ * Perform decoding against a block blockGroup.
+ * @param blockGroup
+ * @return decoding step for caller to do the real work
+ */
+ protected abstract ErasureCodingStep prepareDecodingStep(
+ ECBlockGroup blockGroup);
+
+ /**
+ * We have all the data blocks and parity blocks as input blocks for
+ * recovering by default. It's codec specific
+ * @param blockGroup
+ * @return
+ */
+ protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
+ ECBlock[] inputBlocks = new ECBlock[getNumParityUnits()
+ + getNumDataUnits()];
+
+ System.arraycopy(blockGroup.getParityBlocks(), 0, inputBlocks, 0,
+ getNumParityUnits());
+ System.arraycopy(blockGroup.getDataBlocks(), 0, inputBlocks,
+ getNumParityUnits(), getNumDataUnits());
+
+ return inputBlocks;
+ }
+
+ /**
+ * Which blocks were erased ?
+ * @param blockGroup
+ * @return output blocks to recover
+ */
+ protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
+ ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)];
+
+ int idx = 0;
+
+ for (int i = 0; i < getNumParityUnits(); i++) {
+ if (blockGroup.getParityBlocks()[i].isErased()) {
+ outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
+ }
+ }
+
+ for (int i = 0; i < getNumDataUnits(); i++) {
+ if (blockGroup.getDataBlocks()[i].isErased()) {
+ outputBlocks[idx++] = blockGroup.getDataBlocks()[i];
+ }
+ }
+
+ return outputBlocks;
+ }
+
+ /**
+ * Get the number of erased blocks in the block group.
+ * @param blockGroup
+ * @return number of erased blocks
+ */
+ protected int getNumErasedBlocks(ECBlockGroup blockGroup) {
+ int num = getNumErasedBlocks(blockGroup.getParityBlocks());
+ num += getNumErasedBlocks(blockGroup.getDataBlocks());
+ return num;
+ }
+
+ /**
+ * Find out how many blocks are erased.
+ * @param inputBlocks all the input blocks
+ * @return number of erased blocks
+ */
+ protected static int getNumErasedBlocks(ECBlock[] inputBlocks) {
+ int numErased = 0;
+ for (int i = 0; i < inputBlocks.length; i++) {
+ if (inputBlocks[i].isErased()) {
+ numErased ++;
+ }
+ }
+
+ return numErased;
+ }
+
+ /**
+ * Get indexes of erased blocks from inputBlocks
+ * @param inputBlocks
+ * @return indexes of erased blocks from inputBlocks
+ */
+ protected int[] getErasedIndexes(ECBlock[] inputBlocks) {
+ int numErased = getNumErasedBlocks(inputBlocks);
+ if (numErased == 0) {
+ return new int[0];
+ }
+
+ int[] erasedIndexes = new int[numErased];
+ int i = 0, j = 0;
+ for (; i < inputBlocks.length && j < erasedIndexes.length; i++) {
+ if (inputBlocks[i].isErased()) {
+ erasedIndexes[j++] = i;
+ }
+ }
+
+ return erasedIndexes;
+ }
+
+ /**
+ * Get erased input blocks from inputBlocks
+ * @param inputBlocks
+ * @return an array of erased blocks from inputBlocks
+ */
+ protected ECBlock[] getErasedBlocks(ECBlock[] inputBlocks) {
+ int numErased = getNumErasedBlocks(inputBlocks);
+ if (numErased == 0) {
+ return new ECBlock[0];
+ }
+
+ ECBlock[] erasedBlocks = new ECBlock[numErased];
+ int i = 0, j = 0;
+ for (; i < inputBlocks.length && j < erasedBlocks.length; i++) {
+ if (inputBlocks[i].isErased()) {
+ erasedBlocks[j++] = inputBlocks[i];
+ }
+ }
+
+ return erasedBlocks;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
new file mode 100644
index 0000000..7c887e8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+/**
+ * An abstract erasure encoder that's to be inherited by new encoders.
+ *
+ * It implements the {@link ErasureCoder} interface.
+ */
+public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
+
+ public AbstractErasureEncoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public AbstractErasureEncoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
+ // We may have more than this when considering complicate cases. HADOOP-11550
+ return prepareEncodingStep(blockGroup);
+ }
+
+ /**
+ * Perform encoding against a block group.
+ * @param blockGroup
+ * @return encoding step for caller to do the real work
+ */
+ protected abstract ErasureCodingStep prepareEncodingStep(
+ ECBlockGroup blockGroup);
+
+ protected ECBlock[] getInputBlocks(ECBlockGroup blockGroup) {
+ return blockGroup.getDataBlocks();
+ }
+
+ protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
+ return blockGroup.getParityBlocks();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
new file mode 100644
index 0000000..f05ea41
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+
+/**
+ * An erasure coder to perform encoding or decoding given a group. Generally it
+ * involves calculating necessary internal steps according to codec logic. For
+ * each step,it calculates necessary input blocks to read chunks from and output
+ * parity blocks to write parity chunks into from the group. It also takes care
+ * of appropriate raw coder to use for the step. And encapsulates all the
+ * necessary info (input blocks, output blocks and raw coder) into a step
+ * represented by {@link ErasureCodingStep}. ErasureCoder callers can use the
+ * step to do the real work with retrieved input and output chunks.
+ *
+ * Note, currently only one coding step is supported. Will support complex cases
+ * of multiple coding steps.
+ *
+ */
+public interface ErasureCoder extends Configurable {
+
+ /**
+ * The number of data input units for the coding. A unit can be a byte,
+ * chunk or buffer or even a block.
+ * @return count of data input units
+ */
+ public int getNumDataUnits();
+
+ /**
+ * The number of parity output units for the coding. A unit can be a byte,
+ * chunk, buffer or even a block.
+ * @return count of parity output units
+ */
+ public int getNumParityUnits();
+
+ /**
+ * Calculate the encoding or decoding steps given a block blockGroup.
+ *
+ * Note, currently only one coding step is supported. Will support complex
+ * cases of multiple coding steps.
+ *
+ * @param blockGroup the erasure coding block group containing all necessary
+ * information for codec calculation
+ */
+ public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
+
+ /**
+ * Tell if direct or off-heap buffer is preferred or not. It's for callers to
+ * decide how to allocate coding chunk buffers, either on heap or off heap.
+ * It will return false by default.
+ * @return true if direct buffer is preferred for performance consideration,
+ * otherwise false.
+ */
+ public boolean preferDirectBuffer();
+
+ /**
+ * Release the resources if any. Good chance to invoke RawErasureCoder#release.
+ */
+ public void release();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
new file mode 100644
index 0000000..a3b177f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCodingStep.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+
+/**
+ * Erasure coding step that's involved in encoding/decoding of a block group.
+ */
+public interface ErasureCodingStep {
+
+ /**
+ * Input blocks of readable data involved in this step, may be data blocks
+ * or parity blocks.
+ * @return input blocks
+ */
+ public ECBlock[] getInputBlocks();
+
+ /**
+ * Output blocks of writable buffers involved in this step, may be data
+ * blocks or parity blocks.
+ * @return output blocks
+ */
+ public ECBlock[] getOutputBlocks();
+
+ /**
+ * Perform encoding or decoding given the input chunks, and generated results
+ * will be written to the output chunks.
+ * @param inputChunks
+ * @param outputChunks
+ */
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks);
+
+ /**
+ * Notify erasure coder that all the chunks of input blocks are processed so
+ * the coder can be able to update internal states, considering next step.
+ */
+ public void finish();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
new file mode 100644
index 0000000..980c580
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecodingStep.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+
+/**
+ * Erasure decoding step, a wrapper of all the necessary information to perform
+ * a decoding step involved in the whole process of decoding a block group.
+ */
+public class ErasureDecodingStep extends AbstractErasureCodingStep {
+ private int[] erasedIndexes;
+ private RawErasureDecoder rawDecoder;
+
+ /**
+ * The constructor with all the necessary info.
+ * @param inputBlocks
+ * @param erasedIndexes the indexes of erased blocks in inputBlocks array
+ * @param outputBlocks
+ * @param rawDecoder
+ */
+ public ErasureDecodingStep(ECBlock[] inputBlocks, int[] erasedIndexes,
+ ECBlock[] outputBlocks,
+ RawErasureDecoder rawDecoder) {
+ super(inputBlocks, outputBlocks);
+ this.erasedIndexes = erasedIndexes;
+ this.rawDecoder = rawDecoder;
+ }
+
+ @Override
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ rawDecoder.decode(inputChunks, erasedIndexes, outputChunks);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
new file mode 100644
index 0000000..bd7587f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncodingStep.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+
+/**
+ * Erasure encoding step, a wrapper of all the necessary information to perform
+ * an encoding step involved in the whole process of encoding a block group.
+ */
+public class ErasureEncodingStep extends AbstractErasureCodingStep {
+
+ private RawErasureEncoder rawEncoder;
+
+ /**
+ * The constructor with all the necessary info.
+ * @param inputBlocks
+ * @param outputBlocks
+ * @param rawEncoder
+ */
+ public ErasureEncodingStep(ECBlock[] inputBlocks, ECBlock[] outputBlocks,
+ RawErasureEncoder rawEncoder) {
+ super(inputBlocks, outputBlocks);
+ this.rawEncoder = rawEncoder;
+ }
+
+ @Override
+ public void performCoding(ECChunk[] inputChunks, ECChunk[] outputChunks) {
+ rawEncoder.encode(inputChunks, outputChunks);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
new file mode 100644
index 0000000..f56674d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/RSErasureDecoder.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+
+/**
+ * Reed-Solomon erasure decoder that decodes a block group.
+ *
+ * It implements {@link ErasureCoder}.
+ */
+public class RSErasureDecoder extends AbstractErasureDecoder {
+ private RawErasureDecoder rsRawDecoder;
+
+ public RSErasureDecoder(int numDataUnits, int numParityUnits) {
+ super(numDataUnits, numParityUnits);
+ }
+
+ public RSErasureDecoder(ECSchema schema) {
+ super(schema);
+ }
+
+ @Override
+ protected ErasureCodingStep prepareDecodingStep(final ECBlockGroup blockGroup) {
+
+ ECBlock[] inputBlocks = getInputBlocks(blockGroup);
+ ECBlock[] outputBlocks = getOutputBlocks(blockGroup);
+
+ RawErasureDecoder rawDecoder = checkCreateRSRawDecoder();
+ return new ErasureDecodingStep(inputBlocks,
+ getErasedIndexes(inputBlocks), outputBlocks, rawDecoder);
+ }
+
+ private RawErasureDecoder checkCreateRSRawDecoder() {
+ if (rsRawDecoder == null) {
+ rsRawDecoder = CodecUtil.createRSRawDecoder(getConf(),
+ getNumDataUnits(), getNumParityUnits());
+ }
+ return rsRawDecoder;
+ }
+
+ @Override
+ public void release() {
+ if (rsRawDecoder != null) {
+ rsRawDecoder.release();
+ }
+ }
+}
[09/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
new file mode 100644
index 0000000..9285fd7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRecoverStripedFile.java
@@ -0,0 +1,400 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ThreadLocalRandom;
+
+public class TestRecoverStripedFile {
+ public static final Log LOG = LogFactory.getLog(TestRecoverStripedFile.class);
+
+ private static final int dataBlkNum = HdfsConstants.NUM_DATA_BLOCKS;
+ private static final int parityBlkNum = HdfsConstants.NUM_PARITY_BLOCKS;
+ private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private static final int blockSize = cellSize * 3;
+ private static final int groupSize = dataBlkNum + parityBlkNum;
+ private static final int dnNum = groupSize + parityBlkNum;
+
+ private MiniDFSCluster cluster;
+ private Configuration conf;
+ private DistributedFileSystem fs;
+ // Map: DatanodeID -> datanode index in cluster
+ private Map<DatanodeID, Integer> dnMap = new HashMap<DatanodeID, Integer>();
+
+ @Before
+ public void setup() throws IOException {
+ conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_STRIPED_READ_BUFFER_SIZE_KEY, cellSize - 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(dnNum).build();;
+ cluster.waitActive();
+
+ fs = cluster.getFileSystem();
+ fs.getClient().createErasureCodingZone("/", null, 0);
+
+ List<DataNode> datanodes = cluster.getDataNodes();
+ for (int i = 0; i < dnNum; i++) {
+ dnMap.put(datanodes.get(i).getDatanodeId(), i);
+ }
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverOneParityBlock() throws Exception {
+ int fileLen = 10 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverOneParityBlock", fileLen, 0, 1);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverOneParityBlock1() throws Exception {
+ int fileLen = cellSize + cellSize/10;
+ assertFileBlocksRecovery("/testRecoverOneParityBlock1", fileLen, 0, 1);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverOneParityBlock2() throws Exception {
+ int fileLen = 1;
+ assertFileBlocksRecovery("/testRecoverOneParityBlock2", fileLen, 0, 1);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverOneParityBlock3() throws Exception {
+ int fileLen = 3 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverOneParityBlock3", fileLen, 0, 1);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverThreeParityBlocks() throws Exception {
+ int fileLen = 10 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverThreeParityBlocks", fileLen, 0, 3);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverThreeDataBlocks() throws Exception {
+ int fileLen = 10 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverThreeDataBlocks", fileLen, 1, 3);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverThreeDataBlocks1() throws Exception {
+ int fileLen = 3 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverThreeDataBlocks1", fileLen, 1, 3);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverOneDataBlock() throws Exception {
+ int fileLen = 10 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverOneDataBlock", fileLen, 1, 1);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverOneDataBlock1() throws Exception {
+ int fileLen = cellSize + cellSize/10;
+ assertFileBlocksRecovery("/testRecoverOneDataBlock1", fileLen, 1, 1);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverOneDataBlock2() throws Exception {
+ int fileLen = 1;
+ assertFileBlocksRecovery("/testRecoverOneDataBlock2", fileLen, 1, 1);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverAnyBlocks() throws Exception {
+ int fileLen = 3 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverAnyBlocks", fileLen, 2, 2);
+ }
+
+ @Test(timeout = 120000)
+ public void testRecoverAnyBlocks1() throws Exception {
+ int fileLen = 10 * blockSize + blockSize/10;
+ assertFileBlocksRecovery("/testRecoverAnyBlocks1", fileLen, 2, 3);
+ }
+
+ /**
+ * Test the file blocks recovery.
+ * 1. Check the replica is recovered in the target datanode,
+ * and verify the block replica length, generationStamp and content.
+ * 2. Read the file and verify content.
+ */
+ private void assertFileBlocksRecovery(String fileName, int fileLen,
+ int recovery, int toRecoverBlockNum) throws Exception {
+ if (recovery != 0 && recovery != 1 && recovery != 2) {
+ Assert.fail("Invalid recovery: 0 is to recovery parity blocks,"
+ + "1 is to recovery data blocks, 2 is any.");
+ }
+ if (toRecoverBlockNum < 1 || toRecoverBlockNum > parityBlkNum) {
+ Assert.fail("toRecoverBlockNum should be between 1 ~ " + parityBlkNum);
+ }
+
+ Path file = new Path(fileName);
+
+ testCreateStripedFile(file, fileLen);
+
+ LocatedBlocks locatedBlocks = getLocatedBlocks(file);
+ assertEquals(locatedBlocks.getFileLength(), fileLen);
+
+ LocatedStripedBlock lastBlock =
+ (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+
+ DatanodeInfo[] storageInfos = lastBlock.getLocations();
+ int[] indices = lastBlock.getBlockIndices();
+
+ BitSet bitset = new BitSet(dnNum);
+ for (DatanodeInfo storageInfo : storageInfos) {
+ bitset.set(dnMap.get(storageInfo));
+ }
+
+ int[] toDead = new int[toRecoverBlockNum];
+ int n = 0;
+ for (int i = 0; i < indices.length; i++) {
+ if (n < toRecoverBlockNum) {
+ if (recovery == 0) {
+ if (indices[i] >= dataBlkNum) {
+ toDead[n++] = i;
+ }
+ } else if (recovery == 1) {
+ if (indices[i] < dataBlkNum) {
+ toDead[n++] = i;
+ }
+ } else {
+ toDead[n++] = i;
+ }
+ } else {
+ break;
+ }
+ }
+
+ DatanodeInfo[] dataDNs = new DatanodeInfo[toRecoverBlockNum];
+ int[] deadDnIndices = new int[toRecoverBlockNum];
+ ExtendedBlock[] blocks = new ExtendedBlock[toRecoverBlockNum];
+ File[] replicas = new File[toRecoverBlockNum];
+ File[] metadatas = new File[toRecoverBlockNum];
+ byte[][] replicaContents = new byte[toRecoverBlockNum][];
+ for (int i = 0; i < toRecoverBlockNum; i++) {
+ dataDNs[i] = storageInfos[toDead[i]];
+ deadDnIndices[i] = dnMap.get(dataDNs[i]);
+
+ // Check the block replica file on deadDn before it dead.
+ blocks[i] = StripedBlockUtil.constructInternalBlock(
+ lastBlock.getBlock(), cellSize, dataBlkNum, indices[toDead[i]]);
+ replicas[i] = cluster.getBlockFile(deadDnIndices[i], blocks[i]);
+ metadatas[i] = cluster.getBlockMetadataFile(deadDnIndices[i], blocks[i]);
+ // the block replica on the datanode should be the same as expected
+ assertEquals(replicas[i].length(),
+ StripedBlockUtil.getInternalBlockLength(
+ lastBlock.getBlockSize(), cellSize, dataBlkNum, indices[toDead[i]]));
+ assertTrue(metadatas[i].getName().
+ endsWith(blocks[i].getGenerationStamp() + ".meta"));
+ replicaContents[i] = readReplica(replicas[i]);
+ }
+
+ int cellsNum = (fileLen - 1) / cellSize + 1;
+ int groupSize = Math.min(cellsNum, dataBlkNum) + parityBlkNum;
+
+ try {
+ DatanodeID[] dnIDs = new DatanodeID[toRecoverBlockNum];
+ for (int i = 0; i < toRecoverBlockNum; i++) {
+ /*
+ * Kill the datanode which contains one replica
+ * We need to make sure it dead in namenode: clear its update time and
+ * trigger NN to check heartbeat.
+ */
+ DataNode dn = cluster.getDataNodes().get(deadDnIndices[i]);
+ dn.shutdown();
+ dnIDs[i] = dn.getDatanodeId();
+ }
+ setDataNodesDead(dnIDs);
+
+ // Check the locatedBlocks of the file again
+ locatedBlocks = getLocatedBlocks(file);
+ lastBlock = (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+ storageInfos = lastBlock.getLocations();
+ assertEquals(storageInfos.length, groupSize - toRecoverBlockNum);
+
+ int[] targetDNs = new int[dnNum - groupSize];
+ n = 0;
+ for (int i = 0; i < dnNum; i++) {
+ if (!bitset.get(i)) { // not contain replica of the block.
+ targetDNs[n++] = i;
+ }
+ }
+
+ waitForRecoveryFinished(file, groupSize);
+
+ targetDNs = sortTargetsByReplicas(blocks, targetDNs);
+
+ // Check the replica on the new target node.
+ for (int i = 0; i < toRecoverBlockNum; i++) {
+ File replicaAfterRecovery = cluster.getBlockFile(targetDNs[i], blocks[i]);
+ File metadataAfterRecovery =
+ cluster.getBlockMetadataFile(targetDNs[i], blocks[i]);
+ assertEquals(replicaAfterRecovery.length(), replicas[i].length());
+ assertTrue(metadataAfterRecovery.getName().
+ endsWith(blocks[i].getGenerationStamp() + ".meta"));
+ byte[] replicaContentAfterRecovery = readReplica(replicaAfterRecovery);
+
+ Assert.assertArrayEquals(replicaContents[i], replicaContentAfterRecovery);
+ }
+ } finally {
+ for (int i = 0; i < toRecoverBlockNum; i++) {
+ restartDataNode(toDead[i]);
+ }
+ cluster.waitActive();
+ }
+ fs.delete(file, true);
+ }
+
+ private void setDataNodesDead(DatanodeID[] dnIDs) throws IOException {
+ for (DatanodeID dn : dnIDs) {
+ DatanodeDescriptor dnd =
+ NameNodeAdapter.getDatanode(cluster.getNamesystem(), dn);
+ DFSTestUtil.setDatanodeDead(dnd);
+ }
+
+ BlockManagerTestUtil.checkHeartbeat(cluster.getNamesystem().getBlockManager());
+ }
+
+ private void restartDataNode(int dn) {
+ try {
+ cluster.restartDataNode(dn, true, true);
+ } catch (IOException e) {
+ }
+ }
+
+ private int[] sortTargetsByReplicas(ExtendedBlock[] blocks, int[] targetDNs) {
+ int[] result = new int[blocks.length];
+ for (int i = 0; i < blocks.length; i++) {
+ result[i] = -1;
+ for (int j = 0; j < targetDNs.length; j++) {
+ if (targetDNs[j] != -1) {
+ File replica = cluster.getBlockFile(targetDNs[j], blocks[i]);
+ if (replica != null) {
+ result[i] = targetDNs[j];
+ targetDNs[j] = -1;
+ break;
+ }
+ }
+ }
+ if (result[i] == -1) {
+ Assert.fail("Failed to recover striped block: " + blocks[i].getBlockId());
+ }
+ }
+ return result;
+ }
+
+ private byte[] readReplica(File replica) throws IOException {
+ int length = (int)replica.length();
+ ByteArrayOutputStream content = new ByteArrayOutputStream(length);
+ FileInputStream in = new FileInputStream(replica);
+ try {
+ byte[] buffer = new byte[1024];
+ int total = 0;
+ while (total < length) {
+ int n = in.read(buffer);
+ if (n <= 0) {
+ break;
+ }
+ content.write(buffer, 0, n);
+ total += n;
+ }
+ if (total < length) {
+ Assert.fail("Failed to read all content of replica");
+ }
+ return content.toByteArray();
+ } finally {
+ in.close();
+ }
+ }
+
+ private LocatedBlocks waitForRecoveryFinished(Path file, int groupSize)
+ throws Exception {
+ final int ATTEMPTS = 60;
+ for (int i = 0; i < ATTEMPTS; i++) {
+ LocatedBlocks locatedBlocks = getLocatedBlocks(file);
+ LocatedStripedBlock lastBlock =
+ (LocatedStripedBlock)locatedBlocks.getLastLocatedBlock();
+ DatanodeInfo[] storageInfos = lastBlock.getLocations();
+ if (storageInfos.length >= groupSize) {
+ return locatedBlocks;
+ }
+ Thread.sleep(1000);
+ }
+ throw new IOException ("Time out waiting for EC block recovery.");
+ }
+
+ private LocatedBlocks getLocatedBlocks(Path file) throws IOException {
+ return fs.getClient().getLocatedBlocks(file.toString(), 0, Long.MAX_VALUE);
+ }
+
+ private void testCreateStripedFile(Path file, int dataLen)
+ throws IOException {
+ final byte[] data = new byte[dataLen];
+ ThreadLocalRandom.current().nextBytes(data);
+ writeContents(file, data);
+ }
+
+ void writeContents(Path file, byte[] contents)
+ throws IOException {
+ FSDataOutputStream out = fs.create(file);
+ try {
+ out.write(contents, 0, contents.length);
+ } finally {
+ out.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index 6cea7e8..6b4e46a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -552,7 +552,7 @@ public class TestSafeMode {
if(cluster!= null) cluster.shutdown();
}
}
-
+
void checkGetBlockLocationsWorks(FileSystem fs, Path fileName) throws IOException {
FileStatus stat = fs.getFileStatus(fileName);
try {
@@ -560,7 +560,7 @@ public class TestSafeMode {
} catch (SafeModeException e) {
assertTrue("Should have not got safemode exception", false);
} catch (RemoteException re) {
- assertTrue("Should have not got safemode exception", false);
+ assertTrue("Should have not got remote exception", false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
new file mode 100644
index 0000000..6f0bc71
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestSafeModeWithStripedFile {
+
+ static final short DATA_BLK_NUM = HdfsConstants.NUM_DATA_BLOCKS;
+ static final short PARITY_BLK_NUM = HdfsConstants.NUM_PARITY_BLOCKS;
+ static final int numDNs = DATA_BLK_NUM + PARITY_BLK_NUM;
+ static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ static final int blockSize = cellSize * 2;
+
+ static MiniDFSCluster cluster;
+ static Configuration conf;
+
+ @Before
+ public void setup() throws IOException {
+ conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.getFileSystem().getClient().createErasureCodingZone("/",
+ null, cellSize);
+ cluster.waitActive();
+
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testStripedFile0() throws IOException {
+ doTest(cellSize, 1);
+ }
+
+ @Test
+ public void testStripedFile1() throws IOException {
+ doTest(cellSize * 5, 5);
+ }
+
+ /**
+ * This util writes a small block group whose size is given by caller.
+ * Then write another 2 full stripe blocks.
+ * Then shutdown all DNs and start again one by one. and verify the safemode
+ * status accordingly.
+ *
+ * @param smallSize file size of the small block group
+ * @param minStorages minimum replicas needed by the block so it can be safe
+ */
+ private void doTest(int smallSize, int minStorages) throws IOException {
+ FileSystem fs = cluster.getFileSystem();
+ // add 1 block
+ byte[] data = StripedFileTestUtil.generateBytes(smallSize);
+ Path smallFilePath = new Path("/testStripedFile_" + smallSize);
+ DFSTestUtil.writeFile(fs, smallFilePath, data);
+
+ // If we only have 1 block, NN won't enter safemode in the first place
+ // because the threshold is 0 blocks.
+ // So we need to add another 2 blocks.
+ int bigSize = blockSize * DATA_BLK_NUM * 2;
+ Path bigFilePath = new Path("/testStripedFile_" + bigSize);
+ data = StripedFileTestUtil.generateBytes(bigSize);
+ DFSTestUtil.writeFile(fs, bigFilePath, data);
+ // now we have 3 blocks. NN needs 2 blocks to reach the threshold 0.9 of
+ // total blocks 3.
+
+ // stopping all DNs
+ List<MiniDFSCluster.DataNodeProperties> dnprops = Lists.newArrayList();
+ LocatedBlocks lbs = cluster.getNameNodeRpc()
+ .getBlockLocations(smallFilePath.toString(), 0, smallSize);
+ DatanodeInfo[] locations = lbs.get(0).getLocations();
+ for (DatanodeInfo loc : locations) {
+ // keep the DNs that have smallFile in the head of dnprops
+ dnprops.add(cluster.stopDataNode(loc.getName()));
+ }
+ for (int i = 0; i < numDNs - locations.length; i++) {
+ dnprops.add(cluster.stopDataNode(0));
+ }
+
+ cluster.restartNameNode(0);
+ NameNode nn = cluster.getNameNode();
+ assertTrue(cluster.getNameNode().isInSafeMode());
+ assertEquals(0, NameNodeAdapter.getSafeModeSafeBlocks(nn));
+
+ // the block of smallFile doesn't reach minStorages,
+ // so the safe blocks count doesn't increment.
+ for (int i = 0; i < minStorages - 1; i++) {
+ cluster.restartDataNode(dnprops.remove(0));
+ cluster.triggerBlockReports();
+ assertEquals(0, NameNodeAdapter.getSafeModeSafeBlocks(nn));
+ }
+
+ // the block of smallFile reaches minStorages,
+ // so the safe blocks count increment.
+ cluster.restartDataNode(dnprops.remove(0));
+ cluster.triggerBlockReports();
+ assertEquals(1, NameNodeAdapter.getSafeModeSafeBlocks(nn));
+
+ // the 2 blocks of bigFile need DATA_BLK_NUM storages to be safe
+ for (int i = minStorages; i < DATA_BLK_NUM - 1; i++) {
+ cluster.restartDataNode(dnprops.remove(0));
+ cluster.triggerBlockReports();
+ assertTrue(nn.isInSafeMode());
+ }
+
+ cluster.restartDataNode(dnprops.remove(0));
+ cluster.triggerBlockReports();
+ assertFalse(nn.isInSafeMode());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
new file mode 100644
index 0000000..089a134
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java
@@ -0,0 +1,251 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.web.WebHdfsConstants;
+import org.apache.hadoop.hdfs.web.WebHdfsTestUtil;
+import org.apache.log4j.Level;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.cellSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.stripesPerBlock;
+
+public class TestWriteReadStripedFile {
+ public static final Log LOG = LogFactory.getLog(TestWriteReadStripedFile.class);
+ private static MiniDFSCluster cluster;
+ private static FileSystem fs;
+ private static Configuration conf = new HdfsConfiguration();
+
+ static {
+ ((Log4JLogger)LogFactory.getLog(BlockPlacementPolicy.class))
+ .getLogger().setLevel(Level.ALL);
+ }
+
+ @Before
+ public void setup() throws IOException {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.getFileSystem().getClient().createErasureCodingZone("/",
+ null, cellSize);
+ fs = cluster.getFileSystem();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testFileEmpty() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/EmptyFile", 0);
+ testOneFileUsingDFSStripedInputStream("/EmptyFile2", 0, true);
+ }
+
+ @Test
+ public void testFileSmallerThanOneCell1() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell", 1);
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell2", 1, true);
+ }
+
+ @Test
+ public void testFileSmallerThanOneCell2() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell", cellSize - 1);
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneCell2", cellSize - 1,
+ true);
+ }
+
+ @Test
+ public void testFileEqualsWithOneCell() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/EqualsWithOneCell", cellSize);
+ testOneFileUsingDFSStripedInputStream("/EqualsWithOneCell2", cellSize, true);
+ }
+
+ @Test
+ public void testFileSmallerThanOneStripe1() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe",
+ cellSize * dataBlocks - 1);
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe2",
+ cellSize * dataBlocks - 1, true);
+ }
+
+ @Test
+ public void testFileSmallerThanOneStripe2() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe",
+ cellSize + 123);
+ testOneFileUsingDFSStripedInputStream("/SmallerThanOneStripe2",
+ cellSize + 123, true);
+ }
+
+ @Test
+ public void testFileEqualsWithOneStripe() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/EqualsWithOneStripe",
+ cellSize * dataBlocks);
+ testOneFileUsingDFSStripedInputStream("/EqualsWithOneStripe2",
+ cellSize * dataBlocks, true);
+ }
+
+ @Test
+ public void testFileMoreThanOneStripe1() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe1",
+ cellSize * dataBlocks + 123);
+ testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe12",
+ cellSize * dataBlocks + 123, true);
+ }
+
+ @Test
+ public void testFileMoreThanOneStripe2() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe2",
+ cellSize * dataBlocks + cellSize * dataBlocks + 123);
+ testOneFileUsingDFSStripedInputStream("/MoreThanOneStripe22",
+ cellSize * dataBlocks + cellSize * dataBlocks + 123, true);
+ }
+
+ @Test
+ public void testLessThanFullBlockGroup() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/LessThanFullBlockGroup",
+ cellSize * dataBlocks * (stripesPerBlock - 1) + cellSize);
+ testOneFileUsingDFSStripedInputStream("/LessThanFullBlockGroup2",
+ cellSize * dataBlocks * (stripesPerBlock - 1) + cellSize, true);
+ }
+
+ @Test
+ public void testFileFullBlockGroup() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/FullBlockGroup",
+ blockSize * dataBlocks);
+ testOneFileUsingDFSStripedInputStream("/FullBlockGroup2",
+ blockSize * dataBlocks, true);
+ }
+
+ @Test
+ public void testFileMoreThanABlockGroup1() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup1",
+ blockSize * dataBlocks + 123);
+ testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup12",
+ blockSize * dataBlocks + 123, true);
+ }
+
+ @Test
+ public void testFileMoreThanABlockGroup2() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup2",
+ blockSize * dataBlocks + cellSize + 123);
+ testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup22",
+ blockSize * dataBlocks + cellSize + 123, true);
+ }
+
+
+ @Test
+ public void testFileMoreThanABlockGroup3() throws IOException {
+ testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup3",
+ blockSize * dataBlocks * 3 + cellSize * dataBlocks
+ + cellSize + 123);
+ testOneFileUsingDFSStripedInputStream("/MoreThanABlockGroup32",
+ blockSize * dataBlocks * 3 + cellSize * dataBlocks
+ + cellSize + 123, true);
+ }
+
+ private void testOneFileUsingDFSStripedInputStream(String src, int fileLength)
+ throws IOException {
+ testOneFileUsingDFSStripedInputStream(src, fileLength, false);
+ }
+
+ private void testOneFileUsingDFSStripedInputStream(String src, int fileLength,
+ boolean withDataNodeFailure) throws IOException {
+ final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
+ Path srcPath = new Path(src);
+ DFSTestUtil.writeFile(fs, srcPath, new String(expected));
+
+ StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
+
+ if (withDataNodeFailure) {
+ int dnIndex = 1; // TODO: StripedFileTestUtil.random.nextInt(dataBlocks);
+ LOG.info("stop DataNode " + dnIndex);
+ stopDataNode(srcPath, dnIndex);
+ }
+
+ byte[] smallBuf = new byte[1024];
+ byte[] largeBuf = new byte[fileLength + 100];
+ StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
+
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
+ largeBuf);
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
+ ByteBuffer.allocate(fileLength + 100));
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
+ smallBuf);
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
+ ByteBuffer.allocate(1024));
+ }
+
+ private void stopDataNode(Path path, int failedDNIdx)
+ throws IOException {
+ BlockLocation[] locs = fs.getFileBlockLocations(path, 0, cellSize);
+ if (locs != null && locs.length > 0) {
+ String name = (locs[0].getNames())[failedDNIdx];
+ for (DataNode dn : cluster.getDataNodes()) {
+ int port = dn.getXferPort();
+ if (name.contains(Integer.toString(port))) {
+ dn.shutdown();
+ break;
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testWriteReadUsingWebHdfs() throws Exception {
+ int fileLength = blockSize * dataBlocks + cellSize + 123;
+
+ final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
+ FileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(conf,
+ WebHdfsConstants.WEBHDFS_SCHEME);
+ Path srcPath = new Path("/testWriteReadUsingWebHdfs");
+ DFSTestUtil.writeFile(fs, srcPath, new String(expected));
+
+ StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
+
+ byte[] smallBuf = new byte[1024];
+ byte[] largeBuf = new byte[fileLength + 100];
+ // TODO: HDFS-8797
+ //StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
+
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, largeBuf);
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected, smallBuf);
+ // webhdfs doesn't support bytebuffer read
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
new file mode 100644
index 0000000..3679c5f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.blockSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.cellSize;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.dataBlocks;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.numDNs;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.parityBlocks;
+
+public class TestWriteStripedFileWithFailure {
+ public static final Log LOG = LogFactory
+ .getLog(TestWriteStripedFileWithFailure.class);
+ private static MiniDFSCluster cluster;
+ private static FileSystem fs;
+ private static Configuration conf = new HdfsConfiguration();
+ private final int smallFileLength = blockSize * dataBlocks - 123;
+ private final int largeFileLength = blockSize * dataBlocks + 123;
+ private final int[] fileLengths = {smallFileLength, largeFileLength};
+
+ public void setup() throws IOException {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+ cluster.getFileSystem().getClient().createErasureCodingZone("/",
+ null, cellSize);
+ fs = cluster.getFileSystem();
+ }
+
+ public void tearDown() throws IOException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ // Test writing file with some Datanodes failure
+ @Test(timeout = 300000)
+ public void testWriteStripedFileWithDNFailure() throws IOException {
+ for (int fileLength : fileLengths) {
+ for (int dataDelNum = 1; dataDelNum < 4; dataDelNum++) {
+ for (int parityDelNum = 0; (dataDelNum+parityDelNum) < 4; parityDelNum++) {
+ try {
+ // setup a new cluster with no dead datanode
+ setup();
+ writeFileWithDNFailure(fileLength, dataDelNum, parityDelNum);
+ } catch (IOException ioe) {
+ String fileType = fileLength < (blockSize * dataBlocks) ?
+ "smallFile" : "largeFile";
+ LOG.error("Failed to write file with DN failure:"
+ + " fileType = "+ fileType
+ + ", dataDelNum = " + dataDelNum
+ + ", parityDelNum = " + parityDelNum);
+ throw ioe;
+ } finally {
+ // tear down the cluster
+ tearDown();
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Test writing a file with shutting down some DNs(data DNs or parity DNs or both).
+ * @param fileLength file length
+ * @param dataDNFailureNum the shutdown number of data DNs
+ * @param parityDNFailureNum the shutdown number of parity DNs
+ * @throws IOException
+ */
+ private void writeFileWithDNFailure(int fileLength,
+ int dataDNFailureNum, int parityDNFailureNum) throws IOException {
+ String fileType = fileLength < (blockSize * dataBlocks) ?
+ "smallFile" : "largeFile";
+ String src = "/dnFailure_" + dataDNFailureNum + "_" + parityDNFailureNum
+ + "_" + fileType;
+ LOG.info("writeFileWithDNFailure: file = " + src
+ + ", fileType = " + fileType
+ + ", dataDNFailureNum = " + dataDNFailureNum
+ + ", parityDNFailureNum = " + parityDNFailureNum);
+
+ Path srcPath = new Path(src);
+ final AtomicInteger pos = new AtomicInteger();
+ final FSDataOutputStream out = fs.create(srcPath);
+ final DFSStripedOutputStream stripedOut
+ = (DFSStripedOutputStream)out.getWrappedStream();
+
+ int[] dataDNFailureIndices = StripedFileTestUtil.randomArray(0, dataBlocks,
+ dataDNFailureNum);
+ Assert.assertNotNull(dataDNFailureIndices);
+ int[] parityDNFailureIndices = StripedFileTestUtil.randomArray(dataBlocks,
+ dataBlocks + parityBlocks, parityDNFailureNum);
+ Assert.assertNotNull(parityDNFailureIndices);
+
+ int[] failedDataNodes = new int[dataDNFailureNum + parityDNFailureNum];
+ System.arraycopy(dataDNFailureIndices, 0, failedDataNodes,
+ 0, dataDNFailureIndices.length);
+ System.arraycopy(parityDNFailureIndices, 0, failedDataNodes,
+ dataDNFailureIndices.length, parityDNFailureIndices.length);
+
+ final int killPos = fileLength/2;
+ for (; pos.get() < fileLength; ) {
+ final int i = pos.getAndIncrement();
+ if (i == killPos) {
+ for(int failedDn : failedDataNodes) {
+ StripedFileTestUtil.killDatanode(cluster, stripedOut, failedDn, pos);
+ }
+ }
+ write(out, i);
+ }
+ out.close();
+
+ // make sure the expected number of Datanode have been killed
+ int dnFailureNum = dataDNFailureNum + parityDNFailureNum;
+ Assert.assertEquals(cluster.getDataNodes().size(), numDNs - dnFailureNum);
+
+ byte[] smallBuf = new byte[1024];
+ byte[] largeBuf = new byte[fileLength + 100];
+ final byte[] expected = StripedFileTestUtil.generateBytes(fileLength);
+ StripedFileTestUtil.verifyLength(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifySeek(fs, srcPath, fileLength);
+ StripedFileTestUtil.verifyStatefulRead(fs, srcPath, fileLength, expected,
+ smallBuf);
+ StripedFileTestUtil.verifyPread(fs, srcPath, fileLength, expected, largeBuf);
+
+ // delete the file
+ fs.delete(srcPath, true);
+ }
+
+ void write(FSDataOutputStream out, int i) throws IOException {
+ try {
+ out.write(StripedFileTestUtil.getByte(i));
+ } catch (IOException e) {
+ throw new IOException("Failed at i=" + i, e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
index 9f8aef5..e944b81 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocol/TestLayoutVersion.java
@@ -126,7 +126,8 @@ public class TestLayoutVersion {
EnumSet<NameNodeLayoutVersion.Feature> compatibleFeatures = EnumSet.of(
NameNodeLayoutVersion.Feature.TRUNCATE,
NameNodeLayoutVersion.Feature.APPEND_NEW_BLOCK,
- NameNodeLayoutVersion.Feature.QUOTA_BY_STORAGE_TYPE);
+ NameNodeLayoutVersion.Feature.QUOTA_BY_STORAGE_TYPE,
+ NameNodeLayoutVersion.Feature.ERASURE_CODING);
for (LayoutFeature f : compatibleFeatures) {
assertEquals(String.format("Expected minimum compatible layout version " +
"%d for feature %s.", baseLV, f), baseLV,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index c7233bd..3675e63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTrue;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -40,6 +41,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto;
+import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockECRecoveryCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeRegistrationProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
@@ -63,15 +65,21 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageInfoProto;
import org.apache.hadoop.hdfs.security.token.block.BlockKey;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.CheckpointSignature;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
import org.apache.hadoop.hdfs.server.protocol.BlockCommand;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand.BlockECRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand;
import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations;
import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.BlockWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.BlocksWithLocations.StripedBlockWithLocations;
+import org.apache.hadoop.hdfs.server.protocol.BlockECRecoveryCommand;
import org.apache.hadoop.hdfs.server.protocol.DatanodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
@@ -80,6 +88,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.DataChecksum;
@@ -181,40 +190,58 @@ public class TestPBHelper {
assertEquals(b, b2);
}
- private static BlockWithLocations getBlockWithLocations(int bid) {
+ private static BlockWithLocations getBlockWithLocations(
+ int bid, boolean isStriped) {
final String[] datanodeUuids = {"dn1", "dn2", "dn3"};
final String[] storageIDs = {"s1", "s2", "s3"};
final StorageType[] storageTypes = {
StorageType.DISK, StorageType.DISK, StorageType.DISK};
- return new BlockWithLocations(new Block(bid, 0, 1),
+ final byte[] indices = {0, 1, 2};
+ final short dataBlkNum = 6;
+ BlockWithLocations blkLocs = new BlockWithLocations(new Block(bid, 0, 1),
datanodeUuids, storageIDs, storageTypes);
+ if (isStriped) {
+ blkLocs = new StripedBlockWithLocations(blkLocs, indices, dataBlkNum);
+ }
+ return blkLocs;
}
private void compare(BlockWithLocations locs1, BlockWithLocations locs2) {
assertEquals(locs1.getBlock(), locs2.getBlock());
assertTrue(Arrays.equals(locs1.getStorageIDs(), locs2.getStorageIDs()));
+ if (locs1 instanceof StripedBlockWithLocations) {
+ assertTrue(Arrays.equals(((StripedBlockWithLocations) locs1).getIndices(),
+ ((StripedBlockWithLocations) locs2).getIndices()));
+ }
}
@Test
public void testConvertBlockWithLocations() {
- BlockWithLocations locs = getBlockWithLocations(1);
- BlockWithLocationsProto locsProto = PBHelper.convert(locs);
- BlockWithLocations locs2 = PBHelper.convert(locsProto);
- compare(locs, locs2);
+ boolean[] testSuite = new boolean[]{false, true};
+ for (int i = 0; i < testSuite.length; i++) {
+ BlockWithLocations locs = getBlockWithLocations(1, testSuite[i]);
+ BlockWithLocationsProto locsProto = PBHelper.convert(locs);
+ BlockWithLocations locs2 = PBHelper.convert(locsProto);
+ compare(locs, locs2);
+ }
}
@Test
public void testConvertBlocksWithLocations() {
- BlockWithLocations[] list = new BlockWithLocations[] {
- getBlockWithLocations(1), getBlockWithLocations(2) };
- BlocksWithLocations locs = new BlocksWithLocations(list);
- BlocksWithLocationsProto locsProto = PBHelper.convert(locs);
- BlocksWithLocations locs2 = PBHelper.convert(locsProto);
- BlockWithLocations[] blocks = locs.getBlocks();
- BlockWithLocations[] blocks2 = locs2.getBlocks();
- assertEquals(blocks.length, blocks2.length);
- for (int i = 0; i < blocks.length; i++) {
- compare(blocks[i], blocks2[i]);
+ boolean[] testSuite = new boolean[]{false, true};
+ for (int i = 0; i < testSuite.length; i++) {
+ BlockWithLocations[] list = new BlockWithLocations[]{
+ getBlockWithLocations(1, testSuite[i]),
+ getBlockWithLocations(2, testSuite[i])};
+ BlocksWithLocations locs = new BlocksWithLocations(list);
+ BlocksWithLocationsProto locsProto = PBHelper.convert(locs);
+ BlocksWithLocations locs2 = PBHelper.convert(locsProto);
+ BlockWithLocations[] blocks = locs.getBlocks();
+ BlockWithLocations[] blocks2 = locs2.getBlocks();
+ assertEquals(blocks.length, blocks2.length);
+ for (int j = 0; j < blocks.length; j++) {
+ compare(blocks[j], blocks2[j]);
+ }
}
}
@@ -489,16 +516,16 @@ public class TestPBHelper {
@Test
public void testConvertLocatedBlock() {
LocatedBlock lb = createLocatedBlock();
- LocatedBlockProto lbProto = PBHelper.convert(lb);
- LocatedBlock lb2 = PBHelper.convert(lbProto);
+ LocatedBlockProto lbProto = PBHelper.convertLocatedBlock(lb);
+ LocatedBlock lb2 = PBHelper.convertLocatedBlockProto(lbProto);
compare(lb,lb2);
}
@Test
public void testConvertLocatedBlockNoStorageMedia() {
LocatedBlock lb = createLocatedBlockNoStorageMedia();
- LocatedBlockProto lbProto = PBHelper.convert(lb);
- LocatedBlock lb2 = PBHelper.convert(lbProto);
+ LocatedBlockProto lbProto = PBHelper.convertLocatedBlock(lb);
+ LocatedBlock lb2 = PBHelper.convertLocatedBlockProto(lbProto);
compare(lb,lb2);
}
@@ -508,8 +535,8 @@ public class TestPBHelper {
for (int i=0;i<3;i++) {
lbl.add(createLocatedBlock());
}
- List<LocatedBlockProto> lbpl = PBHelper.convertLocatedBlock2(lbl);
- List<LocatedBlock> lbl2 = PBHelper.convertLocatedBlock(lbpl);
+ List<LocatedBlockProto> lbpl = PBHelper.convertLocatedBlocks2(lbl);
+ List<LocatedBlock> lbl2 = PBHelper.convertLocatedBlocks(lbpl);
assertEquals(lbl.size(), lbl2.size());
for (int i=0;i<lbl.size();i++) {
compare(lbl.get(i), lbl2.get(2));
@@ -522,8 +549,8 @@ public class TestPBHelper {
for (int i=0;i<3;i++) {
lbl[i] = createLocatedBlock();
}
- LocatedBlockProto [] lbpl = PBHelper.convertLocatedBlock(lbl);
- LocatedBlock [] lbl2 = PBHelper.convertLocatedBlock(lbpl);
+ LocatedBlockProto [] lbpl = PBHelper.convertLocatedBlocks(lbl);
+ LocatedBlock [] lbl2 = PBHelper.convertLocatedBlocks(lbpl);
assertEquals(lbl.length, lbl2.length);
for (int i=0;i<lbl.length;i++) {
compare(lbl[i], lbl2[i]);
@@ -639,4 +666,99 @@ public class TestPBHelper {
.build();
Assert.assertEquals(s, PBHelper.convert(PBHelper.convert(s)));
}
+
+ @Test
+ public void testBlockECRecoveryCommand() {
+ DatanodeInfo[] dnInfos0 = new DatanodeInfo[] {
+ DFSTestUtil.getLocalDatanodeInfo(), DFSTestUtil.getLocalDatanodeInfo() };
+ DatanodeStorageInfo targetDnInfos_0 = BlockManagerTestUtil
+ .newDatanodeStorageInfo(DFSTestUtil.getLocalDatanodeDescriptor(),
+ new DatanodeStorage("s00"));
+ DatanodeStorageInfo targetDnInfos_1 = BlockManagerTestUtil
+ .newDatanodeStorageInfo(DFSTestUtil.getLocalDatanodeDescriptor(),
+ new DatanodeStorage("s01"));
+ DatanodeStorageInfo[] targetDnInfos0 = new DatanodeStorageInfo[] {
+ targetDnInfos_0, targetDnInfos_1 };
+ short[] liveBlkIndices0 = new short[2];
+ BlockECRecoveryInfo blkECRecoveryInfo0 = new BlockECRecoveryInfo(
+ new ExtendedBlock("bp1", 1234), dnInfos0, targetDnInfos0,
+ liveBlkIndices0, ErasureCodingSchemaManager.getSystemDefaultSchema(),
+ 64 * 1024);
+ DatanodeInfo[] dnInfos1 = new DatanodeInfo[] {
+ DFSTestUtil.getLocalDatanodeInfo(), DFSTestUtil.getLocalDatanodeInfo() };
+ DatanodeStorageInfo targetDnInfos_2 = BlockManagerTestUtil
+ .newDatanodeStorageInfo(DFSTestUtil.getLocalDatanodeDescriptor(),
+ new DatanodeStorage("s02"));
+ DatanodeStorageInfo targetDnInfos_3 = BlockManagerTestUtil
+ .newDatanodeStorageInfo(DFSTestUtil.getLocalDatanodeDescriptor(),
+ new DatanodeStorage("s03"));
+ DatanodeStorageInfo[] targetDnInfos1 = new DatanodeStorageInfo[] {
+ targetDnInfos_2, targetDnInfos_3 };
+ short[] liveBlkIndices1 = new short[2];
+ BlockECRecoveryInfo blkECRecoveryInfo1 = new BlockECRecoveryInfo(
+ new ExtendedBlock("bp2", 3256), dnInfos1, targetDnInfos1,
+ liveBlkIndices1, ErasureCodingSchemaManager.getSystemDefaultSchema(),
+ 64 * 1024);
+ List<BlockECRecoveryInfo> blkRecoveryInfosList = new ArrayList<BlockECRecoveryInfo>();
+ blkRecoveryInfosList.add(blkECRecoveryInfo0);
+ blkRecoveryInfosList.add(blkECRecoveryInfo1);
+ BlockECRecoveryCommand blkECRecoveryCmd = new BlockECRecoveryCommand(
+ DatanodeProtocol.DNA_ERASURE_CODING_RECOVERY, blkRecoveryInfosList);
+ BlockECRecoveryCommandProto blkECRecoveryCmdProto = PBHelper
+ .convert(blkECRecoveryCmd);
+ blkECRecoveryCmd = PBHelper.convert(blkECRecoveryCmdProto);
+ Iterator<BlockECRecoveryInfo> iterator = blkECRecoveryCmd.getECTasks()
+ .iterator();
+ assertBlockECRecoveryInfoEquals(blkECRecoveryInfo0, iterator.next());
+ assertBlockECRecoveryInfoEquals(blkECRecoveryInfo1, iterator.next());
+ }
+
+ private void assertBlockECRecoveryInfoEquals(
+ BlockECRecoveryInfo blkECRecoveryInfo1,
+ BlockECRecoveryInfo blkECRecoveryInfo2) {
+ assertEquals(blkECRecoveryInfo1.getExtendedBlock(),
+ blkECRecoveryInfo2.getExtendedBlock());
+
+ DatanodeInfo[] sourceDnInfos1 = blkECRecoveryInfo1.getSourceDnInfos();
+ DatanodeInfo[] sourceDnInfos2 = blkECRecoveryInfo2.getSourceDnInfos();
+ assertDnInfosEqual(sourceDnInfos1, sourceDnInfos2);
+
+ DatanodeInfo[] targetDnInfos1 = blkECRecoveryInfo1.getTargetDnInfos();
+ DatanodeInfo[] targetDnInfos2 = blkECRecoveryInfo2.getTargetDnInfos();
+ assertDnInfosEqual(targetDnInfos1, targetDnInfos2);
+
+ String[] targetStorageIDs1 = blkECRecoveryInfo1.getTargetStorageIDs();
+ String[] targetStorageIDs2 = blkECRecoveryInfo2.getTargetStorageIDs();
+ assertEquals(targetStorageIDs1.length, targetStorageIDs2.length);
+ for (int i = 0; i < targetStorageIDs1.length; i++) {
+ assertEquals(targetStorageIDs1[i], targetStorageIDs2[i]);
+ }
+
+ short[] liveBlockIndices1 = blkECRecoveryInfo1.getLiveBlockIndices();
+ short[] liveBlockIndices2 = blkECRecoveryInfo2.getLiveBlockIndices();
+ for (int i = 0; i < liveBlockIndices1.length; i++) {
+ assertEquals(liveBlockIndices1[i], liveBlockIndices2[i]);
+ }
+
+ ECSchema ecSchema1 = blkECRecoveryInfo1.getECSchema();
+ ECSchema ecSchema2 = blkECRecoveryInfo2.getECSchema();
+ // Compare ECSchemas same as default ECSchema as we used system default
+ // ECSchema used in this test
+ compareECSchemas(ErasureCodingSchemaManager.getSystemDefaultSchema(), ecSchema1);
+ compareECSchemas(ErasureCodingSchemaManager.getSystemDefaultSchema(), ecSchema2);
+ }
+
+ private void compareECSchemas(ECSchema ecSchema1, ECSchema ecSchema2) {
+ assertEquals(ecSchema1.getSchemaName(), ecSchema2.getSchemaName());
+ assertEquals(ecSchema1.getNumDataUnits(), ecSchema2.getNumDataUnits());
+ assertEquals(ecSchema1.getNumParityUnits(), ecSchema2.getNumParityUnits());
+ }
+
+ private void assertDnInfosEqual(DatanodeInfo[] dnInfos1,
+ DatanodeInfo[] dnInfos2) {
+ assertEquals(dnInfos1.length, dnInfos2.length);
+ for (int i = 0; i < dnInfos1.length; i++) {
+ compare(dnInfos1[i], dnInfos2[i]);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 194aa0f..3c149d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -73,6 +73,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.balancer.Balancer.Cli;
import org.apache.hadoop.hdfs.server.balancer.Balancer.Parameters;
import org.apache.hadoop.hdfs.server.balancer.Balancer.Result;
@@ -80,6 +81,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.LazyPersistTestCase;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
@@ -144,6 +146,22 @@ public class TestBalancer {
LazyPersistTestCase.initCacheManipulator();
}
+ int dataBlocks = HdfsConstants.NUM_DATA_BLOCKS;
+ int parityBlocks = HdfsConstants.NUM_PARITY_BLOCKS;
+ int groupSize = dataBlocks + parityBlocks;
+ private final static int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final static int stripesPerBlock = 4;
+ static int DEFAULT_STRIPE_BLOCK_SIZE = cellSize * stripesPerBlock;
+
+ static void initConfWithStripe(Configuration conf) {
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, DEFAULT_STRIPE_BLOCK_SIZE);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY, false);
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ SimulatedFSDataset.setFactory(conf);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);
+ conf.setLong(DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY, 2000L);
+ }
+
/* create a file with a length of <code>fileLen</code> */
static void createFile(MiniDFSCluster cluster, Path filePath, long fileLen,
short replicationFactor, int nnIndex)
@@ -1529,6 +1547,75 @@ public class TestBalancer {
}
}
+ public void integrationTestWithStripedFile(Configuration conf) throws Exception {
+ initConfWithStripe(conf);
+ doTestBalancerWithStripedFile(conf);
+ }
+
+ @Test(timeout = 100000)
+ public void testBalancerWithStripedFile() throws Exception {
+ Configuration conf = new Configuration();
+ initConfWithStripe(conf);
+ doTestBalancerWithStripedFile(conf);
+ }
+
+ private void doTestBalancerWithStripedFile(Configuration conf) throws Exception {
+ int numOfDatanodes = dataBlocks + parityBlocks + 2;
+ int numOfRacks = dataBlocks;
+ long capacity = 20 * DEFAULT_STRIPE_BLOCK_SIZE;
+ long[] capacities = new long[numOfDatanodes];
+ for (int i = 0; i < capacities.length; i++) {
+ capacities[i] = capacity;
+ }
+ String[] racks = new String[numOfDatanodes];
+ for (int i = 0; i < numOfDatanodes; i++) {
+ racks[i] = "/rack" + (i % numOfRacks);
+ }
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(numOfDatanodes)
+ .racks(racks)
+ .simulatedCapacities(capacities)
+ .build();
+
+ try {
+ cluster.waitActive();
+ client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(),
+ ClientProtocol.class).getProxy();
+ client.createErasureCodingZone("/", null, 0);
+
+ long totalCapacity = sum(capacities);
+
+ // fill up the cluster with 30% data. It'll be 45% full plus parity.
+ long fileLen = totalCapacity * 3 / 10;
+ long totalUsedSpace = fileLen * (dataBlocks + parityBlocks) / dataBlocks;
+ FileSystem fs = cluster.getFileSystem(0);
+ DFSTestUtil.createFile(fs, filePath, fileLen, (short) 3, r.nextLong());
+
+ // verify locations of striped blocks
+ LocatedBlocks locatedBlocks = client.getBlockLocations(fileName, 0, fileLen);
+ DFSTestUtil.verifyLocatedStripedBlocks(locatedBlocks, groupSize);
+
+ // add one datanode
+ String newRack = "/rack" + (++numOfRacks);
+ cluster.startDataNodes(conf, 1, true, null,
+ new String[]{newRack}, null, new long[]{capacity});
+ totalCapacity += capacity;
+ cluster.triggerHeartbeats();
+
+ // run balancer and validate results
+ Balancer.Parameters p = Balancer.Parameters.DEFAULT;
+ Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
+ runBalancer(conf, totalUsedSpace, totalCapacity, p, 0);
+
+ // verify locations of striped blocks
+ locatedBlocks = client.getBlockLocations(fileName, 0, fileLen);
+ DFSTestUtil.verifyLocatedStripedBlocks(locatedBlocks, groupSize);
+
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
/**
* @param args
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 148135b..64d80bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -161,7 +161,7 @@ public class BlockManagerTestUtil {
*/
public static int computeAllPendingWork(BlockManager bm) {
int work = computeInvalidationWork(bm);
- work += bm.computeReplicationWork(Integer.MAX_VALUE);
+ work += bm.computeBlockRecoveryWork(Integer.MAX_VALUE);
return work;
}
@@ -306,4 +306,12 @@ public class BlockManagerTestUtil {
throws ExecutionException, InterruptedException {
dm.getDecomManager().runMonitor();
}
+
+ /**
+ * add block to the replicateBlocks queue of the Datanode
+ */
+ public static void addBlockToBeReplicated(DatanodeDescriptor node,
+ Block block, DatanodeStorageInfo[] targets) {
+ node.addBlockToBeReplicated(block, targets);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index 5126aa7..bae4f1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -63,7 +63,7 @@ public class TestBlockInfo {
final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
- boolean added = blockInfo.addStorage(storage);
+ boolean added = blockInfo.addStorage(storage, blockInfo);
Assert.assertTrue(added);
Assert.assertEquals(storage, blockInfo.getStorageInfo(0));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
new file mode 100644
index 0000000..6788770
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoStriped.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo.AddBlockResult;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingSchemaManager;
+import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.internal.util.reflection.Whitebox;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
+
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS;
+import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_PARITY_BLOCKS;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Test {@link BlockInfoStriped}
+ */
+public class TestBlockInfoStriped {
+ private static final int TOTAL_NUM_BLOCKS = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+ private static final long BASE_ID = -1600;
+ private static final Block baseBlock = new Block(BASE_ID);
+ private static final ECSchema testSchema
+ = ErasureCodingSchemaManager.getSystemDefaultSchema();
+ private static final int cellSize = HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
+ private final BlockInfoStriped info = new BlockInfoStriped(baseBlock,
+ testSchema, cellSize);
+
+ private Block[] createReportedBlocks(int num) {
+ Block[] blocks = new Block[num];
+ for (int i = 0; i < num; i++) {
+ blocks[i] = new Block(BASE_ID + i);
+ }
+ return blocks;
+ }
+
+ /**
+ * Test adding storage and reported block
+ */
+ @Test
+ public void testAddStorage() {
+ // first add NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS storages, i.e., a complete
+ // group of blocks/storages
+ DatanodeStorageInfo[] storageInfos = DFSTestUtil.createDatanodeStorageInfos(
+ TOTAL_NUM_BLOCKS);
+ Block[] blocks = createReportedBlocks(TOTAL_NUM_BLOCKS);
+ int i = 0;
+ for (; i < storageInfos.length; i += 2) {
+ info.addStorage(storageInfos[i], blocks[i]);
+ Assert.assertEquals(i/2 + 1, info.numNodes());
+ }
+ i /= 2;
+ for (int j = 1; j < storageInfos.length; j += 2) {
+ Assert.assertTrue(info.addStorage(storageInfos[j], blocks[j]));
+ Assert.assertEquals(i + (j+1)/2, info.numNodes());
+ }
+
+ // check
+ byte[] indices = (byte[]) Whitebox.getInternalState(info, "indices");
+ Assert.assertEquals(TOTAL_NUM_BLOCKS, info.getCapacity());
+ Assert.assertEquals(TOTAL_NUM_BLOCKS, indices.length);
+ i = 0;
+ for (DatanodeStorageInfo storage : storageInfos) {
+ int index = info.findStorageInfo(storage);
+ Assert.assertEquals(i++, index);
+ Assert.assertEquals(index, indices[index]);
+ }
+
+ // the same block is reported from the same storage twice
+ i = 0;
+ for (DatanodeStorageInfo storage : storageInfos) {
+ Assert.assertTrue(info.addStorage(storage, blocks[i++]));
+ }
+ Assert.assertEquals(TOTAL_NUM_BLOCKS, info.getCapacity());
+ Assert.assertEquals(TOTAL_NUM_BLOCKS, info.numNodes());
+ Assert.assertEquals(TOTAL_NUM_BLOCKS, indices.length);
+ i = 0;
+ for (DatanodeStorageInfo storage : storageInfos) {
+ int index = info.findStorageInfo(storage);
+ Assert.assertEquals(i++, index);
+ Assert.assertEquals(index, indices[index]);
+ }
+
+ // the same block is reported from another storage
+ DatanodeStorageInfo[] storageInfos2 = DFSTestUtil.createDatanodeStorageInfos(
+ TOTAL_NUM_BLOCKS * 2);
+ // only add the second half of info2
+ for (i = TOTAL_NUM_BLOCKS; i < storageInfos2.length; i++) {
+ info.addStorage(storageInfos2[i], blocks[i % TOTAL_NUM_BLOCKS]);
+ Assert.assertEquals(i + 1, info.getCapacity());
+ Assert.assertEquals(i + 1, info.numNodes());
+ indices = (byte[]) Whitebox.getInternalState(info, "indices");
+ Assert.assertEquals(i + 1, indices.length);
+ }
+ for (i = TOTAL_NUM_BLOCKS; i < storageInfos2.length; i++) {
+ int index = info.findStorageInfo(storageInfos2[i]);
+ Assert.assertEquals(i++, index);
+ Assert.assertEquals(index - TOTAL_NUM_BLOCKS, indices[index]);
+ }
+ }
+
+ @Test
+ public void testRemoveStorage() {
+ // first add TOTAL_NUM_BLOCKS into the BlockInfoStriped
+ DatanodeStorageInfo[] storages = DFSTestUtil.createDatanodeStorageInfos(
+ TOTAL_NUM_BLOCKS);
+ Block[] blocks = createReportedBlocks(TOTAL_NUM_BLOCKS);
+ for (int i = 0; i < storages.length; i++) {
+ info.addStorage(storages[i], blocks[i]);
+ }
+
+ // remove two storages
+ info.removeStorage(storages[0]);
+ info.removeStorage(storages[2]);
+
+ // check
+ Assert.assertEquals(TOTAL_NUM_BLOCKS, info.getCapacity());
+ Assert.assertEquals(TOTAL_NUM_BLOCKS - 2, info.numNodes());
+ byte[] indices = (byte[]) Whitebox.getInternalState(info, "indices");
+ for (int i = 0; i < storages.length; i++) {
+ int index = info.findStorageInfo(storages[i]);
+ if (i != 0 && i != 2) {
+ Assert.assertEquals(i, index);
+ Assert.assertEquals(index, indices[index]);
+ } else {
+ Assert.assertEquals(-1, index);
+ Assert.assertEquals(-1, indices[i]);
+ }
+ }
+
+ // the same block is reported from another storage
+ DatanodeStorageInfo[] storages2 = DFSTestUtil.createDatanodeStorageInfos(
+ TOTAL_NUM_BLOCKS * 2);
+ for (int i = TOTAL_NUM_BLOCKS; i < storages2.length; i++) {
+ info.addStorage(storages2[i], blocks[i % TOTAL_NUM_BLOCKS]);
+ }
+ // now we should have 8 storages
+ Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, info.numNodes());
+ Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, info.getCapacity());
+ indices = (byte[]) Whitebox.getInternalState(info, "indices");
+ Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, indices.length);
+ int j = TOTAL_NUM_BLOCKS;
+ for (int i = TOTAL_NUM_BLOCKS; i < storages2.length; i++) {
+ int index = info.findStorageInfo(storages2[i]);
+ if (i == TOTAL_NUM_BLOCKS || i == TOTAL_NUM_BLOCKS + 2) {
+ Assert.assertEquals(i - TOTAL_NUM_BLOCKS, index);
+ } else {
+ Assert.assertEquals(j++, index);
+ }
+ }
+
+ // remove the storages from storages2
+ for (int i = 0; i < TOTAL_NUM_BLOCKS; i++) {
+ info.removeStorage(storages2[i + TOTAL_NUM_BLOCKS]);
+ }
+ // now we should have 3 storages
+ Assert.assertEquals(TOTAL_NUM_BLOCKS - 2, info.numNodes());
+ Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, info.getCapacity());
+ indices = (byte[]) Whitebox.getInternalState(info, "indices");
+ Assert.assertEquals(TOTAL_NUM_BLOCKS * 2 - 2, indices.length);
+ for (int i = 0; i < TOTAL_NUM_BLOCKS; i++) {
+ if (i == 0 || i == 2) {
+ int index = info.findStorageInfo(storages2[i + TOTAL_NUM_BLOCKS]);
+ Assert.assertEquals(-1, index);
+ } else {
+ int index = info.findStorageInfo(storages[i]);
+ Assert.assertEquals(i, index);
+ }
+ }
+ for (int i = TOTAL_NUM_BLOCKS; i < TOTAL_NUM_BLOCKS * 2 - 2; i++) {
+ Assert.assertEquals(-1, indices[i]);
+ Assert.assertNull(info.getDatanode(i));
+ }
+ }
+
+ @Test
+ public void testReplaceBlock() {
+ DatanodeStorageInfo[] storages = DFSTestUtil.createDatanodeStorageInfos(
+ TOTAL_NUM_BLOCKS);
+ Block[] blocks = createReportedBlocks(TOTAL_NUM_BLOCKS);
+ // add block/storage 0, 2, 4 into the BlockInfoStriped
+ for (int i = 0; i < storages.length; i += 2) {
+ Assert.assertEquals(AddBlockResult.ADDED,
+ storages[i].addBlock(info, blocks[i]));
+ }
+
+ BlockInfoStriped newBlockInfo = new BlockInfoStriped(info);
+ info.replaceBlock(newBlockInfo);
+
+ // make sure the newBlockInfo is correct
+ byte[] indices = (byte[]) Whitebox.getInternalState(newBlockInfo, "indices");
+ for (int i = 0; i < storages.length; i += 2) {
+ int index = newBlockInfo.findStorageInfo(storages[i]);
+ Assert.assertEquals(i, index);
+ Assert.assertEquals(index, indices[i]);
+
+ // make sure the newBlockInfo is added to the linked list of the storage
+ Assert.assertSame(newBlockInfo, storages[i].getBlockListHeadForTesting());
+ Assert.assertEquals(1, storages[i].numBlocks());
+ Assert.assertNull(newBlockInfo.getNext());
+ }
+ }
+
+ @Test
+ public void testWrite() {
+ long blkID = 1;
+ long numBytes = 1;
+ long generationStamp = 1;
+ ByteBuffer byteBuffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE * 3);
+ byteBuffer.putLong(blkID).putLong(numBytes).putLong(generationStamp);
+
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ DataOutput out = new DataOutputStream(byteStream);
+ BlockInfoStriped blk = new BlockInfoStriped(new Block(blkID, numBytes,
+ generationStamp), testSchema, cellSize);
+
+ try {
+ blk.write(out);
+ } catch(Exception ex) {
+ fail("testWrite error:" + ex.getMessage());
+ }
+ assertEquals(byteBuffer.array().length, byteStream.toByteArray().length);
+ assertArrayEquals(byteBuffer.array(), byteStream.toByteArray());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
index a7ba293..a447aaf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfoUnderConstruction.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
-import org.apache.hadoop.util.Time;
import org.junit.Test;
/**
@@ -51,7 +50,7 @@ public class TestBlockInfoUnderConstruction {
DFSTestUtil.resetLastUpdatesWithOffset(dd2, -1 * 1000);
DFSTestUtil.resetLastUpdatesWithOffset(dd3, -2 * 1000);
blockInfo.initializeBlockRecovery(1);
- BlockInfoContiguousUnderConstruction[] blockInfoRecovery = dd2.getLeaseRecoveryCommand(1);
+ BlockInfoUnderConstruction[] blockInfoRecovery = dd2.getLeaseRecoveryCommand(1);
assertEquals(blockInfoRecovery[0], blockInfo);
// Recovery attempt #2.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 2d3d90a..66a4681 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -383,7 +383,7 @@ public class TestBlockManager {
for (int i = 1; i < pipeline.length; i++) {
DatanodeStorageInfo storage = pipeline[i];
bm.addBlock(storage, blockInfo, null);
- blockInfo.addStorage(storage);
+ blockInfo.addStorage(storage, blockInfo);
}
}
@@ -393,7 +393,7 @@ public class TestBlockManager {
for (DatanodeDescriptor dn : nodes) {
for (DatanodeStorageInfo storage : dn.getStorageInfos()) {
- blockInfo.addStorage(storage);
+ blockInfo.addStorage(storage, blockInfo);
}
}
return blockInfo;
@@ -453,8 +453,8 @@ public class TestBlockManager {
assertEquals("Block not initially pending replication", 0,
bm.pendingReplications.getNumReplicas(block));
assertEquals(
- "computeReplicationWork should indicate replication is needed", 1,
- bm.computeReplicationWorkForBlocks(list_all));
+ "computeBlockRecoveryWork should indicate replication is needed", 1,
+ bm.computeRecoveryWorkForBlocks(list_all));
assertTrue("replication is pending after work is computed",
bm.pendingReplications.getNumReplicas(block) > 0);
@@ -508,35 +508,38 @@ public class TestBlockManager {
assertNotNull("Chooses source node for a highest-priority replication"
+ " even if all available source nodes have reached their replication"
+ " limits below the hard limit.",
- bm.chooseSourceDatanode(
- aBlock,
+ bm.chooseSourceDatanodes(
+ bm.getStoredBlock(aBlock),
cntNodes,
liveNodes,
new NumberReplicas(),
- UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY));
-
- assertNull("Does not choose a source node for a less-than-highest-priority"
- + " replication since all available source nodes have reached"
- + " their replication limits.",
- bm.chooseSourceDatanode(
- aBlock,
+ new ArrayList<Short>(),
+ UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY)[0]);
+
+ assertEquals("Does not choose a source node for a less-than-highest-priority"
+ + " replication since all available source nodes have reached"
+ + " their replication limits.", 0,
+ bm.chooseSourceDatanodes(
+ bm.getStoredBlock(aBlock),
cntNodes,
liveNodes,
new NumberReplicas(),
- UnderReplicatedBlocks.QUEUE_VERY_UNDER_REPLICATED));
+ new ArrayList<Short>(),
+ UnderReplicatedBlocks.QUEUE_VERY_UNDER_REPLICATED).length);
// Increase the replication count to test replication count > hard limit
DatanodeStorageInfo targets[] = { origNodes.get(1).getStorageInfos()[0] };
origNodes.get(0).addBlockToBeReplicated(aBlock, targets);
- assertNull("Does not choose a source node for a highest-priority"
- + " replication when all available nodes exceed the hard limit.",
- bm.chooseSourceDatanode(
- aBlock,
+ assertEquals("Does not choose a source node for a highest-priority"
+ + " replication when all available nodes exceed the hard limit.", 0,
+ bm.chooseSourceDatanodes(
+ bm.getStoredBlock(aBlock),
cntNodes,
liveNodes,
new NumberReplicas(),
- UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY));
+ new ArrayList<Short>(),
+ UnderReplicatedBlocks.QUEUE_HIGHEST_PRIORITY).length);
}
@Test
@@ -557,30 +560,28 @@ public class TestBlockManager {
assertNotNull("Chooses decommissioning source node for a normal replication"
+ " if all available source nodes have reached their replication"
+ " limits below the hard limit.",
- bm.chooseSourceDatanode(
- aBlock,
+ bm.chooseSourceDatanodes(
+ bm.getStoredBlock(aBlock),
cntNodes,
liveNodes,
- new NumberReplicas(),
- UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED));
+ new NumberReplicas(), new LinkedList<Short>(),
+ UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED)[0]);
// Increase the replication count to test replication count > hard limit
DatanodeStorageInfo targets[] = { origNodes.get(1).getStorageInfos()[0] };
origNodes.get(0).addBlockToBeReplicated(aBlock, targets);
- assertNull("Does not choose a source decommissioning node for a normal"
- + " replication when all available nodes exceed the hard limit.",
- bm.chooseSourceDatanode(
- aBlock,
+ assertEquals("Does not choose a source decommissioning node for a normal"
+ + " replication when all available nodes exceed the hard limit.", 0,
+ bm.chooseSourceDatanodes(
+ bm.getStoredBlock(aBlock),
cntNodes,
liveNodes,
- new NumberReplicas(),
- UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED));
+ new NumberReplicas(), new LinkedList<Short>(),
+ UnderReplicatedBlocks.QUEUE_UNDER_REPLICATED).length);
}
-
-
@Test
public void testSafeModeIBR() throws Exception {
DatanodeDescriptor node = spy(nodes.get(0));
[21/23] hadoop git commit: HDFS-7285. Erasure Coding Support inside
HDFS.
Posted by zh...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
new file mode 100644
index 0000000..03683b0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/GaloisField.java
@@ -0,0 +1,561 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder.util;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Implementation of Galois field arithmetic with 2^p elements. The input must
+ * be unsigned integers. It's ported from HDFS-RAID, slightly adapted.
+ */
+public class GaloisField {
+
+ // Field size 256 is good for byte based system
+ private static final int DEFAULT_FIELD_SIZE = 256;
+ // primitive polynomial 1 + X^2 + X^3 + X^4 + X^8 (substitute 2)
+ private static final int DEFAULT_PRIMITIVE_POLYNOMIAL = 285;
+ static private final Map<Integer, GaloisField> instances =
+ new HashMap<Integer, GaloisField>();
+ private final int[] logTable;
+ private final int[] powTable;
+ private final int[][] mulTable;
+ private final int[][] divTable;
+ private final int fieldSize;
+ private final int primitivePeriod;
+ private final int primitivePolynomial;
+
+ private GaloisField(int fieldSize, int primitivePolynomial) {
+ assert fieldSize > 0;
+ assert primitivePolynomial > 0;
+
+ this.fieldSize = fieldSize;
+ this.primitivePeriod = fieldSize - 1;
+ this.primitivePolynomial = primitivePolynomial;
+ logTable = new int[fieldSize];
+ powTable = new int[fieldSize];
+ mulTable = new int[fieldSize][fieldSize];
+ divTable = new int[fieldSize][fieldSize];
+ int value = 1;
+ for (int pow = 0; pow < fieldSize - 1; pow++) {
+ powTable[pow] = value;
+ logTable[value] = pow;
+ value = value * 2;
+ if (value >= fieldSize) {
+ value = value ^ primitivePolynomial;
+ }
+ }
+ // building multiplication table
+ for (int i = 0; i < fieldSize; i++) {
+ for (int j = 0; j < fieldSize; j++) {
+ if (i == 0 || j == 0) {
+ mulTable[i][j] = 0;
+ continue;
+ }
+ int z = logTable[i] + logTable[j];
+ z = z >= primitivePeriod ? z - primitivePeriod : z;
+ z = powTable[z];
+ mulTable[i][j] = z;
+ }
+ }
+ // building division table
+ for (int i = 0; i < fieldSize; i++) {
+ for (int j = 1; j < fieldSize; j++) {
+ if (i == 0) {
+ divTable[i][j] = 0;
+ continue;
+ }
+ int z = logTable[i] - logTable[j];
+ z = z < 0 ? z + primitivePeriod : z;
+ z = powTable[z];
+ divTable[i][j] = z;
+ }
+ }
+ }
+
+ /**
+ * Get the object performs Galois field arithmetics
+ *
+ * @param fieldSize size of the field
+ * @param primitivePolynomial a primitive polynomial corresponds to the size
+ */
+ public static GaloisField getInstance(int fieldSize,
+ int primitivePolynomial) {
+ int key = ((fieldSize << 16) & 0xFFFF0000)
+ + (primitivePolynomial & 0x0000FFFF);
+ GaloisField gf;
+ synchronized (instances) {
+ gf = instances.get(key);
+ if (gf == null) {
+ gf = new GaloisField(fieldSize, primitivePolynomial);
+ instances.put(key, gf);
+ }
+ }
+ return gf;
+ }
+
+ /**
+ * Get the object performs Galois field arithmetic with default setting
+ */
+ public static GaloisField getInstance() {
+ return getInstance(DEFAULT_FIELD_SIZE, DEFAULT_PRIMITIVE_POLYNOMIAL);
+ }
+
+ /**
+ * Return number of elements in the field
+ *
+ * @return number of elements in the field
+ */
+ public int getFieldSize() {
+ return fieldSize;
+ }
+
+ /**
+ * Return the primitive polynomial in GF(2)
+ *
+ * @return primitive polynomial as a integer
+ */
+ public int getPrimitivePolynomial() {
+ return primitivePolynomial;
+ }
+
+ /**
+ * Compute the sum of two fields
+ *
+ * @param x input field
+ * @param y input field
+ * @return result of addition
+ */
+ public int add(int x, int y) {
+ assert (x >= 0 && x < getFieldSize() && y >= 0 && y < getFieldSize());
+ return x ^ y;
+ }
+
+ /**
+ * Compute the multiplication of two fields
+ *
+ * @param x input field
+ * @param y input field
+ * @return result of multiplication
+ */
+ public int multiply(int x, int y) {
+ assert (x >= 0 && x < getFieldSize() && y >= 0 && y < getFieldSize());
+ return mulTable[x][y];
+ }
+
+ /**
+ * Compute the division of two fields
+ *
+ * @param x input field
+ * @param y input field
+ * @return x/y
+ */
+ public int divide(int x, int y) {
+ assert (x >= 0 && x < getFieldSize() && y > 0 && y < getFieldSize());
+ return divTable[x][y];
+ }
+
+ /**
+ * Compute power n of a field
+ *
+ * @param x input field
+ * @param n power
+ * @return x^n
+ */
+ public int power(int x, int n) {
+ assert (x >= 0 && x < getFieldSize());
+ if (n == 0) {
+ return 1;
+ }
+ if (x == 0) {
+ return 0;
+ }
+ x = logTable[x] * n;
+ if (x < primitivePeriod) {
+ return powTable[x];
+ }
+ x = x % primitivePeriod;
+ return powTable[x];
+ }
+
+ /**
+ * Given a Vandermonde matrix V[i][j]=x[j]^i and vector y, solve for z such
+ * that Vz=y. The output z will be placed in y.
+ *
+ * @param x the vector which describe the Vandermonde matrix
+ * @param y right-hand side of the Vandermonde system equation. will be
+ * replaced the output in this vector
+ */
+ public void solveVandermondeSystem(int[] x, int[] y) {
+ solveVandermondeSystem(x, y, x.length);
+ }
+
+ /**
+ * Given a Vandermonde matrix V[i][j]=x[j]^i and vector y, solve for z such
+ * that Vz=y. The output z will be placed in y.
+ *
+ * @param x the vector which describe the Vandermonde matrix
+ * @param y right-hand side of the Vandermonde system equation. will be
+ * replaced the output in this vector
+ * @param len consider x and y only from 0...len-1
+ */
+ public void solveVandermondeSystem(int[] x, int[] y, int len) {
+ assert (x.length <= len && y.length <= len);
+ for (int i = 0; i < len - 1; i++) {
+ for (int j = len - 1; j > i; j--) {
+ y[j] = y[j] ^ mulTable[x[i]][y[j - 1]];
+ }
+ }
+ for (int i = len - 1; i >= 0; i--) {
+ for (int j = i + 1; j < len; j++) {
+ y[j] = divTable[y[j]][x[j] ^ x[j - i - 1]];
+ }
+ for (int j = i; j < len - 1; j++) {
+ y[j] = y[j] ^ y[j + 1];
+ }
+ }
+ }
+
+ /**
+ * A "bulk" version to the solving of Vandermonde System
+ */
+ public void solveVandermondeSystem(int[] x, byte[][] y, int[] outputOffsets,
+ int len, int dataLen) {
+ int idx1, idx2;
+ for (int i = 0; i < len - 1; i++) {
+ for (int j = len - 1; j > i; j--) {
+ for (idx2 = outputOffsets[j-1], idx1 = outputOffsets[j];
+ idx1 < outputOffsets[j] + dataLen; idx1++, idx2++) {
+ y[j][idx1] = (byte) (y[j][idx1] ^ mulTable[x[i]][y[j - 1][idx2] &
+ 0x000000FF]);
+ }
+ }
+ }
+ for (int i = len - 1; i >= 0; i--) {
+ for (int j = i + 1; j < len; j++) {
+ for (idx1 = outputOffsets[j];
+ idx1 < outputOffsets[j] + dataLen; idx1++) {
+ y[j][idx1] = (byte) (divTable[y[j][idx1] & 0x000000FF][x[j] ^
+ x[j - i - 1]]);
+ }
+ }
+ for (int j = i; j < len - 1; j++) {
+ for (idx2 = outputOffsets[j+1], idx1 = outputOffsets[j];
+ idx1 < outputOffsets[j] + dataLen; idx1++, idx2++) {
+ y[j][idx1] = (byte) (y[j][idx1] ^ y[j + 1][idx2]);
+ }
+ }
+ }
+ }
+
+ /**
+ * A "bulk" version of the solveVandermondeSystem, using ByteBuffer.
+ */
+ public void solveVandermondeSystem(int[] x, ByteBuffer[] y, int len) {
+ ByteBuffer p;
+ int idx1, idx2;
+ for (int i = 0; i < len - 1; i++) {
+ for (int j = len - 1; j > i; j--) {
+ p = y[j];
+ for (idx1 = p.position(), idx2 = y[j-1].position();
+ idx1 < p.limit(); idx1++, idx2++) {
+ p.put(idx1, (byte) (p.get(idx1) ^ mulTable[x[i]][y[j-1].get(idx2) &
+ 0x000000FF]));
+ }
+ }
+ }
+
+ for (int i = len - 1; i >= 0; i--) {
+ for (int j = i + 1; j < len; j++) {
+ p = y[j];
+ for (idx1 = p.position(); idx1 < p.limit(); idx1++) {
+ p.put(idx1, (byte) (divTable[p.get(idx1) &
+ 0x000000FF][x[j] ^ x[j - i - 1]]));
+ }
+ }
+
+ for (int j = i; j < len - 1; j++) {
+ p = y[j];
+ for (idx1 = p.position(), idx2 = y[j+1].position();
+ idx1 < p.limit(); idx1++, idx2++) {
+ p.put(idx1, (byte) (p.get(idx1) ^ y[j+1].get(idx2)));
+ }
+ }
+ }
+ }
+
+ /**
+ * Compute the multiplication of two polynomials. The index in the array
+ * corresponds to the power of the entry. For example p[0] is the constant
+ * term of the polynomial p.
+ *
+ * @param p input polynomial
+ * @param q input polynomial
+ * @return polynomial represents p*q
+ */
+ public int[] multiply(int[] p, int[] q) {
+ int len = p.length + q.length - 1;
+ int[] result = new int[len];
+ for (int i = 0; i < len; i++) {
+ result[i] = 0;
+ }
+ for (int i = 0; i < p.length; i++) {
+
+ for (int j = 0; j < q.length; j++) {
+ result[i + j] = add(result[i + j], multiply(p[i], q[j]));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Compute the remainder of a dividend and divisor pair. The index in the
+ * array corresponds to the power of the entry. For example p[0] is the
+ * constant term of the polynomial p.
+ *
+ * @param dividend dividend polynomial, the remainder will be placed
+ * here when return
+ * @param divisor divisor polynomial
+ */
+ public void remainder(int[] dividend, int[] divisor) {
+ for (int i = dividend.length - divisor.length; i >= 0; i--) {
+ int ratio = divTable[dividend[i +
+ divisor.length - 1]][divisor[divisor.length - 1]];
+ for (int j = 0; j < divisor.length; j++) {
+ int k = j + i;
+ dividend[k] = dividend[k] ^ mulTable[ratio][divisor[j]];
+ }
+ }
+ }
+
+ /**
+ * Compute the sum of two polynomials. The index in the array corresponds to
+ * the power of the entry. For example p[0] is the constant term of the
+ * polynomial p.
+ *
+ * @param p input polynomial
+ * @param q input polynomial
+ * @return polynomial represents p+q
+ */
+ public int[] add(int[] p, int[] q) {
+ int len = Math.max(p.length, q.length);
+ int[] result = new int[len];
+ for (int i = 0; i < len; i++) {
+ if (i < p.length && i < q.length) {
+ result[i] = add(p[i], q[i]);
+ } else if (i < p.length) {
+ result[i] = p[i];
+ } else {
+ result[i] = q[i];
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Substitute x into polynomial p(x).
+ *
+ * @param p input polynomial
+ * @param x input field
+ * @return p(x)
+ */
+ public int substitute(int[] p, int x) {
+ int result = 0;
+ int y = 1;
+ for (int i = 0; i < p.length; i++) {
+ result = result ^ mulTable[p[i]][y];
+ y = mulTable[x][y];
+ }
+ return result;
+ }
+
+ /**
+ * A "bulk" version of the substitute.
+ * Tends to be 2X faster than the "int" substitute in a loop.
+ *
+ * @param p input polynomial
+ * @param q store the return result
+ * @param x input field
+ */
+ public void substitute(byte[][] p, byte[] q, int x) {
+ int y = 1;
+ for (int i = 0; i < p.length; i++) {
+ byte[] pi = p[i];
+ for (int j = 0; j < pi.length; j++) {
+ int pij = pi[j] & 0x000000FF;
+ q[j] = (byte) (q[j] ^ mulTable[pij][y]);
+ }
+ y = mulTable[x][y];
+ }
+ }
+
+ /**
+ * A "bulk" version of the substitute.
+ * Tends to be 2X faster than the "int" substitute in a loop.
+ *
+ * @param p input polynomial
+ * @param offsets
+ * @param len
+ * @param q store the return result
+ * @param offset
+ * @param x input field
+ */
+ public void substitute(byte[][] p, int[] offsets,
+ int len, byte[] q, int offset, int x) {
+ int y = 1, iIdx, oIdx;
+ for (int i = 0; i < p.length; i++) {
+ byte[] pi = p[i];
+ for (iIdx = offsets[i], oIdx = offset;
+ iIdx < offsets[i] + len; iIdx++, oIdx++) {
+ int pij = pi != null ? pi[iIdx] & 0x000000FF : 0;
+ q[oIdx] = (byte) (q[oIdx] ^ mulTable[pij][y]);
+ }
+ y = mulTable[x][y];
+ }
+ }
+
+ /**
+ * A "bulk" version of the substitute, using ByteBuffer.
+ * Tends to be 2X faster than the "int" substitute in a loop.
+ *
+ * @param p input polynomial
+ * @param q store the return result
+ * @param x input field
+ */
+ public void substitute(ByteBuffer[] p, int len, ByteBuffer q, int x) {
+ int y = 1, iIdx, oIdx;
+ for (int i = 0; i < p.length; i++) {
+ ByteBuffer pi = p[i];
+ int pos = pi != null ? pi.position() : 0;
+ int limit = pi != null ? pi.limit() : len;
+ for (oIdx = q.position(), iIdx = pos;
+ iIdx < limit; iIdx++, oIdx++) {
+ int pij = pi != null ? pi.get(iIdx) & 0x000000FF : 0;
+ q.put(oIdx, (byte) (q.get(oIdx) ^ mulTable[pij][y]));
+ }
+ y = mulTable[x][y];
+ }
+ }
+
+ /**
+ * The "bulk" version of the remainder.
+ * Warning: This function will modify the "dividend" inputs.
+ */
+ public void remainder(byte[][] dividend, int[] divisor) {
+ for (int i = dividend.length - divisor.length; i >= 0; i--) {
+ for (int j = 0; j < divisor.length; j++) {
+ for (int k = 0; k < dividend[i].length; k++) {
+ int ratio = divTable[dividend[i + divisor.length - 1][k] &
+ 0x00FF][divisor[divisor.length - 1]];
+ dividend[j + i][k] = (byte) ((dividend[j + i][k] & 0x00FF) ^
+ mulTable[ratio][divisor[j]]);
+ }
+ }
+ }
+ }
+
+ /**
+ * The "bulk" version of the remainder.
+ * Warning: This function will modify the "dividend" inputs.
+ */
+ public void remainder(byte[][] dividend, int[] offsets,
+ int len, int[] divisor) {
+ int idx1, idx2;
+ for (int i = dividend.length - divisor.length; i >= 0; i--) {
+ for (int j = 0; j < divisor.length; j++) {
+ for (idx2 = offsets[j + i], idx1 = offsets[i + divisor.length - 1];
+ idx1 < offsets[i + divisor.length - 1] + len;
+ idx1++, idx2++) {
+ int ratio = divTable[dividend[i + divisor.length - 1][idx1] &
+ 0x00FF][divisor[divisor.length - 1]];
+ dividend[j + i][idx2] = (byte) ((dividend[j + i][idx2] & 0x00FF) ^
+ mulTable[ratio][divisor[j]]);
+ }
+ }
+ }
+ }
+
+ /**
+ * The "bulk" version of the remainder, using ByteBuffer.
+ * Warning: This function will modify the "dividend" inputs.
+ */
+ public void remainder(ByteBuffer[] dividend, int[] divisor) {
+ int idx1, idx2;
+ ByteBuffer b1, b2;
+ for (int i = dividend.length - divisor.length; i >= 0; i--) {
+ for (int j = 0; j < divisor.length; j++) {
+ b1 = dividend[i + divisor.length - 1];
+ b2 = dividend[j + i];
+ for (idx1 = b1.position(), idx2 = b2.position();
+ idx1 < b1.limit(); idx1++, idx2++) {
+ int ratio = divTable[b1.get(idx1) &
+ 0x00FF][divisor[divisor.length - 1]];
+ b2.put(idx2, (byte) ((b2.get(idx2) & 0x00FF) ^
+ mulTable[ratio][divisor[j]]));
+ }
+ }
+ }
+ }
+
+ /**
+ * Perform Gaussian elimination on the given matrix. This matrix has to be a
+ * fat matrix (number of rows > number of columns).
+ */
+ public void gaussianElimination(int[][] matrix) {
+ assert(matrix != null && matrix.length > 0 && matrix[0].length > 0
+ && matrix.length < matrix[0].length);
+ int height = matrix.length;
+ int width = matrix[0].length;
+ for (int i = 0; i < height; i++) {
+ boolean pivotFound = false;
+ // scan the column for a nonzero pivot and swap it to the diagonal
+ for (int j = i; j < height; j++) {
+ if (matrix[i][j] != 0) {
+ int[] tmp = matrix[i];
+ matrix[i] = matrix[j];
+ matrix[j] = tmp;
+ pivotFound = true;
+ break;
+ }
+ }
+ if (!pivotFound) {
+ continue;
+ }
+ int pivot = matrix[i][i];
+ for (int j = i; j < width; j++) {
+ matrix[i][j] = divide(matrix[i][j], pivot);
+ }
+ for (int j = i + 1; j < height; j++) {
+ int lead = matrix[j][i];
+ for (int k = i; k < width; k++) {
+ matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k]));
+ }
+ }
+ }
+ for (int i = height - 1; i >=0; i--) {
+ for (int j = 0; j < i; j++) {
+ int lead = matrix[j][i];
+ for (int k = i; k < width; k++) {
+ matrix[j][k] = add(matrix[j][k], multiply(lead, matrix[i][k]));
+ }
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java
new file mode 100644
index 0000000..8badf02
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/util/RSUtil.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder.util;
+
+/**
+ * Some utilities for Reed-Solomon coding.
+ */
+public class RSUtil {
+
+ // We always use the byte system (with symbol size 8, field size 256,
+ // primitive polynomial 285, and primitive root 2).
+ public static GaloisField GF = GaloisField.getInstance();
+ public static final int PRIMITIVE_ROOT = 2;
+
+ public static int[] getPrimitivePower(int numDataUnits, int numParityUnits) {
+ int[] primitivePower = new int[numDataUnits + numParityUnits];
+ // compute powers of the primitive root
+ for (int i = 0; i < numDataUnits + numParityUnits; i++) {
+ primitivePower[i] = GF.power(PRIMITIVE_ROOT, i);
+ }
+ return primitivePower;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/BufferAllocator.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/BufferAllocator.java
new file mode 100644
index 0000000..8f552b7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/BufferAllocator.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+
+import java.nio.ByteBuffer;
+
+/**
+ * An abstract buffer allocator used for test.
+ */
+public abstract class BufferAllocator {
+ private boolean usingDirect = false;
+
+ public BufferAllocator(boolean usingDirect) {
+ this.usingDirect = usingDirect;
+ }
+
+ protected boolean isUsingDirect() {
+ return usingDirect;
+ }
+
+ /**
+ * Allocate and return a ByteBuffer of specified length.
+ * @param bufferLen
+ * @return
+ */
+ public abstract ByteBuffer allocate(int bufferLen);
+
+ /**
+ * A simple buffer allocator that just uses ByteBuffer's
+ * allocate/allocateDirect API.
+ */
+ public static class SimpleBufferAllocator extends BufferAllocator {
+
+ public SimpleBufferAllocator(boolean usingDirect) {
+ super(usingDirect);
+ }
+
+ @Override
+ public ByteBuffer allocate(int bufferLen) {
+ return isUsingDirect() ? ByteBuffer.allocateDirect(bufferLen) :
+ ByteBuffer.allocate(bufferLen);
+ }
+ }
+
+ /**
+ * A buffer allocator that allocates a buffer from an existing large buffer by
+ * slice calling, but if no available space just degrades as
+ * SimpleBufferAllocator. So please ensure enough space for it.
+ */
+ public static class SlicedBufferAllocator extends BufferAllocator {
+ private ByteBuffer overallBuffer;
+
+ public SlicedBufferAllocator(boolean usingDirect, int totalBufferLen) {
+ super(usingDirect);
+ overallBuffer = isUsingDirect() ?
+ ByteBuffer.allocateDirect(totalBufferLen) :
+ ByteBuffer.allocate(totalBufferLen);
+ }
+
+ @Override
+ public ByteBuffer allocate(int bufferLen) {
+ if (bufferLen > overallBuffer.capacity() - overallBuffer.position()) {
+ // If no available space for the requested length, then allocate new
+ return isUsingDirect() ? ByteBuffer.allocateDirect(bufferLen) :
+ ByteBuffer.allocate(bufferLen);
+ }
+
+ overallBuffer.limit(overallBuffer.position() + bufferLen);
+ ByteBuffer result = overallBuffer.slice();
+ overallBuffer.position(overallBuffer.position() + bufferLen);
+ return result;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
new file mode 100644
index 0000000..8f277f4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -0,0 +1,500 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.erasurecode.BufferAllocator.SimpleBufferAllocator;
+import org.apache.hadoop.io.erasurecode.BufferAllocator.SlicedBufferAllocator;
+import org.apache.hadoop.io.erasurecode.rawcoder.util.DumpUtil;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Random;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test base of common utilities for tests not only raw coders but also block
+ * coders.
+ */
+public abstract class TestCoderBase {
+ protected static Random RAND = new Random();
+
+ private boolean allowDump = true;
+
+ private Configuration conf;
+ protected int numDataUnits;
+ protected int numParityUnits;
+ protected int baseChunkSize = 513;
+ private int chunkSize = baseChunkSize;
+ private BufferAllocator allocator;
+
+ private byte[] zeroChunkBytes;
+
+ private boolean startBufferWithZero = true;
+
+ // Indexes of erased data units.
+ protected int[] erasedDataIndexes = new int[] {0};
+
+ // Indexes of erased parity units.
+ protected int[] erasedParityIndexes = new int[] {0};
+
+ // Data buffers are either direct or on-heap, for performance the two cases
+ // may go to different coding implementations.
+ protected boolean usingDirectBuffer = true;
+
+ protected boolean usingFixedData = true;
+ // Using this the generated data can be repeatable across multiple calls to
+ // encode(), in order for troubleshooting.
+ private static int FIXED_DATA_GENERATOR = 0;
+ protected byte[][] fixedData;
+
+ protected int getChunkSize() {
+ return chunkSize;
+ }
+
+ protected void setChunkSize(int chunkSize) {
+ this.chunkSize = chunkSize;
+ this.zeroChunkBytes = new byte[chunkSize]; // With ZERO by default
+ }
+
+ protected void prepareBufferAllocator(boolean usingSlicedBuffer) {
+ if (usingSlicedBuffer) {
+ int roughEstimationSpace =
+ chunkSize * (numDataUnits + numParityUnits) * 10;
+ allocator = new SlicedBufferAllocator(usingDirectBuffer,
+ roughEstimationSpace);
+ } else {
+ allocator = new SimpleBufferAllocator(usingDirectBuffer);
+ }
+ }
+
+ /**
+ * Set true during setup if want to dump test settings and coding data,
+ * useful in debugging.
+ * @param allowDump
+ */
+ protected void setAllowDump(boolean allowDump) {
+ this.allowDump = allowDump;
+ }
+
+ /**
+ * Prepare before running the case.
+ * @param conf
+ * @param numDataUnits
+ * @param numParityUnits
+ * @param erasedDataIndexes
+ * @param erasedParityIndexes
+ * @param usingFixedData Using fixed or pre-generated data to test instead of
+ * generating data
+ */
+ protected void prepare(Configuration conf, int numDataUnits,
+ int numParityUnits, int[] erasedDataIndexes,
+ int[] erasedParityIndexes, boolean usingFixedData) {
+ this.conf = conf;
+ this.numDataUnits = numDataUnits;
+ this.numParityUnits = numParityUnits;
+ this.erasedDataIndexes = erasedDataIndexes != null ?
+ erasedDataIndexes : new int[] {0};
+ this.erasedParityIndexes = erasedParityIndexes != null ?
+ erasedParityIndexes : new int[] {0};
+ this.usingFixedData = usingFixedData;
+ if (usingFixedData) {
+ prepareFixedData();
+ }
+ }
+
+ /**
+ * Prepare before running the case.
+ * @param conf
+ * @param numDataUnits
+ * @param numParityUnits
+ * @param erasedDataIndexes
+ * @param erasedParityIndexes
+ */
+ protected void prepare(Configuration conf, int numDataUnits,
+ int numParityUnits, int[] erasedDataIndexes,
+ int[] erasedParityIndexes) {
+ prepare(conf, numDataUnits, numParityUnits, erasedDataIndexes,
+ erasedParityIndexes, false);
+ }
+
+ /**
+ * Prepare before running the case.
+ * @param numDataUnits
+ * @param numParityUnits
+ * @param erasedDataIndexes
+ * @param erasedParityIndexes
+ */
+ protected void prepare(int numDataUnits, int numParityUnits,
+ int[] erasedDataIndexes, int[] erasedParityIndexes) {
+ prepare(null, numDataUnits, numParityUnits, erasedDataIndexes,
+ erasedParityIndexes, false);
+ }
+
+ /**
+ * Get the conf the test.
+ * @return configuration
+ */
+ protected Configuration getConf() {
+ return this.conf;
+ }
+
+ /**
+ * Compare and verify if erased chunks are equal to recovered chunks
+ * @param erasedChunks
+ * @param recoveredChunks
+ */
+ protected void compareAndVerify(ECChunk[] erasedChunks,
+ ECChunk[] recoveredChunks) {
+ byte[][] erased = toArrays(erasedChunks);
+ byte[][] recovered = toArrays(recoveredChunks);
+ boolean result = Arrays.deepEquals(erased, recovered);
+ assertTrue("Decoding and comparing failed.", result);
+ }
+
+ /**
+ * Adjust and return erased indexes altogether, including erased data indexes
+ * and parity indexes.
+ * @return erased indexes altogether
+ */
+ protected int[] getErasedIndexesForDecoding() {
+ int[] erasedIndexesForDecoding =
+ new int[erasedParityIndexes.length + erasedDataIndexes.length];
+
+ int idx = 0;
+
+ for (int i = 0; i < erasedParityIndexes.length; i++) {
+ erasedIndexesForDecoding[idx ++] = erasedParityIndexes[i];
+ }
+
+ for (int i = 0; i < erasedDataIndexes.length; i++) {
+ erasedIndexesForDecoding[idx ++] = erasedDataIndexes[i] + numParityUnits;
+ }
+
+ return erasedIndexesForDecoding;
+ }
+
+ /**
+ * Return input chunks for decoding, which is parityChunks + dataChunks.
+ * @param dataChunks
+ * @param parityChunks
+ * @return
+ */
+ protected ECChunk[] prepareInputChunksForDecoding(ECChunk[] dataChunks,
+ ECChunk[] parityChunks) {
+ ECChunk[] inputChunks = new ECChunk[numParityUnits + numDataUnits];
+
+ int idx = 0;
+ for (int i = 0; i < numParityUnits; i++) {
+ inputChunks[idx ++] = parityChunks[i];
+ }
+ for (int i = 0; i < numDataUnits; i++) {
+ inputChunks[idx ++] = dataChunks[i];
+ }
+
+ return inputChunks;
+ }
+
+ /**
+ * Erase some data chunks to test the recovering of them. As they're erased,
+ * we don't need to read them and will not have the buffers at all, so just
+ * set them as null.
+ * @param dataChunks
+ * @param parityChunks
+ * @return clone of erased chunks
+ */
+ protected ECChunk[] backupAndEraseChunks(ECChunk[] dataChunks,
+ ECChunk[] parityChunks) {
+ ECChunk[] toEraseChunks = new ECChunk[erasedParityIndexes.length +
+ erasedDataIndexes.length];
+
+ int idx = 0;
+
+ for (int i = 0; i < erasedParityIndexes.length; i++) {
+ toEraseChunks[idx ++] = parityChunks[erasedParityIndexes[i]];
+ parityChunks[erasedParityIndexes[i]] = null;
+ }
+
+ for (int i = 0; i < erasedDataIndexes.length; i++) {
+ toEraseChunks[idx ++] = dataChunks[erasedDataIndexes[i]];
+ dataChunks[erasedDataIndexes[i]] = null;
+ }
+
+ return toEraseChunks;
+ }
+
+ /**
+ * Erase data from the specified chunks, just setting them as null.
+ * @param chunks
+ */
+ protected void eraseDataFromChunks(ECChunk[] chunks) {
+ for (int i = 0; i < chunks.length; i++) {
+ chunks[i] = null;
+ }
+ }
+
+ /**
+ * Clone chunks along with copying the associated data. It respects how the
+ * chunk buffer is allocated, direct or non-direct. It avoids affecting the
+ * original chunk buffers.
+ * @param chunks
+ * @return
+ */
+ protected ECChunk[] cloneChunksWithData(ECChunk[] chunks) {
+ ECChunk[] results = new ECChunk[chunks.length];
+ for (int i = 0; i < chunks.length; i++) {
+ results[i] = cloneChunkWithData(chunks[i]);
+ }
+
+ return results;
+ }
+
+ /**
+ * Clone chunk along with copying the associated data. It respects how the
+ * chunk buffer is allocated, direct or non-direct. It avoids affecting the
+ * original chunk.
+ * @param chunk
+ * @return a new chunk
+ */
+ protected ECChunk cloneChunkWithData(ECChunk chunk) {
+ ByteBuffer srcBuffer = chunk.getBuffer();
+
+ byte[] bytesArr = new byte[srcBuffer.remaining()];
+ srcBuffer.mark();
+ srcBuffer.get(bytesArr, 0, bytesArr.length);
+ srcBuffer.reset();
+
+ ByteBuffer destBuffer = allocateOutputBuffer(bytesArr.length);
+ int pos = destBuffer.position();
+ destBuffer.put(bytesArr);
+ destBuffer.flip();
+ destBuffer.position(pos);
+
+ return new ECChunk(destBuffer);
+ }
+
+ /**
+ * Allocate a chunk for output or writing.
+ * @return
+ */
+ protected ECChunk allocateOutputChunk() {
+ ByteBuffer buffer = allocateOutputBuffer(chunkSize);
+
+ return new ECChunk(buffer);
+ }
+
+ /**
+ * Allocate a buffer for output or writing. It can prepare for two kinds of
+ * data buffers: one with position as 0, the other with position > 0
+ * @return a buffer ready to write chunkSize bytes from current position
+ */
+ protected ByteBuffer allocateOutputBuffer(int bufferLen) {
+ /**
+ * When startBufferWithZero, will prepare a buffer as:---------------
+ * otherwise, the buffer will be like: ___TO--BE--WRITTEN___,
+ * and in the beginning, dummy data are prefixed, to simulate a buffer of
+ * position > 0.
+ */
+ int startOffset = startBufferWithZero ? 0 : 11; // 11 is arbitrary
+ int allocLen = startOffset + bufferLen + startOffset;
+ ByteBuffer buffer = allocator.allocate(allocLen);
+ buffer.limit(startOffset + bufferLen);
+ fillDummyData(buffer, startOffset);
+ startBufferWithZero = ! startBufferWithZero;
+
+ return buffer;
+ }
+
+ /**
+ * Prepare data chunks for each data unit, by generating random data.
+ * @return
+ */
+ protected ECChunk[] prepareDataChunksForEncoding() {
+ if (usingFixedData) {
+ ECChunk[] chunks = new ECChunk[numDataUnits];
+ for (int i = 0; i < chunks.length; i++) {
+ chunks[i] = makeChunkUsingData(fixedData[i]);
+ }
+ return chunks;
+ }
+
+ return generateDataChunks();
+ }
+
+ private ECChunk makeChunkUsingData(byte[] data) {
+ ECChunk chunk = allocateOutputChunk();
+ ByteBuffer buffer = chunk.getBuffer();
+ int pos = buffer.position();
+ buffer.put(data, 0, chunkSize);
+ buffer.flip();
+ buffer.position(pos);
+
+ return chunk;
+ }
+
+ private ECChunk[] generateDataChunks() {
+ ECChunk[] chunks = new ECChunk[numDataUnits];
+ for (int i = 0; i < chunks.length; i++) {
+ chunks[i] = generateDataChunk();
+ }
+
+ return chunks;
+ }
+
+ private void prepareFixedData() {
+ // We may load test data from a resource, or just generate randomly.
+ // The generated data will be used across subsequent encode/decode calls.
+ this.fixedData = new byte[numDataUnits][];
+ for (int i = 0; i < numDataUnits; i++) {
+ fixedData[i] = generateFixedData(baseChunkSize * 2);
+ }
+ }
+
+ /**
+ * Generate data chunk by making random data.
+ * @return
+ */
+ protected ECChunk generateDataChunk() {
+ ByteBuffer buffer = allocateOutputBuffer(chunkSize);
+ int pos = buffer.position();
+ buffer.put(generateData(chunkSize));
+ buffer.flip();
+ buffer.position(pos);
+
+ return new ECChunk(buffer);
+ }
+
+ /**
+ * Fill len of dummy data in the buffer at the current position.
+ * @param buffer
+ * @param len
+ */
+ protected void fillDummyData(ByteBuffer buffer, int len) {
+ byte[] dummy = new byte[len];
+ RAND.nextBytes(dummy);
+ buffer.put(dummy);
+ }
+
+ protected byte[] generateData(int len) {
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) RAND.nextInt(256);
+ }
+ return buffer;
+ }
+
+ protected byte[] generateFixedData(int len) {
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) FIXED_DATA_GENERATOR++;
+ if (FIXED_DATA_GENERATOR == 256) {
+ FIXED_DATA_GENERATOR = 0;
+ }
+ }
+ return buffer;
+ }
+
+ /**
+ * Prepare parity chunks for encoding, each chunk for each parity unit.
+ * @return
+ */
+ protected ECChunk[] prepareParityChunksForEncoding() {
+ ECChunk[] chunks = new ECChunk[numParityUnits];
+ for (int i = 0; i < chunks.length; i++) {
+ chunks[i] = allocateOutputChunk();
+ }
+
+ return chunks;
+ }
+
+ /**
+ * Prepare output chunks for decoding, each output chunk for each erased
+ * chunk.
+ * @return
+ */
+ protected ECChunk[] prepareOutputChunksForDecoding() {
+ ECChunk[] chunks = new ECChunk[erasedDataIndexes.length +
+ erasedParityIndexes.length];
+
+ for (int i = 0; i < chunks.length; i++) {
+ chunks[i] = allocateOutputChunk();
+ }
+
+ return chunks;
+ }
+
+ /**
+ * Convert an array of this chunks to an array of byte array.
+ * Note the chunk buffers are not affected.
+ * @param chunks
+ * @return an array of byte array
+ */
+ protected byte[][] toArrays(ECChunk[] chunks) {
+ byte[][] bytesArr = new byte[chunks.length][];
+
+ for (int i = 0; i < chunks.length; i++) {
+ bytesArr[i] = chunks[i].toBytesArray();
+ }
+
+ return bytesArr;
+ }
+
+ /**
+ * Dump all the settings used in the test case if allowDump is enabled.
+ */
+ protected void dumpSetting() {
+ if (allowDump) {
+ StringBuilder sb = new StringBuilder("Erasure coder test settings:\n");
+ sb.append(" numDataUnits=").append(numDataUnits);
+ sb.append(" numParityUnits=").append(numParityUnits);
+ sb.append(" chunkSize=").append(chunkSize).append("\n");
+
+ sb.append(" erasedDataIndexes=").
+ append(Arrays.toString(erasedDataIndexes));
+ sb.append(" erasedParityIndexes=").
+ append(Arrays.toString(erasedParityIndexes));
+ sb.append(" usingDirectBuffer=").append(usingDirectBuffer).append("\n");
+
+ System.out.println(sb.toString());
+ }
+ }
+
+ /**
+ * Dump chunks prefixed with a header if allowDump is enabled.
+ * @param header
+ * @param chunks
+ */
+ protected void dumpChunks(String header, ECChunk[] chunks) {
+ if (allowDump) {
+ DumpUtil.dumpChunks(header, chunks);
+ }
+ }
+
+ /**
+ * Make some chunk messy or not correct any more
+ * @param chunks
+ */
+ protected void corruptSomeChunk(ECChunk[] chunks) {
+ int idx = new Random().nextInt(chunks.length);
+ ByteBuffer buffer = chunks[idx].getBuffer();
+ if (buffer.hasRemaining()) {
+ buffer.position(buffer.position() + 1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestECSchema.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestECSchema.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestECSchema.java
new file mode 100644
index 0000000..c362b96
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestECSchema.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TestECSchema {
+
+ @Test
+ public void testGoodSchema() {
+ String schemaName = "goodSchema";
+ int numDataUnits = 6;
+ int numParityUnits = 3;
+ String codec = "rs";
+ String extraOption = "extraOption";
+ String extraOptionValue = "extraOptionValue";
+
+ Map<String, String> options = new HashMap<String, String>();
+ options.put(ECSchema.NUM_DATA_UNITS_KEY, String.valueOf(numDataUnits));
+ options.put(ECSchema.NUM_PARITY_UNITS_KEY, String.valueOf(numParityUnits));
+ options.put(ECSchema.CODEC_NAME_KEY, codec);
+ options.put(extraOption, extraOptionValue);
+
+ ECSchema schema = new ECSchema(schemaName, options);
+ System.out.println(schema.toString());
+
+ assertEquals(schemaName, schema.getSchemaName());
+ assertEquals(numDataUnits, schema.getNumDataUnits());
+ assertEquals(numParityUnits, schema.getNumParityUnits());
+ assertEquals(codec, schema.getCodecName());
+ assertEquals(extraOptionValue, schema.getExtraOptions().get(extraOption));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestSchemaLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestSchemaLoader.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestSchemaLoader.java
new file mode 100644
index 0000000..50d2091
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestSchemaLoader.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.PrintWriter;
+import java.util.List;
+
+import org.junit.Test;
+
+public class TestSchemaLoader {
+
+ final static String TEST_DIR = new File(System.getProperty(
+ "test.build.data", "/tmp")).getAbsolutePath();
+
+ final static String SCHEMA_FILE = new File(TEST_DIR, "test-ecschema")
+ .getAbsolutePath();
+
+ @Test
+ public void testLoadSchema() throws Exception {
+ PrintWriter out = new PrintWriter(new FileWriter(SCHEMA_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<schemas>");
+ out.println(" <schema name=\"RSk6m3\">");
+ out.println(" <numDataUnits>6</numDataUnits>");
+ out.println(" <numParityUnits>3</numParityUnits>");
+ out.println(" <codec>RS</codec>");
+ out.println(" </schema>");
+ out.println(" <schema name=\"RSk10m4\">");
+ out.println(" <numDataUnits>10</numDataUnits>");
+ out.println(" <numParityUnits>4</numParityUnits>");
+ out.println(" <codec>RS</codec>");
+ out.println(" </schema>");
+ out.println("</schemas>");
+ out.close();
+
+ SchemaLoader schemaLoader = new SchemaLoader();
+ List<ECSchema> schemas = schemaLoader.loadSchema(SCHEMA_FILE);
+
+ assertEquals(2, schemas.size());
+
+ ECSchema schema1 = schemas.get(0);
+ assertEquals("RSk6m3", schema1.getSchemaName());
+ assertEquals(0, schema1.getExtraOptions().size());
+ assertEquals(6, schema1.getNumDataUnits());
+ assertEquals(3, schema1.getNumParityUnits());
+ assertEquals("RS", schema1.getCodecName());
+
+ ECSchema schema2 = schemas.get(1);
+ assertEquals("RSk10m4", schema2.getSchemaName());
+ assertEquals(0, schema2.getExtraOptions().size());
+ assertEquals(10, schema2.getNumDataUnits());
+ assertEquals(4, schema2.getNumParityUnits());
+ assertEquals("RS", schema2.getCodecName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
new file mode 100644
index 0000000..738d28e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
@@ -0,0 +1,297 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.io.erasurecode.ECBlock;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.TestCoderBase;
+
+import java.lang.reflect.Constructor;
+
+/**
+ * Erasure coder test base with utilities.
+ */
+public abstract class TestErasureCoderBase extends TestCoderBase {
+ protected Class<? extends ErasureCoder> encoderClass;
+ protected Class<? extends ErasureCoder> decoderClass;
+
+ private ErasureCoder encoder;
+ private ErasureCoder decoder;
+
+ protected int numChunksInBlock = 16;
+
+ /**
+ * It's just a block for this test purpose. We don't use HDFS block here
+ * at all for simple.
+ */
+ protected static class TestBlock extends ECBlock {
+ private ECChunk[] chunks;
+
+ // For simple, just assume the block have the chunks already ready.
+ // In practice we need to read/write chunks from/to the block via file IO.
+ public TestBlock(ECChunk[] chunks) {
+ this.chunks = chunks;
+ }
+ }
+
+ /**
+ * Generating source data, encoding, recovering and then verifying.
+ * RawErasureCoder mainly uses ECChunk to pass input and output data buffers,
+ * it supports two kinds of ByteBuffers, one is array backed, the other is
+ * direct ByteBuffer. Have usingDirectBuffer to indicate which case to test.
+ * @param usingDirectBuffer
+ */
+ protected void testCoding(boolean usingDirectBuffer) {
+ this.usingDirectBuffer = usingDirectBuffer;
+ prepareCoders();
+
+ /**
+ * The following runs will use 3 different chunkSize for inputs and outputs,
+ * to verify the same encoder/decoder can process variable width of data.
+ */
+ performTestCoding(baseChunkSize, true);
+ performTestCoding(baseChunkSize - 17, false);
+ performTestCoding(baseChunkSize + 16, true);
+ }
+
+ private void performTestCoding(int chunkSize, boolean usingSlicedBuffer) {
+ setChunkSize(chunkSize);
+ prepareBufferAllocator(usingSlicedBuffer);
+
+ // Generate data and encode
+ ECBlockGroup blockGroup = prepareBlockGroupForEncoding();
+ // Backup all the source chunks for later recovering because some coders
+ // may affect the source data.
+ TestBlock[] clonedDataBlocks =
+ cloneBlocksWithData((TestBlock[]) blockGroup.getDataBlocks());
+ TestBlock[] parityBlocks = (TestBlock[]) blockGroup.getParityBlocks();
+
+ ErasureCodingStep codingStep;
+ codingStep = encoder.calculateCoding(blockGroup);
+ performCodingStep(codingStep);
+ // Erase specified sources but return copies of them for later comparing
+ TestBlock[] backupBlocks = backupAndEraseBlocks(clonedDataBlocks, parityBlocks);
+
+ // Decode
+ blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
+ codingStep = decoder.calculateCoding(blockGroup);
+ performCodingStep(codingStep);
+
+ // Compare
+ compareAndVerify(backupBlocks, codingStep.getOutputBlocks());
+ }
+
+ /**
+ * This is typically how a coding step should be performed.
+ * @param codingStep
+ */
+ private void performCodingStep(ErasureCodingStep codingStep) {
+ // Pretend that we're opening these input blocks and output blocks.
+ ECBlock[] inputBlocks = codingStep.getInputBlocks();
+ ECBlock[] outputBlocks = codingStep.getOutputBlocks();
+ // We allocate input and output chunks accordingly.
+ ECChunk[] inputChunks = new ECChunk[inputBlocks.length];
+ ECChunk[] outputChunks = new ECChunk[outputBlocks.length];
+
+ for (int i = 0; i < numChunksInBlock; ++i) {
+ // Pretend that we're reading input chunks from input blocks.
+ for (int j = 0; j < inputBlocks.length; ++j) {
+ inputChunks[j] = ((TestBlock) inputBlocks[j]).chunks[i];
+ }
+
+ // Pretend that we allocate and will write output results to the blocks.
+ for (int j = 0; j < outputBlocks.length; ++j) {
+ outputChunks[j] = allocateOutputChunk();
+ ((TestBlock) outputBlocks[j]).chunks[i] = outputChunks[j];
+ }
+
+ // Given the input chunks and output chunk buffers, just call it !
+ codingStep.performCoding(inputChunks, outputChunks);
+ }
+
+ codingStep.finish();
+ }
+
+ /**
+ * Compare and verify if recovered blocks data are the same with the erased
+ * blocks data.
+ * @param erasedBlocks
+ * @param recoveredBlocks
+ */
+ protected void compareAndVerify(ECBlock[] erasedBlocks,
+ ECBlock[] recoveredBlocks) {
+ for (int i = 0; i < erasedBlocks.length; ++i) {
+ compareAndVerify(((TestBlock) erasedBlocks[i]).chunks, ((TestBlock) recoveredBlocks[i]).chunks);
+ }
+ }
+
+ private void prepareCoders() {
+ if (encoder == null) {
+ encoder = createEncoder();
+ }
+
+ if (decoder == null) {
+ decoder = createDecoder();
+ }
+ }
+
+ /**
+ * Create the raw erasure encoder to test
+ * @return
+ */
+ protected ErasureCoder createEncoder() {
+ ErasureCoder encoder;
+ try {
+ Constructor<? extends ErasureCoder> constructor =
+ (Constructor<? extends ErasureCoder>)
+ encoderClass.getConstructor(int.class, int.class);
+ encoder = constructor.newInstance(numDataUnits, numParityUnits);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create encoder", e);
+ }
+
+ encoder.setConf(getConf());
+ return encoder;
+ }
+
+ /**
+ * create the raw erasure decoder to test
+ * @return
+ */
+ protected ErasureCoder createDecoder() {
+ ErasureCoder decoder;
+ try {
+ Constructor<? extends ErasureCoder> constructor =
+ (Constructor<? extends ErasureCoder>)
+ decoderClass.getConstructor(int.class, int.class);
+ decoder = constructor.newInstance(numDataUnits, numParityUnits);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create decoder", e);
+ }
+
+ decoder.setConf(getConf());
+ return decoder;
+ }
+
+ /**
+ * Prepare a block group for encoding.
+ * @return
+ */
+ protected ECBlockGroup prepareBlockGroupForEncoding() {
+ ECBlock[] dataBlocks = new TestBlock[numDataUnits];
+ ECBlock[] parityBlocks = new TestBlock[numParityUnits];
+
+ for (int i = 0; i < numDataUnits; i++) {
+ dataBlocks[i] = generateDataBlock();
+ }
+
+ for (int i = 0; i < numParityUnits; i++) {
+ parityBlocks[i] = allocateOutputBlock();
+ }
+
+ return new ECBlockGroup(dataBlocks, parityBlocks);
+ }
+
+ /**
+ * Generate random data and return a data block.
+ * @return
+ */
+ protected ECBlock generateDataBlock() {
+ ECChunk[] chunks = new ECChunk[numChunksInBlock];
+
+ for (int i = 0; i < numChunksInBlock; ++i) {
+ chunks[i] = generateDataChunk();
+ }
+
+ return new TestBlock(chunks);
+ }
+
+ /**
+ * Erase blocks to test the recovering of them. Before erasure clone them
+ * first so could return themselves.
+ * @param dataBlocks
+ * @return clone of erased dataBlocks
+ */
+ protected TestBlock[] backupAndEraseBlocks(TestBlock[] dataBlocks,
+ TestBlock[] parityBlocks) {
+ TestBlock[] toEraseBlocks = new TestBlock[erasedDataIndexes.length +
+ erasedParityIndexes.length];
+ int idx = 0;
+ TestBlock block;
+
+ for (int i = 0; i < erasedParityIndexes.length; i++) {
+ block = parityBlocks[erasedParityIndexes[i]];
+ toEraseBlocks[idx ++] = cloneBlockWithData(block);
+ eraseDataFromBlock(block);
+ }
+
+ for (int i = 0; i < erasedDataIndexes.length; i++) {
+ block = dataBlocks[erasedDataIndexes[i]];
+ toEraseBlocks[idx ++] = cloneBlockWithData(block);
+ eraseDataFromBlock(block);
+ }
+
+ return toEraseBlocks;
+ }
+
+ /**
+ * Allocate an output block. Note the chunk buffer will be allocated by the
+ * up caller when performing the coding step.
+ * @return
+ */
+ protected TestBlock allocateOutputBlock() {
+ ECChunk[] chunks = new ECChunk[numChunksInBlock];
+
+ return new TestBlock(chunks);
+ }
+
+ /**
+ * Clone blocks with data copied along with, avoiding affecting the original
+ * blocks.
+ * @param blocks
+ * @return
+ */
+ protected TestBlock[] cloneBlocksWithData(TestBlock[] blocks) {
+ TestBlock[] results = new TestBlock[blocks.length];
+ for (int i = 0; i < blocks.length; ++i) {
+ results[i] = cloneBlockWithData(blocks[i]);
+ }
+
+ return results;
+ }
+
+ /**
+ * Clone exactly a block, avoiding affecting the original block.
+ * @param block
+ * @return a new block
+ */
+ protected TestBlock cloneBlockWithData(TestBlock block) {
+ ECChunk[] newChunks = cloneChunksWithData(block.chunks);
+
+ return new TestBlock(newChunks);
+ }
+
+ /**
+ * Erase data from a block.
+ */
+ protected void eraseDataFromBlock(TestBlock theBlock) {
+ eraseDataFromChunks(theBlock.chunks);
+ theBlock.setErased(true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
new file mode 100644
index 0000000..94f77db
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test Reed-Solomon encoding and decoding.
+ */
+public class TestRSErasureCoder extends TestErasureCoderBase {
+
+ @Before
+ public void setup() {
+ this.encoderClass = RSErasureEncoder.class;
+ this.decoderClass = RSErasureDecoder.class;
+
+ this.numDataUnits = 10;
+ this.numParityUnits = 1;
+
+ this.numChunksInBlock = 10;
+ }
+
+ @Test
+ public void testCodingNoDirectBuffer_10x4_erasing_d0_p0() {
+ prepare(null, 10, 4, new int[] {0}, new int[] {0});
+ /**
+ * Doing twice to test if the coders can be repeatedly reused. This matters
+ * as the underlying coding buffers are shared, which may have bugs.
+ */
+ testCoding(false);
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingDirectBufferWithConf_10x4_erasing_d0() {
+ /**
+ * This tests if the configuration items work or not.
+ */
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+ RSRawErasureCoderFactory.class.getCanonicalName());
+ prepare(conf, 10, 4, new int[]{0}, new int[0]);
+
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_p1() {
+ prepare(null, 10, 4, new int[]{}, new int[]{1});
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_d2() {
+ prepare(null, 10, 4, new int[] {2}, new int[] {});
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_d0_p0() {
+ prepare(null, 10, 4, new int[] {0}, new int[] {0});
+ testCoding(true);
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingBothBuffers_10x4_erasing_d0_p0() {
+ prepare(null, 10, 4, new int[] {0}, new int[] {0});
+
+ /**
+ * Doing in mixed buffer usage model to test if the coders can be repeatedly
+ * reused with different buffer usage model. This matters as the underlying
+ * coding buffers are shared, which may have bugs.
+ */
+ testCoding(true);
+ testCoding(false);
+ testCoding(true);
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() {
+ prepare(null, 10, 4, new int[] {2, 4}, new int[] {0});
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_10x4_erasing_d0_d1_p0_p1() {
+ prepare(null, 10, 4, new int[] {0, 1}, new int[] {0, 1});
+ testCoding(true);
+ }
+
+ @Test
+ public void testCodingNoDirectBuffer_3x3_erasing_d0_p0() {
+ prepare(null, 3, 3, new int[] {0}, new int[] {0});
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingDirectBuffer_3x3_erasing_d0_p0() {
+ prepare(null, 3, 3, new int[] {0}, new int[] {0});
+ testCoding(true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
new file mode 100644
index 0000000..06e0087
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.coder;
+
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test XOR encoding and decoding.
+ */
+public class TestXORCoder extends TestErasureCoderBase {
+
+ @Before
+ public void setup() {
+ this.encoderClass = XORErasureEncoder.class;
+ this.decoderClass = XORErasureDecoder.class;
+
+ this.numDataUnits = 10;
+ this.numParityUnits = 1;
+ this.numChunksInBlock = 10;
+ }
+
+ @Test
+ public void testCodingNoDirectBuffer_erasing_p0() {
+ prepare(null, 10, 1, new int[0], new int[] {0});
+
+ /**
+ * Doing twice to test if the coders can be repeatedly reused. This matters
+ * as the underlying coding buffers are shared, which may have bugs.
+ */
+ testCoding(false);
+ testCoding(false);
+ }
+
+ @Test
+ public void testCodingBothBuffers_erasing_d5() {
+ prepare(null, 10, 1, new int[]{5}, new int[0]);
+
+ /**
+ * Doing in mixed buffer usage model to test if the coders can be repeatedly
+ * reused with different buffer usage model. This matters as the underlying
+ * coding buffers are shared, which may have bugs.
+ */
+ testCoding(true);
+ testCoding(false);
+ testCoding(true);
+ testCoding(false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
new file mode 100644
index 0000000..a35a4dd
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test raw Reed-solomon coder implemented in Java.
+ */
+public class TestRSRawCoder extends TestRSRawCoderBase {
+
+ @Before
+ public void setup() {
+ this.encoderClass = RSRawEncoder.class;
+ this.decoderClass = RSRawDecoder.class;
+ setAllowDump(false); // Change to true to allow verbose dump for debugging
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_all_d() {
+ prepare(null, 6, 3, new int[]{0, 1, 2}, new int[0], true);
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_d0_d2() {
+ prepare(null, 6, 3, new int[] {0, 2}, new int[]{});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_d0() {
+ prepare(null, 6, 3, new int[]{0}, new int[0]);
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_d2() {
+ prepare(null, 6, 3, new int[]{2}, new int[]{});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_d0_p0() {
+ prepare(null, 6, 3, new int[]{0}, new int[]{0});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_all_p() {
+ prepare(null, 6, 3, new int[0], new int[]{0, 1, 2});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_p0() {
+ prepare(null, 6, 3, new int[0], new int[]{0});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_p2() {
+ prepare(null, 6, 3, new int[0], new int[]{2});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasure_p0_p2() {
+ prepare(null, 6, 3, new int[0], new int[]{0, 2});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_d0_p0_p1() {
+ prepare(null, 6, 3, new int[]{0}, new int[]{0, 1});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_6x3_erasing_d0_d2_p2() {
+ prepare(null, 6, 3, new int[]{0, 2}, new int[]{2});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCodingNegative_6x3_erasing_d2_d4() {
+ prepare(null, 6, 3, new int[]{2, 4}, new int[0]);
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCodingNegative_6x3_erasing_too_many() {
+ prepare(null, 6, 3, new int[]{2, 4}, new int[]{0, 1});
+ testCodingWithErasingTooMany();
+ }
+
+ @Test
+ public void testCoding_10x4_erasing_d0_p0() {
+ prepare(null, 10, 4, new int[] {0}, new int[] {0});
+ testCodingDoMixAndTwice();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
new file mode 100644
index 0000000..efde332
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
+
+/**
+ * Test base for raw Reed-solomon coders.
+ */
+public abstract class TestRSRawCoderBase extends TestRawCoderBase {
+
+ private static int symbolSize = 0;
+ private static int symbolMax = 0;
+
+ private static int RS_FIXED_DATA_GENERATOR = 0;
+
+ static {
+ symbolSize = (int) Math.round(Math.log(
+ RSUtil.GF.getFieldSize()) / Math.log(2));
+ symbolMax = (int) Math.pow(2, symbolSize);
+ }
+
+ @Override
+ protected byte[] generateData(int len) {
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) RAND.nextInt(symbolMax);
+ }
+ return buffer;
+ }
+
+ @Override
+ protected byte[] generateFixedData(int len) {
+ byte[] buffer = new byte[len];
+ for (int i = 0; i < buffer.length; i++) {
+ buffer[i] = (byte) RS_FIXED_DATA_GENERATOR++;
+ if (RS_FIXED_DATA_GENERATOR == symbolMax) {
+ RS_FIXED_DATA_GENERATOR = 0;
+ }
+ }
+ return buffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
new file mode 100644
index 0000000..2b7a3c4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.TestCoderBase;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Constructor;
+
+/**
+ * Raw coder test base with utilities.
+ */
+public abstract class TestRawCoderBase extends TestCoderBase {
+ protected Class<? extends RawErasureEncoder> encoderClass;
+ protected Class<? extends RawErasureDecoder> decoderClass;
+ private RawErasureEncoder encoder;
+ private RawErasureDecoder decoder;
+
+ /**
+ * Doing twice to test if the coders can be repeatedly reused. This matters
+ * as the underlying coding buffers are shared, which may have bugs.
+ */
+ protected void testCodingDoMixAndTwice() {
+ testCodingDoMixed();
+ testCodingDoMixed();
+ }
+
+ /**
+ * Doing in mixed buffer usage model to test if the coders can be repeatedly
+ * reused with different buffer usage model. This matters as the underlying
+ * coding buffers are shared, which may have bugs.
+ */
+ protected void testCodingDoMixed() {
+ testCoding(true);
+ testCoding(false);
+ }
+
+ /**
+ * Generating source data, encoding, recovering and then verifying.
+ * RawErasureCoder mainly uses ECChunk to pass input and output data buffers,
+ * it supports two kinds of ByteBuffers, one is array backed, the other is
+ * direct ByteBuffer. Use usingDirectBuffer indicate which case to test.
+ *
+ * @param usingDirectBuffer
+ */
+ protected void testCoding(boolean usingDirectBuffer) {
+ this.usingDirectBuffer = usingDirectBuffer;
+ prepareCoders();
+
+ /**
+ * The following runs will use 3 different chunkSize for inputs and outputs,
+ * to verify the same encoder/decoder can process variable width of data.
+ */
+ performTestCoding(baseChunkSize, true, false, false);
+ performTestCoding(baseChunkSize - 17, false, false, false);
+ performTestCoding(baseChunkSize + 16, true, false, false);
+ }
+
+ /**
+ * Similar to above, but perform negative cases using bad input for encoding.
+ * @param usingDirectBuffer
+ */
+ protected void testCodingWithBadInput(boolean usingDirectBuffer) {
+ this.usingDirectBuffer = usingDirectBuffer;
+ prepareCoders();
+
+ try {
+ performTestCoding(baseChunkSize, false, true, false);
+ Assert.fail("Encoding test with bad input should fail");
+ } catch (Exception e) {
+ // Expected
+ }
+ }
+
+ /**
+ * Similar to above, but perform negative cases using bad output for decoding.
+ * @param usingDirectBuffer
+ */
+ protected void testCodingWithBadOutput(boolean usingDirectBuffer) {
+ this.usingDirectBuffer = usingDirectBuffer;
+ prepareCoders();
+
+ try {
+ performTestCoding(baseChunkSize, false, false, true);
+ Assert.fail("Decoding test with bad output should fail");
+ } catch (Exception e) {
+ // Expected
+ }
+ }
+
+ @Test
+ public void testCodingWithErasingTooMany() {
+ try {
+ testCoding(true);
+ Assert.fail("Decoding test erasing too many should fail");
+ } catch (Exception e) {
+ // Expected
+ }
+
+ try {
+ testCoding(false);
+ Assert.fail("Decoding test erasing too many should fail");
+ } catch (Exception e) {
+ // Expected
+ }
+ }
+
+ private void performTestCoding(int chunkSize, boolean usingSlicedBuffer,
+ boolean useBadInput, boolean useBadOutput) {
+ setChunkSize(chunkSize);
+ prepareBufferAllocator(usingSlicedBuffer);
+
+ dumpSetting();
+
+ // Generate data and encode
+ ECChunk[] dataChunks = prepareDataChunksForEncoding();
+ if (useBadInput) {
+ corruptSomeChunk(dataChunks);
+ }
+ dumpChunks("Testing data chunks", dataChunks);
+
+ ECChunk[] parityChunks = prepareParityChunksForEncoding();
+
+ // Backup all the source chunks for later recovering because some coders
+ // may affect the source data.
+ ECChunk[] clonedDataChunks = cloneChunksWithData(dataChunks);
+
+ encoder.encode(dataChunks, parityChunks);
+ dumpChunks("Encoded parity chunks", parityChunks);
+
+ // Backup and erase some chunks
+ ECChunk[] backupChunks = backupAndEraseChunks(clonedDataChunks, parityChunks);
+
+ // Decode
+ ECChunk[] inputChunks = prepareInputChunksForDecoding(
+ clonedDataChunks, parityChunks);
+
+ // Remove unnecessary chunks, allowing only least required chunks to be read.
+ ensureOnlyLeastRequiredChunks(inputChunks);
+
+ ECChunk[] recoveredChunks = prepareOutputChunksForDecoding();
+ if (useBadOutput) {
+ corruptSomeChunk(recoveredChunks);
+ }
+
+ dumpChunks("Decoding input chunks", inputChunks);
+ decoder.decode(inputChunks, getErasedIndexesForDecoding(), recoveredChunks);
+ dumpChunks("Decoded/recovered chunks", recoveredChunks);
+
+ // Compare
+ compareAndVerify(backupChunks, recoveredChunks);
+ }
+
+ private void prepareCoders() {
+ if (encoder == null) {
+ encoder = createEncoder();
+ }
+
+ if (decoder == null) {
+ decoder = createDecoder();
+ }
+ }
+
+ private void ensureOnlyLeastRequiredChunks(ECChunk[] inputChunks) {
+ int leastRequiredNum = numDataUnits;
+ int erasedNum = erasedDataIndexes.length + erasedParityIndexes.length;
+ int goodNum = inputChunks.length - erasedNum;
+ int redundantNum = goodNum - leastRequiredNum;
+
+ for (int i = 0; i < inputChunks.length && redundantNum > 0; i++) {
+ if (inputChunks[i] != null) {
+ inputChunks[i] = null; // Setting it null, not needing it actually
+ redundantNum--;
+ }
+ }
+ }
+
+ /**
+ * Create the raw erasure encoder to test
+ * @return
+ */
+ protected RawErasureEncoder createEncoder() {
+ RawErasureEncoder encoder;
+ try {
+ Constructor<? extends RawErasureEncoder> constructor =
+ (Constructor<? extends RawErasureEncoder>)
+ encoderClass.getConstructor(int.class, int.class);
+ encoder = constructor.newInstance(numDataUnits, numParityUnits);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create encoder", e);
+ }
+
+ encoder.setConf(getConf());
+ return encoder;
+ }
+
+ /**
+ * create the raw erasure decoder to test
+ * @return
+ */
+ protected RawErasureDecoder createDecoder() {
+ RawErasureDecoder decoder;
+ try {
+ Constructor<? extends RawErasureDecoder> constructor =
+ (Constructor<? extends RawErasureDecoder>)
+ decoderClass.getConstructor(int.class, int.class);
+ decoder = constructor.newInstance(numDataUnits, numParityUnits);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create decoder", e);
+ }
+
+ decoder.setConf(getConf());
+ return decoder;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
new file mode 100644
index 0000000..48463ad
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.rawcoder;
+
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test XOR encoding and decoding.
+ */
+public class TestXORRawCoder extends TestRawCoderBase {
+
+ @Before
+ public void setup() {
+ this.encoderClass = XORRawEncoder.class;
+ this.decoderClass = XORRawDecoder.class;
+ }
+
+ @Test
+ public void testCoding_10x1_erasing_d0() {
+ prepare(null, 10, 1, new int[] {0}, new int[0]);
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_10x1_erasing_p0() {
+ prepare(null, 10, 1, new int[0], new int[] {0});
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCoding_10x1_erasing_d5() {
+ prepare(null, 10, 1, new int[]{5}, new int[0]);
+ testCodingDoMixAndTwice();
+ }
+
+ @Test
+ public void testCodingNegative_10x1_erasing_too_many() {
+ prepare(null, 10, 1, new int[]{2}, new int[]{0});
+ testCodingWithErasingTooMany();
+ }
+
+ @Test
+ public void testCodingNegative_10x1_erasing_d5() {
+ prepare(null, 10, 1, new int[]{5}, new int[0]);
+ testCodingWithBadInput(true);
+ testCodingWithBadOutput(false);
+ testCodingWithBadInput(true);
+ testCodingWithBadOutput(false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index aeaa980..03b4a1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -91,6 +91,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>hdfs.proto</include>
<include>encryption.proto</include>
<include>inotify.proto</include>
+ <include>erasurecoding.proto</include>
</includes>
</source>
<output>${project.build.directory}/generated-sources/java</output>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 600c7ca..214e15d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -179,6 +179,18 @@ public interface HdfsClientConfigKeys {
int THREADPOOL_SIZE_DEFAULT = 0;
}
+ /** dfs.client.read.striped configuration properties */
+ interface StripedRead {
+ String PREFIX = Read.PREFIX + "striped.";
+
+ String THREADPOOL_SIZE_KEY = PREFIX + "threadpool.size";
+ /**
+ * With default 6+3 schema, each normal read could span 6 DNs. So this
+ * default value accommodates 3 read streams
+ */
+ int THREADPOOL_SIZE_DEFAULT = 18;
+ }
+
/** dfs.http.client configuration properties */
interface HttpClient {
String PREFIX = "dfs.http.client.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 8528999..e37f440 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
import org.apache.hadoop.security.KerberosInfo;
@@ -1483,4 +1484,30 @@ public interface ClientProtocol {
*/
@Idempotent
EventBatchList getEditsFromTxid(long txid) throws IOException;
+
+ /**
+ * Create an erasure coding zone with specified schema, if any, otherwise
+ * default
+ */
+ @AtMostOnce
+ void createErasureCodingZone(String src, ECSchema schema, int cellSize)
+ throws IOException;
+
+ /**
+ * Gets list of ECSchemas loaded in Namenode
+ *
+ * @return Returns the list of ECSchemas loaded at Namenode
+ * @throws IOException
+ */
+ @Idempotent
+ ECSchema[] getECSchemas() throws IOException;
+
+ /**
+ * Get the information about the EC zone for the path
+ *
+ * @param src path to get the info for
+ * @throws IOException
+ */
+ @Idempotent
+ ErasureCodingZone getErasureCodingZone(String src) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc2564af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZone.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZone.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZone.java
new file mode 100644
index 0000000..655def3
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingZone.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+/**
+ * Information about the EC Zone at the specified path.
+ */
+public class ErasureCodingZone {
+
+ private String dir;
+ private ECSchema schema;
+ private int cellSize;
+
+ public ErasureCodingZone(String dir, ECSchema schema, int cellSize) {
+ this.dir = dir;
+ this.schema = schema;
+ this.cellSize = cellSize;
+ }
+
+ /**
+ * Get directory of the EC zone.
+ *
+ * @return
+ */
+ public String getDir() {
+ return dir;
+ }
+
+ /**
+ * Get the schema for the EC Zone
+ *
+ * @return
+ */
+ public ECSchema getSchema() {
+ return schema;
+ }
+
+ /**
+ * Get cellSize for the EC Zone
+ */
+ public int getCellSize() {
+ return cellSize;
+ }
+
+ @Override
+ public String toString() {
+ return "Dir: " + getDir() + ", Schema: " + schema + ", cellSize: "
+ + cellSize;
+ }
+}