You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by kk...@apache.org on 2017/03/07 23:10:29 UTC
[15/50] hadoop git commit: HDFS-11416. Refactor out system default
erasure coding policy. Contributed by Andrew Wang.
HDFS-11416. Refactor out system default erasure coding policy. Contributed by Andrew Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3749152b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3749152b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3749152b
Branch: refs/heads/YARN-5972
Commit: 3749152b661d0359b3b941ab1d17177230f3b8dc
Parents: 8f4817f
Author: Rakesh Radhakrishnan <ra...@apache.org>
Authored: Fri Mar 3 09:28:03 2017 +0530
Committer: Rakesh Radhakrishnan <ra...@apache.org>
Committed: Fri Mar 3 09:28:03 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/util/StringUtils.java | 11 ++++++
.../server/namenode/FSDirErasureCodingOp.java | 38 +++++++++-----------
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 16 +++++----
.../hdfs/server/namenode/NamenodeFsck.java | 6 +---
.../server/namenode/TestStripedINodeFile.java | 13 +++++++
6 files changed, 52 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 11a93de..d2b572b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -178,6 +178,17 @@ public class StringUtils {
}
/**
+ * Convert a byte to a hex string.
+ * @see #byteToHexString(byte[])
+ * @see #byteToHexString(byte[], int, int)
+ * @param b byte
+ * @return byte's hex value as a String
+ */
+ public static String byteToHexString(byte b) {
+ return byteToHexString(new byte[] {b});
+ }
+
+ /**
* Given a hexstring this will return the byte array corresponding to the
* string
* @param hex the hex String array
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index f4c4ad7..5240c7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -102,6 +102,7 @@ final class FSDirErasureCodingOp {
FSDirectory fsd = fsn.getFSDirectory();
assert fsd.hasWriteLock();
Preconditions.checkNotNull(srcIIP, "INodes cannot be null");
+ Preconditions.checkNotNull(ecPolicy, "EC policy cannot be null");
String src = srcIIP.getPath();
final INode inode = srcIIP.getLastINode();
if (inode == null) {
@@ -112,29 +113,24 @@ final class FSDirErasureCodingOp {
"for a file " + src);
}
- // System default erasure coding policy will be used since no specified.
- if (ecPolicy == null) {
- ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
- } else {
- // If ecPolicy is specified check if it is one among active policies.
- boolean validPolicy = false;
- ErasureCodingPolicy[] activePolicies =
- FSDirErasureCodingOp.getErasureCodingPolicies(fsd.getFSNamesystem());
- for (ErasureCodingPolicy activePolicy : activePolicies) {
- if (activePolicy.equals(ecPolicy)) {
- validPolicy = true;
- break;
- }
+ // Check that the EC policy is one of the active policies.
+ boolean validPolicy = false;
+ ErasureCodingPolicy[] activePolicies =
+ FSDirErasureCodingOp.getErasureCodingPolicies(fsd.getFSNamesystem());
+ for (ErasureCodingPolicy activePolicy : activePolicies) {
+ if (activePolicy.equals(ecPolicy)) {
+ validPolicy = true;
+ break;
}
- if (!validPolicy) {
- List<String> ecPolicyNames = new ArrayList<String>();
- for (ErasureCodingPolicy activePolicy : activePolicies) {
- ecPolicyNames.add(activePolicy.getName());
- }
- throw new HadoopIllegalArgumentException("Policy [ " +
- ecPolicy.getName() + " ] does not match any of the " +
- "supported policies. Please select any one of " + ecPolicyNames);
+ }
+ if (!validPolicy) {
+ List<String> ecPolicyNames = new ArrayList<String>();
+ for (ErasureCodingPolicy activePolicy : activePolicies) {
+ ecPolicyNames.add(activePolicy.getName());
}
+ throw new HadoopIllegalArgumentException("Policy [ " +
+ ecPolicy.getName() + " ] does not match any of the " +
+ "supported policies. Please select any one of " + ecPolicyNames);
}
final XAttr ecXAttr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 2990344..e3ad076 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -1108,8 +1108,7 @@ public class FSEditLogLoader {
// versions of Hadoop. Current versions always log
// OP_ADD operations as each block is allocated.
if (isStriped) {
- newBI = new BlockInfoStriped(newBlock,
- ErasureCodingPolicyManager.getSystemDefaultPolicy());
+ newBI = new BlockInfoStriped(newBlock, ecPolicy);
} else {
newBI = new BlockInfoContiguous(newBlock,
file.getFileReplication());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 0315150..19480e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.util.LongBitFormat;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.util.StringUtils;
/** I-node for closed file. */
@InterfaceAudience.Private
@@ -190,9 +191,10 @@ public class INodeFile extends INodeWithAdditionalFields
if (blockType == STRIPED) {
Preconditions.checkArgument(replication == null &&
erasureCodingPolicyID != null);
- Preconditions.checkArgument(
- ErasureCodingPolicyManager.getPolicyByPolicyID(
- erasureCodingPolicyID) != null);
+ Preconditions.checkArgument(ErasureCodingPolicyManager
+ .getPolicyByPolicyID(erasureCodingPolicyID) != null,
+ "Could not find EC policy with ID 0x" + StringUtils
+ .byteToHexString(erasureCodingPolicyID));
layoutRedundancy |= BLOCK_TYPE_MASK_STRIPED;
// Following bitwise OR with signed byte erasureCodingPolicyID is safe
// as the PolicyID can never be in negative.
@@ -201,7 +203,8 @@ public class INodeFile extends INodeWithAdditionalFields
Preconditions.checkArgument(replication != null &&
erasureCodingPolicyID == null);
Preconditions.checkArgument(replication >= 0 &&
- replication <= MAX_REDUNDANCY);
+ replication <= MAX_REDUNDANCY,
+ "Invalid replication value " + replication);
layoutRedundancy |= replication;
}
return layoutRedundancy;
@@ -513,9 +516,8 @@ public class INodeFile extends INodeWithAdditionalFields
ErasureCodingPolicy ecPolicy =
ErasureCodingPolicyManager.getPolicyByPolicyID(
getErasureCodingPolicyID());
- if (ecPolicy == null){
- ecPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy();
- }
+ Preconditions.checkNotNull(ecPolicy, "Could not find EC policy with ID 0x"
+ + StringUtils.byteToHexString(getErasureCodingPolicyID()));
return (short) (ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index 7e91935..eddab3f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -1311,11 +1311,8 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
@VisibleForTesting
static class ErasureCodingResult extends Result {
- final String defaultECPolicy;
ErasureCodingResult(Configuration conf) {
- defaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy()
- .getName();
}
@Override
@@ -1392,8 +1389,7 @@ public class NamenodeFsck implements DataEncryptionKeyFactory {
((float) (numMisReplicatedBlocks * 100) / (float) totalBlocks))
.append(" %)");
}
- res.append("\n Default ecPolicy:\t\t").append(defaultECPolicy)
- .append("\n Average block group size:\t").append(
+ res.append("\n Average block group size:\t").append(
getReplicationFactor()).append("\n Missing block groups:\t\t").append(
missingIds.size()).append("\n Corrupt block groups:\t\t").append(
corruptBlocks).append("\n Missing internal blocks:\t").append(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3749152b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index a712498..ae9793a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.junit.rules.Timeout;
/**
@@ -83,6 +84,18 @@ public class TestStripedINodeFile {
HdfsConstants.COLD_STORAGE_POLICY_ID, BlockType.STRIPED);
}
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Test
+ public void testInvalidECPolicy() throws IllegalArgumentException {
+ thrown.expect(IllegalArgumentException.class);
+ thrown.expectMessage("Could not find EC policy with ID 0xbb");
+ new INodeFile(HdfsConstants.GRANDFATHER_INODE_ID, null, perm, 0L, 0L,
+ null, null, (byte) 0xBB, 1024L,
+ HdfsConstants.COLD_STORAGE_POLICY_ID, BlockType.STRIPED);
+ }
+
@Test
public void testBlockStripedFeature()
throws IOException, InterruptedException{
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org