You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dr...@apache.org on 2017/09/06 05:30:59 UTC
hadoop git commit: HDFS-12392. Writing striped file failed due to
different cell size. Contributed by Sammi Chen
Repository: hadoop
Updated Branches:
refs/heads/trunk d4035d42f -> d7f27043c
HDFS-12392. Writing striped file failed due to different cell size. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7f27043
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7f27043
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7f27043
Branch: refs/heads/trunk
Commit: d7f27043ce034d69003b054dfe4f9e8e317ec543
Parents: d4035d4
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Sep 6 13:29:52 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Sep 6 13:29:52 2017 +0800
----------------------------------------------------------------------
.../hadoop/hdfs/DFSStripedOutputStream.java | 2 +
.../hadoop/hdfs/TestErasureCodingPolicies.java | 50 +++++++++++++++++++-
2 files changed, 50 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7f27043/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
index 408b325..7f05338 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java
@@ -215,6 +215,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
buffers = new ByteBuffer[numAllBlocks];
for (int i = 0; i < buffers.length; i++) {
buffers[i] = BUFFER_POOL.getBuffer(useDirectBuffer(), cellSize);
+ buffers[i].limit(cellSize);
}
}
@@ -237,6 +238,7 @@ public class DFSStripedOutputStream extends DFSOutputStream
private void clear() {
for (int i = 0; i< numAllBlocks; i++) {
buffers[i].clear();
+ buffers[i].limit(cellSize);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7f27043/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 0c545be..e095602 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.test.GenericTestUtils;
@@ -60,7 +61,7 @@ public class TestErasureCodingPolicies {
private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
- private static final int BLOCK_SIZE = 1024;
+ private static final int BLOCK_SIZE = 16 * 1024;
private ErasureCodingPolicy ecPolicy;
private FSNamesystem namesystem;
@@ -78,7 +79,8 @@ public class TestErasureCodingPolicies {
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
DFSTestUtil.enableAllECPolicies(conf);
cluster = new MiniDFSCluster.Builder(conf).
- numDataNodes(1).build();
+ numDataNodes(ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits()).
+ build();
cluster.waitActive();
fs = cluster.getFileSystem();
namesystem = cluster.getNamesystem();
@@ -836,4 +838,48 @@ public class TestErasureCodingPolicies {
ecPolicy, fs.getErasureCodingPolicy(subReplicaFile));
fs.delete(subReplicaFile, false);
}
+
+ @Test
+ public void testDifferentErasureCodingPolicyCellSize() throws Exception {
+ // add policy with cell size 8K
+ ErasureCodingPolicy newPolicy1 =
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 8 * 1024);
+ ErasureCodingPolicy[] policyArray =
+ new ErasureCodingPolicy[] {newPolicy1};
+ AddECPolicyResponse[] responses = fs.addErasureCodingPolicies(policyArray);
+ assertEquals(1, responses.length);
+ assertTrue(responses[0].isSucceed());
+ newPolicy1 = responses[0].getPolicy();
+
+ // add policy with cell size 4K
+ ErasureCodingPolicy newPolicy2 =
+ new ErasureCodingPolicy(ErasureCodeConstants.RS_3_2_SCHEMA, 4 * 1024);
+ policyArray = new ErasureCodingPolicy[] {newPolicy2};
+ responses = fs.addErasureCodingPolicies(policyArray);
+ assertEquals(1, responses.length);
+ assertTrue(responses[0].isSucceed());
+ newPolicy2 = responses[0].getPolicy();
+
+ // enable policies
+ fs.enableErasureCodingPolicy(newPolicy1.getName());
+ fs.enableErasureCodingPolicy(newPolicy2.getName());
+
+ final Path stripedDir1 = new Path("/striped1");
+ final Path stripedDir2 = new Path("/striped2");
+ final Path file1 = new Path(stripedDir1, "file");
+ final Path file2 = new Path(stripedDir2, "file");
+
+ fs.mkdirs(stripedDir1);
+ fs.setErasureCodingPolicy(stripedDir1, newPolicy1.getName());
+ fs.mkdirs(stripedDir2);
+ fs.setErasureCodingPolicy(stripedDir2, newPolicy2.getName());
+
+ final int fileLength = BLOCK_SIZE * newPolicy1.getNumDataUnits();
+ final byte[] bytes = StripedFileTestUtil.generateBytes(fileLength);
+ DFSTestUtil.writeFile(fs, file1, bytes);
+ DFSTestUtil.writeFile(fs, file2, bytes);
+
+ fs.delete(stripedDir1, true);
+ fs.delete(stripedDir2, true);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org