You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/05/19 00:09:59 UTC
hadoop git commit: HDFS-8417. Erasure Coding: Pread failed to read
data starting from not-first stripe. Contributed by Walter Su.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 031b4d09d -> a579a19c8
HDFS-8417. Erasure Coding: Pread failed to read data starting from not-first stripe. Contributed by Walter Su.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a579a19c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a579a19c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a579a19c
Branch: refs/heads/HDFS-7285
Commit: a579a19c84893d287bad4408618e96e695178e42
Parents: 031b4d0
Author: Jing Zhao <ji...@apache.org>
Authored: Mon May 18 15:08:30 2015 -0700
Committer: Jing Zhao <ji...@apache.org>
Committed: Mon May 18 15:08:30 2015 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +
.../hadoop/hdfs/util/StripedBlockUtil.java | 3 +-
.../hadoop/hdfs/TestDFSStripedInputStream.java | 72 ++++++++++++--------
.../server/datanode/SimulatedFSDataset.java | 2 +-
4 files changed, 50 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a579a19c/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 333d85f..e016ba0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -217,3 +217,6 @@
assigning new tasks. (umamahesh)
HDFS-8367. BlockInfoStriped uses EC schema. (Kai Sasaki via Kai Zheng)
+
+ HDFS-8417. Erasure Coding: Pread failed to read data starting from not-first stripe.
+ (Walter Su via jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a579a19c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index c95f0b4..81c0c95 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -379,7 +379,8 @@ public class StripedBlockUtil {
int firstCellIdxInBG = (int) (start / cellSize);
int lastCellIdxInBG = (int) (end / cellSize);
int firstCellSize = Math.min(cellSize - (int) (start % cellSize), len);
- long firstCellOffsetInBlk = start % cellSize;
+ long firstCellOffsetInBlk = firstCellIdxInBG / dataBlkNum * cellSize +
+ start % cellSize;
int lastCellSize = lastCellIdxInBG == firstCellIdxInBG ?
firstCellSize : (int) (end % cellSize) + 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a579a19c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 452cc2b..9032d09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hdfs.util.StripedBlockUtil;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.rawcoder.RSRawDecoder;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -115,40 +116,55 @@ public class TestDFSStripedInputStream {
DFSTestUtil.createStripedFile(cluster, filePath, null, numBlocks,
NUM_STRIPE_PER_BLOCK, false);
LocatedBlocks lbs = fs.getClient().namenode.getBlockLocations(
- filePath.toString(), 0, BLOCK_GROUP_SIZE);
+ filePath.toString(), 0, BLOCK_GROUP_SIZE * numBlocks);
+ int fileLen = BLOCK_GROUP_SIZE * numBlocks;
- assert lbs.get(0) instanceof LocatedStripedBlock;
- LocatedStripedBlock bg = (LocatedStripedBlock)(lbs.get(0));
- for (int i = 0; i < DATA_BLK_NUM; i++) {
- Block blk = new Block(bg.getBlock().getBlockId() + i,
- NUM_STRIPE_PER_BLOCK * CELLSIZE,
- bg.getBlock().getGenerationStamp());
- blk.setGenerationStamp(bg.getBlock().getGenerationStamp());
- cluster.injectBlocks(i, Arrays.asList(blk),
- bg.getBlock().getBlockPoolId());
- }
- DFSStripedInputStream in =
- new DFSStripedInputStream(fs.getClient(),
- filePath.toString(), false, schema);
- int readSize = BLOCK_GROUP_SIZE;
- byte[] readBuffer = new byte[readSize];
- int ret = in.read(0, readBuffer, 0, readSize);
+ byte[] expected = new byte[fileLen];
+ assertEquals(numBlocks, lbs.getLocatedBlocks().size());
+ for (int bgIdx = 0; bgIdx < numBlocks; bgIdx++) {
+ LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(bgIdx));
+ for (int i = 0; i < DATA_BLK_NUM; i++) {
+ Block blk = new Block(bg.getBlock().getBlockId() + i,
+ NUM_STRIPE_PER_BLOCK * CELLSIZE,
+ bg.getBlock().getGenerationStamp());
+ blk.setGenerationStamp(bg.getBlock().getGenerationStamp());
+ cluster.injectBlocks(i, Arrays.asList(blk),
+ bg.getBlock().getBlockPoolId());
+ }
- byte[] expected = new byte[readSize];
- /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
- for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
- for (int j = 0; j < DATA_BLK_NUM; j++) {
- for (int k = 0; k < CELLSIZE; k++) {
- int posInBlk = i * CELLSIZE + k;
- int posInFile = i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE + k;
- expected[posInFile] = SimulatedFSDataset.simulatedByte(
- new Block(bg.getBlock().getBlockId() + j), posInBlk);
+ /** A variation of {@link DFSTestUtil#fillExpectedBuf} for striped blocks */
+ for (int i = 0; i < NUM_STRIPE_PER_BLOCK; i++) {
+ for (int j = 0; j < DATA_BLK_NUM; j++) {
+ for (int k = 0; k < CELLSIZE; k++) {
+ int posInBlk = i * CELLSIZE + k;
+ int posInFile = i * CELLSIZE * DATA_BLK_NUM + j * CELLSIZE + k;
+ expected[bgIdx*BLOCK_GROUP_SIZE + posInFile] =
+ SimulatedFSDataset.simulatedByte(
+ new Block(bg.getBlock().getBlockId() + j), posInBlk);
+ }
}
}
}
+ DFSStripedInputStream in = new DFSStripedInputStream(fs.getClient(),
+ filePath.toString(), false, schema);
- assertEquals(readSize, ret);
- assertArrayEquals(expected, readBuffer);
+ int[] startOffsets = {0, 1, CELLSIZE - 102, CELLSIZE, CELLSIZE + 102,
+ CELLSIZE*DATA_BLK_NUM, CELLSIZE*DATA_BLK_NUM + 102,
+ BLOCK_GROUP_SIZE - 102, BLOCK_GROUP_SIZE, BLOCK_GROUP_SIZE + 102,
+ fileLen - 1};
+ for (int startOffset : startOffsets) {
+ startOffset = Math.max(0, Math.min(startOffset, fileLen - 1));
+ int remaining = fileLen - startOffset;
+ byte[] buf = new byte[fileLen];
+ int ret = in.read(startOffset, buf, 0, fileLen);
+ assertEquals(remaining, ret);
+ for (int i = 0; i < remaining; i++) {
+ Assert.assertEquals("Byte at " + (startOffset + i) + " should be the " +
+ "same",
+ expected[startOffset + i], buf[i]);
+ }
+ }
+ in.close();
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a579a19c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 778dd28..dd6b600 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -100,7 +100,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
public static byte simulatedByte(Block b, long offsetInBlk) {
byte firstByte = (byte) (b.getBlockId() & BYTE_MASK);
- return (byte) ((firstByte + offsetInBlk) & BYTE_MASK);
+ return (byte) ((firstByte + offsetInBlk % 29) & BYTE_MASK);
}
public static final String CONFIG_PROPERTY_CAPACITY =