You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/04/18 00:35:41 UTC
[13/50] [abbrv] hadoop git commit: HDFS-8117. More accurate
verification in SimulatedFSDataset: replace DEFAULT_DATABYTE with patterned
data. Contributed by Zhe Zhang.
HDFS-8117. More accurate verification in SimulatedFSDataset: replace DEFAULT_DATABYTE with patterned data. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/45fa3321
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/45fa3321
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/45fa3321
Branch: refs/heads/YARN-2928
Commit: 45fa3321cb0b389c781be11c30c3fb2a280f7a5b
Parents: ad69744
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Apr 13 13:01:10 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri Apr 17 15:29:41 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 25 ++++++++++++++++++++
.../org/apache/hadoop/hdfs/TestFileAppend.java | 12 +++++-----
.../java/org/apache/hadoop/hdfs/TestPread.java | 21 +++++++++-------
.../org/apache/hadoop/hdfs/TestSmallBlock.java | 16 ++++++-------
.../server/datanode/SimulatedFSDataset.java | 25 ++++++++++----------
.../server/datanode/TestSimulatedFSDataset.java | 3 ++-
7 files changed, 70 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45fa3321/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2a26544..1aaf42c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -431,6 +431,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8083. Move dfs.client.write.* conf from DFSConfigKeys to
HdfsClientConfigKeys.Write. (szetszwo)
+ HDFS-8117. More accurate verification in SimulatedFSDataset: replace
+ DEFAULT_DATABYTE with patterned data. (Zhe Zhang via wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45fa3321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 1b3b62d..ae2d403 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster.NameNodeInfo;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -117,6 +118,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion;
+import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.namenode.FSEditLog;
@@ -1769,4 +1771,27 @@ public class DFSTestUtil {
dn.setLastUpdateMonotonic(Time.monotonicNow() + offset);
}
+ /**
+ * This method takes a set of block locations and fills the provided buffer
+ * with expected bytes based on simulated content from
+ * {@link SimulatedFSDataset}.
+ *
+ * @param lbs The block locations of a file
+ * @param expected The buffer to be filled with expected bytes on the above
+ * locations.
+ */
+ public static void fillExpectedBuf(LocatedBlocks lbs, byte[] expected) {
+ Block[] blks = new Block[lbs.getLocatedBlocks().size()];
+ for (int i = 0; i < lbs.getLocatedBlocks().size(); i++) {
+ blks[i] = lbs.getLocatedBlocks().get(i).getBlock().getLocalBlock();
+ }
+ int bufPos = 0;
+ for (Block b : blks) {
+ for (long blkPos = 0; blkPos < b.getNumBytes(); blkPos++) {
+ assert bufPos < expected.length;
+ expected[bufPos++] = SimulatedFSDataset.simulatedByte(b, blkPos);
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45fa3321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
index af404cd..6a7c3ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
@@ -68,7 +68,7 @@ public class TestFileAppend{
//
// verify that the data written to the full blocks are sane
//
- private void checkFile(FileSystem fileSys, Path name, int repl)
+ private void checkFile(DistributedFileSystem fileSys, Path name, int repl)
throws IOException {
boolean done = false;
@@ -96,9 +96,9 @@ public class TestFileAppend{
byte[] expected =
new byte[AppendTestUtil.NUM_BLOCKS * AppendTestUtil.BLOCK_SIZE];
if (simulatedStorage) {
- for (int i= 0; i < expected.length; i++) {
- expected[i] = SimulatedFSDataset.DEFAULT_DATABYTE;
- }
+ LocatedBlocks lbs = fileSys.getClient().getLocatedBlocks(name.toString(),
+ 0, AppendTestUtil.FILE_SIZE);
+ DFSTestUtil.fillExpectedBuf(lbs, expected);
} else {
System.arraycopy(fileContents, 0, expected, 0, expected.length);
}
@@ -193,7 +193,7 @@ public class TestFileAppend{
}
fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- FileSystem fs = cluster.getFileSystem();
+ DistributedFileSystem fs = cluster.getFileSystem();
try {
// create a new file.
@@ -249,7 +249,7 @@ public class TestFileAppend{
}
fileContents = AppendTestUtil.initBuffer(AppendTestUtil.FILE_SIZE);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- FileSystem fs = cluster.getFileSystem();
+ DistributedFileSystem fs = cluster.getFileSystem();
try {
// create a new file.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45fa3321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
index 62f6c06..1a28404 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.io.IOUtils;
@@ -54,6 +55,8 @@ import org.mockito.stubbing.Answer;
public class TestPread {
static final long seed = 0xDEADBEEFL;
static final int blockSize = 4096;
+ static final int numBlocksPerFile = 12;
+ static final int fileSize = numBlocksPerFile * blockSize;
boolean simulatedStorage;
boolean isHedgedRead;
@@ -66,10 +69,10 @@ public class TestPread {
private void writeFile(FileSystem fileSys, Path name) throws IOException {
int replication = 3;// We need > 1 blocks to test out the hedged reads.
// test empty file open and read
- DFSTestUtil.createFile(fileSys, name, 12 * blockSize, 0,
+ DFSTestUtil.createFile(fileSys, name, fileSize, 0,
blockSize, (short)replication, seed);
FSDataInputStream in = fileSys.open(name);
- byte[] buffer = new byte[12 * blockSize];
+ byte[] buffer = new byte[fileSize];
in.readFully(0, buffer, 0, 0);
IOException res = null;
try { // read beyond the end of the file
@@ -84,7 +87,7 @@ public class TestPread {
assertTrue("Cannot delete file", false);
// now create the real file
- DFSTestUtil.createFile(fileSys, name, 12 * blockSize, 12 * blockSize,
+ DFSTestUtil.createFile(fileSys, name, fileSize, fileSize,
blockSize, (short) replication, seed);
}
@@ -128,11 +131,13 @@ public class TestPread {
private void pReadFile(FileSystem fileSys, Path name) throws IOException {
FSDataInputStream stm = fileSys.open(name);
- byte[] expected = new byte[12 * blockSize];
+ byte[] expected = new byte[fileSize];
if (simulatedStorage) {
- for (int i= 0; i < expected.length; i++) {
- expected[i] = SimulatedFSDataset.DEFAULT_DATABYTE;
- }
+ assert fileSys instanceof DistributedFileSystem;
+ DistributedFileSystem dfs = (DistributedFileSystem) fileSys;
+ LocatedBlocks lbs = dfs.getClient().getLocatedBlocks(name.toString(),
+ 0, fileSize);
+ DFSTestUtil.fillExpectedBuf(lbs, expected);
} else {
Random rand = new Random(seed);
rand.nextBytes(expected);
@@ -447,7 +452,7 @@ public class TestPread {
FileSystem fileSys = cluster.getFileSystem();
fileSys.setVerifyChecksum(verifyChecksum);
try {
- Path file1 = new Path("preadtest.dat");
+ Path file1 = new Path("/preadtest.dat");
writeFile(fileSys, file1);
pReadFile(fileSys, file1);
datanodeRestartTest(cluster, fileSys, file1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45fa3321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java
index 90f47e5..6983cde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSmallBlock.java
@@ -25,11 +25,10 @@ import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.junit.Test;
@@ -52,16 +51,17 @@ public class TestSmallBlock {
}
}
- private void checkFile(FileSystem fileSys, Path name) throws IOException {
+ private void checkFile(DistributedFileSystem fileSys, Path name)
+ throws IOException {
BlockLocation[] locations = fileSys.getFileBlockLocations(
fileSys.getFileStatus(name), 0, fileSize);
assertEquals("Number of blocks", fileSize, locations.length);
FSDataInputStream stm = fileSys.open(name);
byte[] expected = new byte[fileSize];
if (simulatedStorage) {
- for (int i = 0; i < expected.length; ++i) {
- expected[i] = SimulatedFSDataset.DEFAULT_DATABYTE;
- }
+ LocatedBlocks lbs = fileSys.getClient().getLocatedBlocks(name.toString(),
+ 0, fileSize);
+ DFSTestUtil.fillExpectedBuf(lbs, expected);
} else {
Random rand = new Random(seed);
rand.nextBytes(expected);
@@ -90,9 +90,9 @@ public class TestSmallBlock {
}
conf.set(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, "1");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
- FileSystem fileSys = cluster.getFileSystem();
+ DistributedFileSystem fileSys = cluster.getFileSystem();
try {
- Path file1 = new Path("smallblocktest.dat");
+ Path file1 = new Path("/smallblocktest.dat");
DFSTestUtil.createFile(fileSys, file1, fileSize, fileSize, blockSize,
(short) 1, seed);
checkFile(fileSys, file1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45fa3321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index a358e22..344d1fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -97,12 +97,16 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
conf.set(DFSConfigKeys.DFS_DATANODE_FSDATASET_FACTORY_KEY,
Factory.class.getName());
}
+
+ public static byte simulatedByte(Block b, long offsetInBlk) {
+ byte firstByte = (byte) (b.getBlockId() % Byte.MAX_VALUE);
+ return (byte) ((firstByte + offsetInBlk) % Byte.MAX_VALUE);
+ }
public static final String CONFIG_PROPERTY_CAPACITY =
"dfs.datanode.simulateddatastorage.capacity";
public static final long DEFAULT_CAPACITY = 2L<<40; // 1 terabyte
- public static final byte DEFAULT_DATABYTE = 9;
public static final String CONFIG_PROPERTY_STATE =
"dfs.datanode.simulateddatastorage.state";
@@ -182,9 +186,9 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
synchronized SimulatedInputStream getIStream() {
if (!finalized) {
// throw new IOException("Trying to read an unfinalized block");
- return new SimulatedInputStream(oStream.getLength(), DEFAULT_DATABYTE);
+ return new SimulatedInputStream(oStream.getLength(), theBlock);
} else {
- return new SimulatedInputStream(theBlock.getNumBytes(), DEFAULT_DATABYTE);
+ return new SimulatedInputStream(theBlock.getNumBytes(), theBlock);
}
}
@@ -991,21 +995,19 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
*
*/
static private class SimulatedInputStream extends java.io.InputStream {
-
-
- byte theRepeatedData = 7;
final long length; // bytes
int currentPos = 0;
byte[] data = null;
+ Block theBlock = null;
/**
* An input stream of size l with repeated bytes
* @param l size of the stream
* @param iRepeatedData byte that is repeated in the stream
*/
- SimulatedInputStream(long l, byte iRepeatedData) {
+ SimulatedInputStream(long l, Block b) {
length = l;
- theRepeatedData = iRepeatedData;
+ theBlock = b;
}
/**
@@ -1031,8 +1033,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
if (data !=null) {
return data[currentPos++];
} else {
- currentPos++;
- return theRepeatedData;
+ return simulatedByte(theBlock, currentPos++);
}
}
@@ -1052,8 +1053,8 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
if (data != null) {
System.arraycopy(data, currentPos, b, 0, bytesRead);
} else { // all data is zero
- for (int i : b) {
- b[i] = theRepeatedData;
+ for (int i = 0; i < bytesRead; i++) {
+ b[i] = simulatedByte(theBlock, currentPos + i);
}
}
currentPos += bytesRead;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/45fa3321/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
index dd24685..f76781d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestSimulatedFSDataset.java
@@ -144,7 +144,8 @@ public class TestSimulatedFSDataset {
long lengthRead = 0;
int data;
while ((data = input.read()) != -1) {
- assertEquals(SimulatedFSDataset.DEFAULT_DATABYTE, data);
+ assertEquals(SimulatedFSDataset.simulatedByte(b.getLocalBlock(),
+ lengthRead), data);
lengthRead++;
}
assertEquals(expectedLen, lengthRead);