You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2011/04/08 19:30:28 UTC
svn commit: r1090357 - in /hadoop/hdfs/trunk: CHANGES.txt
src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java
Author: szetszwo
Date: Fri Apr 8 17:30:28 2011
New Revision: 1090357
URL: http://svn.apache.org/viewvc?rev=1090357&view=rev
Log:
HDFS-1817. Move pipeline_Fi_[39-51] from TestFiDataTransferProtocol to TestFiPipelineClose.
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1090357&r1=1090356&r2=1090357&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Apr 8 17:30:28 2011
@@ -96,6 +96,9 @@ Trunk (unreleased changes)
HDFS-1767. Namenode ignores non-initial block report from datanodes
when in safemode during startup. (Matt Foley via suresh)
+ HDFS-1817. Move pipeline_Fi_[39-51] from TestFiDataTransferProtocol
+ to TestFiPipelineClose. (szetszwo)
+
OPTIMIZATIONS
HDFS-1458. Improve checkpoint performance by avoiding unnecessary image
Modified: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java?rev=1090357&r1=1090356&r2=1090357&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java (original)
+++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java Fri Apr 8 17:30:28 2011
@@ -19,21 +19,15 @@ package org.apache.hadoop.hdfs.server.da
import java.io.IOException;
-import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fi.DataTransferTestUtil;
-import org.apache.hadoop.fi.FiTestUtil;
-import org.apache.hadoop.fi.DataTransferTestUtil.DataNodeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DataTransferTest;
-import org.apache.hadoop.fi.DataTransferTestUtil.DatanodeMarkingAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DoosAction;
-import org.apache.hadoop.fi.DataTransferTestUtil.IoeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.OomAction;
import org.apache.hadoop.fi.DataTransferTestUtil.SleepAction;
import org.apache.hadoop.fi.DataTransferTestUtil.VerificationAction;
+import org.apache.hadoop.fi.FiTestUtil;
import org.apache.hadoop.fi.FiTestUtil.Action;
-import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
-import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -70,9 +64,9 @@ public class TestFiDataTransferProtocol
* 4. open the same file
* 5. read the 1 byte and compare results
*/
- private static void write1byte(String methodName) throws IOException {
- final MiniDFSCluster cluster = new MiniDFSCluster(conf, REPLICATION, true,
- null);
+ static void write1byte(String methodName) throws IOException {
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
+ ).numDataNodes(REPLICATION).build();
final FileSystem dfs = cluster.getFileSystem();
try {
final Path p = new Path("/" + methodName + "/foo");
@@ -305,184 +299,4 @@ public class TestFiDataTransferProtocol
final String methodName = FiTestUtil.getMethodName();
runCallWritePacketToDisk(methodName, 2, new DoosAction(methodName, 2));
}
-
- private static void runPipelineCloseTest(String methodName,
- Action<DatanodeID, IOException> a) throws IOException {
- FiTestUtil.LOG.info("Running " + methodName + " ...");
- final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
- .initTest();
- t.fiPipelineClose.set(a);
- write1byte(methodName);
- }
-
- private static void run41_43(String name, int i) throws IOException {
- runPipelineCloseTest(name, new SleepAction(name, i, 3000));
- }
-
- private static void runPipelineCloseAck(String name, int i, DataNodeAction a
- ) throws IOException {
- FiTestUtil.LOG.info("Running " + name + " ...");
- final DataTransferTest t = (DataTransferTest)DataTransferTestUtil.initTest();
- final MarkerConstraint marker = new MarkerConstraint(name);
- t.fiPipelineClose.set(new DatanodeMarkingAction(name, i, marker));
- t.fiPipelineAck.set(new ConstraintSatisfactionAction<DatanodeID, IOException>(a, marker));
- write1byte(name);
- }
-
- private static void run39_40(String name, int i) throws IOException {
- runPipelineCloseAck(name, i, new SleepAction(name, i, 0));
- }
-
- /**
- * Pipeline close:
- * DN1 never responses after received close ack DN2.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_39() throws IOException {
- run39_40(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close:
- * DN0 never responses after received close ack DN1.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_40() throws IOException {
- run39_40(FiTestUtil.getMethodName(), 0);
- }
-
- /**
- * Pipeline close with DN0 very slow but it won't lead to timeout.
- * Client finishes close successfully.
- */
- @Test
- public void pipeline_Fi_41() throws IOException {
- run41_43(FiTestUtil.getMethodName(), 0);
- }
-
- /**
- * Pipeline close with DN1 very slow but it won't lead to timeout.
- * Client finishes close successfully.
- */
- @Test
- public void pipeline_Fi_42() throws IOException {
- run41_43(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close with DN2 very slow but it won't lead to timeout.
- * Client finishes close successfully.
- */
- @Test
- public void pipeline_Fi_43() throws IOException {
- run41_43(FiTestUtil.getMethodName(), 2);
- }
-
- /**
- * Pipeline close:
- * DN0 throws an OutOfMemoryException
- * right after it received a close request from client.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_44() throws IOException {
- final String methodName = FiTestUtil.getMethodName();
- runPipelineCloseTest(methodName, new OomAction(methodName, 0));
- }
-
- /**
- * Pipeline close:
- * DN1 throws an OutOfMemoryException
- * right after it received a close request from client.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_45() throws IOException {
- final String methodName = FiTestUtil.getMethodName();
- runPipelineCloseTest(methodName, new OomAction(methodName, 1));
- }
-
- /**
- * Pipeline close:
- * DN2 throws an OutOfMemoryException
- * right after it received a close request from client.
- * Client gets an IOException and determine DN2 bad.
- */
- @Test
- public void pipeline_Fi_46() throws IOException {
- final String methodName = FiTestUtil.getMethodName();
- runPipelineCloseTest(methodName, new OomAction(methodName, 2));
- }
-
- private static void run47_48(String name, int i) throws IOException {
- runPipelineCloseAck(name, i, new OomAction(name, i));
- }
-
- /**
- * Pipeline close:
- * DN1 throws an OutOfMemoryException right after
- * it received a close ack from DN2.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_47() throws IOException {
- run47_48(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close:
- * DN0 throws an OutOfMemoryException right after
- * it received a close ack from DN1.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_48() throws IOException {
- run47_48(FiTestUtil.getMethodName(), 0);
- }
-
- private static void runBlockFileCloseTest(String methodName,
- Action<DatanodeID, IOException> a) throws IOException {
- FiTestUtil.LOG.info("Running " + methodName + " ...");
- final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
- .initTest();
- t.fiBlockFileClose.set(a);
- write1byte(methodName);
- }
-
- private static void run49_51(String name, int i) throws IOException {
- runBlockFileCloseTest(name, new IoeAction(name, i, "DISK ERROR"));
- }
-
- /**
- * Pipeline close:
- * DN0 throws a disk error exception when it is closing the block file.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_49() throws IOException {
- run49_51(FiTestUtil.getMethodName(), 0);
- }
-
-
- /**
- * Pipeline close:
- * DN1 throws a disk error exception when it is closing the block file.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_50() throws IOException {
- run49_51(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close:
- * DN2 throws a disk error exception when it is closing the block file.
- * Client gets an IOException and determine DN2 bad.
- */
- @Test
- public void pipeline_Fi_51() throws IOException {
- run49_51(FiTestUtil.getMethodName(), 2);
- }
}
\ No newline at end of file
Modified: hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java?rev=1090357&r1=1090356&r2=1090357&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java (original)
+++ hadoop/hdfs/trunk/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java Fri Apr 8 17:30:28 2011
@@ -19,76 +19,29 @@ package org.apache.hadoop.hdfs.server.da
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fi.DataTransferTestUtil;
-import org.apache.hadoop.fi.FiTestUtil;
+import org.apache.hadoop.fi.DataTransferTestUtil.DataNodeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DataTransferTest;
+import org.apache.hadoop.fi.DataTransferTestUtil.DatanodeMarkingAction;
+import org.apache.hadoop.fi.DataTransferTestUtil.IoeAction;
+import org.apache.hadoop.fi.DataTransferTestUtil.OomAction;
import org.apache.hadoop.fi.DataTransferTestUtil.SleepAction;
+import org.apache.hadoop.fi.FiTestUtil;
import org.apache.hadoop.fi.FiTestUtil.Action;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
+import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.junit.Assert;
import org.junit.Test;
/** Test DataTransferProtocol with fault injection. */
public class TestFiPipelineClose {
- static final short REPLICATION = 3;
- static final long BLOCKSIZE = 1L * (1L << 20);
-
- static final Configuration conf = new HdfsConfiguration();
- static {
- conf.setInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, 1);
- conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, REPLICATION);
- conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 5000);
- }
-
- static private FSDataOutputStream createFile(FileSystem fs, Path p
- ) throws IOException {
- return fs.create(p, true, fs.getConf().getInt("io.file.buffer.size", 4096),
- REPLICATION, BLOCKSIZE);
- }
-
- /**
- * 1. create files with dfs
- * 2. write 1 byte
- * 3. close file
- * 4. open the same file
- * 5. read the 1 byte and compare results
- */
- private static void write1byte(String methodName) throws IOException {
- final MiniDFSCluster cluster = new MiniDFSCluster(conf, REPLICATION, true,
- null);
- final FileSystem dfs = cluster.getFileSystem();
- try {
- final Path p = new Path("/" + methodName + "/foo");
- final FSDataOutputStream out = createFile(dfs, p);
- out.write(1);
- out.close();
-
- final FSDataInputStream in = dfs.open(p);
- final int b = in.read();
- in.close();
- Assert.assertEquals(1, b);
- }
- finally {
- dfs.close();
- cluster.shutdown();
- }
- }
-
- private static void runPipelineCloseTest(String methodName,
+ private static void runPipelineCloseTest(String methodName,
Action<DatanodeID, IOException> a) throws IOException {
FiTestUtil.LOG.info("Running " + methodName + " ...");
final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
.initTest();
t.fiPipelineClose.set(a);
- write1byte(methodName);
+ TestFiDataTransferProtocol.write1byte(methodName);
}
/**
@@ -123,4 +76,175 @@ public class TestFiPipelineClose {
final String methodName = FiTestUtil.getMethodName();
runPipelineCloseTest(methodName, new SleepAction(methodName, 2, 0));
}
+
+ private static void run41_43(String name, int i) throws IOException {
+ runPipelineCloseTest(name, new SleepAction(name, i, 3000));
+ }
+
+ private static void runPipelineCloseAck(String name, int i, DataNodeAction a
+ ) throws IOException {
+ FiTestUtil.LOG.info("Running " + name + " ...");
+ final DataTransferTest t = (DataTransferTest)DataTransferTestUtil.initTest();
+ final MarkerConstraint marker = new MarkerConstraint(name);
+ t.fiPipelineClose.set(new DatanodeMarkingAction(name, i, marker));
+ t.fiPipelineAck.set(new ConstraintSatisfactionAction<DatanodeID, IOException>(a, marker));
+ TestFiDataTransferProtocol.write1byte(name);
+ }
+
+ private static void run39_40(String name, int i) throws IOException {
+ runPipelineCloseAck(name, i, new SleepAction(name, i, 0));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 never responses after received close ack DN2.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_39() throws IOException {
+ run39_40(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 never responses after received close ack DN1.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_40() throws IOException {
+ run39_40(FiTestUtil.getMethodName(), 0);
+ }
+
+ /**
+ * Pipeline close with DN0 very slow but it won't lead to timeout.
+ * Client finishes close successfully.
+ */
+ @Test
+ public void pipeline_Fi_41() throws IOException {
+ run41_43(FiTestUtil.getMethodName(), 0);
+ }
+
+ /**
+ * Pipeline close with DN1 very slow but it won't lead to timeout.
+ * Client finishes close successfully.
+ */
+ @Test
+ public void pipeline_Fi_42() throws IOException {
+ run41_43(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close with DN2 very slow but it won't lead to timeout.
+ * Client finishes close successfully.
+ */
+ @Test
+ public void pipeline_Fi_43() throws IOException {
+ run41_43(FiTestUtil.getMethodName(), 2);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 throws an OutOfMemoryException
+ * right after it received a close request from client.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_44() throws IOException {
+ final String methodName = FiTestUtil.getMethodName();
+ runPipelineCloseTest(methodName, new OomAction(methodName, 0));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 throws an OutOfMemoryException
+ * right after it received a close request from client.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_45() throws IOException {
+ final String methodName = FiTestUtil.getMethodName();
+ runPipelineCloseTest(methodName, new OomAction(methodName, 1));
+ }
+
+ /**
+ * Pipeline close:
+ * DN2 throws an OutOfMemoryException
+ * right after it received a close request from client.
+ * Client gets an IOException and determine DN2 bad.
+ */
+ @Test
+ public void pipeline_Fi_46() throws IOException {
+ final String methodName = FiTestUtil.getMethodName();
+ runPipelineCloseTest(methodName, new OomAction(methodName, 2));
+ }
+
+ private static void run47_48(String name, int i) throws IOException {
+ runPipelineCloseAck(name, i, new OomAction(name, i));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 throws an OutOfMemoryException right after
+ * it received a close ack from DN2.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_47() throws IOException {
+ run47_48(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 throws an OutOfMemoryException right after
+ * it received a close ack from DN1.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_48() throws IOException {
+ run47_48(FiTestUtil.getMethodName(), 0);
+ }
+
+ private static void runBlockFileCloseTest(String methodName,
+ Action<DatanodeID, IOException> a) throws IOException {
+ FiTestUtil.LOG.info("Running " + methodName + " ...");
+ final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
+ .initTest();
+ t.fiBlockFileClose.set(a);
+ TestFiDataTransferProtocol.write1byte(methodName);
+ }
+
+ private static void run49_51(String name, int i) throws IOException {
+ runBlockFileCloseTest(name, new IoeAction(name, i, "DISK ERROR"));
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 throws a disk error exception when it is closing the block file.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_49() throws IOException {
+ run49_51(FiTestUtil.getMethodName(), 0);
+ }
+
+
+ /**
+ * Pipeline close:
+ * DN1 throws a disk error exception when it is closing the block file.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_50() throws IOException {
+ run49_51(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN2 throws a disk error exception when it is closing the block file.
+ * Client gets an IOException and determine DN2 bad.
+ */
+ @Test
+ public void pipeline_Fi_51() throws IOException {
+ run49_51(FiTestUtil.getMethodName(), 2);
+ }
}