You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2009/10/26 19:17:21 UTC
svn commit: r829888 - in /hadoop/hdfs/branches/branch-0.21: ./
src/test/aop/org/apache/hadoop/fi/
src/test/aop/org/apache/hadoop/hdfs/server/datanode/
Author: szetszwo
Date: Mon Oct 26 18:17:21 2009
New Revision: 829888
URL: http://svn.apache.org/viewvc?rev=829888&view=rev
Log:
HDFS-730. Add 4 fault injection tests to simulate non-responsive datanode and out-of-memory problem for pipeline close ack.
Modified:
hadoop/hdfs/branches/branch-0.21/CHANGES.txt
hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java
hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java
hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj
hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
Modified: hadoop/hdfs/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/CHANGES.txt?rev=829888&r1=829887&r2=829888&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.21/CHANGES.txt Mon Oct 26 18:17:21 2009
@@ -283,6 +283,9 @@
HDFS-616. Create functional tests for new design of the block report. (cos)
HDFS-584. Fail the fault-inject build if any advices are mis-bound. (cos)
+
+ HDFS-730. Add 4 fault injection tests to simulate non-responsive datanode
+ and out-of-memory problem for pipeline close ack. (szetszwo)
BUG FIXES
Modified: hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java?rev=829888&r1=829887&r2=829888&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java Mon Oct 26 18:17:21 2009
@@ -25,6 +25,7 @@
import org.apache.hadoop.fi.FiTestUtil.ActionContainer;
import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
import org.apache.hadoop.fi.FiTestUtil.CountdownConstraint;
+import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -137,7 +138,8 @@
/** {@inheritDoc} */
public String toString() {
- return currentTest + ", index=" + index;
+ return getClass().getSimpleName() + ":" + currentTest
+ + ", index=" + index;
}
/** return a String with this object and the datanodeID. */
@@ -147,6 +149,33 @@
}
}
+ /** An action to set a marker if the DatanodeID is matched. */
+ public static class DatanodeMarkingAction extends DataNodeAction {
+ private final MarkerConstraint marker;
+
+ /** Construct an object. */
+ public DatanodeMarkingAction(String currentTest, int index,
+ MarkerConstraint marker) {
+ super(currentTest, index);
+ this.marker = marker;
+ }
+
+ /** Set the marker if the DatanodeID is matched. */
+ @Override
+ public void run(DatanodeID datanodeid) throws IOException {
+ final DataTransferTest test = getDataTransferTest();
+ final Pipeline p = test.getPipeline(datanodeid);
+ if (p.contains(index, datanodeid)) {
+ marker.mark();
+ }
+ }
+
+ /** {@inheritDoc} */
+ public String toString() {
+ return super.toString() + ", " + marker;
+ }
+ }
+
/** Throws OutOfMemoryError. */
public static class OomAction extends DataNodeAction {
/** Create an action for datanode i in the pipeline. */
@@ -273,8 +302,8 @@
/** {@inheritDoc} */
@Override
public String toString() {
- return super.toString()
- + ", duration = [" + minDuration + "," + maxDuration + ")";
+ return super.toString() + ", duration="
+ + (maxDuration <= 0? "infinity": "[" + minDuration + ", " + maxDuration + ")");
}
}
Modified: hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java?rev=829888&r1=829887&r2=829888&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java Mon Oct 26 18:17:21 2009
@@ -174,4 +174,32 @@
action.run(parameter);
}
}
+
+ /** A MarkerConstraint is satisfied if it is marked. */
+ public static class MarkerConstraint implements Constraint {
+ private final String name;
+ private boolean marked = false;
+
+ /** Construct an object. */
+ public MarkerConstraint(String name) {
+ this.name = name;
+ }
+
+ /** Set marker to be marked. */
+ public void mark() {
+ marked = true;
+ LOG.info("Marking this " + this);
+ }
+
+ /** Is the marker marked? */
+ @Override
+ public boolean isSatisfied() {
+ return marked;
+ }
+
+ /** {@inheritDoc} */
+ public String toString() {
+ return getClass().getSimpleName() + "[" + name + ": " + marked + "]";
+ }
+ }
}
\ No newline at end of file
Modified: hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj?rev=829888&r1=829887&r2=829888&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj (original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj Mon Oct 26 18:17:21 2009
@@ -47,11 +47,11 @@
before(BlockReceiver blockreceiver
) throws IOException : callReceivePacket(blockreceiver) {
- LOG.info("FI: callReceivePacket");
+ final DatanodeRegistration dr = blockreceiver.getDataNode().getDatanodeRegistration();
+ LOG.info("FI: callReceivePacket, datanode=" + dr.getName());
DataTransferTest dtTest = DataTransferTestUtil.getDataTransferTest();
if (dtTest != null)
- dtTest.fiCallReceivePacket.run(
- blockreceiver.getDataNode().getDatanodeRegistration());
+ dtTest.fiCallReceivePacket.run(dr);
if (ProbabilityModel.injectCriteria(BlockReceiver.class.getSimpleName())) {
LOG.info("Before the injection point");
@@ -73,7 +73,7 @@
lastPacketInBlock, len, endOfHeader) {
if (len == 0) {
final DatanodeRegistration dr = blockreceiver.getDataNode().getDatanodeRegistration();
- LOG.info("FI: pipelineClose, datanode=" + dr
+ LOG.info("FI: pipelineClose, datanode=" + dr.getName()
+ ", offsetInBlock=" + offsetInBlock
+ ", seqno=" + seqno
+ ", lastPacketInBlock=" + lastPacketInBlock
Modified: hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java?rev=829888&r1=829887&r2=829888&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java (original)
+++ hadoop/hdfs/branches/branch-0.21/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java Mon Oct 26 18:17:21 2009
@@ -22,13 +22,17 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fi.DataTransferTestUtil;
import org.apache.hadoop.fi.FiTestUtil;
+import org.apache.hadoop.fi.DataTransferTestUtil.DataNodeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DataTransferTest;
+import org.apache.hadoop.fi.DataTransferTestUtil.DatanodeMarkingAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DoosAction;
import org.apache.hadoop.fi.DataTransferTestUtil.IoeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.OomAction;
import org.apache.hadoop.fi.DataTransferTestUtil.SleepAction;
import org.apache.hadoop.fi.DataTransferTestUtil.VerificationAction;
import org.apache.hadoop.fi.FiTestUtil.Action;
+import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
+import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -313,6 +317,40 @@
private static void run41_43(String name, int i) throws IOException {
runPipelineCloseTest(name, new SleepAction(name, i, 3000));
}
+
+ private static void runPipelineCloseAck(String name, int i, DataNodeAction a
+ ) throws IOException {
+ FiTestUtil.LOG.info("Running " + name + " ...");
+ final DataTransferTest t = (DataTransferTest)DataTransferTestUtil.initTest();
+ final MarkerConstraint marker = new MarkerConstraint(name);
+ t.fiPipelineClose.set(new DatanodeMarkingAction(name, i, marker));
+ t.fiPipelineAck.set(new ConstraintSatisfactionAction<DatanodeID>(a, marker));
+ write1byte(name);
+ }
+
+ private static void run39_40(String name, int i) throws IOException {
+ runPipelineCloseAck(name, i, new SleepAction(name, i, 0));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 never responses after received close ack DN2.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_39() throws IOException {
+ run39_40(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 never responses after received close ack DN1.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_40() throws IOException {
+ run39_40(FiTestUtil.getMethodName(), 0);
+ }
/**
* Pipeline close with DN0 very slow but it won't lead to timeout.
@@ -377,6 +415,32 @@
runPipelineCloseTest(methodName, new OomAction(methodName, 2));
}
+ private static void run47_48(String name, int i) throws IOException {
+ runPipelineCloseAck(name, i, new OomAction(name, i));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 throws an OutOfMemoryException right after
+ * it received a close ack from DN2.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_47() throws IOException {
+ run47_48(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 throws an OutOfMemoryException right after
+ * it received a close ack from DN1.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_48() throws IOException {
+ run47_48(FiTestUtil.getMethodName(), 0);
+ }
+
private static void runBlockFileCloseTest(String methodName,
Action<DatanodeID> a) throws IOException {
FiTestUtil.LOG.info("Running " + methodName + " ...");