You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/04/20 04:28:21 UTC
svn commit: r1095253 [3/4] - in /hadoop/hdfs/branches/HDFS-1073: ./ bin/
src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/fs/ src/java/org/apache/hadoop/hdfs/
src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/had...
Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Wed Apr 20 02:28:19 2011
@@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement;
import org.apache.hadoop.io.Text;
@@ -121,7 +120,7 @@ class ImageLoaderCurrent implements Imag
protected final DateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
private static int [] versions =
- {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28};
+ {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30, -31};
private int imageVersion = 0;
/* (non-Javadoc)
@@ -156,7 +155,7 @@ class ImageLoaderCurrent implements Imag
v.visit(ImageElement.GENERATION_STAMP, in.readLong());
- if (imageVersion <= -28) {
+ if (imageVersion <= -31) {
v.visit(ImageElement.TRANSACTION_ID, in.readLong());
}
@@ -338,34 +337,105 @@ class ImageLoaderCurrent implements Imag
long numInodes, boolean skipBlocks) throws IOException {
v.visitEnclosingElement(ImageElement.INODES,
ImageElement.NUM_INODES, numInodes);
+
+ if (imageVersion <= -30) { // local file name
+ processLocalNameINodes(in, v, numInodes, skipBlocks);
+ } else { // full path name
+ processFullNameINodes(in, v, numInodes, skipBlocks);
+ }
- for(long i = 0; i < numInodes; i++) {
- v.visitEnclosingElement(ImageElement.INODE);
- v.visit(ImageElement.INODE_PATH, FSImageSerialization.readString(in));
- v.visit(ImageElement.REPLICATION, in.readShort());
- v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
- if(imageVersion <= -17) // added in version -17
- v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
- v.visit(ImageElement.BLOCK_SIZE, in.readLong());
- int numBlocks = in.readInt();
-
- processBlocks(in, v, numBlocks, skipBlocks);
-
- // File or directory
- if (numBlocks > 0 || numBlocks == -1) {
- v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
- if(imageVersion <= -18) // added in version -18
- v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
- }
- if (imageVersion <= -23 && numBlocks == -2) {
- v.visit(ImageElement.SYMLINK, Text.readString(in));
+
+ v.leaveEnclosingElement(); // INodes
+ }
+
+ /**
+ * Process image with full path name
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param numInodes number of indoes to read
+ * @param skipBlocks skip blocks or not
+ * @throws IOException if there is any error occurs
+ */
+ private void processLocalNameINodes(DataInputStream in, ImageVisitor v,
+ long numInodes, boolean skipBlocks) throws IOException {
+ // process root
+ processINode(in, v, skipBlocks, "");
+ numInodes--;
+ while (numInodes > 0) {
+ numInodes -= processDirectory(in, v, skipBlocks);
+ }
+ }
+
+ private int processDirectory(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks) throws IOException {
+ String parentName = FSImageSerialization.readString(in);
+ int numChildren = in.readInt();
+ for (int i=0; i<numChildren; i++) {
+ processINode(in, v, skipBlocks, parentName);
+ }
+ return numChildren;
+ }
+
+ /**
+ * Process image with full path name
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param numInodes number of indoes to read
+ * @param skipBlocks skip blocks or not
+ * @throws IOException if there is any error occurs
+ */
+ private void processFullNameINodes(DataInputStream in, ImageVisitor v,
+ long numInodes, boolean skipBlocks) throws IOException {
+ for(long i = 0; i < numInodes; i++) {
+ processINode(in, v, skipBlocks, null);
+ }
+ }
+
+ /**
+ * Process an INode
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param skipBlocks skip blocks or not
+ * @param parentName the name of its parent node
+ * @return the number of Children
+ * @throws IOException
+ */
+ private void processINode(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks, String parentName) throws IOException {
+ v.visitEnclosingElement(ImageElement.INODE);
+ String pathName = FSImageSerialization.readString(in);
+ if (parentName != null) { // local name
+ pathName = "/" + pathName;
+ if (!"/".equals(parentName)) { // children of non-root directory
+ pathName = parentName + pathName;
}
+ }
- processPermission(in, v);
- v.leaveEnclosingElement(); // INode
+ v.visit(ImageElement.INODE_PATH, pathName);
+ v.visit(ImageElement.REPLICATION, in.readShort());
+ v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
+ if(imageVersion <= -17) // added in version -17
+ v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
+ v.visit(ImageElement.BLOCK_SIZE, in.readLong());
+ int numBlocks = in.readInt();
+
+ processBlocks(in, v, numBlocks, skipBlocks);
+
+ // File or directory
+ if (numBlocks > 0 || numBlocks == -1) {
+ v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
+ if(imageVersion <= -18) // added in version -18
+ v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
}
-
- v.leaveEnclosingElement(); // INodes
+ if (imageVersion <= -23 && numBlocks == -2) {
+ v.visit(ImageElement.SYMLINK, Text.readString(in));
+ }
+
+ processPermission(in, v);
+ v.leaveEnclosingElement(); // INode
}
/**
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/DataTransferTestUtil.java Wed Apr 20 02:28:19 2011
@@ -343,7 +343,7 @@ public class DataTransferTestUtil {
if (!test.isSuccess() && p.contains(index, id)) {
FiTestUtil.LOG.info(toString(id));
if (maxDuration <= 0) {
- for(; true; FiTestUtil.sleep(1000)); //sleep forever
+ for(; FiTestUtil.sleep(1000); ); //sleep forever until interrupt
} else {
FiTestUtil.sleep(minDuration, maxDuration);
}
@@ -391,7 +391,7 @@ public class DataTransferTestUtil {
+ minDuration + "," + maxDuration + ")";
FiTestUtil.LOG.info(s);
if (maxDuration <= 1) {
- for(; true; FiTestUtil.sleep(1000)); //sleep forever
+ for(; FiTestUtil.sleep(1000); ); //sleep forever until interrupt
} else {
FiTestUtil.sleep(minDuration, maxDuration);
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/fi/FiTestUtil.java Wed Apr 20 02:28:19 2011
@@ -73,14 +73,17 @@ public class FiTestUtil {
/**
* Sleep.
- * If there is an InterruptedException, re-throw it as a RuntimeException.
+ * @return true if sleep exits normally; false if InterruptedException.
*/
- public static void sleep(long ms) {
+ public static boolean sleep(long ms) {
+ LOG.info("Sleep " + ms + " ms");
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
- throw new RuntimeException(e);
+ LOG.info("Sleep is interrupted", e);
+ return false;
}
+ return true;
}
/**
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/BlockReceiverAspects.aj Wed Apr 20 02:28:19 2011
@@ -44,6 +44,11 @@ import org.apache.hadoop.util.DiskChecke
privileged public aspect BlockReceiverAspects {
public static final Log LOG = LogFactory.getLog(BlockReceiverAspects.class);
+ BlockReceiver BlockReceiver.PacketResponder.getReceiver(){
+ LOG.info("FI: getReceiver() " + getClass().getName());
+ return BlockReceiver.this;
+ }
+
pointcut callReceivePacket(BlockReceiver blockreceiver) :
call(* receivePacket(..)) && target(blockreceiver);
@@ -80,7 +85,7 @@ privileged public aspect BlockReceiverAs
after(BlockReceiver.PacketResponder responder)
throws IOException: afterDownstreamStatusRead(responder) {
- final DataNode d = responder.receiver.getDataNode();
+ final DataNode d = responder.getReceiver().getDataNode();
DataTransferTest dtTest = DataTransferTestUtil.getDataTransferTest();
if (dtTest != null)
dtTest.fiAfterDownstreamStatusRead.run(d.getDatanodeRegistration());
@@ -124,8 +129,9 @@ privileged public aspect BlockReceiverAs
LOG.debug("FI: no pipeline has been found in acking");
return;
}
- LOG.debug("FI: Acked total bytes from: " +
- pr.receiver.datanode.dnRegistration.getStorageID() + ": " + acked);
+ LOG.debug("FI: Acked total bytes from: "
+ + pr.getReceiver().datanode.dnRegistration.getStorageID()
+ + ": " + acked);
if (pTest instanceof PipelinesTest) {
bytesAckedService((PipelinesTest)pTest, pr, acked);
}
@@ -133,7 +139,7 @@ privileged public aspect BlockReceiverAs
private void bytesAckedService
(final PipelinesTest pTest, final PacketResponder pr, final long acked) {
- NodeBytes nb = new NodeBytes(pr.receiver.datanode.dnRegistration, acked);
+ NodeBytes nb = new NodeBytes(pr.getReceiver().datanode.dnRegistration, acked);
try {
pTest.fiCallSetBytesAcked.run(nb);
} catch (IOException e) {
@@ -201,7 +207,7 @@ privileged public aspect BlockReceiverAs
after(BlockReceiver.PacketResponder packetresponder) throws IOException
: pipelineAck(packetresponder) {
- final DatanodeRegistration dr = packetresponder.receiver.getDataNode().getDatanodeRegistration();
+ final DatanodeRegistration dr = packetresponder.getReceiver().getDataNode().getDatanodeRegistration();
LOG.info("FI: fiPipelineAck, datanode=" + dr);
final DataTransferTest test = DataTransferTestUtil.getDataTransferTest();
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol.java Wed Apr 20 02:28:19 2011
@@ -22,18 +22,13 @@ import java.io.IOException;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fi.DataTransferTestUtil;
-import org.apache.hadoop.fi.FiTestUtil;
-import org.apache.hadoop.fi.DataTransferTestUtil.DataNodeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DataTransferTest;
-import org.apache.hadoop.fi.DataTransferTestUtil.DatanodeMarkingAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DoosAction;
-import org.apache.hadoop.fi.DataTransferTestUtil.IoeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.OomAction;
import org.apache.hadoop.fi.DataTransferTestUtil.SleepAction;
import org.apache.hadoop.fi.DataTransferTestUtil.VerificationAction;
+import org.apache.hadoop.fi.FiTestUtil;
import org.apache.hadoop.fi.FiTestUtil.Action;
-import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
-import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -41,7 +36,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.DataTransferProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
@@ -63,6 +60,10 @@ public class TestFiDataTransferProtocol
REPLICATION, BLOCKSIZE);
}
+ {
+ ((Log4JLogger)DataTransferProtocol.LOG).getLogger().setLevel(Level.ALL);
+ }
+
/**
* 1. create files with dfs
* 2. write 1 byte
@@ -70,9 +71,9 @@ public class TestFiDataTransferProtocol
* 4. open the same file
* 5. read the 1 byte and compare results
*/
- private static void write1byte(String methodName) throws IOException {
- final MiniDFSCluster cluster = new MiniDFSCluster(conf, REPLICATION, true,
- null);
+ static void write1byte(String methodName) throws IOException {
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
+ ).numDataNodes(REPLICATION + 1).build();
final FileSystem dfs = cluster.getFileSystem();
try {
final Path p = new Path("/" + methodName + "/foo");
@@ -305,184 +306,4 @@ public class TestFiDataTransferProtocol
final String methodName = FiTestUtil.getMethodName();
runCallWritePacketToDisk(methodName, 2, new DoosAction(methodName, 2));
}
-
- private static void runPipelineCloseTest(String methodName,
- Action<DatanodeID, IOException> a) throws IOException {
- FiTestUtil.LOG.info("Running " + methodName + " ...");
- final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
- .initTest();
- t.fiPipelineClose.set(a);
- write1byte(methodName);
- }
-
- private static void run41_43(String name, int i) throws IOException {
- runPipelineCloseTest(name, new SleepAction(name, i, 3000));
- }
-
- private static void runPipelineCloseAck(String name, int i, DataNodeAction a
- ) throws IOException {
- FiTestUtil.LOG.info("Running " + name + " ...");
- final DataTransferTest t = (DataTransferTest)DataTransferTestUtil.initTest();
- final MarkerConstraint marker = new MarkerConstraint(name);
- t.fiPipelineClose.set(new DatanodeMarkingAction(name, i, marker));
- t.fiPipelineAck.set(new ConstraintSatisfactionAction<DatanodeID, IOException>(a, marker));
- write1byte(name);
- }
-
- private static void run39_40(String name, int i) throws IOException {
- runPipelineCloseAck(name, i, new SleepAction(name, i, 0));
- }
-
- /**
- * Pipeline close:
- * DN1 never responses after received close ack DN2.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_39() throws IOException {
- run39_40(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close:
- * DN0 never responses after received close ack DN1.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_40() throws IOException {
- run39_40(FiTestUtil.getMethodName(), 0);
- }
-
- /**
- * Pipeline close with DN0 very slow but it won't lead to timeout.
- * Client finishes close successfully.
- */
- @Test
- public void pipeline_Fi_41() throws IOException {
- run41_43(FiTestUtil.getMethodName(), 0);
- }
-
- /**
- * Pipeline close with DN1 very slow but it won't lead to timeout.
- * Client finishes close successfully.
- */
- @Test
- public void pipeline_Fi_42() throws IOException {
- run41_43(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close with DN2 very slow but it won't lead to timeout.
- * Client finishes close successfully.
- */
- @Test
- public void pipeline_Fi_43() throws IOException {
- run41_43(FiTestUtil.getMethodName(), 2);
- }
-
- /**
- * Pipeline close:
- * DN0 throws an OutOfMemoryException
- * right after it received a close request from client.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_44() throws IOException {
- final String methodName = FiTestUtil.getMethodName();
- runPipelineCloseTest(methodName, new OomAction(methodName, 0));
- }
-
- /**
- * Pipeline close:
- * DN1 throws an OutOfMemoryException
- * right after it received a close request from client.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_45() throws IOException {
- final String methodName = FiTestUtil.getMethodName();
- runPipelineCloseTest(methodName, new OomAction(methodName, 1));
- }
-
- /**
- * Pipeline close:
- * DN2 throws an OutOfMemoryException
- * right after it received a close request from client.
- * Client gets an IOException and determine DN2 bad.
- */
- @Test
- public void pipeline_Fi_46() throws IOException {
- final String methodName = FiTestUtil.getMethodName();
- runPipelineCloseTest(methodName, new OomAction(methodName, 2));
- }
-
- private static void run47_48(String name, int i) throws IOException {
- runPipelineCloseAck(name, i, new OomAction(name, i));
- }
-
- /**
- * Pipeline close:
- * DN1 throws an OutOfMemoryException right after
- * it received a close ack from DN2.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_47() throws IOException {
- run47_48(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close:
- * DN0 throws an OutOfMemoryException right after
- * it received a close ack from DN1.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_48() throws IOException {
- run47_48(FiTestUtil.getMethodName(), 0);
- }
-
- private static void runBlockFileCloseTest(String methodName,
- Action<DatanodeID, IOException> a) throws IOException {
- FiTestUtil.LOG.info("Running " + methodName + " ...");
- final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
- .initTest();
- t.fiBlockFileClose.set(a);
- write1byte(methodName);
- }
-
- private static void run49_51(String name, int i) throws IOException {
- runBlockFileCloseTest(name, new IoeAction(name, i, "DISK ERROR"));
- }
-
- /**
- * Pipeline close:
- * DN0 throws a disk error exception when it is closing the block file.
- * Client gets an IOException and determine DN0 bad.
- */
- @Test
- public void pipeline_Fi_49() throws IOException {
- run49_51(FiTestUtil.getMethodName(), 0);
- }
-
-
- /**
- * Pipeline close:
- * DN1 throws a disk error exception when it is closing the block file.
- * Client gets an IOException and determine DN1 bad.
- */
- @Test
- public void pipeline_Fi_50() throws IOException {
- run49_51(FiTestUtil.getMethodName(), 1);
- }
-
- /**
- * Pipeline close:
- * DN2 throws a disk error exception when it is closing the block file.
- * Client gets an IOException and determine DN2 bad.
- */
- @Test
- public void pipeline_Fi_51() throws IOException {
- run49_51(FiTestUtil.getMethodName(), 2);
- }
}
\ No newline at end of file
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java Wed Apr 20 02:28:19 2011
@@ -23,13 +23,13 @@ import java.util.Random;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fi.DataTransferTestUtil;
-import org.apache.hadoop.fi.FiTestUtil;
import org.apache.hadoop.fi.DataTransferTestUtil.CountdownDoosAction;
import org.apache.hadoop.fi.DataTransferTestUtil.CountdownOomAction;
import org.apache.hadoop.fi.DataTransferTestUtil.CountdownSleepAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DataTransferTest;
import org.apache.hadoop.fi.DataTransferTestUtil.SleepAction;
import org.apache.hadoop.fi.DataTransferTestUtil.VerificationAction;
+import org.apache.hadoop.fi.FiTestUtil;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -37,9 +37,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.datanode.BlockReceiver;
+import org.apache.hadoop.hdfs.protocol.DataTransferProtocol;
import org.apache.log4j.Level;
-
import org.junit.Assert;
import org.junit.Test;
@@ -71,6 +70,7 @@ public class TestFiDataTransferProtocol2
{
((Log4JLogger) BlockReceiver.LOG).getLogger().setLevel(Level.ALL);
((Log4JLogger) DFSClient.LOG).getLogger().setLevel(Level.ALL);
+ ((Log4JLogger)DataTransferProtocol.LOG).getLogger().setLevel(Level.ALL);
}
/**
* 1. create files with dfs
@@ -88,7 +88,8 @@ public class TestFiDataTransferProtocol2
FiTestUtil.LOG.info("size=" + size + ", nPackets=" + nPackets
+ ", lastPacketSize=" + lastPacketSize);
- final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION).build();
+ final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf
+ ).numDataNodes(REPLICATION + 1).build();
final FileSystem dfs = cluster.getFileSystem();
try {
final Path p = new Path("/" + methodName + "/foo");
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiPipelineClose.java Wed Apr 20 02:28:19 2011
@@ -19,76 +19,29 @@ package org.apache.hadoop.hdfs.server.da
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fi.DataTransferTestUtil;
-import org.apache.hadoop.fi.FiTestUtil;
+import org.apache.hadoop.fi.DataTransferTestUtil.DataNodeAction;
import org.apache.hadoop.fi.DataTransferTestUtil.DataTransferTest;
+import org.apache.hadoop.fi.DataTransferTestUtil.DatanodeMarkingAction;
+import org.apache.hadoop.fi.DataTransferTestUtil.IoeAction;
+import org.apache.hadoop.fi.DataTransferTestUtil.OomAction;
import org.apache.hadoop.fi.DataTransferTestUtil.SleepAction;
+import org.apache.hadoop.fi.FiTestUtil;
import org.apache.hadoop.fi.FiTestUtil.Action;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.fi.FiTestUtil.ConstraintSatisfactionAction;
+import org.apache.hadoop.fi.FiTestUtil.MarkerConstraint;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.junit.Assert;
import org.junit.Test;
/** Test DataTransferProtocol with fault injection. */
public class TestFiPipelineClose {
- static final short REPLICATION = 3;
- static final long BLOCKSIZE = 1L * (1L << 20);
-
- static final Configuration conf = new HdfsConfiguration();
- static {
- conf.setInt(DFSConfigKeys.DFS_DATANODE_HANDLER_COUNT_KEY, 1);
- conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, REPLICATION);
- conf.setInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, 5000);
- }
-
- static private FSDataOutputStream createFile(FileSystem fs, Path p
- ) throws IOException {
- return fs.create(p, true, fs.getConf().getInt("io.file.buffer.size", 4096),
- REPLICATION, BLOCKSIZE);
- }
-
- /**
- * 1. create files with dfs
- * 2. write 1 byte
- * 3. close file
- * 4. open the same file
- * 5. read the 1 byte and compare results
- */
- private static void write1byte(String methodName) throws IOException {
- final MiniDFSCluster cluster = new MiniDFSCluster(conf, REPLICATION, true,
- null);
- final FileSystem dfs = cluster.getFileSystem();
- try {
- final Path p = new Path("/" + methodName + "/foo");
- final FSDataOutputStream out = createFile(dfs, p);
- out.write(1);
- out.close();
-
- final FSDataInputStream in = dfs.open(p);
- final int b = in.read();
- in.close();
- Assert.assertEquals(1, b);
- }
- finally {
- dfs.close();
- cluster.shutdown();
- }
- }
-
- private static void runPipelineCloseTest(String methodName,
+ private static void runPipelineCloseTest(String methodName,
Action<DatanodeID, IOException> a) throws IOException {
FiTestUtil.LOG.info("Running " + methodName + " ...");
final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
.initTest();
t.fiPipelineClose.set(a);
- write1byte(methodName);
+ TestFiDataTransferProtocol.write1byte(methodName);
}
/**
@@ -123,4 +76,175 @@ public class TestFiPipelineClose {
final String methodName = FiTestUtil.getMethodName();
runPipelineCloseTest(methodName, new SleepAction(methodName, 2, 0));
}
+
+ private static void run41_43(String name, int i) throws IOException {
+ runPipelineCloseTest(name, new SleepAction(name, i, 3000));
+ }
+
+ private static void runPipelineCloseAck(String name, int i, DataNodeAction a
+ ) throws IOException {
+ FiTestUtil.LOG.info("Running " + name + " ...");
+ final DataTransferTest t = (DataTransferTest)DataTransferTestUtil.initTest();
+ final MarkerConstraint marker = new MarkerConstraint(name);
+ t.fiPipelineClose.set(new DatanodeMarkingAction(name, i, marker));
+ t.fiPipelineAck.set(new ConstraintSatisfactionAction<DatanodeID, IOException>(a, marker));
+ TestFiDataTransferProtocol.write1byte(name);
+ }
+
+ private static void run39_40(String name, int i) throws IOException {
+ runPipelineCloseAck(name, i, new SleepAction(name, i, 0));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 never responses after received close ack DN2.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_39() throws IOException {
+ run39_40(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 never responses after received close ack DN1.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_40() throws IOException {
+ run39_40(FiTestUtil.getMethodName(), 0);
+ }
+
+ /**
+ * Pipeline close with DN0 very slow but it won't lead to timeout.
+ * Client finishes close successfully.
+ */
+ @Test
+ public void pipeline_Fi_41() throws IOException {
+ run41_43(FiTestUtil.getMethodName(), 0);
+ }
+
+ /**
+ * Pipeline close with DN1 very slow but it won't lead to timeout.
+ * Client finishes close successfully.
+ */
+ @Test
+ public void pipeline_Fi_42() throws IOException {
+ run41_43(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close with DN2 very slow but it won't lead to timeout.
+ * Client finishes close successfully.
+ */
+ @Test
+ public void pipeline_Fi_43() throws IOException {
+ run41_43(FiTestUtil.getMethodName(), 2);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 throws an OutOfMemoryException
+ * right after it received a close request from client.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_44() throws IOException {
+ final String methodName = FiTestUtil.getMethodName();
+ runPipelineCloseTest(methodName, new OomAction(methodName, 0));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 throws an OutOfMemoryException
+ * right after it received a close request from client.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_45() throws IOException {
+ final String methodName = FiTestUtil.getMethodName();
+ runPipelineCloseTest(methodName, new OomAction(methodName, 1));
+ }
+
+ /**
+ * Pipeline close:
+ * DN2 throws an OutOfMemoryException
+ * right after it received a close request from client.
+ * Client gets an IOException and determine DN2 bad.
+ */
+ @Test
+ public void pipeline_Fi_46() throws IOException {
+ final String methodName = FiTestUtil.getMethodName();
+ runPipelineCloseTest(methodName, new OomAction(methodName, 2));
+ }
+
+ private static void run47_48(String name, int i) throws IOException {
+ runPipelineCloseAck(name, i, new OomAction(name, i));
+ }
+
+ /**
+ * Pipeline close:
+ * DN1 throws an OutOfMemoryException right after
+ * it received a close ack from DN2.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_47() throws IOException {
+ run47_48(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 throws an OutOfMemoryException right after
+ * it received a close ack from DN1.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_48() throws IOException {
+ run47_48(FiTestUtil.getMethodName(), 0);
+ }
+
+ private static void runBlockFileCloseTest(String methodName,
+ Action<DatanodeID, IOException> a) throws IOException {
+ FiTestUtil.LOG.info("Running " + methodName + " ...");
+ final DataTransferTest t = (DataTransferTest) DataTransferTestUtil
+ .initTest();
+ t.fiBlockFileClose.set(a);
+ TestFiDataTransferProtocol.write1byte(methodName);
+ }
+
+ private static void run49_51(String name, int i) throws IOException {
+ runBlockFileCloseTest(name, new IoeAction(name, i, "DISK ERROR"));
+ }
+
+ /**
+ * Pipeline close:
+ * DN0 throws a disk error exception when it is closing the block file.
+ * Client gets an IOException and determine DN0 bad.
+ */
+ @Test
+ public void pipeline_Fi_49() throws IOException {
+ run49_51(FiTestUtil.getMethodName(), 0);
+ }
+
+
+ /**
+ * Pipeline close:
+ * DN1 throws a disk error exception when it is closing the block file.
+ * Client gets an IOException and determine DN1 bad.
+ */
+ @Test
+ public void pipeline_Fi_50() throws IOException {
+ run49_51(FiTestUtil.getMethodName(), 1);
+ }
+
+ /**
+ * Pipeline close:
+ * DN2 throws a disk error exception when it is closing the block file.
+ * Client gets an IOException and determine DN2 bad.
+ */
+ @Test
+ public void pipeline_Fi_51() throws IOException {
+ run49_51(FiTestUtil.getMethodName(), 2);
+ }
}
Propchange: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Apr 20 02:28:19 2011
@@ -2,3 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
+/hadoop/hdfs/trunk/src/test/hdfs:1086482-1095244
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/TestHDFSCLI.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/TestHDFSCLI.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/TestHDFSCLI.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/TestHDFSCLI.java Wed Apr 20 02:28:19 2011
@@ -18,7 +18,7 @@
package org.apache.hadoop.cli;
-import org.apache.hadoop.cli.util.CLITestData.TestCmd;
+import org.apache.hadoop.cli.util.CLICommand;
import org.apache.hadoop.cli.util.CommandExecutor.Result;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -31,7 +31,7 @@ import static org.junit.Assert.assertTru
import org.junit.Before;
import org.junit.Test;
-public class TestHDFSCLI extends CLITestHelper {
+public class TestHDFSCLI extends CLITestHelperDFS {
protected MiniDFSCluster dfsCluster = null;
protected DistributedFileSystem dfs = null;
@@ -85,13 +85,13 @@ public class TestHDFSCLI extends CLITest
protected String expandCommand(final String cmd) {
String expCmd = cmd;
expCmd = expCmd.replaceAll("NAMENODE", namenode);
- expCmd = super.expandCommand(cmd);
+ expCmd = super.expandCommand(expCmd);
return expCmd;
}
@Override
- protected Result execute(TestCmd cmd) throws Exception {
- return CmdFactoryDFS.getCommandExecutor(cmd, namenode).executeCommand(cmd.getCmd());
+ protected Result execute(CLICommand cmd) throws Exception {
+ return cmd.getExecutor(namenode).executeCommand(cmd.getCmd());
}
@Test
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/cli/testHDFSConf.xml Wed Apr 20 02:28:19 2011
@@ -7068,7 +7068,7 @@
<comparators>
<comparator>
<type>TokenComparator</type>
- <expected-output>Can not find listing for /file1</expected-output>
+ <expected-output>count: Can not find listing for /file1</expected-output>
</comparator>
</comparators>
</test>
@@ -7083,7 +7083,7 @@
<comparators>
<comparator>
<type>TokenComparator</type>
- <expected-output>Can not find listing for file1</expected-output>
+ <expected-output>count: Can not find listing for file1</expected-output>
</comparator>
</comparators>
</test>
@@ -7426,7 +7426,7 @@
<comparators>
<comparator>
<type>TokenComparator</type>
- <expected-output>Can not find listing for /file1</expected-output>
+ <expected-output>count: Can not find listing for /file1</expected-output>
</comparator>
</comparators>
</test>
@@ -7441,7 +7441,7 @@
<comparators>
<comparator>
<type>TokenComparator</type>
- <expected-output>Can not find listing for file1</expected-output>
+ <expected-output>count: Can not find listing for file1</expected-output>
</comparator>
</comparators>
</test>
@@ -7778,7 +7778,7 @@
<comparators>
<comparator>
<type>TokenComparator</type>
- <expected-output>Can not find listing for hdfs:///file1</expected-output>
+ <expected-output>count: Can not find listing for hdfs:/file1</expected-output>
</comparator>
</comparators>
</test>
@@ -7957,7 +7957,7 @@
<comparators>
<comparator>
<type>TokenComparator</type>
- <expected-output>Can not find listing for hdfs:///file1</expected-output>
+ <expected-output>count: Can not find listing for hdfs:/file1</expected-output>
</comparator>
</comparators>
</test>
@@ -8150,7 +8150,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>Can not find listing for hdfs://\w+[.a-z]*:[0-9]+/file1</expected-output>
+ <expected-output>count: Can not find listing for hdfs://\w+[.a-z]*:[0-9]+/file1</expected-output>
</comparator>
</comparators>
</test>
@@ -8329,7 +8329,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>Can not find listing for hdfs://\w+[.a-z]*:[0-9]+/file1</expected-output>
+ <expected-output>count: Can not find listing for hdfs://\w+[.a-z]*:[0-9]+/file1</expected-output>
</comparator>
</comparators>
</test>
@@ -15096,741 +15096,6 @@
</comparators>
</test>
- <test> <!-- TESTED -->
- <description>help: help for ls</description>
- <test-commands>
- <command>-fs NAMENODE -help ls</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-ls <path>:( |\t)*List the contents that match the specified file pattern. If( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*path is not specified, the contents of /user/<currentUser>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*path is not specified, the contents of /user/<currentUser>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*will be listed. Directory entries are of the form( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*dirName \(full path\) <dir>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*and file entries are of the form( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*fileName\(full path\) <r n> size( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*where n is the number of replicas specified for the file( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*and size is the size of the file, in bytes.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for lsr</description>
- <test-commands>
- <command>-fs NAMENODE -help lsr</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-lsr <path>:( |\t)*Recursively list the contents that match the specified( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*file pattern.( |\t)*Behaves very similarly to hadoop fs -ls,( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*except that the data is shown for all the entries in the( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*subtree.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for get</description>
- <test-commands>
- <command>-fs NAMENODE -help get</command>
- </test-commands>
- <cleanup-commands>
- <!-- No cleanup -->
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-get( )*\[-ignoreCrc\]( )*\[-crc\]( )*<src> <localdst>:( |\t)*Copy files that match the file pattern <src>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*to the local name.( )*<src> is kept.( )*When copying mutiple,( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*files, the destination must be a directory.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for du</description>
- <test-commands>
- <command>-fs NAMENODE -help du</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-du \[-s\] \[-h\] <path>:\s+Show the amount of space, in bytes, used by the files that\s*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*match the specified file pattern. The following flags are optional:</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*-s\s*Rather than showing the size of each individual file that</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*matches the pattern, shows the total \(summary\) size.</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*-h\s*Formats the sizes of files in a human-readable fashion</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>\s*rather than a number of bytes.</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*Note that, even without the -s option, this only shows size summaries</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*one level deep into a directory.</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*The output is in the form </expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^\s*size\s+name\(full path\)\s*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for dus</description>
- <test-commands>
- <command>-fs NAMENODE -help dus</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-dus <path>:( |\t)*Show the amount of space, in bytes, used by the files that( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*match the specified file pattern. This is equivalent to -du -s above.</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for count</description>
- <test-commands>
- <command>-fs NAMENODE -help count</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-count\[-q\] <path>: Count the number of directories, files and bytes under the paths( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*that match the specified file pattern. The output columns are:( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME or( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*QUOTA REMAINING_QUATA SPACE_QUOTA REMAINING_SPACE_QUOTA( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*DIR_COUNT FILE_COUNT CONTENT_SIZE FILE_NAME( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for mv</description>
- <test-commands>
- <command>-fs NAMENODE -help mv</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-mv <src> <dst>:( |\t)*Move files that match the specified file pattern <src>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*to a destination <dst>. When moving multiple files, the( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*destination must be a directory.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for cp</description>
- <test-commands>
- <command>-fs NAMENODE -help cp</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-cp <src> <dst>:( |\t)*Copy files that match the file pattern <src> to a( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*destination. When copying multiple files, the destination( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*must be a directory.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for rm</description>
- <test-commands>
- <command>-fs NAMENODE -help rm</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-rm \[-skipTrash\] <src>:( |\t)*Delete all files that match the specified file pattern.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*Equivalent to the Unix command "rm <src>"( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*-skipTrash option bypasses trash, if enabled, and immediately( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*deletes <src>( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for rmr</description>
- <test-commands>
- <command>-fs NAMENODE -help rmr</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-rmr \[-skipTrash\] <src>:( |\t)*Remove all directories which match the specified file( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*pattern. Equivalent to the Unix command "rm -rf <src>"( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*-skipTrash option bypasses trash, if enabled, and immediately( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*deletes <src>( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for put</description>
- <test-commands>
- <command>-fs NAMENODE -help put</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-put <localsrc> ... <dst>:( |\t)*Copy files from the local file system( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*into fs.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for copyFromLocal</description>
- <test-commands>
- <command>-fs NAMENODE -help copyFromLocal</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-copyFromLocal <localsrc> ... <dst>:( )*Identical to the -put command.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for moveFromLocal</description>
- <test-commands>
- <command>-fs NAMENODE -help moveFromLocal</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-moveFromLocal <localsrc> ... <dst>: Same as -put, except that the source is( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*deleted after it's copied.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for get</description>
- <test-commands>
- <command>-fs NAMENODE -help get</command>
- </test-commands>
- <cleanup-commands>
- <!-- No cleanup -->
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-get( )*\[-ignoreCrc\]( )*\[-crc\]( )*<src> <localdst>:( |\t)*Copy files that match the file pattern <src>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*to the local name.( )*<src> is kept.( )*When copying mutiple,( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*files, the destination must be a directory.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for getmerge</description>
- <test-commands>
- <command>-fs NAMENODE -help getmerge</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-getmerge <src> <localdst>: Get all the files in the directories that( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*match the source file pattern and merge and sort them to only( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*one file on local fs. <src> is kept.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for cat</description>
- <test-commands>
- <command>-fs NAMENODE -help cat</command>
- </test-commands>
- <cleanup-commands>
- <!-- No cleanup -->
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-cat <src>:( |\t)*Fetch all files that match the file pattern <src>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*and display their content on stdout.</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for copyToLocal</description>
- <test-commands>
- <command>-fs NAMENODE -help copyToLocal</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-copyToLocal \[-ignoreCrc\] \[-crc\] <src> <localdst>:( )*Identical to the -get command.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for moveToLocal</description>
- <test-commands>
- <command>-fs NAMENODE -help moveToLocal</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-moveToLocal <src> <localdst>:( )*Not implemented yet( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for mkdir</description>
- <test-commands>
- <command>-fs NAMENODE -help mkdir</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-mkdir <path>:( |\t)*Create a directory in specified location.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for setrep</description>
- <test-commands>
- <command>-fs NAMENODE -help setrep</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-setrep \[-R\] \[-w\] <rep> <path/file>:( )*Set the replication level of a file.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*The -R flag requests a recursive change of replication level( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*for an entire tree.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for touchz</description>
- <test-commands>
- <command>-fs NAMENODE -help touchz</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-touchz <path>: Creates a file of zero length( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*at <path> with current time as the timestamp of that <path>.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)* An error is returned if the file exists with non-zero length( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for test</description>
- <test-commands>
- <command>-fs NAMENODE -help test</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-test -\[ezd\] <path>: If file \{ exists, has zero length, is a directory( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*then return 0, else return 1.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for stat</description>
- <test-commands>
- <command>-fs NAMENODE -help stat</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-stat \[format\] <path>: Print statistics about the file/directory at <path>( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*in the specified format. Format accepts filesize in blocks \(%b\), filename \(%n\),( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*block size \(%o\), replication \(%r\), modification date \(%y, %Y\)( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for tail</description>
- <test-commands>
- <command>-fs NAMENODE -help tail</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-tail \[-f\] <file>: Show the last 1KB of the file.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*The -f option shows apended data as the file grows.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for chmod</description>
- <test-commands>
- <command>-fs NAMENODE -help chmod</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-chmod \[-R\] <MODE\[,MODE\]... \| OCTALMODE> PATH...( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*Changes permissions of a file.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*This works similar to shell's chmod with a few exceptions.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*-R( |\t)*modifies the files recursively. This is the only option( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*currently supported.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*MODE( |\t)*Mode is same as mode used for chmod shell command.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*Only letters recognized are 'rwxXt'. E.g. \+t,a\+r,g-w,\+rwx,o=r( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*OCTALMODE Mode specifed in 3 or 4 digits. If 4 digits, the first may( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*be 1 or 0 to turn the sticky bit on or off, respectively.( )*Unlike( |\t)*shell command, it is not possible to specify only part of the mode( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*E.g. 754 is same as u=rwx,g=rx,o=r( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*If none of 'augo' is specified, 'a' is assumed and unlike( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*shell command, no umask is applied.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for chown</description>
- <test-commands>
- <command>-fs NAMENODE -help chown</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-chown \[-R\] \[OWNER\]\[:\[GROUP\]\] PATH...( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*Changes owner and group of a file.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*This is similar to shell's chown with a few exceptions.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*-R( |\t)*modifies the files recursively. This is the only option( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*currently supported.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*If only owner or group is specified then only owner or( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*group is modified.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*The owner and group names may only cosists of digits, alphabet,( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*and any of '-_.@/' i.e. \[-_.@/a-zA-Z0-9\]. The names are case( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*sensitive.( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*WARNING: Avoid using '.' to separate user name and group though( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*Linux allows it. If user names have dots in them and you are( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*using local file system, you might see surprising results since( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*shell command 'chown' is used for local files.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for chgrp</description>
- <test-commands>
- <command>-fs NAMENODE -help chgrp</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-chgrp \[-R\] GROUP PATH...( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*This is equivalent to -chown ... :GROUP ...( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
- <test> <!-- TESTED -->
- <description>help: help for help</description>
- <test-commands>
- <command>-fs NAMENODE -help help</command>
- </test-commands>
- <cleanup-commands>
- </cleanup-commands>
- <comparators>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^-help \[cmd\]:( |\t)*Displays help for given command or all commands if none( )*</expected-output>
- </comparator>
- <comparator>
- <type>RegexpComparator</type>
- <expected-output>^( |\t)*is specified.( )*</expected-output>
- </comparator>
- </comparators>
- </test>
-
<test> <!--Tested -->
<description>help: help for dfsadmin report</description>
<test-commands>
@@ -16233,7 +15498,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Can not find listing for /test1</expected-output>
+ <expected-output>setSpaceQuota: Directory does not exist: /test1</expected-output>
</comparator>
</comparators>
</test>
@@ -16299,7 +15564,7 @@
<comparators>
<comparator>
<type>SubstringComparator</type>
- <expected-output>Can not find listing for /test1</expected-output>
+ <expected-output>clrQuota: Directory does not exist: /test1</expected-output>
</comparator>
</comparators>
</test>
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsSymlink.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsSymlink.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsSymlink.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/fs/TestFcHdfsSymlink.java Wed Apr 20 02:28:19 2011
@@ -241,5 +241,17 @@ public class TestFcHdfsSymlink extends F
} catch (IOException x) {
// Expected
}
- }
-}
\ No newline at end of file
+ }
+
+ @Test
+ /** Test symlink owner */
+ public void testLinkOwner() throws IOException {
+ Path file = new Path(testBaseDir1(), "file");
+ Path link = new Path(testBaseDir1(), "symlinkToFile");
+ createAndWriteFile(file);
+ fc.createSymlink(file, link, false);
+ FileStatus stat_file = fc.getFileStatus(file);
+ FileStatus stat_link = fc.getFileStatus(link);
+ assertEquals(stat_link.getOwner(), stat_file.getOwner());
+ }
+}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java Wed Apr 20 02:28:19 2011
@@ -18,13 +18,16 @@
package org.apache.hadoop.hdfs;
+import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
-import java.io.FileInputStream;
-import java.io.DataInputStream;
+import java.net.Socket;
import java.net.URL;
import java.net.URLConnection;
import java.security.PrivilegedExceptionAction;
@@ -40,16 +43,22 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DataTransferProtocol;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.junit.Assert;
/** Utilities for HDFS tests */
public class DFSTestUtil {
@@ -389,4 +398,24 @@ public class DFSTestUtil {
in.readFully(content);
return content;
}
+
+ /** For {@link TestTransferRbw} */
+ public static DataTransferProtocol.Status transferRbw(final Block b,
+ final DFSClient dfsClient, final DatanodeInfo... datanodes) throws IOException {
+ Assert.assertEquals(2, datanodes.length);
+ final Socket s = DFSOutputStream.createSocketForPipeline(datanodes[0],
+ datanodes.length, dfsClient);
+ final long writeTimeout = dfsClient.getDatanodeWriteTimeout(datanodes.length);
+ final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
+ NetUtils.getOutputStream(s, writeTimeout),
+ DataNode.SMALL_BUFFER_SIZE));
+ final DataInputStream in = new DataInputStream(NetUtils.getInputStream(s));
+
+ // send the request
+ DataTransferProtocol.Sender.opTransferBlock(out, b, dfsClient.clientName,
+ new DatanodeInfo[]{datanodes[1]}, new Token<BlockTokenIdentifier>());
+ out.flush();
+
+ return DataTransferProtocol.Status.read(in);
+ }
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSShell.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSShell.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSShell.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSShell.java Wed Apr 20 02:28:19 2011
@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.shell.Count;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.FSDataset;
+import org.apache.hadoop.hdfs.tools.DFSAdmin;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
@@ -1285,4 +1286,18 @@ public class TestDFSShell extends TestCa
System.out.println("results:\n" + results);
return results;
}
+
+ /**
+ * default setting is file:// which is not a DFS
+ * so DFSAdmin should throw and catch InvalidArgumentException
+ * and return -1 exit code.
+ * @throws Exception
+ */
+ public void testInvalidShell() throws Exception {
+ Configuration conf = new Configuration(); // default FS (non-DFS)
+ DFSAdmin admin = new DFSAdmin();
+ admin.setConf(conf);
+ int res = admin.run(new String[] {"-refreshNodes"});
+ assertEquals("expected to fail -1", res , -1);
+ }
}
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend2.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend2.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend2.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend2.java Wed Apr 20 02:28:19 2011
@@ -59,7 +59,7 @@ public class TestFileAppend2 extends Tes
private byte[] fileContents = null;
- int numDatanodes = 5;
+ int numDatanodes = 6;
int numberOfFiles = 50;
int numThreads = 10;
int numAppendsPerThread = 20;
@@ -350,7 +350,7 @@ public class TestFileAppend2 extends Tes
// Insert them into a linked list.
//
for (int i = 0; i < numberOfFiles; i++) {
- short replication = (short)(AppendTestUtil.nextInt(numDatanodes) + 1);
+ final int replication = AppendTestUtil.nextInt(numDatanodes - 2) + 1;
Path testFile = new Path("/" + i + ".dat");
FSDataOutputStream stm =
AppendTestUtil.createFile(fs, testFile, replication);
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend4.java Wed Apr 20 02:28:19 2011
@@ -148,7 +148,7 @@ public class TestFileAppend4 {
*/
@Test(timeout=60000)
public void testRecoverFinalizedBlock() throws Throwable {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(5).build();
try {
cluster.waitActive();
@@ -219,7 +219,7 @@ public class TestFileAppend4 {
*/
@Test(timeout=60000)
public void testCompleteOtherLeaseHoldersFile() throws Throwable {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(5).build();
try {
cluster.waitActive();
@@ -294,8 +294,7 @@ public class TestFileAppend4 {
* Mockito answer helper that triggers one latch as soon as the
* method is called, then waits on another before continuing.
*/
- @SuppressWarnings("unchecked")
- private static class DelayAnswer implements Answer {
+ private static class DelayAnswer implements Answer<Object> {
private final CountDownLatch fireLatch = new CountDownLatch(1);
private final CountDownLatch waitLatch = new CountDownLatch(1);
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestFileCreation.java Wed Apr 20 02:28:19 2011
@@ -632,7 +632,8 @@ public class TestFileCreation extends ju
expectedException != null
&& expectedException instanceof FileNotFoundException);
- EnumSet<CreateFlag> overwriteFlag = EnumSet.of(CreateFlag.OVERWRITE);
+ EnumSet<CreateFlag> overwriteFlag =
+ EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE);
// Overwrite a file in root dir, should succeed
out = createNonRecursive(fs, path, 1, overwriteFlag);
out.close();
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java Wed Apr 20 02:28:19 2011
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
@@ -202,19 +203,12 @@ public class TestLeaseRecovery2 {
try {
dfs2.create(filepath, false, BUF_SIZE, REPLICATION_NUM, BLOCK_SIZE);
fail("Creation of an existing file should never succeed.");
+ } catch (FileAlreadyExistsException ex) {
+ done = true;
+ } catch (AlreadyBeingCreatedException ex) {
+ AppendTestUtil.LOG.info("GOOD! got " + ex.getMessage());
} catch (IOException ioe) {
- final String message = ioe.getMessage();
- if (message.contains("file exists")) {
- AppendTestUtil.LOG.info("done", ioe);
- done = true;
- }
- else if (message.contains(
- AlreadyBeingCreatedException.class.getSimpleName())) {
- AppendTestUtil.LOG.info("GOOD! got " + message);
- }
- else {
- AppendTestUtil.LOG.warn("UNEXPECTED IOException", ioe);
- }
+ AppendTestUtil.LOG.warn("UNEXPECTED IOException", ioe);
}
if (!done) {
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestReadWhileWriting.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestReadWhileWriting.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestReadWhileWriting.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/TestReadWhileWriting.java Wed Apr 20 02:28:19 2011
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs;
import java.io.IOException;
import java.io.OutputStream;
-import java.security.PrivilegedExceptionAction;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
@@ -71,7 +70,7 @@ public class TestReadWhileWriting {
final int half = BLOCK_SIZE/2;
//a. On Machine M1, Create file. Write half block of data.
- // Invoke (DFSOutputStream).fsync() on the dfs file handle.
+ // Invoke DFSOutputStream.hflush() on the dfs file handle.
// Do not close file yet.
{
final FSDataOutputStream out = fs.create(p, true,
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Wed Apr 20 02:28:19 2011
@@ -406,6 +406,12 @@ public class SimulatedFSDataset impleme
return blockMap.get(new Block(blockId));
}
+ @Override
+ public synchronized String getReplicaString(long blockId) {
+ final Replica r = blockMap.get(new Block(blockId));
+ return r == null? "null": r.toString();
+ }
+
@Override
public Block getStoredBlock(long blkid) throws IOException {
Block b = new Block(blkid);
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestBlockReport.java Wed Apr 20 02:28:19 2011
@@ -485,11 +485,11 @@ public class TestBlockReport {
long start = System.currentTimeMillis();
int count = 0;
while (r == null) {
- waitTil(50);
+ waitTil(5);
r = ((FSDataset) cluster.getDataNodes().get(DN_N1).getFSDataset()).
fetchReplicaInfo(bl.getBlockId());
long waiting_period = System.currentTimeMillis() - start;
- if (count++ % 10 == 0)
+ if (count++ % 100 == 0)
if(LOG.isDebugEnabled()) {
LOG.debug("Has been waiting for " + waiting_period + " ms.");
}
@@ -504,7 +504,7 @@ public class TestBlockReport {
}
start = System.currentTimeMillis();
while (state != HdfsConstants.ReplicaState.TEMPORARY) {
- waitTil(100);
+ waitTil(5);
state = r.getState();
if(LOG.isDebugEnabled()) {
LOG.debug("Keep waiting for " + bl.getBlockName() +
Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java?rev=1095253&r1=1095252&r2=1095253&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/TestTransferRbw.java Wed Apr 20 02:28:19 2011
@@ -22,21 +22,29 @@ import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.DataTransferProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.FSConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.server.common.HdfsConstants.ReplicaState;
+import org.apache.log4j.Level;
import org.junit.Assert;
import org.junit.Test;
/** Test transferring RBW between datanodes */
public class TestTransferRbw {
private static final Log LOG = LogFactory.getLog(TestTransferRbw.class);
+
+ {
+ ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
+ }
private static final Random RAN = new Random();
private static final short REPLICATION = (short)1;
@@ -86,7 +94,6 @@ public class TestTransferRbw {
final ReplicaBeingWritten oldrbw;
final DataNode newnode;
final DatanodeInfo newnodeinfo;
- final long visible;
{
final DataNode oldnode = cluster.getDataNodes().get(0);
oldrbw = getRbw(oldnode);
@@ -96,6 +103,7 @@ public class TestTransferRbw {
cluster.startDataNodes(conf, 1, true, null, null);
newnode = cluster.getDataNodes().get(REPLICATION);
+ final DatanodeInfo oldnodeinfo;
{
final DatanodeInfo[] datatnodeinfos = cluster.getNameNode(
).getDatanodeReport(DatanodeReportType.LIVE);
@@ -105,20 +113,17 @@ public class TestTransferRbw {
&& !datatnodeinfos[i].equals(newnode.dnRegistration); i++);
Assert.assertTrue(i < datatnodeinfos.length);
newnodeinfo = datatnodeinfos[i];
+ oldnodeinfo = datatnodeinfos[1 - i];
}
//transfer RBW
- visible = oldnode.transferBlockForPipelineRecovery(oldrbw, new DatanodeInfo[]{newnodeinfo});
+ final Block b = new Block(oldrbw.getBlockId(), oldrbw.getBytesAcked(),
+ oldrbw.getGenerationStamp());
+ final DataTransferProtocol.Status s = DFSTestUtil.transferRbw(
+ b, fs.getClient(), oldnodeinfo, newnodeinfo);
+ Assert.assertEquals(DataTransferProtocol.Status.SUCCESS, s);
}
- //check temporary
- final ReplicaInPipeline temp = getReplica(newnode, ReplicaState.TEMPORARY);
- LOG.info("temp = " + temp);
- Assert.assertEquals(oldrbw.getBlockId(), temp.getBlockId());
- Assert.assertEquals(oldrbw.getGenerationStamp(), temp.getGenerationStamp());
- final Block b = new Block(oldrbw.getBlockId(), visible, oldrbw.getGenerationStamp());
- //convert temporary to rbw
- newnode.convertTemporaryToRbw(b);
//check new rbw
final ReplicaBeingWritten newrbw = getRbw(newnode);
LOG.info("newrbw = " + newrbw);