You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by cu...@apache.org on 2014/08/20 03:34:47 UTC
svn commit: r1619019 [11/11] - in
/hadoop/common/branches/YARN-1051/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/
hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs-httpfs/src/main/j...
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingDataNodeMessages.java Wed Aug 20 01:34:29 2014
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.blockmanagement.PendingDataNodeMessages.ReportedBlockInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.junit.Test;
import com.google.common.base.Joiner;
@@ -43,8 +44,10 @@ public class TestPendingDataNodeMessages
@Test
public void testQueues() {
DatanodeDescriptor fakeDN = DFSTestUtil.getLocalDatanodeDescriptor();
- msgs.enqueueReportedBlock(fakeDN, "STORAGE_ID", block1Gs1, ReplicaState.FINALIZED);
- msgs.enqueueReportedBlock(fakeDN, "STORAGE_ID", block1Gs2, ReplicaState.FINALIZED);
+ DatanodeStorage storage = new DatanodeStorage("STORAGE_ID");
+ DatanodeStorageInfo storageInfo = new DatanodeStorageInfo(fakeDN, storage);
+ msgs.enqueueReportedBlock(storageInfo, block1Gs1, ReplicaState.FINALIZED);
+ msgs.enqueueReportedBlock(storageInfo, block1Gs2, ReplicaState.FINALIZED);
assertEquals(2, msgs.count());
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java Wed Aug 20 01:34:29 2014
@@ -82,7 +82,7 @@ public class TestReplicationPolicy {
private static NameNode namenode;
private static BlockPlacementPolicy replicator;
private static final String filename = "/dummyfile.txt";
- private static DatanodeDescriptor dataNodes[];
+ private static DatanodeDescriptor[] dataNodes;
private static DatanodeStorageInfo[] storages;
// The interval for marking a datanode as stale,
private static final long staleInterval =
@@ -1118,8 +1118,7 @@ public class TestReplicationPolicy {
// Adding this block will increase its current replication, and that will
// remove it from the queue.
bm.addStoredBlockUnderConstruction(new StatefulBlockInfo(info, info,
- ReplicaState.FINALIZED), TestReplicationPolicy.dataNodes[0],
- "STORAGE");
+ ReplicaState.FINALIZED), TestReplicationPolicy.storages[0]);
// Choose 1 block from UnderReplicatedBlocks. Then it should pick 1 block
// from QUEUE_VERY_UNDER_REPLICATED.
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java Wed Aug 20 01:34:29 2014
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -1083,6 +1084,11 @@ public class SimulatedFSDataset implemen
}
@Override
+ public void addVolumes(Collection<StorageLocation> volumes) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public DatanodeStorage getStorage(final String storageUuid) {
return storageUuid.equals(storage.getStorageUuid()) ?
storage.dnStorage :
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java Wed Aug 20 01:34:29 2014
@@ -590,7 +590,6 @@ public class TestBlockRecovery {
Configuration conf = new HdfsConfiguration();
conf.set(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, "1000");
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
- .nnTopology(MiniDFSNNTopology.simpleSingleNN(8020, 50070))
.numDataNodes(1).build();
try {
cluster.waitClusterUp();
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockReplacement.java Wed Aug 20 01:34:29 2014
@@ -272,8 +272,10 @@ public class TestBlockReplacement {
// receiveResponse
DataInputStream reply = new DataInputStream(sock.getInputStream());
- BlockOpResponseProto proto =
- BlockOpResponseProto.parseDelimitedFrom(reply);
+ BlockOpResponseProto proto = BlockOpResponseProto.parseDelimitedFrom(reply);
+ while (proto.getStatus() == Status.IN_PROGRESS) {
+ proto = BlockOpResponseProto.parseDelimitedFrom(reply);
+ }
return proto.getStatus() == Status.SUCCESS;
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeRollingUpgrade.java Wed Aug 20 01:34:29 2014
@@ -27,11 +27,14 @@ import static org.junit.Assert.assertTru
import java.io.File;
import java.io.IOException;
+import java.util.Random;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -67,6 +70,7 @@ public class TestDataNodeRollingUpgrade
private void startCluster() throws IOException {
conf = new HdfsConfiguration();
+ conf.setInt("dfs.blocksize", 1024*1024);
cluster = new Builder(conf).numDataNodes(REPL_FACTOR).build();
cluster.waitActive();
fs = cluster.getFileSystem();
@@ -243,4 +247,48 @@ public class TestDataNodeRollingUpgrade
shutdownCluster();
}
}
+
+ @Test (timeout=600000)
+ // Test DatanodeXceiver has correct peer-dataxceiver pairs for sending OOB message
+ public void testDatanodePeersXceiver() throws Exception {
+ try {
+ startCluster();
+
+ // Create files in DFS.
+ String testFile1 = "/TestDataNodeXceiver1.dat";
+ String testFile2 = "/TestDataNodeXceiver2.dat";
+ String testFile3 = "/TestDataNodeXceiver3.dat";
+
+ DFSClient client1 = new DFSClient(NameNode.getAddress(conf), conf);
+ DFSClient client2 = new DFSClient(NameNode.getAddress(conf), conf);
+ DFSClient client3 = new DFSClient(NameNode.getAddress(conf), conf);
+
+ DFSOutputStream s1 = (DFSOutputStream) client1.create(testFile1, true);
+ DFSOutputStream s2 = (DFSOutputStream) client2.create(testFile2, true);
+ DFSOutputStream s3 = (DFSOutputStream) client3.create(testFile3, true);
+
+ byte[] toWrite = new byte[1024*1024*8];
+ Random rb = new Random(1111);
+ rb.nextBytes(toWrite);
+ s1.write(toWrite, 0, 1024*1024*8);
+ s1.flush();
+ s2.write(toWrite, 0, 1024*1024*8);
+ s2.flush();
+ s3.write(toWrite, 0, 1024*1024*8);
+ s3.flush();
+
+ assertTrue(dn.getXferServer().getNumPeersXceiver() == dn.getXferServer()
+ .getNumPeersXceiver());
+ s1.close();
+ s2.close();
+ s3.close();
+ assertTrue(dn.getXferServer().getNumPeersXceiver() == dn.getXferServer()
+ .getNumPeersXceiver());
+ client1.close();
+ client2.close();
+ client3.close();
+ } finally {
+ shutdownCluster();
+ }
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailure.java Wed Aug 20 01:34:29 2014
@@ -25,6 +25,7 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Socket;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -384,7 +385,7 @@ public class TestDataNodeVolumeFailure {
continue;
}
- String [] res = metaFilesInDir(dir);
+ List<File> res = MiniDFSCluster.getAllBlockMetadataFiles(dir);
if(res == null) {
System.out.println("res is null for dir = " + dir + " i=" + i + " and j=" + j);
continue;
@@ -392,7 +393,8 @@ public class TestDataNodeVolumeFailure {
//System.out.println("for dn" + i + "." + j + ": " + dir + "=" + res.length+ " files");
//int ii = 0;
- for(String s: res) {
+ for(File f: res) {
+ String s = f.getName();
// cut off "blk_-" at the beginning and ".meta" at the end
assertNotNull("Block file name should not be null", s);
String bid = s.substring(s.indexOf("_")+1, s.lastIndexOf("_"));
@@ -408,25 +410,9 @@ public class TestDataNodeVolumeFailure {
//System.out.println("dir1="+dir.getPath() + "blocks=" + res.length);
//System.out.println("dir2="+dir2.getPath() + "blocks=" + res2.length);
- total += res.length;
+ total += res.size();
}
}
return total;
}
-
- /*
- * count how many files *.meta are in the dir
- */
- private String [] metaFilesInDir(File dir) {
- String [] res = dir.list(
- new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith("blk_") &&
- name.endsWith(Block.METADATA_EXTENSION);
- }
- }
- );
- return res;
- }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDeleteBlockPool.java Wed Aug 20 01:34:29 2014
@@ -103,9 +103,10 @@ public class TestDeleteBlockPool {
fs1.delete(new Path("/alpha"), true);
// Wait till all blocks are deleted from the dn2 for bpid1.
- while ((MiniDFSCluster.getFinalizedDir(dn2StorageDir1,
- bpid1).list().length != 0) || (MiniDFSCluster.getFinalizedDir(
- dn2StorageDir2, bpid1).list().length != 0)) {
+ File finalDir1 = MiniDFSCluster.getFinalizedDir(dn2StorageDir1, bpid1);
+ File finalDir2 = MiniDFSCluster.getFinalizedDir(dn2StorageDir1, bpid2);
+ while ((!DatanodeUtil.dirNoFilesRecursive(finalDir1)) ||
+ (!DatanodeUtil.dirNoFilesRecursive(finalDir2))) {
try {
Thread.sleep(3000);
} catch (Exception ignored) {
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDiskError.java Wed Aug 20 01:34:29 2014
@@ -201,7 +201,7 @@ public class TestDiskError {
}
/**
- * Checks whether {@link DataNode#checkDiskError()} is being called or not.
+ * Checks whether {@link DataNode#checkDiskErrorAsync()} is being called or not.
* Before refactoring the code the above function was not getting called
* @throws IOException, InterruptedException
*/
@@ -214,7 +214,7 @@ public class TestDiskError {
DataNode dataNode = cluster.getDataNodes().get(0);
long slackTime = dataNode.checkDiskErrorInterval/2;
//checking for disk error
- dataNode.checkDiskError();
+ dataNode.checkDiskErrorAsync();
Thread.sleep(dataNode.checkDiskErrorInterval);
long lastDiskErrorCheck = dataNode.getLastDiskErrorCheck();
assertTrue("Disk Error check is not performed within " + dataNode.checkDiskErrorInterval + " ms", ((Time.monotonicNow()-lastDiskErrorCheck) < (dataNode.checkDiskErrorInterval + slackTime)));
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java Wed Aug 20 01:34:29 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -1256,6 +1257,33 @@ public abstract class FSAclBaseTest {
fsAsDiana.getAclStatus(bruceFile);
}
+ @Test
+ public void testAccess() throws IOException, InterruptedException {
+ Path p1 = new Path("/p1");
+ fs.mkdirs(p1);
+ fs.setOwner(p1, BRUCE.getShortUserName(), "groupX");
+ fsAsBruce.setAcl(p1, Lists.newArrayList(
+ aclEntry(ACCESS, USER, READ),
+ aclEntry(ACCESS, USER, "bruce", READ),
+ aclEntry(ACCESS, GROUP, NONE),
+ aclEntry(ACCESS, OTHER, NONE)));
+ fsAsBruce.access(p1, FsAction.READ);
+ try {
+ fsAsBruce.access(p1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path("/bad/bad");
+ try {
+ fsAsBruce.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+
/**
* Creates a FileSystem for the super-user.
*
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java Wed Aug 20 01:34:29 2014
@@ -50,6 +50,17 @@ public class TestCommitBlockSynchronizat
FSNamesystem namesystem = new FSNamesystem(conf, image);
namesystem.setImageLoaded(true);
+
+ // set file's parent as root and put the file to inodeMap, so
+ // FSNamesystem's isFileDeleted() method will return false on this file
+ if (file.getParent() == null) {
+ INodeDirectory parent = mock(INodeDirectory.class);
+ parent.setLocalName(new byte[0]);
+ parent.addChild(file);
+ file.setParent(parent);
+ }
+ namesystem.dir.getINodeMap().put(file);
+
FSNamesystem namesystemSpy = spy(namesystem);
BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java Wed Aug 20 01:34:29 2014
@@ -31,19 +31,24 @@ import java.util.Random;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
@@ -89,6 +94,8 @@ public class TestDecommissioningStatus {
4);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1);
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY, 1);
+
writeConfigFile(localFileSys, excludeFile, null);
writeConfigFile(localFileSys, includeFile, null);
@@ -99,6 +106,7 @@ public class TestDecommissioningStatus {
@AfterClass
public static void tearDown() throws Exception {
+ if (localFileSys != null ) cleanupFile(localFileSys, dir);
if(fileSys != null) fileSys.close();
if(cluster != null) cluster.shutdown();
}
@@ -138,7 +146,8 @@ public class TestDecommissioningStatus {
return stm;
}
- private void cleanupFile(FileSystem fileSys, Path name) throws IOException {
+ static private void cleanupFile(FileSystem fileSys, Path name)
+ throws IOException {
assertTrue(fileSys.exists(name));
fileSys.delete(name, true);
assertTrue(!fileSys.exists(name));
@@ -147,19 +156,26 @@ public class TestDecommissioningStatus {
/*
* Decommissions the node at the given index
*/
- private String decommissionNode(FSNamesystem namesystem,
- DFSClient client, FileSystem localFileSys, int nodeIndex)
- throws IOException {
+ private String decommissionNode(FSNamesystem namesystem, DFSClient client,
+ FileSystem localFileSys, int nodeIndex) throws IOException {
DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE);
String nodename = info[nodeIndex].getXferAddr();
- System.out.println("Decommissioning node: " + nodename);
+ decommissionNode(namesystem, localFileSys, nodename);
+ return nodename;
+ }
+
+ /*
+ * Decommissions the node by name
+ */
+ private void decommissionNode(FSNamesystem namesystem,
+ FileSystem localFileSys, String dnName) throws IOException {
+ System.out.println("Decommissioning node: " + dnName);
// write nodename into the exclude file.
ArrayList<String> nodes = new ArrayList<String>(decommissionedNodes);
- nodes.add(nodename);
+ nodes.add(dnName);
writeConfigFile(localFileSys, excludeFile, nodes);
- return nodename;
}
private void checkDecommissionStatus(DatanodeDescriptor decommNode,
@@ -276,6 +292,69 @@ public class TestDecommissioningStatus {
st1.close();
cleanupFile(fileSys, file1);
cleanupFile(fileSys, file2);
- cleanupFile(localFileSys, dir);
+ }
+
+ /**
+ * Verify a DN remains in DECOMMISSION_INPROGRESS state if it is marked
+ * as dead before decommission has completed. That will allow DN to resume
+ * the replication process after it rejoins the cluster.
+ */
+ @Test(timeout=120000)
+ public void testDecommissionStatusAfterDNRestart()
+ throws IOException, InterruptedException {
+ DistributedFileSystem fileSys =
+ (DistributedFileSystem)cluster.getFileSystem();
+
+ // Create a file with one block. That block has one replica.
+ Path f = new Path("decommission.dat");
+ DFSTestUtil.createFile(fileSys, f, fileSize, fileSize, fileSize,
+ (short)1, seed);
+
+ // Find the DN that owns the only replica.
+ RemoteIterator<LocatedFileStatus> fileList = fileSys.listLocatedStatus(f);
+ BlockLocation[] blockLocations = fileList.next().getBlockLocations();
+ String dnName = blockLocations[0].getNames()[0];
+
+ // Decommission the DN.
+ FSNamesystem fsn = cluster.getNamesystem();
+ final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager();
+ decommissionNode(fsn, localFileSys, dnName);
+ dm.refreshNodes(conf);
+
+ // Stop the DN when decommission is in progress.
+ // Given DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY is to 1 and the size of
+ // the block, it will take much longer time that test timeout value for
+ // the decommission to complete. So when stopDataNode is called,
+ // decommission should be in progress.
+ DataNodeProperties dataNodeProperties = cluster.stopDataNode(dnName);
+ final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
+ while (true) {
+ dm.fetchDatanodes(null, dead, false);
+ if (dead.size() == 1) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+
+ // Force removal of the dead node's blocks.
+ BlockManagerTestUtil.checkHeartbeat(fsn.getBlockManager());
+
+ // Force DatanodeManager to check decommission state.
+ BlockManagerTestUtil.checkDecommissionState(dm, dead.get(0));
+
+ // Verify that the DN remains in DECOMMISSION_INPROGRESS state.
+ assertTrue("the node is in decommissioned state ",
+ !dead.get(0).isDecommissioned());
+
+ // Add the node back
+ cluster.restartDataNode(dataNodeProperties, true);
+ cluster.waitActive();
+
+ // Call refreshNodes on FSNamesystem with empty exclude file.
+ // This will remove the datanodes from decommissioning list and
+ // make them available again.
+ writeConfigFile(localFileSys, excludeFile, null);
+ dm.refreshNodes(conf);
+ cleanupFile(fileSys, f);
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java Wed Aug 20 01:34:29 2014
@@ -18,7 +18,9 @@
package org.apache.hadoop.hdfs.server.namenode;
import java.io.FileNotFoundException;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -27,19 +29,30 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.DelayAnswer;
import org.junit.Assert;
import org.junit.Test;
+import org.mockito.Mockito;
import org.mockito.internal.util.reflection.Whitebox;
@@ -49,6 +62,7 @@ import org.mockito.internal.util.reflect
* whole duration.
*/
public class TestDeleteRace {
+ private static final int BLOCK_SIZE = 4096;
private static final Log LOG = LogFactory.getLog(TestDeleteRace.class);
private static final Configuration conf = new HdfsConfiguration();
private MiniDFSCluster cluster;
@@ -201,7 +215,126 @@ public class TestDeleteRace {
cluster.shutdown();
}
}
+ }
+
+ /**
+ * Test race between delete operation and commitBlockSynchronization method.
+ * See HDFS-6825.
+ * @param hasSnapshot
+ * @throws Exception
+ */
+ private void testDeleteAndCommitBlockSynchronizationRace(boolean hasSnapshot)
+ throws Exception {
+ LOG.info("Start testing, hasSnapshot: " + hasSnapshot);
+ final String testPaths[] = {
+ "/test-file",
+ "/testdir/testdir1/test-file"
+ };
+ final Path rootPath = new Path("/");
+ final Configuration conf = new Configuration();
+ // Disable permissions so that another user can recover the lease.
+ conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
+ conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+ FSDataOutputStream stm = null;
+ Map<DataNode, DatanodeProtocolClientSideTranslatorPB> dnMap =
+ new HashMap<DataNode, DatanodeProtocolClientSideTranslatorPB>();
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(3)
+ .build();
+ cluster.waitActive();
+
+ DistributedFileSystem fs = cluster.getFileSystem();
+ int stId = 0;
+ for (String testPath : testPaths) {
+ LOG.info("test on " + testPath + " snapshot: " + hasSnapshot);
+ Path fPath = new Path(testPath);
+ //find grandest non-root parent
+ Path grandestNonRootParent = fPath;
+ while (!grandestNonRootParent.getParent().equals(rootPath)) {
+ grandestNonRootParent = grandestNonRootParent.getParent();
+ }
+ stm = fs.create(fPath);
+ LOG.info("test on " + testPath + " created " + fPath);
+
+ // write a half block
+ AppendTestUtil.write(stm, 0, BLOCK_SIZE / 2);
+ stm.hflush();
+
+ if (hasSnapshot) {
+ SnapshotTestHelper.createSnapshot(fs, rootPath,
+ "st" + String.valueOf(stId));
+ ++stId;
+ }
+
+ // Look into the block manager on the active node for the block
+ // under construction.
+ NameNode nn = cluster.getNameNode();
+ ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, fPath);
+ DatanodeDescriptor expectedPrimary =
+ DFSTestUtil.getExpectedPrimaryNode(nn, blk);
+ LOG.info("Expecting block recovery to be triggered on DN " +
+ expectedPrimary);
+
+ // Find the corresponding DN daemon, and spy on its connection to the
+ // active.
+ DataNode primaryDN = cluster.getDataNode(expectedPrimary.getIpcPort());
+ DatanodeProtocolClientSideTranslatorPB nnSpy = dnMap.get(primaryDN);
+ if (nnSpy == null) {
+ nnSpy = DataNodeTestUtils.spyOnBposToNN(primaryDN, nn);
+ dnMap.put(primaryDN, nnSpy);
+ }
+
+ // Delay the commitBlockSynchronization call
+ DelayAnswer delayer = new DelayAnswer(LOG);
+ Mockito.doAnswer(delayer).when(nnSpy).commitBlockSynchronization(
+ Mockito.eq(blk),
+ Mockito.anyInt(), // new genstamp
+ Mockito.anyLong(), // new length
+ Mockito.eq(true), // close file
+ Mockito.eq(false), // delete block
+ (DatanodeID[]) Mockito.anyObject(), // new targets
+ (String[]) Mockito.anyObject()); // new target storages
+
+ fs.recoverLease(fPath);
+
+ LOG.info("Waiting for commitBlockSynchronization call from primary");
+ delayer.waitForCall();
+
+ LOG.info("Deleting recursively " + grandestNonRootParent);
+ fs.delete(grandestNonRootParent, true);
+
+ delayer.proceed();
+ LOG.info("Now wait for result");
+ delayer.waitForResult();
+ Throwable t = delayer.getThrown();
+ if (t != null) {
+ LOG.info("Result exception (snapshot: " + hasSnapshot + "): " + t);
+ }
+ } // end of loop each fPath
+ LOG.info("Now check we can restart");
+ cluster.restartNameNodes();
+ LOG.info("Restart finished");
+ } finally {
+ if (stm != null) {
+ IOUtils.closeStream(stm);
+ }
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
+ @Test(timeout=600000)
+ public void testDeleteAndCommitBlockSynchonizationRaceNoSnapshot()
+ throws Exception {
+ testDeleteAndCommitBlockSynchronizationRace(false);
+ }
+ @Test(timeout=600000)
+ public void testDeleteAndCommitBlockSynchronizationRaceHasSnapshot()
+ throws Exception {
+ testDeleteAndCommitBlockSynchronizationRace(true);
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystemMBean.java Wed Aug 20 01:34:29 2014
@@ -94,6 +94,8 @@ public class TestFSNamesystemMBean {
"SnapshotStats"));
Long MaxObjects = (Long) (mbs.getAttribute(mxbeanNameFsns,
"MaxObjects"));
+ Integer numStaleStorages = (Integer) (mbs.getAttribute(
+ mxbeanNameFsns, "NumStaleStorages"));
// Metrics that belong to "NameNodeInfo".
// These are metrics that FSNamesystem registers directly with MBeanServer.
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java Wed Aug 20 01:34:29 2014
@@ -41,6 +41,7 @@ import java.net.InetSocketAddress;
import java.nio.channels.FileChannel;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -63,6 +64,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CorruptFileBlocks;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -750,15 +752,14 @@ public class TestFsck {
for (int j=0; j<=1; j++) {
File storageDir = cluster.getInstanceStorageDir(i, j);
File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
- File[] blocks = data_dir.listFiles();
- if (blocks == null)
+ List<File> metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles(
+ data_dir);
+ if (metadataFiles == null)
continue;
-
- for (int idx = 0; idx < blocks.length; idx++) {
- if (!blocks[idx].getName().startsWith("blk_")) {
- continue;
- }
- assertTrue("Cannot remove file.", blocks[idx].delete());
+ for (File metadataFile : metadataFiles) {
+ File blockFile = Block.metaToBlockFile(metadataFile);
+ assertTrue("Cannot remove file.", blockFile.delete());
+ assertTrue("Cannot remove file.", metadataFile.delete());
}
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestINodeFile.java Wed Aug 20 01:34:29 2014
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSClient;
@@ -521,6 +522,7 @@ public class TestINodeFile {
Configuration conf = new Configuration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY,
DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
@@ -568,6 +570,20 @@ public class TestINodeFile {
// ClientProtocol#getPreferredBlockSize
assertEquals(testFileBlockSize,
nnRpc.getPreferredBlockSize(testFileInodePath.toString()));
+
+ /*
+ * HDFS-6749 added missing calls to FSDirectory.resolvePath in the
+ * following four methods. The calls below ensure that
+ * /.reserved/.inodes paths work properly. No need to check return
+ * values as these methods are tested elsewhere.
+ */
+ {
+ fs.isFileClosed(testFileInodePath);
+ fs.getAclStatus(testFileInodePath);
+ fs.getXAttrs(testFileInodePath);
+ fs.listXAttrs(testFileInodePath);
+ fs.access(testFileInodePath, FsAction.READ_WRITE);
+ }
// symbolic link related tests
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestListCorruptFileBlocks.java Wed Aug 20 01:34:29 2014
@@ -25,6 +25,7 @@ import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Collection;
+import java.util.List;
import java.util.Random;
import org.apache.commons.logging.Log;
@@ -39,7 +40,11 @@ import org.apache.hadoop.hdfs.DFSTestUti
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.TestFileCorruption;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.util.StringUtils;
import org.junit.Test;
@@ -87,36 +92,29 @@ public class TestListCorruptFileBlocks {
File storageDir = cluster.getInstanceStorageDir(0, 1);
File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
assertTrue("data directory does not exist", data_dir.exists());
- File[] blocks = data_dir.listFiles();
- assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
- for (int idx = 0; idx < blocks.length; idx++) {
- if (blocks[idx].getName().startsWith("blk_") &&
- blocks[idx].getName().endsWith(".meta")) {
- //
- // shorten .meta file
- //
- RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
- FileChannel channel = file.getChannel();
- long position = channel.size() - 2;
- int length = 2;
- byte[] buffer = new byte[length];
- random.nextBytes(buffer);
- channel.write(ByteBuffer.wrap(buffer), position);
- file.close();
- LOG.info("Deliberately corrupting file " + blocks[idx].getName() +
- " at offset " + position + " length " + length);
-
- // read all files to trigger detection of corrupted replica
- try {
- util.checkFiles(fs, "/srcdat10");
- } catch (BlockMissingException e) {
- System.out.println("Received BlockMissingException as expected.");
- } catch (IOException e) {
- assertTrue("Corrupted replicas not handled properly. Expecting BlockMissingException " +
- " but received IOException " + e, false);
- }
- break;
- }
+ List<File> metaFiles = MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
+ assertTrue("Data directory does not contain any blocks or there was an "
+ + "IO error", metaFiles != null && !metaFiles.isEmpty());
+ File metaFile = metaFiles.get(0);
+ RandomAccessFile file = new RandomAccessFile(metaFile, "rw");
+ FileChannel channel = file.getChannel();
+ long position = channel.size() - 2;
+ int length = 2;
+ byte[] buffer = new byte[length];
+ random.nextBytes(buffer);
+ channel.write(ByteBuffer.wrap(buffer), position);
+ file.close();
+ LOG.info("Deliberately corrupting file " + metaFile.getName() +
+ " at offset " + position + " length " + length);
+
+ // read all files to trigger detection of corrupted replica
+ try {
+ util.checkFiles(fs, "/srcdat10");
+ } catch (BlockMissingException e) {
+ System.out.println("Received BlockMissingException as expected.");
+ } catch (IOException e) {
+ assertTrue("Corrupted replicas not handled properly. Expecting BlockMissingException " +
+ " but received IOException " + e, false);
}
// fetch bad file list from namenode. There should be one file.
@@ -174,38 +172,30 @@ public class TestListCorruptFileBlocks {
File data_dir = MiniDFSCluster.getFinalizedDir(storageDir,
cluster.getNamesystem().getBlockPoolId());
assertTrue("data directory does not exist", data_dir.exists());
- File[] blocks = data_dir.listFiles();
- assertTrue("Blocks do not exist in data-dir", (blocks != null) &&
- (blocks.length > 0));
- for (int idx = 0; idx < blocks.length; idx++) {
- if (blocks[idx].getName().startsWith("blk_") &&
- blocks[idx].getName().endsWith(".meta")) {
- //
- // shorten .meta file
- //
- RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
- FileChannel channel = file.getChannel();
- long position = channel.size() - 2;
- int length = 2;
- byte[] buffer = new byte[length];
- random.nextBytes(buffer);
- channel.write(ByteBuffer.wrap(buffer), position);
- file.close();
- LOG.info("Deliberately corrupting file " + blocks[idx].getName() +
- " at offset " + position + " length " + length);
-
- // read all files to trigger detection of corrupted replica
- try {
- util.checkFiles(fs, "/srcdat10");
- } catch (BlockMissingException e) {
- System.out.println("Received BlockMissingException as expected.");
- } catch (IOException e) {
- assertTrue("Corrupted replicas not handled properly. " +
- "Expecting BlockMissingException " +
- " but received IOException " + e, false);
- }
- break;
- }
+ List<File> metaFiles = MiniDFSCluster.getAllBlockMetadataFiles(data_dir);
+ assertTrue("Data directory does not contain any blocks or there was an "
+ + "IO error", metaFiles != null && !metaFiles.isEmpty());
+ File metaFile = metaFiles.get(0);
+ RandomAccessFile file = new RandomAccessFile(metaFile, "rw");
+ FileChannel channel = file.getChannel();
+ long position = channel.size() - 2;
+ int length = 2;
+ byte[] buffer = new byte[length];
+ random.nextBytes(buffer);
+ channel.write(ByteBuffer.wrap(buffer), position);
+ file.close();
+ LOG.info("Deliberately corrupting file " + metaFile.getName() +
+ " at offset " + position + " length " + length);
+
+ // read all files to trigger detection of corrupted replica
+ try {
+ util.checkFiles(fs, "/srcdat10");
+ } catch (BlockMissingException e) {
+ System.out.println("Received BlockMissingException as expected.");
+ } catch (IOException e) {
+ assertTrue("Corrupted replicas not handled properly. " +
+ "Expecting BlockMissingException " +
+ " but received IOException " + e, false);
}
// fetch bad file list from namenode. There should be one file.
@@ -295,17 +285,18 @@ public class TestListCorruptFileBlocks {
for (int j = 0; j <= 1; j++) {
File storageDir = cluster.getInstanceStorageDir(i, j);
File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
- File[] blocks = data_dir.listFiles();
- if (blocks == null)
+ List<File> metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles(
+ data_dir);
+ if (metadataFiles == null)
continue;
// assertTrue("Blocks do not exist in data-dir", (blocks != null) &&
// (blocks.length > 0));
- for (int idx = 0; idx < blocks.length; idx++) {
- if (!blocks[idx].getName().startsWith("blk_")) {
- continue;
- }
- LOG.info("Deliberately removing file " + blocks[idx].getName());
- assertTrue("Cannot remove file.", blocks[idx].delete());
+ for (File metadataFile : metadataFiles) {
+ File blockFile = Block.metaToBlockFile(metadataFile);
+ LOG.info("Deliberately removing file " + blockFile.getName());
+ assertTrue("Cannot remove file.", blockFile.delete());
+ LOG.info("Deliberately removing file " + metadataFile.getName());
+ assertTrue("Cannot remove file.", metadataFile.delete());
// break;
}
}
@@ -405,17 +396,18 @@ public class TestListCorruptFileBlocks {
for (int i = 0; i < 2; i++) {
File storageDir = cluster.getInstanceStorageDir(0, i);
File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
- File[] blocks = data_dir.listFiles();
- if (blocks == null)
+ List<File> metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles(
+ data_dir);
+ if (metadataFiles == null)
continue;
// assertTrue("Blocks do not exist in data-dir", (blocks != null) &&
// (blocks.length > 0));
- for (int idx = 0; idx < blocks.length; idx++) {
- if (!blocks[idx].getName().startsWith("blk_")) {
- continue;
- }
- LOG.info("Deliberately removing file " + blocks[idx].getName());
- assertTrue("Cannot remove file.", blocks[idx].delete());
+ for (File metadataFile : metadataFiles) {
+ File blockFile = Block.metaToBlockFile(metadataFile);
+ LOG.info("Deliberately removing file " + blockFile.getName());
+ assertTrue("Cannot remove file.", blockFile.delete());
+ LOG.info("Deliberately removing file " + metadataFile.getName());
+ assertTrue("Cannot remove file.", metadataFile.delete());
// break;
}
}
@@ -482,15 +474,14 @@ public class TestListCorruptFileBlocks {
File storageDir = cluster.getInstanceStorageDir(i, j);
File data_dir = MiniDFSCluster.getFinalizedDir(storageDir, bpid);
LOG.info("Removing files from " + data_dir);
- File[] blocks = data_dir.listFiles();
- if (blocks == null)
+ List<File> metadataFiles = MiniDFSCluster.getAllBlockMetadataFiles(
+ data_dir);
+ if (metadataFiles == null)
continue;
-
- for (int idx = 0; idx < blocks.length; idx++) {
- if (!blocks[idx].getName().startsWith("blk_")) {
- continue;
- }
- assertTrue("Cannot remove file.", blocks[idx].delete());
+ for (File metadataFile : metadataFiles) {
+ File blockFile = Block.metaToBlockFile(metadataFile);
+ assertTrue("Cannot remove file.", blockFile.delete());
+ assertTrue("Cannot remove file.", metadataFile.delete());
}
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java Wed Aug 20 01:34:29 2014
@@ -30,9 +30,13 @@ import javax.management.MBeanServer;
import javax.management.ObjectName;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.NoMlockCacheManipulator;
import org.apache.hadoop.util.VersionInfo;
@@ -58,11 +62,14 @@ public class TestNameNodeMXBean {
public void testNameNodeMXBeanInfo() throws Exception {
Configuration conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_DATANODE_MAX_LOCKED_MEMORY_KEY,
- NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
+ NativeIO.POSIX.getCacheManipulator().getMemlockLimit());
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1);
+
MiniDFSCluster cluster = null;
try {
- cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
FSNamesystem fsn = cluster.getNameNode().namesystem;
@@ -70,6 +77,29 @@ public class TestNameNodeMXBean {
MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
ObjectName mxbeanName = new ObjectName(
"Hadoop:service=NameNode,name=NameNodeInfo");
+
+ // Define include file to generate deadNodes metrics
+ FileSystem localFileSys = FileSystem.getLocal(conf);
+ Path workingDir = localFileSys.getWorkingDirectory();
+ Path dir = new Path(workingDir,
+ "build/test/data/temp/TestNameNodeMXBean");
+ Path includeFile = new Path(dir, "include");
+ assertTrue(localFileSys.mkdirs(dir));
+ StringBuilder includeHosts = new StringBuilder();
+ for(DataNode dn : cluster.getDataNodes()) {
+ includeHosts.append(dn.getDisplayName()).append("\n");
+ }
+ DFSTestUtil.writeFile(localFileSys, includeFile, includeHosts.toString());
+ conf.set(DFSConfigKeys.DFS_HOSTS, includeFile.toUri().getPath());
+ fsn.getBlockManager().getDatanodeManager().refreshNodes(conf);
+
+ cluster.stopDataNode(0);
+ while (fsn.getNumDatanodesInService() != 2) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {}
+ }
+
// get attribute "ClusterId"
String clusterId = (String) mbs.getAttribute(mxbeanName, "ClusterId");
assertEquals(fsn.getClusterId(), clusterId);
@@ -121,6 +151,15 @@ public class TestNameNodeMXBean {
String deadnodeinfo = (String) (mbs.getAttribute(mxbeanName,
"DeadNodes"));
assertEquals(fsn.getDeadNodes(), deadnodeinfo);
+ Map<String, Map<String, Object>> deadNodes =
+ (Map<String, Map<String, Object>>) JSON.parse(deadnodeinfo);
+ assertTrue(deadNodes.size() > 0);
+ for (Map<String, Object> deadNode : deadNodes.values()) {
+ assertTrue(deadNode.containsKey("lastContact"));
+ assertTrue(deadNode.containsKey("decommissioned"));
+ assertTrue(deadNode.containsKey("xferaddr"));
+ }
+
// get attribute NodeUsage
String nodeUsage = (String) (mbs.getAttribute(mxbeanName,
"NodeUsage"));
@@ -181,7 +220,7 @@ public class TestNameNodeMXBean {
assertEquals(1, statusMap.get("active").size());
assertEquals(1, statusMap.get("failed").size());
assertEquals(0L, mbs.getAttribute(mxbeanName, "CacheUsed"));
- assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() *
+ assertEquals(NativeIO.POSIX.getCacheManipulator().getMemlockLimit() *
cluster.getDataNodes().size(),
mbs.getAttribute(mxbeanName, "CacheCapacity"));
} finally {
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStartup.java Wed Aug 20 01:34:29 2014
@@ -26,6 +26,7 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
+import java.lang.management.ManagementFactory;
import java.net.InetAddress;
import java.net.URI;
import java.util.ArrayList;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
@@ -64,6 +66,9 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+
/**
* Startup and checkpoint tests
*
@@ -684,4 +689,40 @@ public class TestStartup {
}
}
}
+
+
+ /**
+ * Verify the following scenario.
+ * 1. NN restarts.
+ * 2. Heartbeat RPC will retry and succeed. NN asks DN to reregister.
+ * 3. After reregistration completes, DN will send Heartbeat, followed by
+ * Blockreport.
+ * 4. NN will mark DatanodeStorageInfo#blockContentsStale to false.
+ * @throws Exception
+ */
+ @Test(timeout = 60000)
+ public void testStorageBlockContentsStaleAfterNNRestart() throws Exception {
+ MiniDFSCluster dfsCluster = null;
+ try {
+ Configuration config = new Configuration();
+ dfsCluster = new MiniDFSCluster.Builder(config).numDataNodes(1).build();
+ dfsCluster.waitActive();
+ dfsCluster.restartNameNode(true);
+ BlockManagerTestUtil.checkHeartbeat(
+ dfsCluster.getNamesystem().getBlockManager());
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ ObjectName mxbeanNameFsns = new ObjectName(
+ "Hadoop:service=NameNode,name=FSNamesystemState");
+ Integer numStaleStorages = (Integer) (mbs.getAttribute(
+ mxbeanNameFsns, "NumStaleStorages"));
+ assertEquals(0, numStaleStorages.intValue());
+ } finally {
+ if (dfsCluster != null) {
+ dfsCluster.shutdown();
+ }
+ }
+
+ return;
+ }
+
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPipelinesFailover.java Wed Aug 20 01:34:29 2014
@@ -356,7 +356,8 @@ public class TestPipelinesFailover {
NameNode nn0 = cluster.getNameNode(0);
ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
- DatanodeDescriptor expectedPrimary = getExpectedPrimaryNode(nn0, blk);
+ DatanodeDescriptor expectedPrimary =
+ DFSTestUtil.getExpectedPrimaryNode(nn0, blk);
LOG.info("Expecting block recovery to be triggered on DN " +
expectedPrimary);
@@ -506,37 +507,6 @@ public class TestPipelinesFailover {
}
}
-
-
- /**
- * @return the node which is expected to run the recovery of the
- * given block, which is known to be under construction inside the
- * given NameNOde.
- */
- private DatanodeDescriptor getExpectedPrimaryNode(NameNode nn,
- ExtendedBlock blk) {
- BlockManager bm0 = nn.getNamesystem().getBlockManager();
- BlockInfo storedBlock = bm0.getStoredBlock(blk.getLocalBlock());
- assertTrue("Block " + blk + " should be under construction, " +
- "got: " + storedBlock,
- storedBlock instanceof BlockInfoUnderConstruction);
- BlockInfoUnderConstruction ucBlock =
- (BlockInfoUnderConstruction)storedBlock;
- // We expect that the replica with the most recent heart beat will be
- // the one to be in charge of the synchronization / recovery protocol.
- final DatanodeStorageInfo[] storages = ucBlock.getExpectedStorageLocations();
- DatanodeStorageInfo expectedPrimary = storages[0];
- long mostRecentLastUpdate = expectedPrimary.getDatanodeDescriptor().getLastUpdate();
- for (int i = 1; i < storages.length; i++) {
- final long lastUpdate = storages[i].getDatanodeDescriptor().getLastUpdate();
- if (lastUpdate > mostRecentLastUpdate) {
- expectedPrimary = storages[i];
- mostRecentLastUpdate = lastUpdate;
- }
- }
- return expectedPrimary.getDatanodeDescriptor();
- }
-
private DistributedFileSystem createFsAsOtherUser(
final MiniDFSCluster cluster, final Configuration conf)
throws IOException, InterruptedException {
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRetryCacheWithHA.java Wed Aug 20 01:34:29 2014
@@ -50,6 +50,7 @@ import org.apache.hadoop.fs.XAttrSetFlag
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -725,7 +726,12 @@ public class TestRetryCacheWithHA {
client.getNamenode().updatePipeline(client.getClientName(), oldBlock,
newBlock, newNodes, storageIDs);
- out.close();
+ // close can fail if the out.close() commit the block after block received
+ // notifications from Datanode.
+ // Since datanodes and output stream have still old genstamps, these
+ // blocks will be marked as corrupt after HDFS-5723 if RECEIVED
+ // notifications reaches namenode first and close() will fail.
+ DFSTestUtil.abortStream((DFSOutputStream) out.getWrappedStream());
}
@Override
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestAclWithSnapshot.java Wed Aug 20 01:34:29 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -674,6 +675,13 @@ public class TestAclWithSnapshot {
} catch (AccessControlException e) {
// expected
}
+
+ try {
+ fs.access(pathToCheck, FsAction.READ);
+ fail("The access call should have failed for "+pathToCheck);
+ } catch (AccessControlException e) {
+ // expected
+ }
}
/**
@@ -689,6 +697,7 @@ public class TestAclWithSnapshot {
UserGroupInformation user, Path pathToCheck) throws Exception {
try {
fs.listStatus(pathToCheck);
+ fs.access(pathToCheck, FsAction.READ);
} catch (AccessControlException e) {
fail("expected permission granted for user " + user + ", path = " +
pathToCheck);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/shortcircuit/TestShortCircuitCache.java Wed Aug 20 01:34:29 2014
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY;
import static org.hamcrest.CoreMatchers.equalTo;
import java.io.DataOutputStream;
@@ -30,7 +31,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import org.apache.commons.lang.mutable.MutableBoolean;
@@ -462,6 +465,7 @@ public class TestShortCircuitCache {
}
}, 10, 60000);
cluster.shutdown();
+ sockDir.close();
}
@Test(timeout=60000)
@@ -516,4 +520,98 @@ public class TestShortCircuitCache {
});
cluster.shutdown();
}
+
+ /**
+ * Test unlinking a file whose blocks we are caching in the DFSClient.
+ * The DataNode will notify the DFSClient that the replica is stale via the
+ * ShortCircuitShm.
+ */
+ @Test(timeout=60000)
+ public void testUnlinkingReplicasInFileDescriptorCache() throws Exception {
+ BlockReaderTestUtil.enableShortCircuitShmTracing();
+ TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
+ Configuration conf = createShortCircuitConf(
+ "testUnlinkingReplicasInFileDescriptorCache", sockDir);
+ // We don't want the CacheCleaner to time out short-circuit shared memory
+ // segments during the test, so set the timeout really high.
+ conf.setLong(DFS_CLIENT_READ_SHORTCIRCUIT_STREAMS_CACHE_EXPIRY_MS_KEY,
+ 1000000000L);
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ final ShortCircuitCache cache =
+ fs.getClient().getClientContext().getShortCircuitCache();
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info)
+ throws IOException {
+ // The ClientShmManager starts off empty.
+ Assert.assertEquals(0, info.size());
+ }
+ });
+ final Path TEST_PATH = new Path("/test_file");
+ final int TEST_FILE_LEN = 8193;
+ final int SEED = 0xFADE0;
+ DFSTestUtil.createFile(fs, TEST_PATH, TEST_FILE_LEN,
+ (short)1, SEED);
+ byte contents[] = DFSTestUtil.readFileBuffer(fs, TEST_PATH);
+ byte expected[] = DFSTestUtil.
+ calculateFileContentsFromSeed(SEED, TEST_FILE_LEN);
+ Assert.assertTrue(Arrays.equals(contents, expected));
+ // Loading this file brought the ShortCircuitReplica into our local
+ // replica cache.
+ final DatanodeInfo datanode =
+ new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap<DatanodeInfo, PerDatanodeVisitorInfo> info)
+ throws IOException {
+ Assert.assertTrue(info.get(datanode).full.isEmpty());
+ Assert.assertFalse(info.get(datanode).disabled);
+ Assert.assertEquals(1, info.get(datanode).notFull.values().size());
+ DfsClientShm shm =
+ info.get(datanode).notFull.values().iterator().next();
+ Assert.assertFalse(shm.isDisconnected());
+ }
+ });
+ // Remove the file whose blocks we just read.
+ fs.delete(TEST_PATH, false);
+
+ // Wait for the replica to be purged from the DFSClient's cache.
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ MutableBoolean done = new MutableBoolean(true);
+ @Override
+ public Boolean get() {
+ try {
+ done.setValue(true);
+ cache.getDfsClientShmManager().visit(new Visitor() {
+ @Override
+ public void visit(HashMap<DatanodeInfo,
+ PerDatanodeVisitorInfo> info) throws IOException {
+ Assert.assertTrue(info.get(datanode).full.isEmpty());
+ Assert.assertFalse(info.get(datanode).disabled);
+ Assert.assertEquals(1,
+ info.get(datanode).notFull.values().size());
+ DfsClientShm shm = info.get(datanode).notFull.values().
+ iterator().next();
+ // Check that all slots have been invalidated.
+ for (Iterator<Slot> iter = shm.slotIterator();
+ iter.hasNext(); ) {
+ Slot slot = iter.next();
+ if (slot.isValid()) {
+ done.setValue(false);
+ }
+ }
+ }
+ });
+ } catch (IOException e) {
+ LOG.error("error running visitor", e);
+ }
+ return done.booleanValue();
+ }
+ }, 10, 60000);
+ cluster.shutdown();
+ sockDir.close();
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java Wed Aug 20 01:34:29 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AppendTestUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -49,6 +50,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
+import org.junit.Test;
public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest {
private static final Configuration conf = new Configuration();
@@ -530,4 +532,35 @@ public class TestWebHdfsFileSystemContra
}
}
}
+
+ @Test
+ public void testAccess() throws IOException, InterruptedException {
+ Path p1 = new Path("/pathX");
+ try {
+ UserGroupInformation ugi = UserGroupInformation.createUserForTesting("alpha",
+ new String[]{"beta"});
+ WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystemAs(ugi, conf,
+ WebHdfsFileSystem.SCHEME);
+
+ fs.mkdirs(p1);
+ fs.setPermission(p1, new FsPermission((short) 0444));
+ fs.access(p1, FsAction.READ);
+ try {
+ fs.access(p1, FsAction.WRITE);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path("/bad");
+ try {
+ fs.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ } finally {
+ fs.delete(p1, true);
+ }
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java Wed Aug 20 01:34:29 2014
@@ -31,6 +31,7 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
@@ -40,6 +41,7 @@ import org.apache.hadoop.hdfs.web.resour
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
+import org.apache.hadoop.hdfs.web.resources.FsActionParam;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
@@ -283,6 +285,28 @@ public class TestWebHdfsUrl {
},
fileStatusUrl);
}
+
+ @Test(timeout=60000)
+ public void testCheckAccessUrl() throws IOException {
+ Configuration conf = new Configuration();
+
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("test-user");
+ UserGroupInformation.setLoginUser(ugi);
+
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
+ Path fsPath = new Path("/p1");
+
+ URL checkAccessUrl = webhdfs.toUrl(GetOpParam.Op.CHECKACCESS,
+ fsPath, new FsActionParam(FsAction.READ_WRITE));
+ checkQueryParams(
+ new String[]{
+ GetOpParam.Op.CHECKACCESS.toQueryString(),
+ new UserParam(ugi.getShortUserName()).toString(),
+ FsActionParam.NAME + "=" + FsAction.READ_WRITE.SYMBOL
+ },
+ checkAccessUrl);
+ }
private void checkQueryParams(String[] expected, URL url) {
Arrays.sort(expected);
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/security/TestPermissionSymlinks.java Wed Aug 20 01:34:29 2014
@@ -27,6 +27,7 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.io.FileNotFoundException;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@@ -39,6 +40,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestWrapper;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -393,4 +395,37 @@ public class TestPermissionSymlinks {
GenericTestUtils.assertExceptionContains("Permission denied", e);
}
}
+
+ @Test
+ public void testAccess() throws Exception {
+ fs.setPermission(target, new FsPermission((short) 0002));
+ fs.setAcl(target, Arrays.asList(
+ aclEntry(ACCESS, USER, ALL),
+ aclEntry(ACCESS, GROUP, NONE),
+ aclEntry(ACCESS, USER, user.getShortUserName(), WRITE),
+ aclEntry(ACCESS, OTHER, WRITE)));
+ FileContext myfc = user.doAs(new PrivilegedExceptionAction<FileContext>() {
+ @Override
+ public FileContext run() throws IOException {
+ return FileContext.getFileContext(conf);
+ }
+ });
+
+ // Path to targetChild via symlink
+ myfc.access(link, FsAction.WRITE);
+ try {
+ myfc.access(link, FsAction.ALL);
+ fail("The access call should have failed.");
+ } catch (AccessControlException e) {
+ // expected
+ }
+
+ Path badPath = new Path(link, "bad");
+ try {
+ myfc.access(badPath, FsAction.READ);
+ fail("The access call should have failed");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml (original)
+++ hadoop/common/branches/YARN-1051/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml Wed Aug 20 01:34:29 2014
@@ -8655,6 +8655,50 @@
</comparators>
</test>
+ <test> <!-- TESTED -->
+ <description>count: file using -h option</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p dir</command> <!-- make sure user home dir exists -->
+ <command>-fs NAMENODE -put CLITEST_DATA/data15bytes file1</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data1k file2</command>
+ <command>-fs NAMENODE -count -h file1 file2</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm file1 file2</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( |\t)*0( |\t)*1( |\t)*15 file1</expected-output>
+ </comparator>
+ </comparators>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( |\t)*0( |\t)*1( |\t)*1\.0 K file2</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>count: directory using -q and -h options</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <dfs-admin-command>-fs NAMENODE -setQuota 10 /dir1 </dfs-admin-command>
+ <dfs-admin-command>-fs NAMENODE -setSpaceQuota 1m /dir1 </dfs-admin-command>
+ <command>-fs NAMENODE -count -q -h /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>( |\t)*10( |\t)*9( |\t)*1 M( |\t)*1 M( |\t)*1( |\t)*0( |\t)*0 /dir1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
<!-- Tests for chmod -->
<test> <!-- TESTED -->
<description>chmod: change permission(octal mode) of file in absolute path</description>