You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2013/04/14 01:06:00 UTC
svn commit: r1467713 [3/4] - in
/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs: ./
dev-support/ src/main/java/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/net/
src/main/java/org/apache/hadoop/hdfs/proto...
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Sat Apr 13 23:05:54 2013
@@ -1221,7 +1221,6 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
-
if (isInSafeMode()) {
throw new SafeModeException("Cannot set permission for " + src, safeMode);
}
@@ -1259,7 +1258,6 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
-
if (isInSafeMode()) {
throw new SafeModeException("Cannot set owner for " + src, safeMode);
}
@@ -1313,9 +1311,8 @@ public class FSNamesystem implements Nam
LocatedBlocks getBlockLocations(String src, long offset, long length,
boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode)
throws FileNotFoundException, UnresolvedLinkException, IOException {
- FSPermissionChecker pc = getPermissionChecker();
try {
- return getBlockLocationsInt(pc, src, offset, length, doAccessTime,
+ return getBlockLocationsInt(src, offset, length, doAccessTime,
needBlockToken, checkSafeMode);
} catch (AccessControlException e) {
logAuditEvent(false, "open", src);
@@ -1323,14 +1320,10 @@ public class FSNamesystem implements Nam
}
}
- private LocatedBlocks getBlockLocationsInt(FSPermissionChecker pc,
- String src, long offset, long length, boolean doAccessTime,
- boolean needBlockToken, boolean checkSafeMode)
+ private LocatedBlocks getBlockLocationsInt(String src, long offset,
+ long length, boolean doAccessTime, boolean needBlockToken,
+ boolean checkSafeMode)
throws FileNotFoundException, UnresolvedLinkException, IOException {
- if (isPermissionEnabled) {
- checkPathAccess(pc, src, FsAction.READ);
- }
-
if (offset < 0) {
throw new HadoopIllegalArgumentException(
"Negative offset is not supported. File: " + src);
@@ -1358,13 +1351,11 @@ public class FSNamesystem implements Nam
* Get block locations within the specified range, updating the
* access times if necessary.
*/
- private LocatedBlocks getBlockLocationsUpdateTimes(String src,
- long offset,
- long length,
- boolean doAccessTime,
- boolean needBlockToken)
- throws FileNotFoundException, UnresolvedLinkException, IOException {
-
+ private LocatedBlocks getBlockLocationsUpdateTimes(String src, long offset,
+ long length, boolean doAccessTime, boolean needBlockToken)
+ throws FileNotFoundException,
+ UnresolvedLinkException, IOException {
+ FSPermissionChecker pc = getPermissionChecker();
for (int attempt = 0; attempt < 2; attempt++) {
boolean isReadOp = (attempt == 0);
if (isReadOp) { // first attempt is with readlock
@@ -1380,6 +1371,9 @@ public class FSNamesystem implements Nam
} else {
checkOperation(OperationCategory.WRITE);
}
+ if (isPermissionEnabled) {
+ checkPathAccess(pc, src, FsAction.READ);
+ }
// if the namenode is in safemode, then do not update access time
if (isInSafeMode()) {
@@ -1426,6 +1420,10 @@ public class FSNamesystem implements Nam
*/
void concat(String target, String [] srcs)
throws IOException, UnresolvedLinkException {
+ if(FSNamesystem.LOG.isDebugEnabled()) {
+ FSNamesystem.LOG.debug("concat " + Arrays.toString(srcs) +
+ " to " + target);
+ }
try {
concatInt(target, srcs);
} catch (AccessControlException e) {
@@ -1436,11 +1434,6 @@ public class FSNamesystem implements Nam
private void concatInt(String target, String [] srcs)
throws IOException, UnresolvedLinkException {
- if(FSNamesystem.LOG.isDebugEnabled()) {
- FSNamesystem.LOG.debug("concat " + Arrays.toString(srcs) +
- " to " + target);
- }
-
// verify args
if(target.isEmpty()) {
throw new IllegalArgumentException("Target file name is empty");
@@ -1594,6 +1587,10 @@ public class FSNamesystem implements Nam
*/
void setTimes(String src, long mtime, long atime)
throws IOException, UnresolvedLinkException {
+ if (!isAccessTimeSupported() && atime != -1) {
+ throw new IOException("Access time for hdfs is not configured. " +
+ " Please set " + DFS_NAMENODE_ACCESSTIME_PRECISION_KEY + " configuration parameter.");
+ }
try {
setTimesInt(src, mtime, atime);
} catch (AccessControlException e) {
@@ -1604,16 +1601,15 @@ public class FSNamesystem implements Nam
private void setTimesInt(String src, long mtime, long atime)
throws IOException, UnresolvedLinkException {
- if (!isAccessTimeSupported() && atime != -1) {
- throw new IOException("Access time for hdfs is not configured. " +
- " Please set " + DFS_NAMENODE_ACCESSTIME_PRECISION_KEY + " configuration parameter.");
- }
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
+ if (isInSafeMode()) {
+ throw new SafeModeException("Cannot set times " + src, safeMode);
+ }
// Write access is required to set access and modification times
if (isPermissionEnabled) {
@@ -1639,6 +1635,9 @@ public class FSNamesystem implements Nam
void createSymlink(String target, String link,
PermissionStatus dirPerms, boolean createParent)
throws IOException, UnresolvedLinkException {
+ if (!DFSUtil.isValidName(link)) {
+ throw new InvalidPathException("Invalid file name: " + link);
+ }
try {
createSymlinkInt(target, link, dirPerms, createParent);
} catch (AccessControlException e) {
@@ -1650,17 +1649,34 @@ public class FSNamesystem implements Nam
private void createSymlinkInt(String target, String link,
PermissionStatus dirPerms, boolean createParent)
throws IOException, UnresolvedLinkException {
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.createSymlink: target="
+ + target + " link=" + link);
+ }
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
-
+ if (isInSafeMode()) {
+ throw new SafeModeException("Cannot create symlink " + link, safeMode);
+ }
if (!createParent) {
verifyParentDir(link);
}
- createSymlinkInternal(pc, target, link, dirPerms, createParent);
+ if (!dir.isValidToCreate(link)) {
+ throw new IOException("failed to create link " + link
+ +" either because the filename is invalid or the file exists");
+ }
+ if (isPermissionEnabled) {
+ checkAncestorAccess(pc, link, FsAction.WRITE);
+ }
+ // validate that we have enough inodes.
+ checkFsObjectLimit();
+
+ // add symbolic link to namespace
+ dir.addSymlink(link, target, dirPerms, createParent);
resultingStat = getAuditFileInfo(link, false);
} finally {
writeUnlock();
@@ -1670,37 +1686,6 @@ public class FSNamesystem implements Nam
}
/**
- * Create a symbolic link.
- */
- private void createSymlinkInternal(FSPermissionChecker pc, String target,
- String link, PermissionStatus dirPerms, boolean createParent)
- throws IOException, UnresolvedLinkException {
- assert hasWriteLock();
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* NameSystem.createSymlink: target=" +
- target + " link=" + link);
- }
- if (isInSafeMode()) {
- throw new SafeModeException("Cannot create symlink " + link, safeMode);
- }
- if (!DFSUtil.isValidName(link)) {
- throw new InvalidPathException("Invalid file name: " + link);
- }
- if (!dir.isValidToCreate(link)) {
- throw new IOException("failed to create link " + link
- +" either because the filename is invalid or the file exists");
- }
- if (isPermissionEnabled) {
- checkAncestorAccess(pc, link, FsAction.WRITE);
- }
- // validate that we have enough inodes.
- checkFsObjectLimit();
-
- // add symbolic link to namespace
- dir.addSymlink(link, target, dirPerms, createParent);
- }
-
- /**
* Set replication for an existing file.
*
* The NameNode sets new replication and schedules either replication of
@@ -1819,13 +1804,24 @@ public class FSNamesystem implements Nam
throws AccessControlException, SafeModeException,
FileAlreadyExistsException, UnresolvedLinkException,
FileNotFoundException, ParentNotDirectoryException, IOException {
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: src=" + src
+ + ", holder=" + holder
+ + ", clientMachine=" + clientMachine
+ + ", createParent=" + createParent
+ + ", replication=" + replication
+ + ", createFlag=" + flag.toString());
+ }
+ if (!DFSUtil.isValidName(src)) {
+ throw new InvalidPathException(src);
+ }
+
boolean skipSync = false;
final HdfsFileStatus stat;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
writeLock();
try {
- checkOperation(OperationCategory.WRITE);
startFileInternal(pc, src, permissions, holder, clientMachine, flag,
createParent, replication, blockSize);
stat = dir.getFileInfo(src, false);
@@ -1868,21 +1864,10 @@ public class FSNamesystem implements Nam
AccessControlException, UnresolvedLinkException, FileNotFoundException,
ParentNotDirectoryException, IOException {
assert hasWriteLock();
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* NameSystem.startFile: src=" + src
- + ", holder=" + holder
- + ", clientMachine=" + clientMachine
- + ", createParent=" + createParent
- + ", replication=" + replication
- + ", createFlag=" + flag.toString());
- }
+ checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
throw new SafeModeException("Cannot create file" + src, safeMode);
}
- if (!DFSUtil.isValidName(src)) {
- throw new InvalidPathException(src);
- }
-
// Verify that the destination does not exist as a directory already.
final INodesInPath iip = dir.getINodesInPath4Write(src);
final INode inode = iip.getLastINode();
@@ -1945,9 +1930,8 @@ public class FSNamesystem implements Nam
checkFsObjectLimit();
// increment global generation stamp
- long genstamp = nextGenerationStamp();
INodeFileUnderConstruction newNode = dir.addFile(src, permissions,
- replication, blockSize, holder, clientMachine, clientNode, genstamp);
+ replication, blockSize, holder, clientMachine, clientNode);
if (newNode == null) {
throw new IOException("DIR* NameSystem.startFile: " +
"Unable to add file to namespace.");
@@ -2013,21 +1997,20 @@ public class FSNamesystem implements Nam
*/
boolean recoverLease(String src, String holder, String clientMachine)
throws IOException {
+ if (!DFSUtil.isValidName(src)) {
+ throw new IOException("Invalid file name: " + src);
+ }
+
boolean skipSync = false;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
-
if (isInSafeMode()) {
throw new SafeModeException(
"Cannot recover the lease of " + src, safeMode);
}
- if (!DFSUtil.isValidName(src)) {
- throw new IOException("Invalid file name: " + src);
- }
-
final INodeFile inode = INodeFile.valueOf(dir.getINode(src), src);
if (!inode.isUnderConstruction()) {
return true;
@@ -2151,13 +2134,20 @@ public class FSNamesystem implements Nam
"Append is not enabled on this NameNode. Use the " +
DFS_SUPPORT_APPEND_KEY + " configuration option to enable it.");
}
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.appendFile: src=" + src
+ + ", holder=" + holder
+ + ", clientMachine=" + clientMachine);
+ }
+ if (!DFSUtil.isValidName(src)) {
+ throw new InvalidPathException(src);
+ }
+
LocatedBlock lb = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
writeLock();
try {
- checkOperation(OperationCategory.WRITE);
-
lb = startFileInternal(pc, src, null, holder, clientMachine,
EnumSet.of(CreateFlag.APPEND),
false, blockManager.maxReplication, 0);
@@ -2448,21 +2438,21 @@ public class FSNamesystem implements Nam
boolean abandonBlock(ExtendedBlock b, String src, String holder)
throws LeaseExpiredException, FileNotFoundException,
UnresolvedLinkException, IOException {
+ if(NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: " + b
+ + "of file " + src);
+ }
checkOperation(OperationCategory.WRITE);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
- //
- // Remove the block from the pending creates list
- //
- if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("BLOCK* NameSystem.abandonBlock: "
- +b+"of file "+src);
- }
if (isInSafeMode()) {
throw new SafeModeException("Cannot abandon block " + b +
" for fle" + src, safeMode);
}
+ //
+ // Remove the block from the pending creates list
+ //
INodeFileUnderConstruction file = checkLease(src, holder);
dir.removeBlock(src, file, ExtendedBlock.getLocalBlock(b));
if(NameNode.stateChangeLog.isDebugEnabled()) {
@@ -2524,19 +2514,23 @@ public class FSNamesystem implements Nam
*/
boolean completeFile(String src, String holder, ExtendedBlock last)
throws SafeModeException, UnresolvedLinkException, IOException {
+ if (NameNode.stateChangeLog.isDebugEnabled()) {
+ NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " +
+ src + " for " + holder);
+ }
checkBlock(last);
boolean success = false;
checkOperation(OperationCategory.WRITE);
writeLock();
try {
- checkOperation(OperationCategory.WRITE);
-
- success = completeFileInternal(src, holder,
- ExtendedBlock.getLocalBlock(last));
+ success = completeFileInternal(src, holder,
+ ExtendedBlock.getLocalBlock(last));
} finally {
writeUnlock();
}
getEditLog().logSync();
+ NameNode.stateChangeLog.info("DIR* completeFile: " + src + " is closed by "
+ + holder);
return success;
}
@@ -2544,10 +2538,7 @@ public class FSNamesystem implements Nam
String holder, Block last) throws SafeModeException,
UnresolvedLinkException, IOException {
assert hasWriteLock();
- if (NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " +
- src + " for " + holder);
- }
+ checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
throw new SafeModeException("Cannot complete file " + src, safeMode);
}
@@ -2588,9 +2579,6 @@ public class FSNamesystem implements Nam
finalizeINodeFileUnderConstruction(src, pendingFile,
iip.getLatestSnapshot());
-
- NameNode.stateChangeLog.info("DIR* completeFile: " + src + " is closed by "
- + holder);
return true;
}
@@ -2691,18 +2679,19 @@ public class FSNamesystem implements Nam
private boolean renameToInt(String src, String dst)
throws IOException, UnresolvedLinkException {
- boolean status = false;
- HdfsFileStatus resultingStat = null;
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: " + src +
" to " + dst);
}
+ if (!DFSUtil.isValidName(dst)) {
+ throw new IOException("Invalid name: " + dst);
+ }
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ boolean status = false;
+ HdfsFileStatus resultingStat = null;
writeLock();
try {
- checkOperation(OperationCategory.WRITE);
-
status = renameToInternal(pc, src, dst);
if (status) {
resultingStat = getAuditFileInfo(dst, false);
@@ -2722,12 +2711,10 @@ public class FSNamesystem implements Nam
private boolean renameToInternal(FSPermissionChecker pc, String src, String dst)
throws IOException, UnresolvedLinkException {
assert hasWriteLock();
+ checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
throw new SafeModeException("Cannot rename " + src, safeMode);
}
- if (!DFSUtil.isValidName(dst)) {
- throw new IOException("Invalid name: " + dst);
- }
if (isPermissionEnabled) {
//We should not be doing this. This is move() not renameTo().
//but for now,
@@ -2749,16 +2736,18 @@ public class FSNamesystem implements Nam
/** Rename src to dst */
void renameTo(String src, String dst, Options.Rename... options)
throws IOException, UnresolvedLinkException {
- HdfsFileStatus resultingStat = null;
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: with options - "
+ src + " to " + dst);
}
+ if (!DFSUtil.isValidName(dst)) {
+ throw new InvalidPathException("Invalid name: " + dst);
+ }
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ HdfsFileStatus resultingStat = null;
writeLock();
try {
- checkOperation(OperationCategory.WRITE);
renameToInternal(pc, src, dst, options);
resultingStat = getAuditFileInfo(dst, false);
} finally {
@@ -2777,12 +2766,10 @@ public class FSNamesystem implements Nam
private void renameToInternal(FSPermissionChecker pc, String src, String dst,
Options.Rename... options) throws IOException {
assert hasWriteLock();
+ checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
throw new SafeModeException("Cannot rename " + src, safeMode);
}
- if (!DFSUtil.isValidName(dst)) {
- throw new InvalidPathException("Invalid name: " + dst);
- }
if (isPermissionEnabled) {
checkParentAccess(pc, src, FsAction.WRITE);
checkAncestorAccess(pc, dst, FsAction.WRITE);
@@ -2969,16 +2956,15 @@ public class FSNamesystem implements Nam
HdfsFileStatus getFileInfo(String src, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException,
StandbyException, IOException {
+ if (!DFSUtil.isValidName(src)) {
+ throw new InvalidPathException("Invalid file name: " + src);
+ }
HdfsFileStatus stat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.READ);
readLock();
try {
checkOperation(OperationCategory.READ);
-
- if (!DFSUtil.isValidName(src)) {
- throw new InvalidPathException("Invalid file name: " + src);
- }
if (isPermissionEnabled) {
checkTraverse(pc, src);
}
@@ -3035,16 +3021,18 @@ public class FSNamesystem implements Nam
private boolean mkdirsInt(String src, PermissionStatus permissions,
boolean createParent) throws IOException, UnresolvedLinkException {
- HdfsFileStatus resultingStat = null;
- boolean status = false;
if(NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src);
}
+ if (!DFSUtil.isValidName(src)) {
+ throw new InvalidPathException(src);
+ }
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ HdfsFileStatus resultingStat = null;
+ boolean status = false;
writeLock();
try {
- checkOperation(OperationCategory.WRITE);
status = mkdirsInternal(pc, src, permissions, createParent);
if (status) {
resultingStat = dir.getFileInfo(src, false);
@@ -3066,6 +3054,7 @@ public class FSNamesystem implements Nam
PermissionStatus permissions, boolean createParent)
throws IOException, UnresolvedLinkException {
assert hasWriteLock();
+ checkOperation(OperationCategory.WRITE);
if (isInSafeMode()) {
throw new SafeModeException("Cannot create directory " + src, safeMode);
}
@@ -3077,9 +3066,6 @@ public class FSNamesystem implements Nam
// a new directory is not created.
return true;
}
- if (!DFSUtil.isValidName(src)) {
- throw new InvalidPathException(src);
- }
if (isPermissionEnabled) {
checkAncestorAccess(pc, src, FsAction.WRITE);
}
@@ -3355,8 +3341,15 @@ public class FSNamesystem implements Nam
boolean closeFile, boolean deleteblock, DatanodeID[] newtargets,
String[] newtargetstorages)
throws IOException, UnresolvedLinkException {
- String src = "";
+ LOG.info("commitBlockSynchronization(lastblock=" + lastblock
+ + ", newgenerationstamp=" + newgenerationstamp
+ + ", newlength=" + newlength
+ + ", newtargets=" + Arrays.asList(newtargets)
+ + ", closeFile=" + closeFile
+ + ", deleteBlock=" + deleteblock
+ + ")");
checkOperation(OperationCategory.WRITE);
+ String src = "";
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3368,13 +3361,6 @@ public class FSNamesystem implements Nam
"Cannot commitBlockSynchronization while in safe mode",
safeMode);
}
- LOG.info("commitBlockSynchronization(lastblock=" + lastblock
- + ", newgenerationstamp=" + newgenerationstamp
- + ", newlength=" + newlength
- + ", newtargets=" + Arrays.asList(newtargets)
- + ", closeFile=" + closeFile
- + ", deleteBlock=" + deleteblock
- + ")");
final BlockInfo storedBlock = blockManager.getStoredBlock(ExtendedBlock
.getLocalBlock(lastblock));
if (storedBlock == null) {
@@ -3465,7 +3451,6 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
-
if (isInSafeMode()) {
throw new SafeModeException("Cannot renew lease for " + holder, safeMode);
}
@@ -4951,11 +4936,10 @@ public class FSNamesystem implements Nam
*/
void reportBadBlocks(LocatedBlock[] blocks) throws IOException {
checkOperation(OperationCategory.WRITE);
+ NameNode.stateChangeLog.info("*DIR* reportBadBlocks");
writeLock();
try {
checkOperation(OperationCategory.WRITE);
-
- NameNode.stateChangeLog.info("*DIR* reportBadBlocks");
for (int i = 0; i < blocks.length; i++) {
ExtendedBlock blk = blocks[i].getBlock();
DatanodeInfo[] nodes = blocks[i].getLocations();
@@ -5018,6 +5002,12 @@ public class FSNamesystem implements Nam
ExtendedBlock newBlock, DatanodeID[] newNodes)
throws IOException {
checkOperation(OperationCategory.WRITE);
+ LOG.info("updatePipeline(block=" + oldBlock
+ + ", newGenerationStamp=" + newBlock.getGenerationStamp()
+ + ", newLength=" + newBlock.getNumBytes()
+ + ", newNodes=" + Arrays.asList(newNodes)
+ + ", clientName=" + clientName
+ + ")");
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -5027,12 +5017,6 @@ public class FSNamesystem implements Nam
}
assert newBlock.getBlockId()==oldBlock.getBlockId() : newBlock + " and "
+ oldBlock + " has different block identifier";
- LOG.info("updatePipeline(block=" + oldBlock
- + ", newGenerationStamp=" + newBlock.getGenerationStamp()
- + ", newLength=" + newBlock.getNumBytes()
- + ", newNodes=" + Arrays.asList(newNodes)
- + ", clientName=" + clientName
- + ")");
updatePipelineInternal(clientName, oldBlock, newBlock, newNodes);
} finally {
writeUnlock();
@@ -5284,7 +5268,6 @@ public class FSNamesystem implements Nam
writeLock();
try {
checkOperation(OperationCategory.WRITE);
-
if (isInSafeMode()) {
throw new SafeModeException("Cannot issue delegation token", safeMode);
}
@@ -5409,6 +5392,21 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
+ /**
+ * Log the cancellation of expired tokens to edit logs
+ *
+ * @param id token identifier to cancel
+ */
+ public void logExpireDelegationToken(DelegationTokenIdentifier id) {
+ assert !isInSafeMode() :
+ "this should never be called while in safemode, since we stop " +
+ "the DT manager before entering safemode!";
+ // No need to hold FSN lock since we don't access any internal
+ // structures, and this is stopped before the FSN shuts itself
+ // down, etc.
+ getEditLog().logCancelDelegationToken(id);
+ }
+
private void logReassignLease(String leaseHolder, String src,
String newHolder) {
assert hasWriteLock();
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Sat Apr 13 23:05:54 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.BlockReade
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.net.TcpPeerServer;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -559,9 +560,10 @@ public class NamenodeFsck {
String file = BlockReaderFactory.getFileName(targetAddr, block.getBlockPoolId(),
block.getBlockId());
blockReader = BlockReaderFactory.newBlockReader(
- conf, s, file, block, lblock
- .getBlockToken(), 0, -1,
- namenode.getRpcServer().getDataEncryptionKey());
+ conf, file, block, lblock.getBlockToken(), 0, -1, true, "fsck",
+ TcpPeerServer.peerFromSocketAndKey(s, namenode.getRpcServer().
+ getDataEncryptionKey()),
+ chosenNode, null, false);
} catch (IOException ex) {
// Put chosen node into dead list, continue
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-347/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1430995-1467533
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1466653-1467712
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/ClientDatanodeProtocol.proto Sat Apr 13 23:05:54 2013
@@ -74,6 +74,8 @@ message DeleteBlockPoolResponseProto {
* Gets the file information where block and its metadata is stored
* block - block for which path information is being requested
* token - block token
+ *
+ * This message is deprecated in favor of file descriptor passing.
*/
message GetBlockLocalPathInfoRequestProto {
required ExtendedBlockProto block = 1;
@@ -84,6 +86,8 @@ message GetBlockLocalPathInfoRequestProt
* block - block for which file path information is being returned
* localPath - file path where the block data is stored
* localMetaPath - file path where the block meta data is stored
+ *
+ * This message is deprecated in favor of file descriptor passing.
*/
message GetBlockLocalPathInfoResponseProto {
required ExtendedBlockProto block = 1;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/datatransfer.proto Sat Apr 13 23:05:54 2013
@@ -115,6 +115,16 @@ message OpBlockChecksumProto {
required BaseHeaderProto header = 1;
}
+message OpRequestShortCircuitAccessProto {
+ required BaseHeaderProto header = 1;
+
+ /** In order to get short-circuit access to block data, clients must set this
+ * to the highest version of the block data that they can understand.
+ * Currently 1 is the only version, but more versions may exist in the future
+ * if the on-disk format changes.
+ */
+ required uint32 maxVersion = 2;
+}
message PacketHeaderProto {
// All fields must be fixed-length!
@@ -133,6 +143,7 @@ enum Status {
ERROR_EXISTS = 4;
ERROR_ACCESS_TOKEN = 5;
CHECKSUM_OK = 6;
+ ERROR_UNSUPPORTED = 7;
}
message PipelineAckProto {
@@ -165,6 +176,16 @@ message BlockOpResponseProto {
/** explanatory text which may be useful to log on the client side */
optional string message = 5;
+
+ /** If the server chooses to agree to the request of a client for
+ * short-circuit access, it will send a response message with the relevant
+ * file descriptors attached.
+ *
+ * In the body of the message, this version number will be set to the
+ * specific version number of the block data that the client is about to
+ * read.
+ */
+ optional uint32 shortCircuitAccessVersion = 6;
}
/**
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml Sat Apr 13 23:05:54 2013
@@ -1232,6 +1232,17 @@
</property>
<property>
+ <name>dfs.domain.socket.path</name>
+ <value></value>
+ <description>
+ Optional. This is a path to a UNIX domain socket that will be used for
+ communication between the DataNode and local HDFS clients.
+ If the string "_PORT" is present in this path, it will be replaced by the
+ TCP port of the DataNode.
+ </description>
+</property>
+
+<property>
<name>dfs.datanode.fsdataset.volume.choosing.balanced-space-threshold</name>
<value>10737418240</value> <!-- 10 GB -->
<description>
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1466653-1467712
Merged /hadoop/common/branches/HDFS-347/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1430995-1467533
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1466653-1467712
Merged /hadoop/common/branches/HDFS-347/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1430995-1467533
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/HDFS-347/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1430995-1467533
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1466653-1467712
Propchange: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1466653-1467712
Merged /hadoop/common/branches/HDFS-347/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1430995-1467533
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/BlockReaderTestUtil.java Sat Apr 13 23:05:54 2013
@@ -28,9 +28,9 @@ import java.net.Socket;
import java.util.List;
import java.util.Random;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.net.TcpPeerServer;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -150,12 +150,12 @@ public class BlockReaderTestUtil {
sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
return BlockReaderFactory.newBlockReader(
- new DFSClient.Conf(conf),
- sock, targetAddr.toString()+ ":" + block.getBlockId(), block,
+ conf,
+ targetAddr.toString()+ ":" + block.getBlockId(), block,
testBlock.getBlockToken(),
offset, lenToRead,
- conf.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
- true, "", null, null);
+ true, "BlockReaderTestUtil", TcpPeerServer.peerFromSocket(sock),
+ nodes[0], null, false);
}
/**
@@ -166,5 +166,4 @@ public class BlockReaderTestUtil {
int ipcport = nodes[0].getIpcPort();
return cluster.getDataNode(ipcport);
}
-
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Sat Apr 13 23:05:54 2013
@@ -2188,8 +2188,8 @@ public class MiniDFSCluster {
/**
* Get file correpsonding to a block
* @param storageDir storage directory
- * @param blk block to be corrupted
- * @return file corresponding to the block
+ * @param blk the block
+ * @return data file corresponding to the block
*/
public static File getBlockFile(File storageDir, ExtendedBlock blk) {
return new File(getFinalizedDir(storageDir, blk.getBlockPoolId()),
@@ -2197,6 +2197,19 @@ public class MiniDFSCluster {
}
/**
+ * Get the latest metadata file correpsonding to a block
+ * @param storageDir storage directory
+ * @param blk the block
+ * @return metadata file corresponding to the block
+ */
+ public static File getBlockMetadataFile(File storageDir, ExtendedBlock blk) {
+ return new File(getFinalizedDir(storageDir, blk.getBlockPoolId()),
+ blk.getBlockName() + "_" + blk.getGenerationStamp() +
+ Block.METADATA_EXTENSION);
+
+ }
+
+ /**
* Shut down a cluster if it is not null
* @param cluster cluster reference or null
*/
@@ -2223,7 +2236,7 @@ public class MiniDFSCluster {
}
/**
- * Get files related to a block for a given datanode
+ * Get the block data file for a block from a given datanode
* @param dnIndex Index of the datanode to get block files for
* @param block block for which corresponding files are needed
*/
@@ -2238,6 +2251,24 @@ public class MiniDFSCluster {
}
return null;
}
+
+ /**
+ * Get the block metadata file for a block from a given datanode
+ *
+ * @param dnIndex Index of the datanode to get block files for
+ * @param block block for which corresponding files are needed
+ */
+ public static File getBlockMetadataFile(int dnIndex, ExtendedBlock block) {
+ // Check for block file in the two storage directories of the datanode
+ for (int i = 0; i <=1 ; i++) {
+ File storageDir = MiniDFSCluster.getStorageDir(dnIndex, i);
+ File blockMetaFile = getBlockMetadataFile(storageDir, block);
+ if (blockMetaFile.exists()) {
+ return blockMetaFile;
+ }
+ }
+ return null;
+ }
/**
* Throw an exception if the MiniDFSCluster is not started with a single
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockReaderLocal.java Sat Apr 13 23:05:54 2013
@@ -17,90 +17,333 @@
*/
package org.apache.hadoop.hdfs;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
+import java.util.concurrent.TimeoutException;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.Assert;
import org.junit.Test;
public class TestBlockReaderLocal {
- static MiniDFSCluster cluster;
- static HdfsConfiguration conf;
+ public static void assertArrayRegionsEqual(byte []buf1, int off1, byte []buf2,
+ int off2, int len) {
+ for (int i = 0; i < len; i++) {
+ if (buf1[off1 + i] != buf2[off2 + i]) {
+ Assert.fail("arrays differ at byte " + i + ". " +
+ "The first array has " + (int)buf1[off1 + i] +
+ ", but the second array has " + (int)buf2[off2 + i]);
+ }
+ }
+ }
- @BeforeClass
- public static void setupCluster() throws IOException {
- conf = new HdfsConfiguration();
+ /**
+ * Similar to IOUtils#readFully(). Reads bytes in a loop.
+ *
+ * @param reader The BlockReaderLocal to read bytes from
+ * @param buf The ByteBuffer to read into
+ * @param off The offset in the buffer to read into
+ * @param len The number of bytes to read.
+ *
+ * @throws IOException If it could not read the requested number of bytes
+ */
+ private static void readFully(BlockReaderLocal reader,
+ ByteBuffer buf, int off, int len) throws IOException {
+ int amt = len;
+ while (amt > 0) {
+ buf.limit(off + len);
+ buf.position(off);
+ long ret = reader.read(buf);
+ if (ret < 0) {
+ throw new EOFException( "Premature EOF from BlockReaderLocal " +
+ "after reading " + (len - amt) + " byte(s).");
+ }
+ amt -= ret;
+ off += ret;
+ }
+ }
+
+ private static interface BlockReaderLocalTest {
+ final int TEST_LENGTH = 12345;
+ public void setup(File blockFile, boolean usingChecksums)
+ throws IOException;
+ public void doTest(BlockReaderLocal reader, byte original[])
+ throws IOException;
+ }
+
+ public void runBlockReaderLocalTest(BlockReaderLocalTest test,
+ boolean checksum) throws IOException {
+ MiniDFSCluster cluster = null;
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.
+ DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY, !checksum);
+ conf.set(DFSConfigKeys.DFS_CHECKSUM_TYPE_KEY, "CRC32C");
+ FileInputStream dataIn = null, checkIn = null;
+ final Path TEST_PATH = new Path("/a");
+ final long RANDOM_SEED = 4567L;
+ BlockReaderLocal blockReaderLocal = null;
+ FSDataInputStream fsIn = null;
+ byte original[] = new byte[BlockReaderLocalTest.TEST_LENGTH];
+
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+ DFSTestUtil.createFile(fs, TEST_PATH,
+ BlockReaderLocalTest.TEST_LENGTH, (short)1, RANDOM_SEED);
+ try {
+ DFSTestUtil.waitReplication(fs, TEST_PATH, (short)1);
+ } catch (InterruptedException e) {
+ Assert.fail("unexpected InterruptedException during " +
+ "waitReplication: " + e);
+ } catch (TimeoutException e) {
+ Assert.fail("unexpected TimeoutException during " +
+ "waitReplication: " + e);
+ }
+ fsIn = fs.open(TEST_PATH);
+ IOUtils.readFully(fsIn, original, 0,
+ BlockReaderLocalTest.TEST_LENGTH);
+ fsIn.close();
+ fsIn = null;
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, TEST_PATH);
+ File dataFile = MiniDFSCluster.getBlockFile(0, block);
+ File metaFile = MiniDFSCluster.getBlockMetadataFile(0, block);
+
+ DatanodeID datanodeID = cluster.getDataNodes().get(0).getDatanodeId();
+ cluster.shutdown();
+ cluster = null;
+ test.setup(dataFile, checksum);
+ dataIn = new FileInputStream(dataFile);
+ checkIn = new FileInputStream(metaFile);
+ blockReaderLocal = new BlockReaderLocal(conf,
+ TEST_PATH.getName(), block, 0, -1,
+ dataIn, checkIn, datanodeID, checksum);
+ dataIn = null;
+ checkIn = null;
+ test.doTest(blockReaderLocal, original);
+ } finally {
+ if (fsIn != null) fsIn.close();
+ if (cluster != null) cluster.shutdown();
+ if (dataIn != null) dataIn.close();
+ if (checkIn != null) checkIn.close();
+ if (blockReaderLocal != null) blockReaderLocal.close(null, null);
+ }
+ }
+
+ private static class TestBlockReaderLocalImmediateClose
+ implements BlockReaderLocalTest {
+ @Override
+ public void setup(File blockFile, boolean usingChecksums)
+ throws IOException { }
+ @Override
+ public void doTest(BlockReaderLocal reader, byte original[])
+ throws IOException { }
+ }
+
+ @Test
+ public void testBlockReaderLocalImmediateClose() throws IOException {
+ runBlockReaderLocalTest(new TestBlockReaderLocalImmediateClose(), true);
+ runBlockReaderLocalTest(new TestBlockReaderLocalImmediateClose(), false);
+ }
+
+ private static class TestBlockReaderSimpleReads
+ implements BlockReaderLocalTest {
+ @Override
+ public void setup(File blockFile, boolean usingChecksums)
+ throws IOException { }
+ @Override
+ public void doTest(BlockReaderLocal reader, byte original[])
+ throws IOException {
+ byte buf[] = new byte[TEST_LENGTH];
+ reader.readFully(buf, 0, 512);
+ assertArrayRegionsEqual(original, 0, buf, 0, 512);
+ reader.readFully(buf, 512, 512);
+ assertArrayRegionsEqual(original, 512, buf, 512, 512);
+ reader.readFully(buf, 1024, 513);
+ assertArrayRegionsEqual(original, 1024, buf, 1024, 513);
+ reader.readFully(buf, 1537, 514);
+ assertArrayRegionsEqual(original, 1537, buf, 1537, 514);
+ }
+ }
+
+ @Test
+ public void testBlockReaderSimpleReads() throws IOException {
+ runBlockReaderLocalTest(new TestBlockReaderSimpleReads(), true);
+ }
+
+ @Test
+ public void testBlockReaderSimpleReadsNoChecksum() throws IOException {
+ runBlockReaderLocalTest(new TestBlockReaderSimpleReads(), false);
+ }
+
+ private static class TestBlockReaderLocalArrayReads2
+ implements BlockReaderLocalTest {
+ @Override
+ public void setup(File blockFile, boolean usingChecksums)
+ throws IOException { }
+ @Override
+ public void doTest(BlockReaderLocal reader, byte original[])
+ throws IOException {
+ byte buf[] = new byte[TEST_LENGTH];
+ reader.readFully(buf, 0, 10);
+ assertArrayRegionsEqual(original, 0, buf, 0, 10);
+ reader.readFully(buf, 10, 100);
+ assertArrayRegionsEqual(original, 10, buf, 10, 100);
+ reader.readFully(buf, 110, 700);
+ assertArrayRegionsEqual(original, 110, buf, 110, 700);
+ reader.readFully(buf, 810, 1); // from offset 810 to offset 811
+ reader.readFully(buf, 811, 5);
+ assertArrayRegionsEqual(original, 811, buf, 811, 5);
+ reader.readFully(buf, 816, 900); // skip from offset 816 to offset 1716
+ reader.readFully(buf, 1716, 5);
+ assertArrayRegionsEqual(original, 1716, buf, 1716, 5);
+ }
+ }
+
+ @Test
+ public void testBlockReaderLocalArrayReads2() throws IOException {
+ runBlockReaderLocalTest(new TestBlockReaderLocalArrayReads2(),
+ true);
+ }
- conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
- conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_SKIP_CHECKSUM_KEY,
+ @Test
+ public void testBlockReaderLocalArrayReads2NoChecksum()
+ throws IOException {
+ runBlockReaderLocalTest(new TestBlockReaderLocalArrayReads2(),
false);
- conf.set(DFSConfigKeys.DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
- UserGroupInformation.getCurrentUser().getShortUserName());
+ }
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ private static class TestBlockReaderLocalByteBufferReads
+ implements BlockReaderLocalTest {
+ @Override
+ public void setup(File blockFile, boolean usingChecksums)
+ throws IOException { }
+ @Override
+ public void doTest(BlockReaderLocal reader, byte original[])
+ throws IOException {
+ ByteBuffer buf = ByteBuffer.wrap(new byte[TEST_LENGTH]);
+ readFully(reader, buf, 0, 10);
+ assertArrayRegionsEqual(original, 0, buf.array(), 0, 10);
+ readFully(reader, buf, 10, 100);
+ assertArrayRegionsEqual(original, 10, buf.array(), 10, 100);
+ readFully(reader, buf, 110, 700);
+ assertArrayRegionsEqual(original, 110, buf.array(), 110, 700);
+ reader.skip(1); // skip from offset 810 to offset 811
+ readFully(reader, buf, 811, 5);
+ assertArrayRegionsEqual(original, 811, buf.array(), 811, 5);
+ }
+ }
+
+ @Test
+ public void testBlockReaderLocalByteBufferReads()
+ throws IOException {
+ runBlockReaderLocalTest(
+ new TestBlockReaderLocalByteBufferReads(), true);
}
- @AfterClass
- public static void teardownCluster() {
- cluster.shutdown();
+ @Test
+ public void testBlockReaderLocalByteBufferReadsNoChecksum()
+ throws IOException {
+ runBlockReaderLocalTest(
+ new TestBlockReaderLocalByteBufferReads(), false);
}
+
+ private static class TestBlockReaderLocalReadCorruptStart
+ implements BlockReaderLocalTest {
+ boolean usingChecksums = false;
+ @Override
+ public void setup(File blockFile, boolean usingChecksums)
+ throws IOException {
+ RandomAccessFile bf = null;
+ this.usingChecksums = usingChecksums;
+ try {
+ bf = new RandomAccessFile(blockFile, "rw");
+ bf.write(new byte[] {0,0,0,0,0,0,0,0,0,0,0,0,0,0});
+ } finally {
+ if (bf != null) bf.close();
+ }
+ }
- /**
- * Test that, in the case of an error, the position and limit of a ByteBuffer
- * are left unchanged. This is not mandated by ByteBufferReadable, but clients
- * of this class might immediately issue a retry on failure, so it's polite.
- */
+ public void doTest(BlockReaderLocal reader, byte original[])
+ throws IOException {
+ byte buf[] = new byte[TEST_LENGTH];
+ if (usingChecksums) {
+ try {
+ reader.readFully(buf, 0, 10);
+ Assert.fail("did not detect corruption");
+ } catch (IOException e) {
+ // expected
+ }
+ } else {
+ reader.readFully(buf, 0, 10);
+ }
+ }
+ }
+
@Test
- public void testStablePositionAfterCorruptRead() throws Exception {
- final short REPL_FACTOR = 1;
- final long FILE_LENGTH = 512L;
- cluster.waitActive();
- FileSystem fs = cluster.getFileSystem();
-
- Path path = new Path("/corrupted");
-
- DFSTestUtil.createFile(fs, path, FILE_LENGTH, REPL_FACTOR, 12345L);
- DFSTestUtil.waitReplication(fs, path, REPL_FACTOR);
-
- ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, path);
- int blockFilesCorrupted = cluster.corruptBlockOnDataNodes(block);
- assertEquals("All replicas not corrupted", REPL_FACTOR, blockFilesCorrupted);
-
- FSDataInputStream dis = cluster.getFileSystem().open(path);
- ByteBuffer buf = ByteBuffer.allocateDirect((int)FILE_LENGTH);
- boolean sawException = false;
- try {
- dis.read(buf);
- } catch (ChecksumException ex) {
- sawException = true;
+ public void testBlockReaderLocalReadCorruptStart()
+ throws IOException {
+ runBlockReaderLocalTest(new TestBlockReaderLocalReadCorruptStart(), true);
+ }
+
+ private static class TestBlockReaderLocalReadCorrupt
+ implements BlockReaderLocalTest {
+ boolean usingChecksums = false;
+ @Override
+ public void setup(File blockFile, boolean usingChecksums)
+ throws IOException {
+ RandomAccessFile bf = null;
+ this.usingChecksums = usingChecksums;
+ try {
+ bf = new RandomAccessFile(blockFile, "rw");
+ bf.seek(1539);
+ bf.write(new byte[] {0,0,0,0,0,0,0,0,0,0,0,0,0,0});
+ } finally {
+ if (bf != null) bf.close();
+ }
}
- assertTrue(sawException);
- assertEquals(0, buf.position());
- assertEquals(buf.capacity(), buf.limit());
-
- dis = cluster.getFileSystem().open(path);
- buf.position(3);
- buf.limit(25);
- sawException = false;
- try {
- dis.read(buf);
- } catch (ChecksumException ex) {
- sawException = true;
+ public void doTest(BlockReaderLocal reader, byte original[])
+ throws IOException {
+ byte buf[] = new byte[TEST_LENGTH];
+ try {
+ reader.readFully(buf, 0, 10);
+ assertArrayRegionsEqual(original, 0, buf, 0, 10);
+ reader.readFully(buf, 10, 100);
+ assertArrayRegionsEqual(original, 10, buf, 10, 100);
+ reader.readFully(buf, 110, 700);
+ assertArrayRegionsEqual(original, 110, buf, 110, 700);
+ reader.skip(1); // skip from offset 810 to offset 811
+ reader.readFully(buf, 811, 5);
+ assertArrayRegionsEqual(original, 811, buf, 811, 5);
+ reader.readFully(buf, 816, 900);
+ if (usingChecksums) {
+ // We should detect the corruption when using a checksum file.
+ Assert.fail("did not detect corruption");
+ }
+ } catch (ChecksumException e) {
+ if (!usingChecksums) {
+ Assert.fail("didn't expect to get ChecksumException: not " +
+ "using checksums.");
+ }
+ }
}
+ }
- assertTrue(sawException);
- assertEquals(3, buf.position());
- assertEquals(25, buf.limit());
+ @Test
+ public void testBlockReaderLocalReadCorrupt()
+ throws IOException {
+ runBlockReaderLocalTest(new TestBlockReaderLocalReadCorrupt(), true);
+ runBlockReaderLocalTest(new TestBlockReaderLocalReadCorrupt(), false);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientBlockVerification.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientBlockVerification.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientBlockVerification.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientBlockVerification.java Sat Apr 13 23:05:54 2013
@@ -61,7 +61,7 @@ public class TestClientBlockVerification
util.getBlockReader(testBlock, 0, FILE_SIZE_K * 1024));
util.readAndCheckEOS(reader, FILE_SIZE_K * 1024, true);
verify(reader).sendReadResult(Status.CHECKSUM_OK);
- reader.close();
+ reader.close(null, null);
}
/**
@@ -76,7 +76,7 @@ public class TestClientBlockVerification
// We asked the blockreader for the whole file, and only read
// half of it, so no CHECKSUM_OK
verify(reader, never()).sendReadResult(Status.CHECKSUM_OK);
- reader.close();
+ reader.close(null, null);
}
/**
@@ -92,7 +92,7 @@ public class TestClientBlockVerification
// And read half the file
util.readAndCheckEOS(reader, FILE_SIZE_K * 1024 / 2, true);
verify(reader).sendReadResult(Status.CHECKSUM_OK);
- reader.close();
+ reader.close(null, null);
}
/**
@@ -111,7 +111,7 @@ public class TestClientBlockVerification
util.getBlockReader(testBlock, startOffset, length));
util.readAndCheckEOS(reader, length, true);
verify(reader).sendReadResult(Status.CHECKSUM_OK);
- reader.close();
+ reader.close(null, null);
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestConnCache.java Sat Apr 13 23:05:54 2013
@@ -18,28 +18,20 @@
package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.spy;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.security.PrivilegedExceptionAction;
+
+import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.security.token.Token;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.Matchers;
import org.mockito.Mockito;
@@ -55,59 +47,31 @@ public class TestConnCache {
static final int BLOCK_SIZE = 4096;
static final int FILE_SIZE = 3 * BLOCK_SIZE;
- final static int CACHE_SIZE = 4;
- final static long CACHE_EXPIRY_MS = 200;
- static Configuration conf = null;
- static MiniDFSCluster cluster = null;
- static FileSystem fs = null;
- static SocketCache cache;
-
- static final Path testFile = new Path("/testConnCache.dat");
- static byte authenticData[] = null;
-
- static BlockReaderTestUtil util = null;
-
/**
* A mock Answer to remember the BlockReader used.
*
* It verifies that all invocation to DFSInputStream.getBlockReader()
- * use the same socket.
+ * use the same peer.
*/
private class MockGetBlockReader implements Answer<RemoteBlockReader2> {
public RemoteBlockReader2 reader = null;
- private Socket sock = null;
+ private Peer peer = null;
@Override
public RemoteBlockReader2 answer(InvocationOnMock invocation) throws Throwable {
RemoteBlockReader2 prevReader = reader;
reader = (RemoteBlockReader2) invocation.callRealMethod();
- if (sock == null) {
- sock = reader.dnSock;
+ if (peer == null) {
+ peer = reader.getPeer();
} else if (prevReader != null) {
- assertSame("DFSInputStream should use the same socket",
- sock, reader.dnSock);
+ Assert.assertSame("DFSInputStream should use the same peer",
+ peer, reader.getPeer());
}
return reader;
}
}
- @BeforeClass
- public static void setupCluster() throws Exception {
- final int REPLICATION_FACTOR = 1;
-
- /* create a socket cache. There is only one socket cache per jvm */
- cache = SocketCache.getInstance(CACHE_SIZE, CACHE_EXPIRY_MS);
-
- util = new BlockReaderTestUtil(REPLICATION_FACTOR);
- cluster = util.getCluster();
- conf = util.getConf();
- fs = cluster.getFileSystem();
-
- authenticData = util.writeFile(testFile, FILE_SIZE / 1024);
- }
-
-
/**
* (Optionally) seek to position, read and verify data.
*
@@ -117,9 +81,10 @@ public class TestConnCache {
long pos,
byte[] buffer,
int offset,
- int length)
+ int length,
+ byte[] authenticData)
throws IOException {
- assertTrue("Test buffer too small", buffer.length >= offset + length);
+ Assert.assertTrue("Test buffer too small", buffer.length >= offset + length);
if (pos >= 0)
in.seek(pos);
@@ -129,7 +94,7 @@ public class TestConnCache {
while (length > 0) {
int cnt = in.read(buffer, offset, length);
- assertTrue("Error in read", cnt > 0);
+ Assert.assertTrue("Error in read", cnt > 0);
offset += cnt;
length -= cnt;
}
@@ -145,115 +110,22 @@ public class TestConnCache {
}
/**
- * Test the SocketCache itself.
- */
- @Test
- public void testSocketCache() throws Exception {
- // Make a client
- InetSocketAddress nnAddr =
- new InetSocketAddress("localhost", cluster.getNameNodePort());
- DFSClient client = new DFSClient(nnAddr, conf);
-
- // Find out the DN addr
- LocatedBlock block =
- client.getNamenode().getBlockLocations(
- testFile.toString(), 0, FILE_SIZE)
- .getLocatedBlocks().get(0);
- DataNode dn = util.getDataNode(block);
- InetSocketAddress dnAddr = dn.getXferAddress();
-
-
- // Make some sockets to the DN
- Socket[] dnSockets = new Socket[CACHE_SIZE];
- for (int i = 0; i < dnSockets.length; ++i) {
- dnSockets[i] = client.socketFactory.createSocket(
- dnAddr.getAddress(), dnAddr.getPort());
- }
-
-
- // Insert a socket to the NN
- Socket nnSock = new Socket(nnAddr.getAddress(), nnAddr.getPort());
- cache.put(nnSock, null);
- assertSame("Read the write", nnSock, cache.get(nnAddr).sock);
- cache.put(nnSock, null);
-
- // Insert DN socks
- for (Socket dnSock : dnSockets) {
- cache.put(dnSock, null);
- }
-
- assertEquals("NN socket evicted", null, cache.get(nnAddr));
- assertTrue("Evicted socket closed", nnSock.isClosed());
-
- // Lookup the DN socks
- for (Socket dnSock : dnSockets) {
- assertEquals("Retrieve cached sockets", dnSock, cache.get(dnAddr).sock);
- dnSock.close();
- }
-
- assertEquals("Cache is empty", 0, cache.size());
- }
-
-
- /**
- * Test the SocketCache expiry.
- * Verify that socket cache entries expire after the set
- * expiry time.
- */
- @Test
- public void testSocketCacheExpiry() throws Exception {
- // Make a client
- InetSocketAddress nnAddr =
- new InetSocketAddress("localhost", cluster.getNameNodePort());
- DFSClient client = new DFSClient(nnAddr, conf);
-
- // Find out the DN addr
- LocatedBlock block =
- client.getNamenode().getBlockLocations(
- testFile.toString(), 0, FILE_SIZE)
- .getLocatedBlocks().get(0);
- DataNode dn = util.getDataNode(block);
- InetSocketAddress dnAddr = dn.getXferAddress();
-
-
- // Make some sockets to the DN and put in cache
- Socket[] dnSockets = new Socket[CACHE_SIZE];
- for (int i = 0; i < dnSockets.length; ++i) {
- dnSockets[i] = client.socketFactory.createSocket(
- dnAddr.getAddress(), dnAddr.getPort());
- cache.put(dnSockets[i], null);
- }
-
- // Client side still has the sockets cached
- assertEquals(CACHE_SIZE, client.socketCache.size());
-
- //sleep for a second and see if it expired
- Thread.sleep(CACHE_EXPIRY_MS + 1000);
-
- // Client side has no sockets cached
- assertEquals(0, client.socketCache.size());
-
- //sleep for another second and see if
- //the daemon thread runs fine on empty cache
- Thread.sleep(CACHE_EXPIRY_MS + 1000);
- }
-
-
- /**
* Read a file served entirely from one DN. Seek around and read from
* different offsets. And verify that they all use the same socket.
- *
- * @throws java.io.IOException
+ * @throws Exception
*/
@Test
@SuppressWarnings("unchecked")
- public void testReadFromOneDN() throws IOException {
- LOG.info("Starting testReadFromOneDN()");
+ public void testReadFromOneDN() throws Exception {
+ BlockReaderTestUtil util = new BlockReaderTestUtil(1,
+ new HdfsConfiguration());
+ final Path testFile = new Path("/testConnCache.dat");
+ byte authenticData[] = util.writeFile(testFile, FILE_SIZE / 1024);
DFSClient client = new DFSClient(
- new InetSocketAddress("localhost", cluster.getNameNodePort()), conf);
- DFSInputStream in = spy(client.open(testFile.toString()));
+ new InetSocketAddress("localhost",
+ util.getCluster().getNameNodePort()), util.getConf());
+ DFSInputStream in = Mockito.spy(client.open(testFile.toString()));
LOG.info("opened " + testFile.toString());
-
byte[] dataBuf = new byte[BLOCK_SIZE];
MockGetBlockReader answer = new MockGetBlockReader();
@@ -270,18 +142,15 @@ public class TestConnCache {
Matchers.anyString());
// Initial read
- pread(in, 0, dataBuf, 0, dataBuf.length);
+ pread(in, 0, dataBuf, 0, dataBuf.length, authenticData);
// Read again and verify that the socket is the same
- pread(in, FILE_SIZE - dataBuf.length, dataBuf, 0, dataBuf.length);
- pread(in, 1024, dataBuf, 0, dataBuf.length);
- pread(in, -1, dataBuf, 0, dataBuf.length); // No seek; just read
- pread(in, 64, dataBuf, 0, dataBuf.length / 2);
+ pread(in, FILE_SIZE - dataBuf.length, dataBuf, 0, dataBuf.length,
+ authenticData);
+ pread(in, 1024, dataBuf, 0, dataBuf.length, authenticData);
+ // No seek; just read
+ pread(in, -1, dataBuf, 0, dataBuf.length, authenticData);
+ pread(in, 64, dataBuf, 0, dataBuf.length / 2, authenticData);
in.close();
}
-
- @AfterClass
- public static void teardownCluster() throws Exception {
- util.shutdown();
- }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java Sat Apr 13 23:05:54 2013
@@ -53,7 +53,7 @@ import com.google.common.base.Joiner;
*/
public class TestDFSUpgrade {
- private static final int EXPECTED_TXID = 49;
+ private static final int EXPECTED_TXID = 45;
private static final Log LOG = LogFactory.getLog(TestDFSUpgrade.class.getName());
private Configuration conf;
private int testCounter = 0;
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDataTransferKeepalive.java Sat Apr 13 23:05:54 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.FSDataInputS
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
+import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
@@ -70,7 +71,7 @@ public class TestDataTransferKeepalive {
.numDataNodes(1).build();
fs = cluster.getFileSystem();
dfsClient = ((DistributedFileSystem)fs).dfs;
- dfsClient.socketCache.clear();
+ dfsClient.peerCache.clear();
String poolId = cluster.getNamesystem().getBlockPoolId();
dn = cluster.getDataNodes().get(0);
@@ -93,13 +94,13 @@ public class TestDataTransferKeepalive {
DFSTestUtil.createFile(fs, TEST_FILE, 1L, (short)1, 0L);
// Clients that write aren't currently re-used.
- assertEquals(0, dfsClient.socketCache.size());
+ assertEquals(0, dfsClient.peerCache.size());
assertXceiverCount(0);
// Reads the file, so we should get a
// cached socket, and should have an xceiver on the other side.
DFSTestUtil.readFile(fs, TEST_FILE);
- assertEquals(1, dfsClient.socketCache.size());
+ assertEquals(1, dfsClient.peerCache.size());
assertXceiverCount(1);
// Sleep for a bit longer than the keepalive timeout
@@ -110,13 +111,13 @@ public class TestDataTransferKeepalive {
// The socket is still in the cache, because we don't
// notice that it's closed until we try to read
// from it again.
- assertEquals(1, dfsClient.socketCache.size());
+ assertEquals(1, dfsClient.peerCache.size());
// Take it out of the cache - reading should
// give an EOF.
- Socket s = dfsClient.socketCache.get(dnAddr).sock;
- assertNotNull(s);
- assertEquals(-1, NetUtils.getInputStream(s).read());
+ Peer peer = dfsClient.peerCache.get(dn.getDatanodeId(), false);
+ assertNotNull(peer);
+ assertEquals(-1, peer.getInputStream().read());
}
/**
@@ -175,14 +176,14 @@ public class TestDataTransferKeepalive {
}
DFSClient client = ((DistributedFileSystem)fs).dfs;
- assertEquals(5, client.socketCache.size());
+ assertEquals(5, client.peerCache.size());
// Let all the xceivers timeout
Thread.sleep(1500);
assertXceiverCount(0);
// Client side still has the sockets cached
- assertEquals(5, client.socketCache.size());
+ assertEquals(5, client.peerCache.size());
// Reading should not throw an exception.
DFSTestUtil.readFile(fs, TEST_FILE);
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelRead.java Sat Apr 13 23:05:54 2013
@@ -17,53 +17,30 @@
*/
package org.apache.hadoop.hdfs;
-import java.io.IOException;
-
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Test;
public class TestParallelRead extends TestParallelReadUtil {
-
@BeforeClass
static public void setupCluster() throws Exception {
- setupCluster(DEFAULT_REPLICATION_FACTOR, new HdfsConfiguration());
+ // This is a test of the normal (TCP) read path. For this reason, we turn
+ // off both short-circuit local reads and UNIX domain socket data traffic.
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, false);
+ conf.setBoolean(DFSConfigKeys.DFS_CLIENT_DOMAIN_SOCKET_DATA_TRAFFIC,
+ false);
+ // dfs.domain.socket.path should be ignored because the previous two keys
+ // were set to false. This is a regression test for HDFS-4473.
+ conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY, "/will/not/be/created");
+
+ setupCluster(DEFAULT_REPLICATION_FACTOR, conf);
}
@AfterClass
static public void teardownCluster() throws Exception {
TestParallelReadUtil.teardownCluster();
}
-
- /**
- * Do parallel read several times with different number of files and threads.
- *
- * Note that while this is the only "test" in a junit sense, we're actually
- * dispatching a lot more. Failures in the other methods (and other threads)
- * need to be manually collected, which is inconvenient.
- */
- @Test
- public void testParallelReadCopying() throws IOException {
- runTestWorkload(new CopyingReadWorkerHelper());
- }
-
- @Test
- public void testParallelReadByteBuffer() throws IOException {
- runTestWorkload(new DirectReadWorkerHelper());
- }
-
- @Test
- public void testParallelReadMixed() throws IOException {
- runTestWorkload(new MixedWorkloadHelper());
- }
-
- @Test
- public void testParallelNoChecksums() throws IOException {
- verifyChecksums = false;
- runTestWorkload(new MixedWorkloadHelper());
- }
-
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestParallelReadUtil.java Sat Apr 13 23:05:54 2013
@@ -32,12 +32,18 @@ import org.apache.hadoop.hdfs.server.dat
import org.apache.hadoop.util.Time;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
+import org.junit.Assume;
+import org.junit.Ignore;
+import org.junit.Test;
/**
* Driver class for testing the use of DFSInputStream by multiple concurrent
- * readers, using the different read APIs. See subclasses for the actual test
- * cases.
+ * readers, using the different read APIs.
+ *
+ * This class is marked as @Ignore so that junit doesn't try to execute the
+ * tests in here directly. They are executed from subclasses.
*/
+@Ignore
public class TestParallelReadUtil {
static final Log LOG = LogFactory.getLog(TestParallelReadUtil.class);
@@ -388,4 +394,31 @@ public class TestParallelReadUtil {
util.shutdown();
}
+ /**
+ * Do parallel read several times with different number of files and threads.
+ *
+ * Note that while this is the only "test" in a junit sense, we're actually
+ * dispatching a lot more. Failures in the other methods (and other threads)
+ * need to be manually collected, which is inconvenient.
+ */
+ @Test
+ public void testParallelReadCopying() throws IOException {
+ runTestWorkload(new CopyingReadWorkerHelper());
+ }
+
+ @Test
+ public void testParallelReadByteBuffer() throws IOException {
+ runTestWorkload(new DirectReadWorkerHelper());
+ }
+
+ @Test
+ public void testParallelReadMixed() throws IOException {
+ runTestWorkload(new MixedWorkloadHelper());
+ }
+
+ @Test
+ public void testParallelNoChecksums() throws IOException {
+ verifyChecksums = false;
+ runTestWorkload(new MixedWorkloadHelper());
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java Sat Apr 13 23:05:54 2013
@@ -99,7 +99,7 @@ public class TestSafeMode {
*/
@Test
public void testManualSafeMode() throws IOException {
- fs = (DistributedFileSystem)cluster.getFileSystem();
+ fs = cluster.getFileSystem();
Path file1 = new Path("/tmp/testManualSafeMode/file1");
Path file2 = new Path("/tmp/testManualSafeMode/file2");
@@ -112,7 +112,7 @@ public class TestSafeMode {
// now bring up just the NameNode.
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(false).build();
cluster.waitActive();
- dfs = (DistributedFileSystem)cluster.getFileSystem();
+ dfs = cluster.getFileSystem();
assertTrue("No datanode is started. Should be in SafeMode",
dfs.setSafeMode(SafeModeAction.SAFEMODE_GET));
@@ -322,11 +322,11 @@ public class TestSafeMode {
fs.rename(file1, new Path("file2"));
}});
- try {
- fs.setTimes(file1, 0, 0);
- } catch (IOException ioe) {
- fail("Set times failed while in SM");
- }
+ runFsFun("Set time while in SM", new FSRun() {
+ @Override
+ public void run(FileSystem fs) throws IOException {
+ fs.setTimes(file1, 0, 0);
+ }});
try {
DFSTestUtil.readFile(fs, file1);
@@ -350,7 +350,7 @@ public class TestSafeMode {
conf.setInt(DFSConfigKeys.DFS_NAMENODE_SAFEMODE_MIN_DATANODES_KEY, 1);
cluster.restartNameNode();
- fs = (DistributedFileSystem)cluster.getFileSystem();
+ fs = cluster.getFileSystem();
String tipMsg = cluster.getNamesystem().getSafemode();
assertTrue("Safemode tip message looks right: " + tipMsg,
@@ -375,7 +375,7 @@ public class TestSafeMode {
* @throws IOException when there's an issue connecting to the test DFS.
*/
public void testSafeModeUtils() throws IOException {
- dfs = (DistributedFileSystem)cluster.getFileSystem();
+ dfs = cluster.getFileSystem();
// Enter safemode.
dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);