You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by su...@apache.org on 2011/03/08 02:17:09 UTC
svn commit: r1079067 [2/2] - in /hadoop/hdfs/branches/HDFS-1052: ./ bin/
src/c++/libhdfs/ src/c++/libhdfs/m4/ src/contrib/hdfsproxy/
src/docs/src/documentation/content/xdocs/ src/java/
src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apach...
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Tue Mar 8 01:17:07 2011
@@ -34,7 +34,6 @@ import org.apache.hadoop.hdfs.security.t
import static org.apache.hadoop.hdfs.server.common.Util.now;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLog.Ops;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
@@ -122,11 +121,12 @@ public class FSEditLogLoader {
long mtime = 0;
long atime = 0;
long blockSize = 0;
- byte opcode = -1;
+ FSEditLogOpCodes opCode;
try {
in.mark(1);
- opcode = in.readByte();
- if (opcode == Ops.OP_INVALID) {
+ byte opCodeByte = in.readByte();
+ opCode = FSEditLogOpCodes.fromByte(opCodeByte);
+ if (opCode == FSEditLogOpCodes.OP_INVALID) {
in.reset(); // reset back to end of file if somebody reads it again
break; // no more transactions
}
@@ -134,9 +134,9 @@ public class FSEditLogLoader {
break; // no more transactions
}
numEdits++;
- switch (opcode) {
- case Ops.OP_ADD:
- case Ops.OP_CLOSE: {
+ switch (opCode) {
+ case OP_ADD:
+ case OP_CLOSE: {
// versions > 0 support per file replication
// get name and replication
int length = in.readInt();
@@ -158,7 +158,7 @@ public class FSEditLogLoader {
blockSize = readLong(in);
}
// get blocks
- boolean isFileUnderConstruction = (opcode == Ops.OP_ADD);
+ boolean isFileUnderConstruction = (opCode == FSEditLogOpCodes.OP_ADD);
BlockInfo blocks[] =
readBlocks(in, logVersion, isFileUnderConstruction, replication);
@@ -181,7 +181,7 @@ public class FSEditLogLoader {
}
// clientname, clientMachine and block locations of last block.
- if (opcode == Ops.OP_ADD && logVersion <= -12) {
+ if (opCode == FSEditLogOpCodes.OP_ADD && logVersion <= -12) {
clientName = FSImageSerialization.readString(in);
clientMachine = FSImageSerialization.readString(in);
if (-13 <= logVersion) {
@@ -195,7 +195,7 @@ public class FSEditLogLoader {
// The open lease transaction re-creates a file if necessary.
// Delete the file if it already exists.
if (FSNamesystem.LOG.isDebugEnabled()) {
- FSNamesystem.LOG.debug(opcode + ": " + path +
+ FSNamesystem.LOG.debug(opCode + ": " + path +
" numblocks : " + blocks.length +
" clientHolder " + clientName +
" clientMachine " + clientMachine);
@@ -229,16 +229,16 @@ public class FSEditLogLoader {
}
break;
}
- case Ops.OP_SET_REPLICATION: {
+ case OP_SET_REPLICATION: {
numOpSetRepl++;
path = FSImageSerialization.readString(in);
short replication = fsNamesys.adjustReplication(readShort(in));
fsDir.unprotectedSetReplication(path, replication, null);
break;
}
- case Ops.OP_CONCAT_DELETE: {
+ case OP_CONCAT_DELETE: {
if (logVersion > -22) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpConcatDelete++;
@@ -257,7 +257,7 @@ public class FSEditLogLoader {
fsDir.unprotectedConcat(trg, srcs);
break;
}
- case Ops.OP_RENAME_OLD: {
+ case OP_RENAME_OLD: {
numOpRenameOld++;
int length = in.readInt();
if (length != 3) {
@@ -272,7 +272,7 @@ public class FSEditLogLoader {
fsNamesys.changeLease(s, d, dinfo);
break;
}
- case Ops.OP_DELETE: {
+ case OP_DELETE: {
numOpDelete++;
int length = in.readInt();
if (length != 2) {
@@ -284,7 +284,7 @@ public class FSEditLogLoader {
fsDir.unprotectedDelete(path, timestamp);
break;
}
- case Ops.OP_MKDIR: {
+ case OP_MKDIR: {
numOpMkDir++;
PermissionStatus permissions = fsNamesys.getUpgradePermission();
int length = in.readInt();
@@ -309,47 +309,47 @@ public class FSEditLogLoader {
fsDir.unprotectedMkdir(path, permissions, timestamp);
break;
}
- case Ops.OP_SET_GENSTAMP: {
+ case OP_SET_GENSTAMP: {
numOpSetGenStamp++;
long lw = in.readLong();
fsNamesys.setGenerationStamp(lw);
break;
}
- case Ops.OP_DATANODE_ADD: {
+ case OP_DATANODE_ADD: {
numOpOther++;
//Datanodes are not persistent any more.
FSImageSerialization.DatanodeImage.skipOne(in);
break;
}
- case Ops.OP_DATANODE_REMOVE: {
+ case OP_DATANODE_REMOVE: {
numOpOther++;
DatanodeID nodeID = new DatanodeID();
nodeID.readFields(in);
//Datanodes are not persistent any more.
break;
}
- case Ops.OP_SET_PERMISSIONS: {
+ case OP_SET_PERMISSIONS: {
numOpSetPerm++;
if (logVersion > -11)
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
fsDir.unprotectedSetPermission(
FSImageSerialization.readString(in), FsPermission.read(in));
break;
}
- case Ops.OP_SET_OWNER: {
+ case OP_SET_OWNER: {
numOpSetOwner++;
if (logVersion > -11)
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
fsDir.unprotectedSetOwner(FSImageSerialization.readString(in),
FSImageSerialization.readString_EmptyAsNull(in),
FSImageSerialization.readString_EmptyAsNull(in));
break;
}
- case Ops.OP_SET_NS_QUOTA: {
+ case OP_SET_NS_QUOTA: {
if (logVersion > -16) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
@@ -357,9 +357,9 @@ public class FSEditLogLoader {
FSConstants.QUOTA_DONT_SET);
break;
}
- case Ops.OP_CLEAR_NS_QUOTA: {
+ case OP_CLEAR_NS_QUOTA: {
if (logVersion > -16) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
@@ -368,14 +368,14 @@ public class FSEditLogLoader {
break;
}
- case Ops.OP_SET_QUOTA:
+ case OP_SET_QUOTA:
fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
readLongWritable(in),
readLongWritable(in));
break;
- case Ops.OP_TIMES: {
+ case OP_TIMES: {
numOpTimes++;
int length = in.readInt();
if (length != 3) {
@@ -388,7 +388,7 @@ public class FSEditLogLoader {
fsDir.unprotectedSetTimes(path, mtime, atime, true);
break;
}
- case Ops.OP_SYMLINK: {
+ case OP_SYMLINK: {
numOpSymlink++;
int length = in.readInt();
if (length != 4) {
@@ -403,9 +403,9 @@ public class FSEditLogLoader {
fsDir.unprotectedSymlink(path, value, mtime, atime, perm);
break;
}
- case Ops.OP_RENAME: {
+ case OP_RENAME: {
if (logVersion > -21) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpRename++;
@@ -423,9 +423,9 @@ public class FSEditLogLoader {
fsNamesys.changeLease(s, d, dinfo);
break;
}
- case Ops.OP_GET_DELEGATION_TOKEN: {
+ case OP_GET_DELEGATION_TOKEN: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpGetDelegationToken++;
@@ -437,9 +437,9 @@ public class FSEditLogLoader {
.addPersistedDelegationToken(delegationTokenId, expiryTime);
break;
}
- case Ops.OP_RENEW_DELEGATION_TOKEN: {
+ case OP_RENEW_DELEGATION_TOKEN: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpRenewDelegationToken++;
@@ -451,9 +451,9 @@ public class FSEditLogLoader {
.updatePersistedTokenRenewal(delegationTokenId, expiryTime);
break;
}
- case Ops.OP_CANCEL_DELEGATION_TOKEN: {
+ case OP_CANCEL_DELEGATION_TOKEN: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpCancelDelegationToken++;
@@ -464,9 +464,9 @@ public class FSEditLogLoader {
.updatePersistedTokenCancellation(delegationTokenId);
break;
}
- case Ops.OP_UPDATE_MASTER_KEY: {
+ case OP_UPDATE_MASTER_KEY: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpUpdateMasterKey++;
@@ -477,7 +477,7 @@ public class FSEditLogLoader {
break;
}
default: {
- throw new IOException("Never seen opcode " + opcode);
+ throw new IOException("Never seen opCode " + opCode);
}
}
}
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Mar 8 01:17:07 2011
@@ -100,6 +100,7 @@ import java.lang.management.ManagementFa
import java.net.InetAddress;
import java.net.URI;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.Map.Entry;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -186,7 +187,8 @@ public class FSNamesystem implements FSC
private long blockTokenLifetime;
// Scan interval is not configurable.
- private final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL = 3600000; // 1 hour
+ private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL =
+ TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
private DelegationTokenSecretManager dtSecretManager;
//
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue Mar 8 01:17:07 2011
@@ -85,6 +85,7 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetUtils;
@@ -162,6 +163,14 @@ public class NameNode implements Namenod
}
}
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ return ProtocolSignature.getProtocolSignature(
+ this, protocol, clientVersion, clientMethodsHash);
+ }
+
public static final int DEFAULT_PORT = 8020;
public static final Log LOG = LogFactory.getLog(NameNode.class.getName());
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Tue Mar 8 01:17:07 2011
@@ -198,7 +198,7 @@ public class SecondaryNameNode implement
if(UserGroupInformation.isSecurityEnabled()) {
System.setProperty("https.cipherSuites",
- Krb5AndCertsSslSocketConnector.KRB5_CIPHER_SUITES[0]);
+ Krb5AndCertsSslSocketConnector.KRB5_CIPHER_SUITES.get(0));
InetSocketAddress secInfoSocAddr =
NetUtils.createSocketAddr(infoBindAddress + ":"+ conf.get(
"dfs.secondary.https.port", infoBindAddress + ":" + 0));
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java Tue Mar 8 01:17:07 2011
@@ -54,8 +54,8 @@ public interface NamenodeProtocol extend
// Journal action codes. See journal().
public static byte JA_IS_ALIVE = 100; // check whether the journal is alive
public static byte JA_JOURNAL = 101; // just journal
- public static byte JA_JSPOOL_START = 102; // = FSEditLog.OP_JSPOOL_START
- public static byte JA_CHECKPOINT_TIME = 103; // = FSEditLog.OP_CHECKPOINT_TIME
+ public static byte JA_JSPOOL_START = 102; // = FSEditLogOpCodes.OP_JSPOOL_START
+ public static byte JA_CHECKPOINT_TIME = 103; // = FSEditLogOpCodes.OP_CHECKPOINT_TIME
public final static int ACT_UNKNOWN = 0; // unknown action
public final static int ACT_SHUTDOWN = 50; // shutdown node
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/DFSck.java Tue Mar 8 01:17:07 2011
@@ -268,7 +268,7 @@ public class DFSck extends Configured im
String proto = "http://";
if (UserGroupInformation.isSecurityEnabled()) {
System.setProperty("https.cipherSuites",
- Krb5AndCertsSslSocketConnector.KRB5_CIPHER_SUITES[0]);
+ Krb5AndCertsSslSocketConnector.KRB5_CIPHER_SUITES.get(0));
proto = "https://";
}
final StringBuilder url = new StringBuilder(proto);
Modified: hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java Tue Mar 8 01:17:07 2011
@@ -75,17 +75,6 @@ abstract class ImageVisitor {
}
/**
- * Utility class for tracking descent into the structure of the fsimage.
- */
- static protected class DepthCounter {
- private int depth = 0;
-
- public void incLevel() { depth++; }
- public void decLevel() { if(depth >= 1) depth--; }
- public int getLevel() { return depth; }
- }
-
- /**
* Begin visiting the fsimage structure. Opportunity to perform
* any initialization necessary for the implementing visitor.
*/
Propchange: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 01:17:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066986
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java Tue Mar 8 01:17:07 2011
@@ -23,6 +23,8 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.io.FileInputStream;
+import java.io.DataInputStream;
import java.net.URL;
import java.net.URLConnection;
import java.security.PrivilegedExceptionAction;
@@ -376,4 +378,15 @@ public class DFSTestUtil {
public static Statistics getStatistics(FileSystem fs) {
return FileSystem.getStatistics(fs.getUri().getScheme(), fs.getClass());
}
+
+ /**
+ * Load file into byte[]
+ */
+ public static byte[] loadFile(String filename) throws IOException {
+ File file = new File(filename);
+ DataInputStream in = new DataInputStream(new FileInputStream(file));
+ byte[] content = new byte[(int)file.length()];
+ in.readFully(content);
+ return content;
+ }
}
Copied: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java (from r1065960, hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java)
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java?p2=hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java&p1=hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java&r1=1065960&r2=1079067&rev=1079067&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java Tue Mar 8 01:17:07 2011
@@ -138,7 +138,7 @@ public class OfflineEditsViewerHelper {
// no check, if it's not it throws an exception which is what we want
DistributedFileSystem dfs =
(DistributedFileSystem)cluster.getFileSystem();
- FileContext fc = FileContext.getFileContext(cluster.getURI(), config);
+ FileContext fc = FileContext.getFileContext(cluster.getURI(0), config);
// OP_ADD 0, OP_SET_GENSTAMP 10
Path pathFileCreate = new Path("/file_create");
FSDataOutputStream s = dfs.create(pathFileCreate);
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestHeartbeatHandling.java Tue Mar 8 01:17:07 2011
@@ -64,43 +64,48 @@ public class TestHeartbeatHandling exten
final int MAX_REPLICATE_BLOCKS = 2*MAX_REPLICATE_LIMIT+REMAINING_BLOCKS;
final DatanodeDescriptor[] ONE_TARGET = new DatanodeDescriptor[1];
- synchronized (namesystem.heartbeats) {
- for (int i=0; i<MAX_REPLICATE_BLOCKS; i++) {
- dd.addBlockToBeReplicated(
- new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP), ONE_TARGET);
- }
- DatanodeCommand[]cmds = sendHeartBeat(nodeReg, dd, namesystem);
- assertEquals(1, cmds.length);
- assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
- assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length);
-
- ArrayList<Block> blockList = new ArrayList<Block>(MAX_INVALIDATE_BLOCKS);
- for (int i=0; i<MAX_INVALIDATE_BLOCKS; i++) {
- blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
- }
- dd.addBlocksToBeInvalidated(blockList);
-
- cmds = sendHeartBeat(nodeReg, dd, namesystem);
- assertEquals(2, cmds.length);
- assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
- assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length);
- assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction());
- assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length);
-
- cmds = sendHeartBeat(nodeReg, dd, namesystem);
- assertEquals(2, cmds.length);
- assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
- assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
- assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction());
- assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length);
-
- cmds = sendHeartBeat(nodeReg, dd, namesystem);
- assertEquals(1, cmds.length);
- assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[0].getAction());
- assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
+ try {
+ namesystem.writeLock();
+ synchronized (namesystem.heartbeats) {
+ for (int i=0; i<MAX_REPLICATE_BLOCKS; i++) {
+ dd.addBlockToBeReplicated(
+ new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP), ONE_TARGET);
+ }
+ DatanodeCommand[]cmds = sendHeartBeat(nodeReg, dd, namesystem);
+ assertEquals(1, cmds.length);
+ assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
+ assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length);
+
+ ArrayList<Block> blockList = new ArrayList<Block>(MAX_INVALIDATE_BLOCKS);
+ for (int i=0; i<MAX_INVALIDATE_BLOCKS; i++) {
+ blockList.add(new Block(i, 0, GenerationStamp.FIRST_VALID_STAMP));
+ }
+ dd.addBlocksToBeInvalidated(blockList);
+
+ cmds = sendHeartBeat(nodeReg, dd, namesystem);
+ assertEquals(2, cmds.length);
+ assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
+ assertEquals(MAX_REPLICATE_LIMIT, ((BlockCommand)cmds[0]).getBlocks().length);
+ assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction());
+ assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length);
+
+ cmds = sendHeartBeat(nodeReg, dd, namesystem);
+ assertEquals(2, cmds.length);
+ assertEquals(DatanodeProtocol.DNA_TRANSFER, cmds[0].getAction());
+ assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
+ assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[1].getAction());
+ assertEquals(MAX_INVALIDATE_LIMIT, ((BlockCommand)cmds[1]).getBlocks().length);
+
+ cmds = sendHeartBeat(nodeReg, dd, namesystem);
+ assertEquals(1, cmds.length);
+ assertEquals(DatanodeProtocol.DNA_INVALIDATE, cmds[0].getAction());
+ assertEquals(REMAINING_BLOCKS, ((BlockCommand)cmds[0]).getBlocks().length);
- cmds = sendHeartBeat(nodeReg, dd, namesystem);
- assertEquals(null, cmds);
+ cmds = sendHeartBeat(nodeReg, dd, namesystem);
+ assertEquals(null, cmds);
+ }
+ } finally {
+ namesystem.writeUnlock();
}
} finally {
cluster.shutdown();
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNodeCount.java Tue Mar 8 01:17:07 2011
@@ -65,11 +65,18 @@ public class TestNodeCount extends TestC
// bring down first datanode
DatanodeDescriptor datanode = datanodes[0];
DataNodeProperties dnprop = cluster.stopDataNode(datanode.getName());
+
// make sure that NN detects that the datanode is down
- synchronized (namesystem.heartbeats) {
- datanode.setLastUpdate(0); // mark it dead
- namesystem.heartbeatCheck();
+ try {
+ namesystem.writeLock();
+ synchronized (namesystem.heartbeats) {
+ datanode.setLastUpdate(0); // mark it dead
+ namesystem.heartbeatCheck();
+ }
+ } finally {
+ namesystem.writeUnlock();
}
+
// the block will be replicated
DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
@@ -102,20 +109,26 @@ public class TestNodeCount extends TestC
// bring down non excessive datanode
dnprop = cluster.stopDataNode(nonExcessDN.getName());
// make sure that NN detects that the datanode is down
- synchronized (namesystem.heartbeats) {
- nonExcessDN.setLastUpdate(0); // mark it dead
- namesystem.heartbeatCheck();
- }
+ try {
+ namesystem.writeLock();
+ synchronized (namesystem.heartbeats) {
+ nonExcessDN.setLastUpdate(0); // mark it dead
+ namesystem.heartbeatCheck();
+ }
+ } finally {
+ namesystem.writeUnlock();
+ }
+
// The block should be replicated
do {
num = namesystem.blockManager.countNodes(block.getLocalBlock());
} while (num.liveReplicas() != REPLICATION_FACTOR);
-
+
// restart the first datanode
cluster.restartDataNode(dnprop);
cluster.waitActive();
-
+
// check if excessive replica is detected
do {
num = namesystem.blockManager.countNodes(block.getLocalBlock());
Modified: hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java?rev=1079067&r1=1079066&r2=1079067&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java (original)
+++ hadoop/hdfs/branches/HDFS-1052/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestOverReplicatedBlocks.java Tue Mar 8 01:17:07 2011
@@ -82,24 +82,30 @@ public class TestOverReplicatedBlocks ex
cluster.getDataNodes().get(2), blockPoolId);
final FSNamesystem namesystem = cluster.getNamesystem();
- synchronized (namesystem.heartbeats) {
- // set live datanode's remaining space to be 0
- // so they will be chosen to be deleted when over-replication occurs
- String corruptMachineName = corruptDataNode.getName();
- for (DatanodeDescriptor datanode : namesystem.heartbeats) {
- if (!corruptMachineName.equals(datanode.getName())) {
- datanode.updateHeartbeat(100L, 100L, 0L, 100L, 0);
+ try {
+ namesystem.writeLock();
+ synchronized (namesystem.heartbeats) {
+ // set live datanode's remaining space to be 0
+ // so they will be chosen to be deleted when over-replication occurs
+ String corruptMachineName = corruptDataNode.getName();
+ for (DatanodeDescriptor datanode : namesystem.heartbeats) {
+ if (!corruptMachineName.equals(datanode.getName())) {
+ datanode.updateHeartbeat(100L, 100L, 0L, 100L, 0);
+ }
}
- }
-
- // decrease the replication factor to 1;
- namesystem.setReplication(fileName.toString(), (short)1);
- // corrupt one won't be chosen to be excess one
- // without 4910 the number of live replicas would be 0: block gets lost
- assertEquals(1, namesystem.blockManager.countNodes(block.getLocalBlock())
- .liveReplicas());
+ // decrease the replication factor to 1;
+ namesystem.setReplication(fileName.toString(), (short)1);
+
+ // corrupt one won't be chosen to be excess one
+ // without 4910 the number of live replicas would be 0: block gets lost
+ assertEquals(1, namesystem.blockManager.countNodes(block.getLocalBlock())
+ .liveReplicas());
+ }
+ } finally {
+ namesystem.writeUnlock();
}
+
} finally {
cluster.shutdown();
}
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 01:17:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066986
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 01:17:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066986
Propchange: hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Mar 8 01:17:07 2011
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062020
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1034932,1035141,1035143,1035145,1035163,1035386,1035410,1035508,1035515,1035552,1035718,1035795,1035841-1035842,1035890,1035920,1035924,1036132,1036213,1036303,1036310,1036631,1036692,1038222-1038859,1039957,1040005,1040027-1060619,1061067,1062011-1065960,1066986