You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by to...@apache.org on 2011/06/07 00:24:56 UTC
svn commit: r1132808 - in /hadoop/hdfs/trunk: ./
src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/protocol/
src/java/org/apache/hadoop/hdfs/server/common/
src/java/org/apache/hadoop/hdfs/server/datanode/
src/java/org/apache/hadoop/hdfs/...
Author: todd
Date: Mon Jun 6 22:24:55 2011
New Revision: 1132808
URL: http://svn.apache.org/viewvc?rev=1132808&view=rev
Log:
HDFS-1149. Lease reassignment should be persisted to the edit log. Contributed by Aaron T. Myers.
Added:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClientAdapter.java
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Mon Jun 6 22:24:55 2011
@@ -690,6 +690,9 @@ Trunk (unreleased changes)
HDFS-1923. In TestFiDataTransferProtocol2, reduce random sleep time period
and increase the number of datanodes. (szetszwo)
+ HDFS-1149. Lease reassignment should be persisted to the edit log.
+ (Aaron T. Myers via todd)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClientAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClientAdapter.java?rev=1132808&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClientAdapter.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/DFSClientAdapter.java Mon Jun 6 22:24:55 2011
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+
+public class DFSClientAdapter {
+
+ public static void stopLeaseRenewer(DFSClient dfsClient) throws IOException {
+ try {
+ dfsClient.leaserenewer.interruptAndJoin();
+ } catch (InterruptedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ public static LocatedBlocks callGetBlockLocations(ClientProtocol namenode,
+ String src, long start, long length) throws IOException {
+ return DFSClient.callGetBlockLocations(namenode, src, start, length);
+ }
+}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/LayoutVersion.java Mon Jun 6 22:24:55 2011
@@ -77,7 +77,8 @@ public class LayoutVersion {
RESERVED_REL20_204(-32, "Reserved for release 0.20.204"),
RESERVED_REL22(-33, -27, "Reserved for release 0.22"),
RESERVED_REL23(-34, -30, "Reserved for release 0.23"),
- FEDERATION(-35, "Support for namenode federation");
+ FEDERATION(-35, "Support for namenode federation"),
+ LEASE_REASSIGNMENT(-36, "Support for persisting lease holder reassignment");
final int lv;
final int ancestorLV;
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/HdfsConstants.java Mon Jun 6 22:24:55 2011
@@ -172,5 +172,8 @@ public interface HdfsConstants {
*/
COMMITTED;
}
+
+ public static final String NAMENODE_LEASE_HOLDER = "HDFS_NameNode";
+ public static final long NAMENODE_LEASE_RECHECK_INTERVAL = 2000;
}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Mon Jun 6 22:24:55 2011
@@ -347,6 +347,7 @@ public class DataNode extends Configured
boolean resetBlockReportTime = true;
long initialBlockReportDelay = BLOCKREPORT_INITIAL_DELAY * 1000L;
long heartBeatInterval;
+ private boolean heartbeatsDisabledForTests = false;
private DataStorage storage = null;
private HttpServer infoServer = null;
DataNodeMetrics metrics;
@@ -644,6 +645,12 @@ public class DataNode extends Configured
bpos.reportBadBlocks(block);
}
+ // used only for testing
+ void setHeartbeatsDisabledForTests(
+ boolean heartbeatsDisabledForTests) {
+ this.heartbeatsDisabledForTests = heartbeatsDisabledForTests;
+ }
+
/**
* A thread per namenode to perform:
* <ul>
@@ -1034,10 +1041,12 @@ public class DataNode extends Configured
// -- Bytes remaining
//
lastHeartbeat = startTime;
- DatanodeCommand[] cmds = sendHeartBeat();
- metrics.addHeartbeat(now() - startTime);
- if (!processCommand(cmds))
- continue;
+ if (!heartbeatsDisabledForTests) {
+ DatanodeCommand[] cmds = sendHeartBeat();
+ metrics.addHeartbeat(now() - startTime);
+ if (!processCommand(cmds))
+ continue;
+ }
}
reportReceivedBlocks();
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Mon Jun 6 22:24:55 2011
@@ -768,6 +768,12 @@ public class FSEditLog implements NNStor
void logUpdateMasterKey(DelegationKey key) {
logEdit(OP_UPDATE_MASTER_KEY, key);
}
+
+ void logReassignLease(String leaseHolder, String src, String newHolder) {
+ logEdit(OP_REASSIGN_LEASE, new DeprecatedUTF8(leaseHolder),
+ new DeprecatedUTF8(src),
+ new DeprecatedUTF8(newHolder));
+ }
static private DeprecatedUTF8 toLogReplication(short replication) {
return new DeprecatedUTF8(Short.toString(replication));
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Mon Jun 6 22:24:55 2011
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.security.t
import static org.apache.hadoop.hdfs.server.common.Util.now;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
@@ -142,7 +143,7 @@ public class FSEditLogLoader {
numOpTimes = 0, numOpRename = 0, numOpConcatDelete = 0,
numOpSymlink = 0, numOpGetDelegationToken = 0,
numOpRenewDelegationToken = 0, numOpCancelDelegationToken = 0,
- numOpUpdateMasterKey = 0, numOpOther = 0;
+ numOpUpdateMasterKey = 0, numOpReassignLease = 0, numOpOther = 0;
// Keep track of the file offsets of the last several opcodes.
// This is handy when manually recovering corrupted edits files.
@@ -483,6 +484,17 @@ public class FSEditLogLoader {
delegationKey);
break;
}
+ case OP_REASSIGN_LEASE: {
+ numOpReassignLease++;
+ String leaseHolder = FSImageSerialization.readString(in);
+ path = FSImageSerialization.readString(in);
+ String newHolder = FSImageSerialization.readString(in);
+ Lease lease = fsNamesys.leaseManager.getLease(leaseHolder);
+ INodeFileUnderConstruction pendingFile =
+ (INodeFileUnderConstruction) fsDir.getFileINode(path);
+ fsNamesys.reassignLeaseInternal(lease, path, newHolder, pendingFile);
+ break;
+ }
default: {
throw new IOException("Never seen opCode " + opCode);
}
@@ -528,6 +540,7 @@ public class FSEditLogLoader {
+ " numOpRenewDelegationToken = " + numOpRenewDelegationToken
+ " numOpCancelDelegationToken = " + numOpCancelDelegationToken
+ " numOpUpdateMasterKey = " + numOpUpdateMasterKey
+ + " numOpReassignLease = " + numOpReassignLease
+ " numOpOther = " + numOpOther);
}
return numEdits;
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java Mon Jun 6 22:24:55 2011
@@ -53,6 +53,7 @@ public enum FSEditLogOpCodes {
OP_RENEW_DELEGATION_TOKEN ((byte) 19),
OP_CANCEL_DELEGATION_TOKEN ((byte) 20),
OP_UPDATE_MASTER_KEY ((byte) 21),
+ OP_REASSIGN_LEASE ((byte) 22),
// must be same as NamenodeProtocol.JA_JSPOOL_START
OP_JSPOOL_START ((byte)102),
// must be same as NamenodeProtocol.JA_CHECKPOINT_TIME
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Jun 6 22:24:55 2011
@@ -2287,6 +2287,8 @@ public class FSNamesystem implements FSC
String recoveryLeaseHolder) throws AlreadyBeingCreatedException,
IOException, UnresolvedLinkException {
LOG.info("Recovering lease=" + lease + ", src=" + src);
+
+ assert !isInSafeMode();
INodeFile iFile = dir.getFileINode(src);
if (iFile == null) {
@@ -2408,9 +2410,15 @@ public class FSNamesystem implements FSC
}
Lease reassignLease(Lease lease, String src, String newHolder,
- INodeFileUnderConstruction pendingFile) {
+ INodeFileUnderConstruction pendingFile) throws IOException {
if(newHolder == null)
return lease;
+ logReassignLease(lease.getHolder(), src, newHolder);
+ return reassignLeaseInternal(lease, src, newHolder, pendingFile);
+ }
+
+ Lease reassignLeaseInternal(Lease lease, String src, String newHolder,
+ INodeFileUnderConstruction pendingFile) throws IOException {
pendingFile.setClientName(newHolder);
return leaseManager.reassignLease(lease, src, newHolder);
}
@@ -5386,6 +5394,17 @@ public class FSNamesystem implements FSC
getEditLog().logSync();
}
+ private void logReassignLease(String leaseHolder, String src,
+ String newHolder) throws IOException {
+ writeLock();
+ try {
+ getEditLog().logReassignLease(leaseHolder, src, newHolder);
+ } finally {
+ writeUnlock();
+ }
+ getEditLog().logSync();
+ }
+
/**
*
* @return true if delegation token operation is allowed
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java Mon Jun 6 22:24:55 2011
@@ -33,6 +33,8 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.hdfs.protocol.FSConstants;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants;
+
import static org.apache.hadoop.hdfs.server.common.Util.now;
/**
@@ -45,8 +47,8 @@ import static org.apache.hadoop.hdfs.ser
* 2.1) Get the datanodes which contains b
* 2.2) Assign one of the datanodes as the primary datanode p
- * 2.3) p obtains a new generation stamp form the namenode
- * 2.4) p get the block info from each datanode
+ * 2.3) p obtains a new generation stamp from the namenode
+ * 2.4) p gets the block info from each datanode
* 2.5) p computes the minimum block length
* 2.6) p updates the datanodes, which have a valid generation stamp,
* with the new generation stamp and the minimum block length
@@ -377,7 +379,7 @@ public class LeaseManager {
try {
- Thread.sleep(2000);
+ Thread.sleep(HdfsConstants.NAMENODE_LEASE_RECHECK_INTERVAL);
} catch(InterruptedException ie) {
if (LOG.isDebugEnabled()) {
LOG.debug(name + " is interrupted", ie);
@@ -406,13 +408,14 @@ public class LeaseManager {
oldest.getPaths().toArray(leasePaths);
for(String p : leasePaths) {
try {
- if(fsnamesystem.internalReleaseLease(oldest, p, "HDFS_NameNode")) {
+ if(fsnamesystem.internalReleaseLease(oldest, p, HdfsConstants.NAMENODE_LEASE_HOLDER)) {
LOG.info("Lease recovery for file " + p +
" is complete. File closed.");
removing.add(p);
- } else
+ } else {
LOG.info("Started block recovery for file " + p +
" lease " + oldest);
+ }
} catch (IOException e) {
LOG.error("Cannot release the path "+p+" in the lease "+oldest, e);
removing.add(p);
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Mon Jun 6 22:24:55 2011
@@ -40,7 +40,7 @@ import static org.apache.hadoop.hdfs.too
class EditsLoaderCurrent implements EditsLoader {
private static int[] supportedVersions = { -18, -19, -20, -21, -22, -23, -24,
- -25, -26, -27, -28, -30, -31, -32, -33, -34, -35 };
+ -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36 };
private EditsVisitor v;
private int editsVersion = 0;
@@ -319,6 +319,13 @@ class EditsLoaderCurrent implements Edit
VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH);
v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value);
}
+
+ private void visit_OP_REASSIGN_LEASE()
+ throws IOException {
+ v.visitStringUTF8(EditsElement.CLIENT_NAME);
+ v.visitStringUTF8(EditsElement.PATH);
+ v.visitStringUTF8(EditsElement.CLIENT_NAME);
+ }
private void visitOpCode(FSEditLogOpCodes editsOpCode)
throws IOException {
@@ -381,6 +388,9 @@ class EditsLoaderCurrent implements Edit
case OP_UPDATE_MASTER_KEY: // 21
visit_OP_UPDATE_MASTER_KEY();
break;
+ case OP_REASSIGN_LEASE: // 22
+ visit_OP_REASSIGN_LEASE();
+ break;
default:
{
throw new IOException("Unknown op code " + editsOpCode);
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Mon Jun 6 22:24:55 2011
@@ -122,7 +122,7 @@ class ImageLoaderCurrent implements Imag
protected final DateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23,
- -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35 };
+ -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36 };
private int imageVersion = 0;
/* (non-Javadoc)
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/MiniDFSCluster.java Mon Jun 6 22:24:55 2011
@@ -1072,34 +1072,47 @@ public class MiniDFSCluster {
nameNodes[nnIndex] = new NameNodeInfo(null, conf);
}
}
-
+
+ /**
+ * Restart the namenode.
+ */
+ public synchronized void restartNameNode() throws IOException {
+ checkSingleNameNode();
+ restartNameNode(true);
+ }
+
+ /**
+ * Restart the namenode. Optionally wait for the cluster to become active.
+ */
+ public synchronized void restartNameNode(boolean waitActive)
+ throws IOException {
+ checkSingleNameNode();
+ restartNameNode(0, waitActive);
+ }
+
/**
- * Restart namenode at a given index.
+ * Restart the namenode at a given index.
*/
public synchronized void restartNameNode(int nnIndex) throws IOException {
+ restartNameNode(nnIndex, true);
+ }
+
+ /**
+ * Restart the namenode at a given index. Optionally wait for the cluster
+ * to become active.
+ */
+ public synchronized void restartNameNode(int nnIndex, boolean waitActive)
+ throws IOException {
Configuration conf = nameNodes[nnIndex].conf;
shutdownNameNode(nnIndex);
NameNode nn = NameNode.createNameNode(new String[] {}, conf);
nameNodes[nnIndex] = new NameNodeInfo(nn, conf);
- waitClusterUp();
- System.out.println("Restarted the namenode");
- int failedCount = 0;
- while (true) {
- try {
- waitActive();
- break;
- } catch (IOException e) {
- failedCount++;
- // Cached RPC connection to namenode, if any, is expected to fail once
- if (failedCount > 1) {
- System.out.println("Tried waitActive() " + failedCount
- + " time(s) and failed, giving up. "
- + StringUtils.stringifyException(e));
- throw e;
- }
- }
+ if (waitActive) {
+ waitClusterUp();
+ System.out.println("Restarted the namenode");
+ waitActive();
+ System.out.println("Cluster is active");
}
- System.out.println("Cluster is active");
}
/**
@@ -1420,7 +1433,22 @@ public class MiniDFSCluster {
*/
public void waitActive() throws IOException {
for (int index = 0; index < nameNodes.length; index++) {
- waitActive(index);
+ int failedCount = 0;
+ while (true) {
+ try {
+ waitActive(index);
+ break;
+ } catch (IOException e) {
+ failedCount++;
+ // Cached RPC connection to namenode, if any, is expected to fail once
+ if (failedCount > 1) {
+ System.out.println("Tried waitActive() " + failedCount
+ + " time(s) and failed, giving up. "
+ + StringUtils.stringifyException(e));
+ throw e;
+ }
+ }
+ }
}
}
@@ -1565,7 +1593,7 @@ public class MiniDFSCluster {
/**
* Set the softLimit and hardLimit of client lease periods
*/
- void setLeasePeriod(long soft, long hard) {
+ public void setLeasePeriod(long soft, long hard) {
final FSNamesystem namesystem = getNamesystem();
namesystem.leaseManager.setLeasePeriod(soft, hard);
namesystem.lmthread.interrupt();
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestLeaseRecovery2.java Mon Jun 6 22:24:55 2011
@@ -23,6 +23,8 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -31,9 +33,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeAdapter;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Level;
@@ -42,6 +47,9 @@ import org.junit.BeforeClass;
import org.junit.Test;
public class TestLeaseRecovery2 {
+
+ public static final Log LOG = LogFactory.getLog(TestLeaseRecovery2.class);
+
{
((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
@@ -232,4 +240,126 @@ public class TestLeaseRecovery2 {
"Now validating data and sizes from datanodes...");
AppendTestUtil.checkFullFile(dfs, filepath, size, buffer, filestr);
}
+
+ /**
+ * This test makes it so the client does not renew its lease and also
+ * set the hard lease expiration period to be short, thus triggering
+ * lease expiration to happen while the client is still alive. The test
+ * also causes the NN to restart after lease recovery has begun, but before
+ * the DNs have completed the blocks. This test verifies that when the NN
+ * comes back up, the client no longer holds the lease.
+ *
+ * The test makes sure that the lease recovery completes and the client
+ * fails if it continues to write to the file, even after NN restart.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testHardLeaseRecoveryAfterNameNodeRestart() throws Exception {
+ hardLeaseRecoveryRestartHelper(false);
+ }
+
+ @Test
+ public void testHardLeaseRecoveryWithRenameAfterNameNodeRestart()
+ throws Exception {
+ hardLeaseRecoveryRestartHelper(true);
+ }
+
+ public void hardLeaseRecoveryRestartHelper(boolean doRename)
+ throws Exception {
+ //create a file
+ String fileStr = "/hardLeaseRecovery";
+ AppendTestUtil.LOG.info("filestr=" + fileStr);
+ Path filePath = new Path(fileStr);
+ FSDataOutputStream stm = dfs.create(filePath, true,
+ BUF_SIZE, REPLICATION_NUM, BLOCK_SIZE);
+ assertTrue(dfs.dfs.exists(fileStr));
+
+ // write bytes into the file.
+ int size = AppendTestUtil.nextInt(FILE_SIZE);
+ AppendTestUtil.LOG.info("size=" + size);
+ stm.write(buffer, 0, size);
+
+ String originalLeaseHolder = NameNodeAdapter.getLeaseHolderForPath(
+ cluster.getNameNode(), fileStr);
+
+ assertFalse("original lease holder should not be the NN",
+ originalLeaseHolder.equals(HdfsConstants.NAMENODE_LEASE_HOLDER));
+
+ // hflush file
+ AppendTestUtil.LOG.info("hflush");
+ stm.hflush();
+
+ if (doRename) {
+ fileStr += ".renamed";
+ Path renamedPath = new Path(fileStr);
+ assertTrue(dfs.rename(filePath, renamedPath));
+ filePath = renamedPath;
+ }
+
+ // kill the lease renewal thread
+ AppendTestUtil.LOG.info("leasechecker.interruptAndJoin()");
+ dfs.dfs.leaserenewer.interruptAndJoin();
+
+ // Make sure the DNs don't send a heartbeat for a while, so the blocks
+ // won't actually get completed during lease recovery.
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeAdapter.setHeartbeatsDisabledForTests(dn, true);
+ }
+
+ // set the hard limit to be 1 second
+ cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
+
+ // Make sure lease recovery begins.
+ Thread.sleep(HdfsConstants.NAMENODE_LEASE_RECHECK_INTERVAL * 2);
+
+ assertEquals("lease holder should now be the NN", HdfsConstants.NAMENODE_LEASE_HOLDER,
+ NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), fileStr));
+
+ cluster.restartNameNode(false);
+
+ assertEquals("lease holder should still be the NN after restart",
+ HdfsConstants.NAMENODE_LEASE_HOLDER,
+ NameNodeAdapter.getLeaseHolderForPath(cluster.getNameNode(), fileStr));
+
+ // Let the DNs send heartbeats again.
+ for (DataNode dn : cluster.getDataNodes()) {
+ DataNodeAdapter.setHeartbeatsDisabledForTests(dn, false);
+ }
+
+ cluster.waitActive();
+
+ // set the hard limit to be 1 second, to initiate lease recovery.
+ cluster.setLeasePeriod(LONG_LEASE_PERIOD, SHORT_LEASE_PERIOD);
+
+ // wait for lease recovery to complete
+ LocatedBlocks locatedBlocks;
+ do {
+ Thread.sleep(SHORT_LEASE_PERIOD);
+ locatedBlocks = DFSClient.callGetBlockLocations(dfs.dfs.namenode,
+ fileStr, 0L, size);
+ } while (locatedBlocks.isUnderConstruction());
+ assertEquals(size, locatedBlocks.getFileLength());
+
+ // make sure that the client can't write data anymore.
+ stm.write('b');
+ try {
+ stm.hflush();
+ fail("Should not be able to flush after we've lost the lease");
+ } catch (IOException e) {
+ LOG.info("Expceted exception on hflush", e);
+ }
+
+ try {
+ stm.close();
+ fail("Should not be able to close after we've lost the lease");
+ } catch (IOException e) {
+ LOG.info("Expected exception on close", e);
+ }
+
+ // verify data
+ AppendTestUtil.LOG.info(
+ "File size is good. Now validating sizes from datanodes...");
+ AppendTestUtil.checkFullFile(dfs, filePath, size, buffer, fileStr);
+ }
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/datanode/DataNodeAdapter.java Mon Jun 6 22:24:55 2011
@@ -37,4 +37,9 @@ public class DataNodeAdapter {
final long blkId) {
return ((FSDataset)dn.data).fetchReplicaInfo(bpid, blkId);
}
+
+ public static void setHeartbeatsDisabledForTests(DataNode dn,
+ boolean heartbeatsDisabledForTests) {
+ dn.setHeartbeatsDisabledForTests(heartbeatsDisabledForTests);
+ }
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Mon Jun 6 22:24:55 2011
@@ -73,4 +73,8 @@ public class NameNodeAdapter {
ns.readUnlock();
return r;
}
+
+ public static String getLeaseHolderForPath(NameNode namenode, String path) {
+ return namenode.getNamesystem().leaseManager.getLeaseByPath(path).getHolder();
+ }
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java Mon Jun 6 22:24:55 2011
@@ -30,6 +30,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSClientAdapter;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.common.Util;
import org.apache.hadoop.hdfs.server.namenode.FSImage;
@@ -38,6 +40,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.protocol.FSConstants;
+import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -222,5 +225,26 @@ public class OfflineEditsViewerHelper {
// sync to disk, otherwise we parse partial edits
cluster.getNameNode().getFSImage().getEditLog().logSync();
+
+ // OP_REASSIGN_LEASE 22
+ String filePath = "/hard-lease-recovery-test";
+ byte[] bytes = "foo-bar-baz".getBytes();
+ DFSClientAdapter.stopLeaseRenewer(dfs.getClient());
+ FSDataOutputStream leaseRecoveryPath = dfs.create(new Path(filePath));
+ leaseRecoveryPath.write(bytes);
+ leaseRecoveryPath.hflush();
+ // Set the hard lease timeout to 1 second.
+ cluster.setLeasePeriod(60 * 1000, 1000);
+ // wait for lease recovery to complete
+ LocatedBlocks locatedBlocks;
+ do {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ LOG.info("Innocuous exception", e);
+ }
+ locatedBlocks = DFSClientAdapter.callGetBlockLocations(
+ cluster.getNameNode(), filePath, 0L, bytes.length);
+ } while (locatedBlocks.isUnderConstruction());
}
}
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
Binary files - no diff available.
Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml?rev=1132808&r1=1132807&r2=1132808&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml Mon Jun 6 22:24:55 2011
@@ -421,6 +421,33 @@
</DATA>
</RECORD>
<RECORD>
+ <OPCODE>0</OPCODE>
+ <DATA>
+ <LENGTH>5</LENGTH>
+ <PATH>/reassign-lease-test</PATH>
+ <REPLICATION>1</REPLICATION>
+ <MTIME>1286491964741</MTIME>
+ <ATIME>1286491964741</ATIME>
+ <BLOCKSIZE>512</BLOCKSIZE>
+ <NUMBLOCKS>0</NUMBLOCKS>
+ <PERMISSION_STATUS>
+ <USERNAME>atm</USERNAME>
+ <GROUPNAME>supergroup</GROUPNAME>
+ <FS_PERMISSIONS>420</FS_PERMISSIONS>
+ </PERMISSION_STATUS>
+ <CLIENT_NAME>DFSClient_871171074</CLIENT_NAME>
+ <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>22</OPCODE>
+ <DATA>
+ <CLIENT_NAME>DFSClient_871171074</CLIENT_NAME>
+ <PATH>/reassign-lease-test</PATH>
+ <CLIENT_NAME>HDFS_NameNode</CLIENT_NAME>
+ </DATA>
+ </RECORD>
+ <RECORD>
<OPCODE>-1</OPCODE>
<DATA>
</DATA>