You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/04/12 17:20:00 UTC
svn commit: r1467308 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/security/token/delegation/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/se...
Author: kihwal
Date: Fri Apr 12 15:19:59 2013
New Revision: 1467308
URL: http://svn.apache.org/r1467308
Log:
svn merge -c 1467307 Merging from trunk to branch-2 to fix HDFS-4477.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1467308&r1=1467307&r2=1467308&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Apr 12 15:19:59 2013
@@ -2210,6 +2210,8 @@ Release 0.23.8 - UNRELEASED
BUG FIXES
+ HDFS-4477. Secondary namenode may retain old tokens (daryn via kihwal)
+
Release 0.23.7 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1467308&r1=1467307&r2=1467308&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java Fri Apr 12 15:19:59 2013
@@ -310,6 +310,23 @@ public class DelegationTokenSecretManage
namesystem.logUpdateMasterKey(key);
}
}
+
+ @Override //AbstractDelegationTokenManager
+ protected void logExpireToken(final DelegationTokenIdentifier dtId)
+ throws IOException {
+ synchronized (noInterruptsLock) {
+ // The edit logging code will fail catastrophically if it
+ // is interrupted during a logSync, since the interrupt
+ // closes the edit log files. Doing this inside the
+ // above lock and then checking interruption status
+ // prevents this bug.
+ if (Thread.interrupted()) {
+ throw new InterruptedIOException(
+ "Interrupted before expiring delegation token");
+ }
+ namesystem.logExpireDelegationToken(dtId);
+ }
+ }
/** A utility method for creating credentials. */
public static Credentials createCredentials(final NameNode namenode,
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1467308&r1=1467307&r2=1467308&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Apr 12 15:19:59 2013
@@ -5289,6 +5289,21 @@ public class FSNamesystem implements Nam
getEditLog().logSync();
}
+ /**
+ * Log the cancellation of expired tokens to edit logs
+ *
+ * @param id token identifier to cancel
+ */
+ public void logExpireDelegationToken(DelegationTokenIdentifier id) {
+ assert !isInSafeMode() :
+ "this should never be called while in safemode, since we stop " +
+ "the DT manager before entering safemode!";
+ // No need to hold FSN lock since we don't access any internal
+ // structures, and this is stopped before the FSN shuts itself
+ // down, etc.
+ getEditLog().logCancelDelegationToken(id);
+ }
+
private void logReassignLease(String leaseHolder, String src,
String newHolder) {
assert hasWriteLock();
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java?rev=1467308&r1=1467307&r2=1467308&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecurityTokenEditLog.java Fri Apr 12 15:19:59 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.*;
import static org.junit.Assert.assertEquals;
import java.io.File;
@@ -30,12 +31,14 @@ import org.apache.hadoop.hdfs.DFSConfigK
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.junit.Test;
+import static org.mockito.Mockito.*;
/**
* This class tests the creation and validation of a checkpoint.
@@ -163,4 +166,70 @@ public class TestSecurityTokenEditLog {
if(cluster != null) cluster.shutdown();
}
}
+
+ @Test(timeout=10000)
+ public void testEditsForCancelOnTokenExpire() throws IOException,
+ InterruptedException {
+ long renewInterval = 2000;
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ conf.setLong(DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, renewInterval);
+ conf.setLong(DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, renewInterval*2);
+
+ Text renewer = new Text(UserGroupInformation.getCurrentUser().getUserName());
+ FSImage fsImage = mock(FSImage.class);
+ FSEditLog log = mock(FSEditLog.class);
+ doReturn(log).when(fsImage).getEditLog();
+ FSNamesystem fsn = new FSNamesystem(conf, fsImage);
+
+ DelegationTokenSecretManager dtsm = fsn.getDelegationTokenSecretManager();
+ try {
+ dtsm.startThreads();
+
+ // get two tokens
+ Token<DelegationTokenIdentifier> token1 = fsn.getDelegationToken(renewer);
+ Token<DelegationTokenIdentifier> token2 = fsn.getDelegationToken(renewer);
+ DelegationTokenIdentifier ident1 =
+ (DelegationTokenIdentifier)token1.decodeIdentifier();
+ DelegationTokenIdentifier ident2 =
+ (DelegationTokenIdentifier)token2.decodeIdentifier();
+
+ // verify we got the tokens
+ verify(log, times(1)).logGetDelegationToken(eq(ident1), anyLong());
+ verify(log, times(1)).logGetDelegationToken(eq(ident2), anyLong());
+
+ // this is a little tricky because DTSM doesn't let us set scan interval
+ // so need to periodically sleep, then stop/start threads to force scan
+
+ // renew first token 1/2 to expire
+ Thread.sleep(renewInterval/2);
+ fsn.renewDelegationToken(token2);
+ verify(log, times(1)).logRenewDelegationToken(eq(ident2), anyLong());
+ // force scan and give it a little time to complete
+ dtsm.stopThreads(); dtsm.startThreads();
+ Thread.sleep(250);
+ // no token has expired yet
+ verify(log, times(0)).logCancelDelegationToken(eq(ident1));
+ verify(log, times(0)).logCancelDelegationToken(eq(ident2));
+
+ // sleep past expiration of 1st non-renewed token
+ Thread.sleep(renewInterval/2);
+ dtsm.stopThreads(); dtsm.startThreads();
+ Thread.sleep(250);
+ // non-renewed token should have implicitly been cancelled
+ verify(log, times(1)).logCancelDelegationToken(eq(ident1));
+ verify(log, times(0)).logCancelDelegationToken(eq(ident2));
+
+ // sleep past expiration of 2nd renewed token
+ Thread.sleep(renewInterval/2);
+ dtsm.stopThreads(); dtsm.startThreads();
+ Thread.sleep(250);
+ // both tokens should have been implicitly cancelled by now
+ verify(log, times(1)).logCancelDelegationToken(eq(ident1));
+ verify(log, times(1)).logCancelDelegationToken(eq(ident2));
+ } finally {
+ dtsm.stopThreads();
+ }
+ }
}