You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/04/15 22:37:18 UTC
svn commit: r1468216 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/security/token/delegation/
src/main/java/org/apache/hadoop/hdfs/server/namenode/
Author: kihwal
Date: Mon Apr 15 20:37:17 2013
New Revision: 1468216
URL: http://svn.apache.org/r1468216
Log:
HDFS-4690. Namenode exits if entering safemode while secret manager is edit logging. Contributed by Daryn Sharp.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1468216&r1=1468215&r2=1468216&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Apr 15 20:37:17 2013
@@ -11,6 +11,8 @@ Release 0.23.8 - UNRELEASED
OPTIMIZATIONS
BUG FIXES
+ HDFS-4690. Namenode exits if entering safemode while secret manager is
+ edit logging (daryn via kihwal)
Release 0.23.7 - UNRELEASED
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1468216&r1=1468215&r2=1468216&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Mon Apr 15 20:37:17 2013
@@ -154,6 +154,8 @@ public class DFSConfigKeys extends Commo
public static final long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24*60*60*1000; // 1 day
public static final String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY = "dfs.namenode.delegation.token.max-lifetime";
public static final long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7*24*60*60*1000; // 7 days
+ public static final String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY = "dfs.namenode.delegation.token.always-use"; // for tests
+ public static final boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
//Filesystem limit keys
public static final String DFS_NAMENODE_MAX_COMPONENT_LENGTH_KEY = "dfs.namenode.fs-limits.max-component-length";
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1468216&r1=1468215&r2=1468216&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java Mon Apr 15 20:37:17 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.security.
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.InterruptedIOException;
import java.net.InetSocketAddress;
import java.util.Iterator;
@@ -283,7 +284,18 @@ public class DelegationTokenSecretManage
@Override //AbstractDelegationTokenManager
protected void logUpdateMasterKey(DelegationKey key)
throws IOException {
- namesystem.logUpdateMasterKey(key);
+ synchronized (noInterruptsLock) {
+ // The edit logging code will fail catastrophically if it
+ // is interrupted during a logSync, since the interrupt
+ // closes the edit log files. Doing this inside the
+ // above lock and then checking interruption status
+ // prevents this bug.
+ if (Thread.interrupted()) {
+ throw new InterruptedIOException(
+ "Interrupted before updating master key");
+ }
+ namesystem.logUpdateMasterKey(key);
+ }
}
/** A utility method for creating credentials. */
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1468216&r1=1468215&r2=1468216&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Mon Apr 15 20:37:17 2013
@@ -34,6 +34,8 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_MAX_OBJECTS_KEY;
@@ -271,6 +273,7 @@ public class FSNamesystem implements Nam
private static final long DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL =
TimeUnit.MILLISECONDS.convert(1, TimeUnit.HOURS);
private DelegationTokenSecretManager dtSecretManager;
+ private boolean alwaysUseDelegationTokensForTests;
//
// Stores the correct file name hierarchy
@@ -348,6 +351,11 @@ public class FSNamesystem implements Nam
this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics();
this.fsLock = new ReentrantReadWriteLock(true); // fair locking
setConfigurationParameters(conf);
+ // For testing purposes, allow the DT secret manager to be started regardless
+ // of whether security is enabled.
+ alwaysUseDelegationTokensForTests =
+ conf.getBoolean(DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
+ DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT);
dtSecretManager = createDelegationTokenSecretManager(conf);
this.registerMBean(); // register the MBean for the FSNamesystemState
if(fsImage == null) {
@@ -620,6 +628,11 @@ public class FSNamesystem implements Nam
}
}
+ private boolean shouldUseDelegationTokens() {
+ return UserGroupInformation.isSecurityEnabled() ||
+ alwaysUseDelegationTokensForTests;
+ }
+
long getDefaultBlockSize() {
return serverDefaults.getBlockSize();
}
@@ -3430,21 +3443,21 @@ public class FSNamesystem implements Nam
void enterSafeMode(boolean resourcesLow) throws IOException {
writeLock();
try {
- // Ensure that any concurrent operations have been fully synced
- // before entering safe mode. This ensures that the FSImage
- // is entirely stable on disk as soon as we're in safe mode.
- getEditLog().logSyncAll();
- if (!isInSafeMode()) {
- safeMode = new SafeModeInfo(resourcesLow);
- return;
- }
- if (resourcesLow) {
- safeMode.setResourcesLow();
- }
- safeMode.setManual();
- getEditLog().logSyncAll();
- NameNode.stateChangeLog.info("STATE* Safe mode is ON. "
- + safeMode.getTurnOffTip());
+ // Ensure that any concurrent operations have been fully synced
+ // before entering safe mode. This ensures that the FSImage
+ // is entirely stable on disk as soon as we're in safe mode.
+ getEditLog().logSyncAll();
+ if (!isInSafeMode()) {
+ safeMode = new SafeModeInfo(resourcesLow);
+ return;
+ }
+ if (resourcesLow) {
+ safeMode.setResourcesLow();
+ }
+ safeMode.setManual();
+ getEditLog().logSyncAll();
+ NameNode.stateChangeLog.info("STATE* Safe mode is ON. "
+ + safeMode.getTurnOffTip());
} finally {
writeUnlock();
}
@@ -4197,16 +4210,13 @@ public class FSNamesystem implements Nam
* @param key new delegation key.
*/
public void logUpdateMasterKey(DelegationKey key) throws IOException {
- writeLock();
- try {
- if (isInSafeMode()) {
- throw new SafeModeException(
- "Cannot log master key update in safe mode", safeMode);
- }
- getEditLog().logUpdateMasterKey(key);
- } finally {
- writeUnlock();
- }
+ assert !isInSafeMode() :
+ "this should never be called while in safemode, since we stop " +
+ "the DT manager before entering safemode!";
+ // No need to hold FSN lock since we don't access any internal
+ // structures, and this is stopped before the FSN shuts itself
+ // down, etc.
+ getEditLog().logUpdateMasterKey(key);
getEditLog().logSync();
}