You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2014/08/07 22:26:55 UTC
svn commit: r1616589 - in
/hadoop/common/branches/fs-encryption/hadoop-hdfs-project:
hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/
hadoop-hdfs/ hadoop-hdfs/src/main/conf/ hadoop-hdfs/src/main/java/
hadoop-hdfs/src/main/java/org/apa...
Author: wang
Date: Thu Aug 7 20:26:52 2014
New Revision: 1616589
URL: http://svn.apache.org/r1616589
Log:
Merge from trunk to branch.
Added:
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
- copied unchanged from r1616586, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/KeyManager.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Matcher.java
- copied unchanged from r1616586, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Matcher.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/MovedBlocks.java
- copied unchanged from r1616586, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/MovedBlocks.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
- copied unchanged from r1616586, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
Removed:
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/hadoop-metrics2.properties
Modified:
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1615844-1616586
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/server/TestHttpFSServer.java Thu Aug 7 20:26:52 2014
@@ -65,6 +65,7 @@ import org.mortbay.jetty.Server;
import org.mortbay.jetty.webapp.WebAppContext;
import com.google.common.collect.Maps;
+import org.apache.hadoop.security.authentication.util.StringSignerSecretProvider;
public class TestHttpFSServer extends HFSTestCase {
@@ -683,7 +684,7 @@ public class TestHttpFSServer extends HF
new AuthenticationToken("u", "p",
HttpFSKerberosAuthenticationHandlerForTesting.TYPE);
token.setExpires(System.currentTimeMillis() + 100000000);
- Signer signer = new Signer("secret".getBytes());
+ Signer signer = new Signer(new StringSignerSecretProvider("secret"));
String tokenSigned = signer.sign(token.toString());
url = new URL(TestJettyHelper.getJettyURL(),
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Aug 7 20:26:52 2014
@@ -136,6 +136,9 @@ Trunk (Unreleased)
OPTIMIZATIONS
BUG FIXES
+
+ HDFS-6517. Remove hadoop-metrics2.properties from hdfs project (Akira
+ AJISAKA via aw)
HADOOP-9635 Fix potential Stack Overflow in DomainSocket.c (V. Karthik Kumar
via cmccabe)
@@ -357,6 +360,16 @@ Release 2.6.0 - UNRELEASED
HDFS-6787. Remove duplicate code in FSDirectory#unprotectedConcat. (Yi Liu via umamahesh)
+ HDFS-6809. Move Balancer's inner classes MovedBlocks and Matcher as to
+ standalone classes and separates KeyManager from NameNodeConnector.
+ (szetszwo)
+
+ HDFS-6812. Remove addBlock and replaceBlock from DatanodeDescriptor.
+ (szetszwo)
+
+ HDFS-6781. Separate HDFS commands from CommandsManual.apt.vm. (Akira
+ Ajisaka via Arpit Agarwal)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
@@ -440,6 +453,15 @@ Release 2.6.0 - UNRELEASED
HDFS-6451. NFS should not return NFS3ERR_IO for AccessControlException
(Abhiraj Butala via brandonli)
+ HDFS-6717. JIRA HDFS-5804 breaks default nfs-gateway behavior for unsecured config
+ (brandonli)
+
+ HDFS-6790. DFSUtil Should Use configuration.getPassword for SSL passwords
+ (Larry McCay via brandonli)
+
+ HDFS-6791. A block could remain under replicated if all of its replicas are on
+ decommissioned nodes. (Ming Ma via jing9)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -992,9 +1014,6 @@ Release 2.5.0 - UNRELEASED
HDFS-6723. New NN webUI no longer displays decommissioned state for dead node.
(Ming Ma via wheat9)
- HDFS-6717. JIRA HDFS-5804 breaks default nfs-gateway behavior for unsecured config
- (brandonli)
-
HDFS-6768. Fix a few unit tests that use hard-coded port numbers. (Arpit
Agarwal)
Propchange: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1615844-1616586
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Thu Aug 7 20:26:52 2014
@@ -201,6 +201,9 @@ public class DFSConfigKeys extends Commo
public static final String DFS_ADMIN = "dfs.cluster.administrators";
public static final String DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_KEY = "dfs.https.server.keystore.resource";
public static final String DFS_SERVER_HTTPS_KEYSTORE_RESOURCE_DEFAULT = "ssl-server.xml";
+ public static final String DFS_SERVER_HTTPS_KEYPASSWORD_KEY = "ssl.server.keystore.keypassword";
+ public static final String DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY = "ssl.server.keystore.password";
+ public static final String DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY = "ssl.server.truststore.password";
public static final String DFS_NAMENODE_NAME_DIR_RESTORE_KEY = "dfs.namenode.name.dir.restore";
public static final boolean DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT = false;
public static final String DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY = "dfs.namenode.support.allow.format";
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java Thu Aug 7 20:26:52 2014
@@ -33,6 +33,9 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
import java.io.IOException;
import java.io.PrintStream;
@@ -1534,16 +1537,38 @@ public class DFSUtil {
.needsClientAuth(
sslConf.getBoolean(DFS_CLIENT_HTTPS_NEED_AUTH_KEY,
DFS_CLIENT_HTTPS_NEED_AUTH_DEFAULT))
- .keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
+ .keyPassword(getPassword(sslConf, DFS_SERVER_HTTPS_KEYPASSWORD_KEY))
.keyStore(sslConf.get("ssl.server.keystore.location"),
- sslConf.get("ssl.server.keystore.password"),
+ getPassword(sslConf, DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY),
sslConf.get("ssl.server.keystore.type", "jks"))
.trustStore(sslConf.get("ssl.server.truststore.location"),
- sslConf.get("ssl.server.truststore.password"),
+ getPassword(sslConf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY),
sslConf.get("ssl.server.truststore.type", "jks"));
}
/**
+ * Leverages the Configuration.getPassword method to attempt to get
+ * passwords from the CredentialProvider API before falling back to
+ * clear text in config - if falling back is allowed.
+ * @param conf Configuration instance
+ * @param alias name of the credential to retreive
+ * @return String credential value or null
+ */
+ static String getPassword(Configuration conf, String alias) {
+ String password = null;
+ try {
+ char[] passchars = conf.getPassword(alias);
+ if (passchars != null) {
+ password = new String(passchars);
+ }
+ }
+ catch (IOException ioe) {
+ password = null;
+ }
+ return password;
+ }
+
+ /**
* Converts a Date into an ISO-8601 formatted datetime string.
*/
public static String dateToIso8601String(Date date) {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java Thu Aug 7 20:26:52 2014
@@ -58,6 +58,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -85,7 +86,6 @@ import org.apache.hadoop.hdfs.server.pro
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.HostsFileReader;
import org.apache.hadoop.util.StringUtils;
@@ -195,10 +195,12 @@ import com.google.common.base.Preconditi
@InterfaceAudience.Private
public class Balancer {
static final Log LOG = LogFactory.getLog(Balancer.class);
- final private static long GB = 1L << 30; //1GB
- final private static long MAX_SIZE_TO_MOVE = 10*GB;
- final private static long MAX_BLOCKS_SIZE_TO_FETCH = 2*GB;
- private static long WIN_WIDTH = 5400*1000L; // 1.5 hour
+
+ private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
+
+ private static final long GB = 1L << 30; //1GB
+ private static final long MAX_SIZE_TO_MOVE = 10*GB;
+ private static final long MAX_BLOCKS_SIZE_TO_FETCH = 2*GB;
/** The maximum number of concurrent blocks moves for
* balancing purpose at a datanode
@@ -219,6 +221,8 @@ public class Balancer {
+ "\tIncludes only the specified datanodes.";
private final NameNodeConnector nnc;
+ private final KeyManager keyManager;
+
private final BalancingPolicy policy;
private final SaslDataTransferClient saslClient;
private final double threshold;
@@ -241,7 +245,8 @@ public class Balancer {
private final Map<Block, BalancerBlock> globalBlockList
= new HashMap<Block, BalancerBlock>();
- private final MovedBlocks movedBlocks = new MovedBlocks();
+ private final MovedBlocks<BalancerDatanode.StorageGroup> movedBlocks;
+
/** Map (datanodeUuid,storageType -> StorageGroup) */
private final StorageGroupMap storageGroupMap = new StorageGroupMap();
@@ -326,7 +331,7 @@ public class Balancer {
if (isGoodBlockCandidate(source, target, block)) {
this.block = block;
if ( chooseProxySource() ) {
- movedBlocks.add(block);
+ movedBlocks.put(block);
if (LOG.isDebugEnabled()) {
LOG.debug("Decided to move " + this);
}
@@ -399,10 +404,10 @@ public class Balancer {
OutputStream unbufOut = sock.getOutputStream();
InputStream unbufIn = sock.getInputStream();
- ExtendedBlock eb = new ExtendedBlock(nnc.blockpoolID, block.getBlock());
- Token<BlockTokenIdentifier> accessToken = nnc.getAccessToken(eb);
+ ExtendedBlock eb = new ExtendedBlock(nnc.getBlockpoolID(), block.getBlock());
+ Token<BlockTokenIdentifier> accessToken = keyManager.getAccessToken(eb);
IOStreamPair saslStreams = saslClient.socketSend(sock, unbufOut,
- unbufIn, nnc, accessToken, target.getDatanode());
+ unbufIn, keyManager, accessToken, target.getDatanode());
unbufOut = saslStreams.out;
unbufIn = saslStreams.in;
out = new DataOutputStream(new BufferedOutputStream(unbufOut,
@@ -483,47 +488,9 @@ public class Balancer {
}
/* A class for keeping track of blocks in the Balancer */
- static private class BalancerBlock {
- private final Block block; // the block
- /** The locations of the replicas of the block. */
- private final List<BalancerDatanode.StorageGroup> locations
- = new ArrayList<BalancerDatanode.StorageGroup>(3);
-
- /* Constructor */
- private BalancerBlock(Block block) {
- this.block = block;
- }
-
- /* clean block locations */
- private synchronized void clearLocations() {
- locations.clear();
- }
-
- /* add a location */
- private synchronized void addLocation(BalancerDatanode.StorageGroup g) {
- if (!locations.contains(g)) {
- locations.add(g);
- }
- }
-
- /** @return if the block is located on the given storage group. */
- private synchronized boolean isLocatedOn(BalancerDatanode.StorageGroup g) {
- return locations.contains(g);
- }
-
- /* Return its locations */
- private synchronized List<BalancerDatanode.StorageGroup> getLocations() {
- return locations;
- }
-
- /* Return the block */
- private Block getBlock() {
- return block;
- }
-
- /* Return the length of the block */
- private long getNumBytes() {
- return block.getNumBytes();
+ static class BalancerBlock extends MovedBlocks.Locations<BalancerDatanode.StorageGroup> {
+ BalancerBlock(Block block) {
+ super(block);
}
}
@@ -735,7 +702,7 @@ public class Balancer {
*/
private long getBlockList() throws IOException {
final long size = Math.min(MAX_BLOCKS_SIZE_TO_FETCH, blocksToReceive);
- final BlockWithLocations[] newBlocks = nnc.namenode.getBlocks(
+ final BlockWithLocations[] newBlocks = nnc.getNamenode().getBlocks(
getDatanode(), size).getBlocks();
long bytesReceived = 0;
@@ -819,7 +786,7 @@ public class Balancer {
private void filterMovedBlocks() {
for (Iterator<BalancerBlock> blocks=getBlockIterator();
blocks.hasNext();) {
- if (movedBlocks.contains(blocks.next())) {
+ if (movedBlocks.contains(blocks.next().getBlock())) {
blocks.remove();
}
}
@@ -925,6 +892,13 @@ public class Balancer {
this.nodesToBeExcluded = p.nodesToBeExcluded;
this.nodesToBeIncluded = p.nodesToBeIncluded;
this.nnc = theblockpool;
+ this.keyManager = nnc.getKeyManager();
+
+ final long movedWinWidth = conf.getLong(
+ DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY,
+ DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_DEFAULT);
+ movedBlocks = new MovedBlocks<BalancerDatanode.StorageGroup>(movedWinWidth);
+
cluster = NetworkTopology.getInstance(conf);
this.moverExecutor = Executors.newFixedThreadPool(
@@ -1094,36 +1068,6 @@ public class Balancer {
LOG.info(items.size() + " " + name + ": " + items);
}
- /** A matcher interface for matching nodes. */
- private interface Matcher {
- /** Given the cluster topology, does the left node match the right node? */
- boolean match(NetworkTopology cluster, Node left, Node right);
- }
-
- /** Match datanodes in the same node group. */
- static final Matcher SAME_NODE_GROUP = new Matcher() {
- @Override
- public boolean match(NetworkTopology cluster, Node left, Node right) {
- return cluster.isOnSameNodeGroup(left, right);
- }
- };
-
- /** Match datanodes in the same rack. */
- static final Matcher SAME_RACK = new Matcher() {
- @Override
- public boolean match(NetworkTopology cluster, Node left, Node right) {
- return cluster.isOnSameRack(left, right);
- }
- };
-
- /** Match any datanode with any other datanode. */
- static final Matcher ANY_OTHER = new Matcher() {
- @Override
- public boolean match(NetworkTopology cluster, Node left, Node right) {
- return left != right;
- }
- };
-
/**
* Decide all <source, target> pairs and
* the number of bytes to move from a source to a target
@@ -1134,13 +1078,13 @@ public class Balancer {
private long chooseStorageGroups() {
// First, match nodes on the same node group if cluster is node group aware
if (cluster.isNodeGroupAware()) {
- chooseStorageGroups(SAME_NODE_GROUP);
+ chooseStorageGroups(Matcher.SAME_NODE_GROUP);
}
// Then, match nodes on the same rack
- chooseStorageGroups(SAME_RACK);
+ chooseStorageGroups(Matcher.SAME_RACK);
// At last, match all remaining nodes
- chooseStorageGroups(ANY_OTHER);
+ chooseStorageGroups(Matcher.ANY_OTHER);
Preconditions.checkState(storageGroupMap.size() >= sources.size() + targets.size(),
"Mismatched number of datanodes (" + storageGroupMap.size() + " < "
@@ -1307,56 +1251,6 @@ public class Balancer {
} while (shouldWait);
}
- /** This window makes sure to keep blocks that have been moved within 1.5 hour.
- * Old window has blocks that are older;
- * Current window has blocks that are more recent;
- * Cleanup method triggers the check if blocks in the old window are
- * more than 1.5 hour old. If yes, purge the old window and then
- * move blocks in current window to old window.
- */
- private static class MovedBlocks {
- private long lastCleanupTime = Time.now();
- final private static int CUR_WIN = 0;
- final private static int OLD_WIN = 1;
- final private static int NUM_WINS = 2;
- final private List<HashMap<Block, BalancerBlock>> movedBlocks =
- new ArrayList<HashMap<Block, BalancerBlock>>(NUM_WINS);
-
- /* initialize the moved blocks collection */
- private MovedBlocks() {
- movedBlocks.add(new HashMap<Block,BalancerBlock>());
- movedBlocks.add(new HashMap<Block,BalancerBlock>());
- }
-
- /* add a block thus marking a block to be moved */
- synchronized private void add(BalancerBlock block) {
- movedBlocks.get(CUR_WIN).put(block.getBlock(), block);
- }
-
- /* check if a block is marked as moved */
- synchronized private boolean contains(BalancerBlock block) {
- return contains(block.getBlock());
- }
-
- /* check if a block is marked as moved */
- synchronized private boolean contains(Block block) {
- return movedBlocks.get(CUR_WIN).containsKey(block) ||
- movedBlocks.get(OLD_WIN).containsKey(block);
- }
-
- /* remove old blocks */
- synchronized private void cleanup() {
- long curTime = Time.now();
- // check if old win is older than winWidth
- if (lastCleanupTime + WIN_WIDTH <= curTime) {
- // purge the old window
- movedBlocks.set(OLD_WIN, movedBlocks.get(CUR_WIN));
- movedBlocks.set(CUR_WIN, new HashMap<Block, BalancerBlock>());
- lastCleanupTime = curTime;
- }
- }
- }
-
/* Decide if it is OK to move the given block from source to target
* A block is a good candidate if
* 1. the block is not in the process of being moved/has not been moved;
@@ -1369,7 +1263,7 @@ public class Balancer {
return false;
}
// check if the block is moved or not
- if (movedBlocks.contains(block)) {
+ if (movedBlocks.contains(block.getBlock())) {
return false;
}
if (block.isLocatedOn(target)) {
@@ -1387,7 +1281,7 @@ public class Balancer {
} else {
boolean notOnSameRack = true;
synchronized (block) {
- for (BalancerDatanode.StorageGroup loc : block.locations) {
+ for (BalancerDatanode.StorageGroup loc : block.getLocations()) {
if (cluster.isOnSameRack(loc.getDatanode(), target.getDatanode())) {
notOnSameRack = false;
break;
@@ -1399,7 +1293,7 @@ public class Balancer {
goodBlock = true;
} else {
// good if source is on the same rack as on of the replicas
- for (BalancerDatanode.StorageGroup loc : block.locations) {
+ for (BalancerDatanode.StorageGroup loc : block.getLocations()) {
if (loc != source &&
cluster.isOnSameRack(loc.getDatanode(), source.getDatanode())) {
goodBlock = true;
@@ -1425,7 +1319,7 @@ public class Balancer {
private boolean isOnSameNodeGroupWithReplicas(BalancerDatanode.StorageGroup target,
BalancerBlock block, Source source) {
final DatanodeInfo targetDn = target.getDatanode();
- for (BalancerDatanode.StorageGroup loc : block.locations) {
+ for (BalancerDatanode.StorageGroup loc : block.getLocations()) {
if (loc != source &&
cluster.isOnSameNodeGroup(loc.getDatanode(), targetDn)) {
return true;
@@ -1489,7 +1383,7 @@ public class Balancer {
* decide the number of bytes need to be moved
*/
final long bytesLeftToMove = init(
- nnc.client.getDatanodeStorageReport(DatanodeReportType.LIVE));
+ nnc.getClient().getDatanodeStorageReport(DatanodeReportType.LIVE));
if (bytesLeftToMove == 0) {
System.out.println("The cluster is balanced. Exiting...");
return ReturnStatus.SUCCESS;
@@ -1558,8 +1452,8 @@ public class Balancer {
final long sleeptime = 2000*conf.getLong(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
- LOG.info("namenodes = " + namenodes);
- LOG.info("p = " + p);
+ LOG.info("namenodes = " + namenodes);
+ LOG.info("parameters = " + p);
final Formatter formatter = new Formatter(System.out);
System.out.println("Time Stamp Iteration# Bytes Already Moved Bytes Left To Move Bytes Being Moved");
@@ -1568,7 +1462,10 @@ public class Balancer {
= new ArrayList<NameNodeConnector>(namenodes.size());
try {
for (URI uri : namenodes) {
- connectors.add(new NameNodeConnector(uri, conf));
+ final NameNodeConnector nnc = new NameNodeConnector(
+ Balancer.class.getSimpleName(), uri, BALANCER_ID_PATH, conf);
+ nnc.getKeyManager().startBlockKeyUpdater();
+ connectors.add(nnc);
}
boolean done = false;
@@ -1730,9 +1627,6 @@ public class Balancer {
public int run(String[] args) {
final long startTime = Time.now();
final Configuration conf = getConf();
- WIN_WIDTH = conf.getLong(
- DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_KEY,
- DFSConfigKeys.DFS_BALANCER_MOVEDWINWIDTH_DEFAULT);
try {
checkReplicationPolicyCompatibility(conf);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java Thu Aug 7 20:26:52 2014
@@ -17,113 +17,96 @@
*/
package org.apache.hadoop.hdfs.server.balancer;
+import java.io.Closeable;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.URI;
-import java.util.EnumSet;
import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataEncryptionKeyFactory;
-import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.block.BlockTokenSecretManager;
-import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.util.Daemon;
/**
- * The class provides utilities for {@link Balancer} to access a NameNode
+ * The class provides utilities for accessing a NameNode.
*/
@InterfaceAudience.Private
-class NameNodeConnector implements DataEncryptionKeyFactory {
- private static final Log LOG = Balancer.LOG;
- private static final Path BALANCER_ID_PATH = new Path("/system/balancer.id");
+public class NameNodeConnector implements Closeable {
+ private static final Log LOG = LogFactory.getLog(NameNodeConnector.class);
+
private static final int MAX_NOT_CHANGED_ITERATIONS = 5;
- final URI nameNodeUri;
- final String blockpoolID;
+ private final URI nameNodeUri;
+ private final String blockpoolID;
+
+ private final NamenodeProtocol namenode;
+ private final ClientProtocol client;
+ private final KeyManager keyManager;
+
+ private final FileSystem fs;
+ private final Path idPath;
+ private final OutputStream out;
- final NamenodeProtocol namenode;
- final ClientProtocol client;
- final FileSystem fs;
- final OutputStream out;
-
- private final boolean isBlockTokenEnabled;
- private final boolean encryptDataTransfer;
- private boolean shouldRun;
- private long keyUpdaterInterval;
- // used for balancer
private int notChangedIterations = 0;
- private BlockTokenSecretManager blockTokenSecretManager;
- private Daemon keyupdaterthread; // AccessKeyUpdater thread
- private DataEncryptionKey encryptionKey;
- NameNodeConnector(URI nameNodeUri,
+ public NameNodeConnector(String name, URI nameNodeUri, Path idPath,
Configuration conf) throws IOException {
this.nameNodeUri = nameNodeUri;
+ this.idPath = idPath;
- this.namenode =
- NameNodeProxies.createProxy(conf, nameNodeUri, NamenodeProtocol.class)
- .getProxy();
- this.client =
- NameNodeProxies.createProxy(conf, nameNodeUri, ClientProtocol.class)
- .getProxy();
+ this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
+ NamenodeProtocol.class).getProxy();
+ this.client = NameNodeProxies.createProxy(conf, nameNodeUri,
+ ClientProtocol.class).getProxy();
this.fs = FileSystem.get(nameNodeUri, conf);
final NamespaceInfo namespaceinfo = namenode.versionRequest();
this.blockpoolID = namespaceinfo.getBlockPoolID();
- final ExportedBlockKeys keys = namenode.getBlockKeys();
- this.isBlockTokenEnabled = keys.isBlockTokenEnabled();
- if (isBlockTokenEnabled) {
- long blockKeyUpdateInterval = keys.getKeyUpdateInterval();
- long blockTokenLifetime = keys.getTokenLifetime();
- LOG.info("Block token params received from NN: keyUpdateInterval="
- + blockKeyUpdateInterval / (60 * 1000) + " min(s), tokenLifetime="
- + blockTokenLifetime / (60 * 1000) + " min(s)");
- String encryptionAlgorithm = conf.get(
- DFSConfigKeys.DFS_DATA_ENCRYPTION_ALGORITHM_KEY);
- this.blockTokenSecretManager = new BlockTokenSecretManager(
- blockKeyUpdateInterval, blockTokenLifetime, blockpoolID,
- encryptionAlgorithm);
- this.blockTokenSecretManager.addKeys(keys);
- /*
- * Balancer should sync its block keys with NN more frequently than NN
- * updates its block keys
- */
- this.keyUpdaterInterval = blockKeyUpdateInterval / 4;
- LOG.info("Balancer will update its block keys every "
- + keyUpdaterInterval / (60 * 1000) + " minute(s)");
- this.keyupdaterthread = new Daemon(new BlockKeyUpdater());
- this.shouldRun = true;
- this.keyupdaterthread.start();
- }
- this.encryptDataTransfer = fs.getServerDefaults(new Path("/"))
- .getEncryptDataTransfer();
- // Check if there is another balancer running.
+ final FsServerDefaults defaults = fs.getServerDefaults(new Path("/"));
+ this.keyManager = new KeyManager(blockpoolID, namenode,
+ defaults.getEncryptDataTransfer(), conf);
// Exit if there is another one running.
- out = checkAndMarkRunningBalancer();
+ out = checkAndMarkRunning();
if (out == null) {
- throw new IOException("Another balancer is running");
+ throw new IOException("Another " + name + " is running.");
}
}
- boolean shouldContinue(long dispatchBlockMoveBytes) {
+ /** @return the block pool ID */
+ public String getBlockpoolID() {
+ return blockpoolID;
+ }
+
+ /** @return the namenode proxy. */
+ public NamenodeProtocol getNamenode() {
+ return namenode;
+ }
+
+ /** @return the client proxy. */
+ public ClientProtocol getClient() {
+ return client;
+ }
+
+ /** @return the key manager */
+ public KeyManager getKeyManager() {
+ return keyManager;
+ }
+
+ /** Should the instance continue running? */
+ public boolean shouldContinue(long dispatchBlockMoveBytes) {
if (dispatchBlockMoveBytes > 0) {
notChangedIterations = 0;
} else {
@@ -137,53 +120,25 @@ class NameNodeConnector implements DataE
return true;
}
- /** Get an access token for a block. */
- Token<BlockTokenIdentifier> getAccessToken(ExtendedBlock eb
- ) throws IOException {
- if (!isBlockTokenEnabled) {
- return BlockTokenSecretManager.DUMMY_TOKEN;
- } else {
- if (!shouldRun) {
- throw new IOException(
- "Can not get access token. BlockKeyUpdater is not running");
- }
- return blockTokenSecretManager.generateToken(null, eb,
- EnumSet.of(BlockTokenSecretManager.AccessMode.REPLACE,
- BlockTokenSecretManager.AccessMode.COPY));
- }
- }
-
- @Override
- public DataEncryptionKey newDataEncryptionKey() {
- if (encryptDataTransfer) {
- synchronized (this) {
- if (encryptionKey == null) {
- encryptionKey = blockTokenSecretManager.generateDataEncryptionKey();
- }
- return encryptionKey;
- }
- } else {
- return null;
- }
- }
- /* The idea for making sure that there is no more than one balancer
+ /**
+ * The idea for making sure that there is no more than one instance
* running in an HDFS is to create a file in the HDFS, writes the hostname
- * of the machine on which the balancer is running to the file, but did not
- * close the file until the balancer exits.
- * This prevents the second balancer from running because it can not
+ * of the machine on which the instance is running to the file, but did not
+ * close the file until it exits.
+ *
+ * This prevents the second instance from running because it can not
* creates the file while the first one is running.
*
- * This method checks if there is any running balancer and
- * if no, mark yes if no.
+ * This method checks if there is any running instance. If no, mark yes.
* Note that this is an atomic operation.
*
- * Return null if there is a running balancer; otherwise the output stream
- * to the newly created file.
+ * @return null if there is a running instance;
+ * otherwise, the output stream to the newly created file.
*/
- private OutputStream checkAndMarkRunningBalancer() throws IOException {
+ private OutputStream checkAndMarkRunning() throws IOException {
try {
- final DataOutputStream out = fs.create(BALANCER_ID_PATH);
+ final DataOutputStream out = fs.create(idPath);
out.writeBytes(InetAddress.getLocalHost().getHostName());
out.flush();
return out;
@@ -196,24 +151,17 @@ class NameNodeConnector implements DataE
}
}
- /** Close the connection. */
- void close() {
- shouldRun = false;
- try {
- if (keyupdaterthread != null) {
- keyupdaterthread.interrupt();
- }
- } catch(Exception e) {
- LOG.warn("Exception shutting down access key updater thread", e);
- }
+ @Override
+ public void close() {
+ keyManager.close();
// close the output file
IOUtils.closeStream(out);
if (fs != null) {
try {
- fs.delete(BALANCER_ID_PATH, true);
+ fs.delete(idPath, true);
} catch(IOException ioe) {
- LOG.warn("Failed to delete " + BALANCER_ID_PATH, ioe);
+ LOG.warn("Failed to delete " + idPath, ioe);
}
}
}
@@ -221,31 +169,6 @@ class NameNodeConnector implements DataE
@Override
public String toString() {
return getClass().getSimpleName() + "[namenodeUri=" + nameNodeUri
- + ", id=" + blockpoolID
- + "]";
- }
-
- /**
- * Periodically updates access keys.
- */
- class BlockKeyUpdater implements Runnable {
- @Override
- public void run() {
- try {
- while (shouldRun) {
- try {
- blockTokenSecretManager.addKeys(namenode.getBlockKeys());
- } catch (IOException e) {
- LOG.error("Failed to set keys", e);
- }
- Thread.sleep(keyUpdaterInterval);
- }
- } catch (InterruptedException e) {
- LOG.debug("InterruptedException in block key updater thread", e);
- } catch (Throwable e) {
- LOG.error("Exception in block key updater thread", e);
- shouldRun = false;
- }
- }
+ + ", bpid=" + blockpoolID + "]";
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java Thu Aug 7 20:26:52 2014
@@ -21,7 +21,6 @@ import java.util.LinkedList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.util.LightWeightGSet;
@@ -254,18 +253,18 @@ public class BlockInfo extends Block imp
}
/**
* Find specified DatanodeStorageInfo.
- * @return index or -1 if not found.
+ * @return DatanodeStorageInfo or null if not found.
*/
- int findStorageInfo(DatanodeInfo dn) {
+ DatanodeStorageInfo findStorageInfo(DatanodeDescriptor dn) {
int len = getCapacity();
for(int idx = 0; idx < len; idx++) {
DatanodeStorageInfo cur = getStorageInfo(idx);
if(cur == null)
break;
if(cur.getDatanodeDescriptor() == dn)
- return idx;
+ return cur;
}
- return -1;
+ return null;
}
/**
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java Thu Aug 7 20:26:52 2014
@@ -3177,6 +3177,15 @@ public class BlockManager {
}
}
}
+
+ if (!status && !srcNode.isAlive) {
+ LOG.warn("srcNode " + srcNode + " is dead " +
+ "when decommission is in progress. Continue to mark " +
+ "it as decommission in progress. In that way, when it rejoins the " +
+ "cluster it can continue the decommission process.");
+ status = true;
+ }
+
srcNode.decommissioningStatus.set(underReplicatedBlocks,
decommissionOnlyReplicas,
underReplicatedInOpenFiles);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java Thu Aug 7 20:26:52 2014
@@ -23,8 +23,8 @@ import org.apache.hadoop.hdfs.protocol.B
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
-import org.apache.hadoop.util.LightWeightGSet.SetIterator;
+import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
@@ -217,9 +217,14 @@ class BlocksMap {
BlockInfo currentBlock = blocks.get(newBlock);
assert currentBlock != null : "the block if not in blocksMap";
// replace block in data-node lists
- for(int idx = currentBlock.numNodes()-1; idx >= 0; idx--) {
- DatanodeDescriptor dn = currentBlock.getDatanode(idx);
- dn.replaceBlock(currentBlock, newBlock);
+ for (int i = currentBlock.numNodes() - 1; i >= 0; i--) {
+ final DatanodeDescriptor dn = currentBlock.getDatanode(i);
+ final DatanodeStorageInfo storage = currentBlock.findStorageInfo(dn);
+ final boolean removed = storage.removeBlock(currentBlock);
+ Preconditions.checkState(removed, "currentBlock not found.");
+
+ final boolean added = storage.addBlock(newBlock);
+ Preconditions.checkState(added, "newBlock already exists.");
}
// replace block in the map itself
blocks.put(newBlock);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CorruptReplicasMap.java Thu Aug 7 20:26:52 2014
@@ -48,18 +48,6 @@ public class CorruptReplicasMap{
private final SortedMap<Block, Map<DatanodeDescriptor, Reason>> corruptReplicasMap =
new TreeMap<Block, Map<DatanodeDescriptor, Reason>>();
-
- /**
- * Mark the block belonging to datanode as corrupt.
- *
- * @param blk Block to be added to CorruptReplicasMap
- * @param dn DatanodeDescriptor which holds the corrupt replica
- * @param reason a textual reason (for logging purposes)
- */
- public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
- String reason) {
- addToCorruptReplicasMap(blk, dn, reason, Reason.NONE);
- }
/**
* Mark the block belonging to datanode as corrupt.
@@ -69,7 +57,7 @@ public class CorruptReplicasMap{
* @param reason a textual reason (for logging purposes)
* @param reasonCode the enum representation of the reason
*/
- public void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
+ void addToCorruptReplicasMap(Block blk, DatanodeDescriptor dn,
String reason, Reason reasonCode) {
Map <DatanodeDescriptor, Reason> nodes = corruptReplicasMap.get(blk);
if (nodes == null) {
@@ -127,7 +115,6 @@ public class CorruptReplicasMap{
boolean removeFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode,
Reason reason) {
Map <DatanodeDescriptor, Reason> datanodes = corruptReplicasMap.get(blk);
- boolean removed = false;
if (datanodes==null)
return false;
@@ -174,12 +161,12 @@ public class CorruptReplicasMap{
return ((nodes != null) && (nodes.contains(node)));
}
- public int numCorruptReplicas(Block blk) {
+ int numCorruptReplicas(Block blk) {
Collection<DatanodeDescriptor> nodes = getNodes(blk);
return (nodes == null) ? 0 : nodes.size();
}
- public int size() {
+ int size() {
return corruptReplicasMap.size();
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java Thu Aug 7 20:26:52 2014
@@ -234,18 +234,6 @@ public class DatanodeDescriptor extends
updateHeartbeat(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0);
}
- /**
- * Add data-node to the block. Add block to the head of the list of blocks
- * belonging to the data-node.
- */
- public boolean addBlock(String storageID, BlockInfo b) {
- DatanodeStorageInfo s = getStorageInfo(storageID);
- if (s != null) {
- return s.addBlock(b);
- }
- return false;
- }
-
@VisibleForTesting
public DatanodeStorageInfo getStorageInfo(String storageID) {
synchronized (storageMap) {
@@ -284,13 +272,10 @@ public class DatanodeDescriptor extends
* data-node from the block.
*/
boolean removeBlock(BlockInfo b) {
- int index = b.findStorageInfo(this);
+ final DatanodeStorageInfo s = b.findStorageInfo(this);
// if block exists on this datanode
- if (index >= 0) {
- DatanodeStorageInfo s = b.getStorageInfo(index);
- if (s != null) {
- return s.removeBlock(b);
- }
+ if (s != null) {
+ return s.removeBlock(b);
}
return false;
}
@@ -307,24 +292,6 @@ public class DatanodeDescriptor extends
return false;
}
- /**
- * Replace specified old block with a new one in the DataNodeDescriptor.
- *
- * @param oldBlock - block to be replaced
- * @param newBlock - a replacement block
- * @return the new block
- */
- public BlockInfo replaceBlock(BlockInfo oldBlock, BlockInfo newBlock) {
- int index = oldBlock.findStorageInfo(this);
- DatanodeStorageInfo s = oldBlock.getStorageInfo(index);
- boolean done = s.removeBlock(oldBlock);
- assert done : "Old block should belong to the data-node when replacing";
-
- done = s.addBlock(newBlock);
- assert done : "New block should not belong to the data-node when replacing";
- return newBlock;
- }
-
public void resetBlocks() {
setCapacity(0);
setRemaining(0);
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/PendingDataNodeMessages.java Thu Aug 7 20:26:52 2014
@@ -23,9 +23,9 @@ import java.util.Queue;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
+
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
/**
* In the Standby Node, we can receive messages about blocks
@@ -123,7 +123,7 @@ class PendingDataNodeMessages {
return queue;
}
- public int count() {
+ int count() {
return count ;
}
@@ -140,7 +140,7 @@ class PendingDataNodeMessages {
return sb.toString();
}
- public Iterable<ReportedBlockInfo> takeAll() {
+ Iterable<ReportedBlockInfo> takeAll() {
List<ReportedBlockInfo> rbis = Lists.newArrayListWithCapacity(
count);
for (Queue<ReportedBlockInfo> q : queueByBlockId.values()) {
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm Thu Aug 7 20:26:52 2014
@@ -143,8 +143,8 @@ HDFS Users Guide
** DFSAdmin Command
- The <<<bin/hadoop dfsadmin>>> command supports a few HDFS administration
- related operations. The <<<bin/hadoop dfsadmin -help>>> command lists all the
+ The <<<bin/hdfs dfsadmin>>> command supports a few HDFS administration
+ related operations. The <<<bin/hdfs dfsadmin -help>>> command lists all the
commands currently supported. For e.g.:
* <<<-report>>>: reports basic statistics of HDFS. Some of this
@@ -172,7 +172,7 @@ HDFS Users Guide
of racks and datanodes attached to the tracks as viewed by the
NameNode.
- For command usage, see {{{../hadoop-common/CommandsManual.html#dfsadmin}dfsadmin}}.
+ For command usage, see {{{./HDFSCommands.html#dfsadmin}dfsadmin}}.
* Secondary NameNode
@@ -207,7 +207,7 @@ HDFS Users Guide
primary NameNode if necessary.
For command usage,
- see {{{../hadoop-common/CommandsManual.html#secondarynamenode}secondarynamenode}}.
+ see {{{./HDFSCommands.html#secondarynamenode}secondarynamenode}}.
* Checkpoint Node
@@ -249,7 +249,7 @@ HDFS Users Guide
Multiple checkpoint nodes may be specified in the cluster configuration
file.
- For command usage, see {{{../hadoop-common/CommandsManual.html#namenode}namenode}}.
+ For command usage, see {{{./HDFSCommands.html#namenode}namenode}}.
* Backup Node
@@ -291,7 +291,7 @@ HDFS Users Guide
For a complete discussion of the motivation behind the creation of the
Backup node and Checkpoint node, see {{{https://issues.apache.org/jira/browse/HADOOP-4539}HADOOP-4539}}.
- For command usage, see {{{../hadoop-common/CommandsManual.html#namenode}namenode}}.
+ For command usage, see {{{./HDFSCommands.html#namenode}namenode}}.
* Import Checkpoint
@@ -314,7 +314,7 @@ HDFS Users Guide
verifies that the image in <<<dfs.namenode.checkpoint.dir>>> is consistent,
but does not modify it in any way.
- For command usage, see {{{../hadoop-common/CommandsManual.html#namenode}namenode}}.
+ For command usage, see {{{./HDFSCommands.html#namenode}namenode}}.
* Balancer
@@ -341,7 +341,7 @@ HDFS Users Guide
A brief administrator's guide for balancer is available at
{{{https://issues.apache.org/jira/browse/HADOOP-1652}HADOOP-1652}}.
- For command usage, see {{{../hadoop-common/CommandsManual.html#balancer}balancer}}.
+ For command usage, see {{{./HDFSCommands.html#balancer}balancer}}.
* Rack Awareness
@@ -368,7 +368,7 @@ HDFS Users Guide
allow any modifications to file system or blocks. Normally the NameNode
leaves Safemode automatically after the DataNodes have reported that
most file system blocks are available. If required, HDFS could be
- placed in Safemode explicitly using <<<bin/hadoop dfsadmin -safemode>>>
+ placed in Safemode explicitly using <<<bin/hdfs dfsadmin -safemode>>>
command. NameNode front page shows whether Safemode is on or off. A
more detailed description and configuration is maintained as JavaDoc
for <<<setSafeMode()>>>.
@@ -383,8 +383,8 @@ HDFS Users Guide
most of the recoverable failures. By default fsck ignores open files
but provides an option to select all files during reporting. The HDFS
fsck command is not a Hadoop shell command. It can be run as
- <<<bin/hadoop fsck>>>. For command usage, see
- {{{../hadoop-common/CommandsManual.html#fsck}fsck}}. fsck can be run on
+ <<<bin/hdfs fsck>>>. For command usage, see
+ {{{./HDFSCommands.html#fsck}fsck}}. fsck can be run on
the whole file system or on a subset of files.
* fetchdt
@@ -395,11 +395,11 @@ HDFS Users Guide
Utility uses either RPC or HTTPS (over Kerberos) to get the token, and
thus requires kerberos tickets to be present before the run (run kinit
to get the tickets). The HDFS fetchdt command is not a Hadoop shell
- command. It can be run as <<<bin/hadoop fetchdt DTfile>>>. After you got
+ command. It can be run as <<<bin/hdfs fetchdt DTfile>>>. After you got
the token you can run an HDFS command without having Kerberos tickets,
by pointing <<<HADOOP_TOKEN_FILE_LOCATION>>> environmental variable to the
delegation token file. For command usage, see
- {{{../hadoop-common/CommandsManual.html#fetchdt}fetchdt}} command.
+ {{{./HDFSCommands.html#fetchdt}fetchdt}} command.
* Recovery Mode
@@ -533,5 +533,4 @@ HDFS Users Guide
* Explore {{{./hdfs-default.xml}hdfs-default.xml}}. It includes
brief description of most of the configuration variables available.
- * {{{../hadoop-common/CommandsManual.html}Hadoop Commands Guide}}:
- Hadoop commands usage.
+ * {{{./HDFSCommands.html}HDFS Commands Guide}}: HDFS commands usage.
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java Thu Aug 7 20:26:52 2014
@@ -30,8 +30,12 @@ import static org.apache.hadoop.hdfs.DFS
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICES;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -39,6 +43,7 @@ import static org.junit.Assert.assertTha
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
@@ -61,8 +66,12 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.JavaKeyStoreProvider;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
+import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -792,4 +801,68 @@ public class TestDFSUtil {
}
}
}
+
+ @Test
+ public void testGetPassword() throws Exception {
+ File testDir = new File(System.getProperty("test.build.data",
+ "target/test-dir"));
+
+ Configuration conf = new Configuration();
+ final String ourUrl =
+ JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks";
+
+ File file = new File(testDir, "test.jks");
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ char[] keypass = {'k', 'e', 'y', 'p', 'a', 's', 's'};
+ char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'};
+ char[] trustpass = {'t', 'r', 'u', 's', 't', 'p', 'a', 's', 's'};
+
+ // ensure that we get nulls when the key isn't there
+ assertEquals(null, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
+ assertEquals(null, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
+ assertEquals(null, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));
+
+ // create new aliases
+ try {
+ provider.createCredentialEntry(
+ DFS_SERVER_HTTPS_KEYPASSWORD_KEY, keypass);
+
+ provider.createCredentialEntry(
+ DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY, storepass);
+
+ provider.createCredentialEntry(
+ DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY, trustpass);
+
+ // write out so that it can be found in checks
+ provider.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ // make sure we get back the right key directly from api
+ assertArrayEquals(keypass, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYPASSWORD_KEY).getCredential());
+ assertArrayEquals(storepass, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY).getCredential());
+ assertArrayEquals(trustpass, provider.getCredentialEntry(
+ DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY).getCredential());
+
+ // use WebAppUtils as would be used by loadSslConfiguration
+ Assert.assertEquals("keypass",
+ DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_KEYPASSWORD_KEY));
+ Assert.assertEquals("storepass",
+ DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY));
+ Assert.assertEquals("trustpass",
+ DFSUtil.getPassword(conf, DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY));
+
+ // let's make sure that a password that doesn't exist returns null
+ Assert.assertEquals(null, DFSUtil.getPassword(conf,"invalid-alias"));
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java Thu Aug 7 20:26:52 2014
@@ -268,4 +268,14 @@ public class BlockManagerTestUtil {
}
return reports.toArray(StorageReport.EMPTY_ARRAY);
}
+
+ /**
+ * Have DatanodeManager check decommission state.
+ * @param dm the DatanodeManager to manipulate
+ */
+ public static void checkDecommissionState(DatanodeManager dm,
+ DatanodeDescriptor node) {
+ dm.checkDecommissionState(node);
+ }
+
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestCorruptReplicaInfo.java Thu Aug 7 20:26:52 2014
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.server.blockmanagement.CorruptReplicasMap.Reason;
import org.junit.Test;
@@ -89,14 +90,14 @@ public class TestCorruptReplicaInfo {
DatanodeDescriptor dn1 = DFSTestUtil.getLocalDatanodeDescriptor();
DatanodeDescriptor dn2 = DFSTestUtil.getLocalDatanodeDescriptor();
- crm.addToCorruptReplicasMap(getBlock(0), dn1, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(0), dn1);
assertEquals("Number of corrupt blocks not returning correctly",
1, crm.size());
- crm.addToCorruptReplicasMap(getBlock(1), dn1, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(1), dn1);
assertEquals("Number of corrupt blocks not returning correctly",
2, crm.size());
- crm.addToCorruptReplicasMap(getBlock(1), dn2, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(1), dn2);
assertEquals("Number of corrupt blocks not returning correctly",
2, crm.size());
@@ -109,7 +110,7 @@ public class TestCorruptReplicaInfo {
0, crm.size());
for (Long block_id: block_ids) {
- crm.addToCorruptReplicasMap(getBlock(block_id), dn1, "TEST");
+ addToCorruptReplicasMap(crm, getBlock(block_id), dn1);
}
assertEquals("Number of corrupt blocks not returning correctly",
@@ -127,4 +128,9 @@ public class TestCorruptReplicaInfo {
crm.getCorruptReplicaBlockIds(10, 7L)));
}
+
+ private static void addToCorruptReplicasMap(CorruptReplicasMap crm,
+ Block blk, DatanodeDescriptor dn) {
+ crm.addToCorruptReplicasMap(blk, dn, "TEST", Reason.NONE);
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDecommissioningStatus.java Thu Aug 7 20:26:52 2014
@@ -31,19 +31,24 @@ import java.util.Random;
import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
@@ -89,6 +94,8 @@ public class TestDecommissioningStatus {
4);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
conf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY, 1);
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY, 1);
+
writeConfigFile(localFileSys, excludeFile, null);
writeConfigFile(localFileSys, includeFile, null);
@@ -99,6 +106,7 @@ public class TestDecommissioningStatus {
@AfterClass
public static void tearDown() throws Exception {
+ if (localFileSys != null ) cleanupFile(localFileSys, dir);
if(fileSys != null) fileSys.close();
if(cluster != null) cluster.shutdown();
}
@@ -138,7 +146,8 @@ public class TestDecommissioningStatus {
return stm;
}
- private void cleanupFile(FileSystem fileSys, Path name) throws IOException {
+ static private void cleanupFile(FileSystem fileSys, Path name)
+ throws IOException {
assertTrue(fileSys.exists(name));
fileSys.delete(name, true);
assertTrue(!fileSys.exists(name));
@@ -147,19 +156,26 @@ public class TestDecommissioningStatus {
/*
* Decommissions the node at the given index
*/
- private String decommissionNode(FSNamesystem namesystem,
- DFSClient client, FileSystem localFileSys, int nodeIndex)
- throws IOException {
+ private String decommissionNode(FSNamesystem namesystem, DFSClient client,
+ FileSystem localFileSys, int nodeIndex) throws IOException {
DatanodeInfo[] info = client.datanodeReport(DatanodeReportType.LIVE);
String nodename = info[nodeIndex].getXferAddr();
- System.out.println("Decommissioning node: " + nodename);
+ decommissionNode(namesystem, localFileSys, nodename);
+ return nodename;
+ }
+
+ /*
+ * Decommissions the node by name
+ */
+ private void decommissionNode(FSNamesystem namesystem,
+ FileSystem localFileSys, String dnName) throws IOException {
+ System.out.println("Decommissioning node: " + dnName);
// write nodename into the exclude file.
ArrayList<String> nodes = new ArrayList<String>(decommissionedNodes);
- nodes.add(nodename);
+ nodes.add(dnName);
writeConfigFile(localFileSys, excludeFile, nodes);
- return nodename;
}
private void checkDecommissionStatus(DatanodeDescriptor decommNode,
@@ -276,6 +292,69 @@ public class TestDecommissioningStatus {
st1.close();
cleanupFile(fileSys, file1);
cleanupFile(fileSys, file2);
- cleanupFile(localFileSys, dir);
+ }
+
+ /**
+ * Verify a DN remains in DECOMMISSION_INPROGRESS state if it is marked
+ * as dead before decommission has completed. That will allow DN to resume
+ * the replication process after it rejoins the cluster.
+ */
+ @Test(timeout=120000)
+ public void testDecommissionStatusAfterDNRestart()
+ throws IOException, InterruptedException {
+ DistributedFileSystem fileSys =
+ (DistributedFileSystem)cluster.getFileSystem();
+
+ // Create a file with one block. That block has one replica.
+ Path f = new Path("decommission.dat");
+ DFSTestUtil.createFile(fileSys, f, fileSize, fileSize, fileSize,
+ (short)1, seed);
+
+ // Find the DN that owns the only replica.
+ RemoteIterator<LocatedFileStatus> fileList = fileSys.listLocatedStatus(f);
+ BlockLocation[] blockLocations = fileList.next().getBlockLocations();
+ String dnName = blockLocations[0].getNames()[0];
+
+ // Decommission the DN.
+ FSNamesystem fsn = cluster.getNamesystem();
+ final DatanodeManager dm = fsn.getBlockManager().getDatanodeManager();
+ decommissionNode(fsn, localFileSys, dnName);
+ dm.refreshNodes(conf);
+
+ // Stop the DN when decommission is in progress.
+ // Given DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY is to 1 and the size of
+ // the block, it will take much longer time that test timeout value for
+ // the decommission to complete. So when stopDataNode is called,
+ // decommission should be in progress.
+ DataNodeProperties dataNodeProperties = cluster.stopDataNode(dnName);
+ final List<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
+ while (true) {
+ dm.fetchDatanodes(null, dead, false);
+ if (dead.size() == 1) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+
+ // Force removal of the dead node's blocks.
+ BlockManagerTestUtil.checkHeartbeat(fsn.getBlockManager());
+
+ // Force DatanodeManager to check decommission state.
+ BlockManagerTestUtil.checkDecommissionState(dm, dead.get(0));
+
+ // Verify that the DN remains in DECOMMISSION_INPROGRESS state.
+ assertTrue("the node is in decommissioned state ",
+ !dead.get(0).isDecommissioned());
+
+ // Add the node back
+ cluster.restartDataNode(dataNodeProperties, true);
+ cluster.waitActive();
+
+ // Call refreshNodes on FSNamesystem with empty exclude file.
+ // This will remove the datanodes from decommissioning list and
+ // make them available again.
+ writeConfigFile(localFileSys, excludeFile, null);
+ dm.refreshNodes(conf);
+ cleanupFile(fileSys, f);
}
}