You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2015/06/20 23:25:13 UTC
[08/50] [abbrv] hadoop git commit: HDFS-8591. Remove support for
deprecated configuration key dfs.namenode.decommission.nodes.per.interval.
HDFS-8591. Remove support for deprecated configuration key dfs.namenode.decommission.nodes.per.interval.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3990ca4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3990ca4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3990ca4
Branch: refs/heads/HDFS-7240
Commit: a3990ca41415515b986a41dacefceee1f05622f8
Parents: b039e69
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Jun 16 10:03:34 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Jun 16 10:03:34 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../blockmanagement/DecommissionManager.java | 42 +++++---------------
.../apache/hadoop/hdfs/TestDecommission.java | 26 ------------
3 files changed, 12 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3990ca4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0ae2882..e0ef52f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -31,6 +31,9 @@ Trunk (Unreleased)
HDFS-8135. Remove the deprecated FSConstants class. (Li Lu via wheat9)
+ HDFS-8591. Remove support for deprecated configuration key
+ dfs.namenode.decommission.nodes.per.interval. (wang)
+
NEW FEATURES
HDFS-3125. Add JournalService to enable Journal Daemon. (suresh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3990ca4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index 639d2b0..797d031 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -136,29 +136,20 @@ public class DecommissionManager {
checkArgument(intervalSecs >= 0, "Cannot set a negative " +
"value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY);
- // By default, the new configuration key overrides the deprecated one.
- // No # node limit is set.
int blocksPerInterval = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_DEFAULT);
- int nodesPerInterval = Integer.MAX_VALUE;
- // If the expected key isn't present and the deprecated one is,
- // use the deprecated one into the new one. This overrides the
- // default.
- //
- // Also print a deprecation warning.
final String deprecatedKey =
"dfs.namenode.decommission.nodes.per.interval";
final String strNodes = conf.get(deprecatedKey);
if (strNodes != null) {
- nodesPerInterval = Integer.parseInt(strNodes);
- blocksPerInterval = Integer.MAX_VALUE;
- LOG.warn("Using deprecated configuration key {} value of {}.",
- deprecatedKey, nodesPerInterval);
+ LOG.warn("Deprecated configuration key {} will be ignored.",
+ deprecatedKey);
LOG.warn("Please update your configuration to use {} instead.",
DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY);
}
+
checkArgument(blocksPerInterval > 0,
"Must set a positive value for "
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_BLOCKS_PER_INTERVAL_KEY);
@@ -170,15 +161,14 @@ public class DecommissionManager {
"value for "
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_MAX_CONCURRENT_TRACKED_NODES);
- monitor = new Monitor(blocksPerInterval,
- nodesPerInterval, maxConcurrentTrackedNodes);
+ monitor = new Monitor(blocksPerInterval, maxConcurrentTrackedNodes);
executor.scheduleAtFixedRate(monitor, intervalSecs, intervalSecs,
TimeUnit.SECONDS);
LOG.debug("Activating DecommissionManager with interval {} seconds, " +
- "{} max blocks per interval, {} max nodes per interval, " +
+ "{} max blocks per interval, " +
"{} max concurrently tracked nodes.", intervalSecs,
- blocksPerInterval, nodesPerInterval, maxConcurrentTrackedNodes);
+ blocksPerInterval, maxConcurrentTrackedNodes);
}
/**
@@ -334,10 +324,6 @@ public class DecommissionManager {
*/
private final int numBlocksPerCheck;
/**
- * The maximum number of nodes to check per tick.
- */
- private final int numNodesPerCheck;
- /**
* The maximum number of nodes to track in decomNodeBlocks. A value of 0
* means no limit.
*/
@@ -348,7 +334,7 @@ public class DecommissionManager {
private int numBlocksChecked = 0;
/**
* The number of nodes that have been checked on this tick. Used for
- * testing.
+ * statistics.
*/
private int numNodesChecked = 0;
/**
@@ -357,10 +343,8 @@ public class DecommissionManager {
private DatanodeDescriptor iterkey = new DatanodeDescriptor(new
DatanodeID("", "", "", 0, 0, 0, 0));
- Monitor(int numBlocksPerCheck, int numNodesPerCheck, int
- maxConcurrentTrackedNodes) {
+ Monitor(int numBlocksPerCheck, int maxConcurrentTrackedNodes) {
this.numBlocksPerCheck = numBlocksPerCheck;
- this.numNodesPerCheck = numNodesPerCheck;
this.maxConcurrentTrackedNodes = maxConcurrentTrackedNodes;
}
@@ -369,12 +353,6 @@ public class DecommissionManager {
return numBlocksChecked >= numBlocksPerCheck;
}
- @Deprecated
- private boolean exceededNumNodesPerCheck() {
- LOG.trace("Processed {} nodes so far this tick", numNodesChecked);
- return numNodesChecked >= numNodesPerCheck;
- }
-
@Override
public void run() {
if (!namesystem.isRunning()) {
@@ -416,9 +394,7 @@ public class DecommissionManager {
it = new CyclicIteration<>(decomNodeBlocks, iterkey).iterator();
final LinkedList<DatanodeDescriptor> toRemove = new LinkedList<>();
- while (it.hasNext()
- && !exceededNumBlocksPerCheck()
- && !exceededNumNodesPerCheck()) {
+ while (it.hasNext() && !exceededNumBlocksPerCheck()) {
numNodesChecked++;
final Map.Entry<DatanodeDescriptor, AbstractList<BlockInfo>>
entry = it.next();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3990ca4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 1f18014..8f965ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -1045,32 +1045,6 @@ public class TestDecommission {
doDecomCheck(datanodeManager, decomManager, 1);
}
- @Deprecated
- @Test(timeout=120000)
- public void testNodesPerInterval() throws Exception {
- Configuration newConf = new Configuration(conf);
- org.apache.log4j.Logger.getLogger(DecommissionManager.class)
- .setLevel(Level.TRACE);
- // Set the deprecated configuration key which limits the # of nodes per
- // interval
- newConf.setInt("dfs.namenode.decommission.nodes.per.interval", 1);
- // Disable the normal monitor runs
- newConf.setInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
- Integer.MAX_VALUE);
- startCluster(1, 3, newConf);
- final FileSystem fs = cluster.getFileSystem();
- final DatanodeManager datanodeManager =
- cluster.getNamesystem().getBlockManager().getDatanodeManager();
- final DecommissionManager decomManager = datanodeManager.getDecomManager();
-
- // Write a 3 block file, so each node has one block. Should scan 1 node
- // each time.
- DFSTestUtil.createFile(fs, new Path("/file1"), 64, (short) 3, 0xBAD1DEA);
- for (int i=0; i<3; i++) {
- doDecomCheck(datanodeManager, decomManager, 1);
- }
- }
-
private void doDecomCheck(DatanodeManager datanodeManager,
DecommissionManager decomManager, int expectedNumCheckedNodes)
throws IOException, ExecutionException, InterruptedException {