You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2014/09/19 01:15:26 UTC
[49/50] git commit: HDFS-7088. Archival Storage: fix TestBalancer and
TestBalancerWithMultipleNameNodes. Contributed by Tsz Wo Nicholas Sze.
HDFS-7088. Archival Storage: fix TestBalancer and TestBalancerWithMultipleNameNodes. Contributed by Tsz Wo Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ccb80f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ccb80f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ccb80f
Branch: refs/heads/trunk
Commit: 76ccb80fadad50a4465e7e26bc21ae7016036207
Parents: 2d2b000
Author: Jing Zhao <ji...@hortonworks.com>
Authored: Thu Sep 18 10:54:01 2014 -0700
Committer: Jing Zhao <ji...@hortonworks.com>
Committed: Thu Sep 18 10:54:01 2014 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/balancer/NameNodeConnector.java | 27 ++++++++++++--------
.../hdfs/server/balancer/TestBalancer.java | 7 +++++
.../balancer/TestBalancerWithHANameNodes.java | 2 +-
.../TestBalancerWithMultipleNameNodes.java | 2 +-
.../balancer/TestBalancerWithNodeGroup.java | 2 +-
.../hdfs/server/mover/TestStorageMover.java | 7 ++---
7 files changed, 31 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ccb80f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a888da2..8719849 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -73,6 +73,9 @@ HDFS-6584: Archival Storage
HDFS-6864. Archival Storage: add user documentation. (szetszwo via jing9)
+ HDFS-7088. Archival Storage: fix TestBalancer and
+ TestBalancerWithMultipleNameNodes. (szetszwo via jing9)
+
Trunk (Unreleased)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ccb80f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
index 9e08d51..d27f33f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
@@ -18,14 +18,16 @@
package org.apache.hadoop.hdfs.server.balancer;
import java.io.Closeable;
-import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.URI;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -34,7 +36,6 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsServerDefaults;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.NameNodeProxies;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
@@ -48,6 +49,8 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* The class provides utilities for accessing a NameNode.
*/
@@ -56,7 +59,7 @@ public class NameNodeConnector implements Closeable {
private static final Log LOG = LogFactory.getLog(NameNodeConnector.class);
private static final int MAX_NOT_CHANGED_ITERATIONS = 5;
- private static boolean createIdFile = true;
+ private static boolean write2IdFile = true;
/** Create {@link NameNodeConnector} for the given namenodes. */
public static List<NameNodeConnector> newNameNodeConnectors(
@@ -88,8 +91,8 @@ public class NameNodeConnector implements Closeable {
}
@VisibleForTesting
- public static void setCreateIdFile(boolean create) {
- createIdFile = create;
+ public static void setWrite2IdFile(boolean write2IdFile) {
+ NameNodeConnector.write2IdFile = write2IdFile;
}
private final URI nameNodeUri;
@@ -127,8 +130,8 @@ public class NameNodeConnector implements Closeable {
this.keyManager = new KeyManager(blockpoolID, namenode,
defaults.getEncryptDataTransfer(), conf);
// if it is for test, we do not create the id file
- out = createIdFile ? checkAndMarkRunning() : null;
- if (createIdFile && out == null) {
+ out = checkAndMarkRunning();
+ if (out == null) {
// Exit if there is another one running.
throw new IOException("Another " + name + " is running.");
}
@@ -199,8 +202,10 @@ public class NameNodeConnector implements Closeable {
private OutputStream checkAndMarkRunning() throws IOException {
try {
final FSDataOutputStream out = fs.create(idPath);
- out.writeBytes(InetAddress.getLocalHost().getHostName());
- out.hflush();
+ if (write2IdFile) {
+ out.writeBytes(InetAddress.getLocalHost().getHostName());
+ out.hflush();
+ }
return out;
} catch(RemoteException e) {
if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ccb80f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 72597d2..dbc3212 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -89,7 +89,14 @@ public class TestBalancer {
private static final Random r = new Random();
static {
+ initTestSetup();
+ }
+
+ public static void initTestSetup() {
Dispatcher.setBlockMoveWaitTime(1000L) ;
+
+ // do not create id file since it occupies the disk space
+ NameNodeConnector.setWrite2IdFile(false);
}
static void initConf(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ccb80f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
index d9d70d1..bd91366 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithHANameNodes.java
@@ -44,7 +44,7 @@ public class TestBalancerWithHANameNodes {
ClientProtocol client;
static {
- Dispatcher.setBlockMoveWaitTime(1000L);
+ TestBalancer.initTestSetup();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ccb80f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
index a16a979..6ee6e54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithMultipleNameNodes.java
@@ -73,7 +73,7 @@ public class TestBalancerWithMultipleNameNodes {
private static final Random RANDOM = new Random();
static {
- Dispatcher.setBlockMoveWaitTime(1000L) ;
+ TestBalancer.initTestSetup();
}
/** Common objects used in various methods. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ccb80f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
index 9961a2e..7af3a0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancerWithNodeGroup.java
@@ -75,7 +75,7 @@ public class TestBalancerWithNodeGroup {
static final int DEFAULT_BLOCK_SIZE = 100;
static {
- Dispatcher.setBlockMoveWaitTime(1000L) ;
+ TestBalancer.initTestSetup();
}
static Configuration createConf() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ccb80f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index eeea62b..ad813cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -30,7 +30,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -52,7 +51,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
import org.apache.hadoop.hdfs.server.balancer.Dispatcher;
import org.apache.hadoop.hdfs.server.balancer.ExitStatus;
-import org.apache.hadoop.hdfs.server.balancer.NameNodeConnector;
+import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
@@ -99,10 +98,8 @@ public class TestStorageMover {
HOT = DEFAULT_POLICIES.getPolicy("HOT");
WARM = DEFAULT_POLICIES.getPolicy("WARM");
COLD = DEFAULT_POLICIES.getPolicy("COLD");
- Dispatcher.setBlockMoveWaitTime(1000L);
+ TestBalancer.initTestSetup();
Dispatcher.setDelayAfterErrors(1000L);
- // do not create id file since we will eat up all the disk space
- NameNodeConnector.setCreateIdFile(false);
}
/**