You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/06/24 01:36:41 UTC
[45/49] hadoop git commit: HDFS-10557. Fix handling of the -fs
Generic option. (Arpit Agarwal)
HDFS-10557. Fix handling of the -fs Generic option. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66fa34c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66fa34c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66fa34c8
Branch: refs/heads/HDFS-1312
Commit: 66fa34c839c89733839cb67878fdfdc4b1f65ab8
Parents: 7b23ad1
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Jun 22 08:23:45 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jun 23 18:27:01 2016 -0700
----------------------------------------------------------------------
.../server/diskbalancer/command/Command.java | 27 +++-----------------
.../diskbalancer/command/PlanCommand.java | 11 ++++----
.../diskbalancer/command/ReportCommand.java | 6 ++---
.../diskbalancer/planner/GreedyPlanner.java | 8 +++---
.../apache/hadoop/hdfs/tools/DiskBalancer.java | 18 -------------
.../src/site/markdown/HDFSDiskbalancer.md | 6 ++---
.../command/TestDiskBalancerCommand.java | 23 +++++++----------
7 files changed, 27 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66fa34c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index 19f9945..3ea1b03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -45,9 +45,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
import java.net.URI;
-import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Files;
@@ -93,8 +91,7 @@ public abstract class Command extends Configured {
* Executes the Client Calls.
*
* @param cmd - CommandLine
- * @throws IOException
- * @throws URISyntaxException
+ * @throws Exception
*/
public abstract void execute(CommandLine cmd) throws Exception;
@@ -104,22 +101,6 @@ public abstract class Command extends Configured {
public abstract void printHelp();
/**
- * verifies user provided URL.
- *
- * @param uri - UrlString
- * @return URL
- * @throws URISyntaxException, MalformedURLException
- */
- protected URI verifyURI(String uri)
- throws URISyntaxException, MalformedURLException {
- if ((uri == null) || uri.isEmpty()) {
- throw new MalformedURLException(
- "A valid URI is needed to execute this command.");
- }
- return new URI(uri);
- }
-
- /**
* Process the URI and return the cluster with nodes setup. This is used in
* all commands.
*
@@ -130,11 +111,8 @@ public abstract class Command extends Configured {
protected DiskBalancerCluster readClusterInfo(CommandLine cmd) throws
Exception {
Preconditions.checkNotNull(cmd);
- Preconditions
- .checkState(cmd.getOptionValue(DiskBalancer.NAMENODEURI) != null,
- "Required argument missing : uri");
- setClusterURI(verifyURI(cmd.getOptionValue(DiskBalancer.NAMENODEURI)));
+ setClusterURI(FileSystem.getDefaultUri(getConf()));
LOG.debug("using name node URI : {}", this.getClusterURI());
ClusterConnector connector = ConnectorFactory.getCluster(this.clusterURI,
getConf());
@@ -346,6 +324,7 @@ public abstract class Command extends Configured {
*
* @param fileName - fileName to open.
* @return OutputStream.
+ * @throws IOException
*/
protected FSDataOutputStream create(String fileName) throws IOException {
Preconditions.checkNotNull(fileName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66fa34c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
index c13399b..20b4c6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/PlanCommand.java
@@ -63,10 +63,9 @@ public class PlanCommand extends Command {
this.thresholdPercentage = 1;
this.bandwidth = 0;
this.maxError = 0;
- addValidCommandParameters(DiskBalancer.NAMENODEURI, "Name Node URI or " +
- "file URI for cluster");
-
- addValidCommandParameters(DiskBalancer.OUTFILE, "Output file");
+ addValidCommandParameters(DiskBalancer.OUTFILE, "Output directory in " +
+ "HDFS. The generated plan will be written to a file in this " +
+ "directory.");
addValidCommandParameters(DiskBalancer.BANDWIDTH, "Maximum Bandwidth to " +
"be used while copying.");
addValidCommandParameters(DiskBalancer.THRESHOLD, "Percentage skew that " +
@@ -188,7 +187,7 @@ public class PlanCommand extends Command {
*/
@Override
public void printHelp() {
- String header = "creates a plan that describes how much data should be " +
+ String header = "Creates a plan that describes how much data should be " +
"moved between disks.\n\n";
String footer = "\nPlan command creates a set of steps that represent a " +
@@ -196,7 +195,7 @@ public class PlanCommand extends Command {
" will balance the data.";
HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp("hdfs diskbalancer -uri <namenode> -plan " +
+ helpFormatter.printHelp("hdfs diskbalancer -plan " +
"<hostname> [options]", header, DiskBalancer.getPlanOptions(), footer);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66fa34c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
index 40729f8..9f8e399 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
@@ -201,9 +201,9 @@ public class ReportCommand extends Command {
" datanode, or prints out the list of nodes that will benefit from " +
"running disk balancer. Top defaults to " + getDefaultTop();
String footer = ". E.g.:\n"
- + "hdfs diskbalancer -fs http://namenode.uri -report\n"
- + "hdfs diskbalancer -fs http://namenode.uri -report -top 5\n"
- + "hdfs diskbalancer -fs http://namenode.uri -report "
+ + "hdfs diskbalancer -report\n"
+ + "hdfs diskbalancer -report -top 5\n"
+ + "hdfs diskbalancer -report "
+ "-node {DataNodeID | IP | Hostname}";
HelpFormatter helpFormatter = new HelpFormatter();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66fa34c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
index b3d51c4..0df9843 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/GreedyPlanner.java
@@ -67,7 +67,8 @@ public class GreedyPlanner implements Planner {
long startTime = Time.monotonicNow();
NodePlan plan = new NodePlan(node.getDataNodeName(),
node.getDataNodePort());
- LOG.info("Starting plan for Node : " + node.getDataNodeUUID());
+ LOG.info("Starting plan for Node : {}:{}",
+ node.getDataNodeName(), node.getDataNodePort());
while (node.isBalancingNeeded(this.threshold)) {
for (DiskBalancerVolumeSet vSet : node.getVolumeSets().values()) {
balanceVolumeSet(node, vSet, plan);
@@ -76,8 +77,9 @@ public class GreedyPlanner implements Planner {
long endTime = Time.monotonicNow();
String message = String
- .format("Compute Plan for Node : %s took %d ms ",
- node.getDataNodeUUID(), endTime - startTime);
+ .format("Compute Plan for Node : %s:%d took %d ms ",
+ node.getDataNodeName(), node.getDataNodePort(),
+ endTime - startTime);
LOG.info(message);
return plan;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66fa34c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
index 8a900ba..612aa2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
@@ -52,16 +52,6 @@ import java.io.PrintStream;
*/
public class DiskBalancer extends Configured implements Tool {
/**
- * NameNodeURI can point to either a real namenode, or a json file that
- * contains the diskBalancer data in json form, that jsonNodeConnector knows
- * how to deserialize.
- * <p>
- * Expected formats are :
- * <p>
- * hdfs://namenode.uri or file:///data/myCluster.json
- */
- public static final String NAMENODEURI = "fs";
- /**
* Computes a plan for a given set of nodes.
*/
public static final String PLAN = "plan";
@@ -275,13 +265,6 @@ public class DiskBalancer extends Configured implements Tool {
*/
private void addPlanCommands(Options opt) {
- Option uri = OptionBuilder.withLongOpt(NAMENODEURI)
- .withDescription("Address of the Namenode. e,g. hdfs://namenode")
- .hasArg()
- .create();
- getPlanOptions().addOption(uri);
- opt.addOption(uri);
-
Option plan = OptionBuilder.withLongOpt(PLAN)
.withDescription("creates a plan for datanode.")
.hasArg()
@@ -336,7 +319,6 @@ public class DiskBalancer extends Configured implements Tool {
private void addHelpCommands(Options opt) {
Option help = OptionBuilder.withLongOpt(HELP)
.hasOptionalArg()
- .withArgName(HELP)
.withDescription("valid commands are plan | execute | query | cancel" +
" | report")
.create();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66fa34c8/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
index 522dc5a..c5f73cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSDiskbalancer.md
@@ -53,11 +53,9 @@ The following sections discusses what commands are supported by disk balancer
The plan command can be run against a given datanode by running
- `hdfs diskbalancer -uri hdfs://mycluster.com -plan node1.mycluster.com`
+ `hdfs diskbalancer -plan node1.mycluster.com`
- uri is the address of the namenode and -plan points to the datanode that we
- need to plan for. By deafult, plan command writes its output to
- **/system/diskbalancer**.
+ The command accepts [Generic Options](../hadoop-common/CommandsManual.html#Generic_Options).
The plan command also has a set of parameters that allows user to control
the output and execution of the plan.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66fa34c8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
index 3accbc2..c1c137d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -31,6 +31,7 @@ import java.util.Scanner;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -74,8 +75,7 @@ public class TestDiskBalancerCommand {
/* test basic report */
@Test(timeout=60000)
public void testReportSimple() throws Exception {
- final String cmdLine = String.format("hdfs diskbalancer -fs %s -report",
- clusterJson.toString());
+ final String cmdLine = "hdfs diskbalancer -report";
final List<String> outputs = runCommand(cmdLine);
assertThat(
@@ -103,8 +103,7 @@ public class TestDiskBalancerCommand {
/* test less than 64 DataNode(s) as total, e.g., -report -top 32 */
@Test(timeout=60000)
public void testReportLessThanTotal() throws Exception {
- final String cmdLine = String.format(
- "hdfs diskbalancer -fs %s -report -top 32", clusterJson.toString());
+ final String cmdLine = "hdfs diskbalancer -report -top 32";
final List<String> outputs = runCommand(cmdLine);
assertThat(
@@ -127,8 +126,7 @@ public class TestDiskBalancerCommand {
/* test more than 64 DataNode(s) as total, e.g., -report -top 128 */
@Test(timeout=60000)
public void testReportMoreThanTotal() throws Exception {
- final String cmdLine = String.format(
- "hdfs diskbalancer -fs %s -report -top 128", clusterJson.toString());
+ final String cmdLine = "hdfs diskbalancer -report -top 128";
final List<String> outputs = runCommand(cmdLine);
assertThat(
@@ -152,8 +150,7 @@ public class TestDiskBalancerCommand {
/* test invalid top limit, e.g., -report -top xx */
@Test(timeout=60000)
public void testReportInvalidTopLimit() throws Exception {
- final String cmdLine = String.format(
- "hdfs diskbalancer -fs %s -report -top xx", clusterJson.toString());
+ final String cmdLine = "hdfs diskbalancer -report -top xx";
final List<String> outputs = runCommand(cmdLine);
assertThat(
@@ -177,14 +174,11 @@ public class TestDiskBalancerCommand {
containsString("9 volumes with node data density 1.97"))));
}
- /* test -report -node DataNodeID */
@Test(timeout=60000)
public void testReportNode() throws Exception {
- final String cmdLine = String
- .format(
- "hdfs diskbalancer -fs %s -report -node "
- + "a87654a9-54c7-4693-8dd9-c9c7021dc340",
- clusterJson.toString());
+ final String cmdLine =
+ "hdfs diskbalancer -report -node " +
+ "a87654a9-54c7-4693-8dd9-c9c7021dc340";
final List<String> outputs = runCommand(cmdLine);
assertThat(
@@ -275,6 +269,7 @@ public class TestDiskBalancerCommand {
org.apache.hadoop.hdfs.tools.DiskBalancer db =
new org.apache.hadoop.hdfs.tools.DiskBalancer(conf);
+ FileSystem.setDefaultUri(conf, clusterJson);
ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
PrintStream out = new PrintStream(bufOut);
db.run(cmds, out);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org