You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/06/24 06:05:57 UTC
[26/49] hadoop git commit: HDFS-9461. DiskBalancer: Add Report
Command. Contributed by Xiaobing Zhou.
HDFS-9461. DiskBalancer: Add Report Command. Contributed by Xiaobing Zhou.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b502102b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b502102b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b502102b
Branch: refs/heads/trunk
Commit: b502102bb1a1f416f43dd1227886c57ccad70fcc
Parents: 121142c
Author: Anu Engineer <ae...@apache.org>
Authored: Fri Jun 10 21:15:54 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Thu Jun 23 18:21:08 2016 -0700
----------------------------------------------------------------------
.../server/diskbalancer/command/Command.java | 67 +
.../diskbalancer/command/ReportCommand.java | 197 +
.../datamodel/DiskBalancerVolume.java | 30 +
.../apache/hadoop/hdfs/tools/DiskBalancer.java | 61 +-
.../command/TestDiskBalancerCommand.java | 299 +
.../diskBalancer/data-cluster-64node-3disk.json | 9484 ++++++++++++++++++
6 files changed, 10136 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b502102b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
index 94a21d1..bbf91ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/Command.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.hdfs.server.diskbalancer.command;
import com.google.common.base.Preconditions;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.text.StrBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -70,6 +72,7 @@ public abstract class Command extends Configured {
private URI clusterURI;
private FileSystem fs = null;
private DiskBalancerCluster cluster = null;
+ private int topNodes;
private static final Path DEFAULT_LOG_DIR = new Path("/system/diskbalancer");
@@ -83,6 +86,7 @@ public abstract class Command extends Configured {
// These arguments are valid for all commands.
addValidCommandParameters(DiskBalancer.HELP, "Help for this command");
addValidCommandParameters("arg", "");
+ topNodes = 0;
}
/**
@@ -391,4 +395,67 @@ public abstract class Command extends Configured {
protected DiskBalancerCluster getCluster() {
return cluster;
}
+
+ /**
+ * returns default top number of nodes.
+ * @return default top number of nodes.
+ */
+ protected int getDefaultTop() {
+ return DiskBalancer.DEFAULT_TOP;
+ }
+
+ /**
+ * Put output line to log and string buffer.
+ * */
+ protected void recordOutput(final StrBuilder result,
+ final String outputLine) {
+ LOG.info(outputLine);
+ result.appendln(outputLine);
+ }
+
+ /**
+ * Parse top number of nodes to be processed.
+ * @return top number of nodes to be processed.
+ */
+ protected int parseTopNodes(final CommandLine cmd, final StrBuilder result) {
+ String outputLine = "";
+ int nodes = 0;
+ final String topVal = cmd.getOptionValue(DiskBalancer.TOP);
+ if (StringUtils.isBlank(topVal)) {
+ outputLine = String.format(
+ "No top limit specified, using default top value %d.",
+ getDefaultTop());
+ LOG.info(outputLine);
+ result.appendln(outputLine);
+ nodes = getDefaultTop();
+ } else {
+ try {
+ nodes = Integer.parseInt(topVal);
+ } catch (NumberFormatException nfe) {
+ outputLine = String.format(
+ "Top limit input is not numeric, using default top value %d.",
+ getDefaultTop());
+ LOG.info(outputLine);
+ result.appendln(outputLine);
+ nodes = getDefaultTop();
+ }
+ }
+
+ return Math.min(nodes, cluster.getNodes().size());
+ }
+
+ /**
+ * Set top number of nodes to be processed.
+ * */
+ public void setTopNodes(int topNodes) {
+ this.topNodes = topNodes;
+ }
+
+ /**
+ * Get top number of nodes to be processed.
+ * @return top number of nodes to be processed.
+ * */
+ public int getTopNodes() {
+ return topNodes;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b502102b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
new file mode 100644
index 0000000..acf9ff2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/command/ReportCommand.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdfs.server.diskbalancer.command;
+
+import java.io.PrintStream;
+import java.util.Collections;
+import java.util.ListIterator;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.text.StrBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerDataNode;
+import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
+import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolumeSet;
+import org.apache.hadoop.hdfs.tools.DiskBalancer;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Executes the report command.
+ *
+ * This command will report volume information for a specific DataNode or top X
+ * DataNode(s) benefiting from running DiskBalancer.
+ *
+ * This is done by reading the cluster info, sorting the DiskbalancerNodes by
+ * their NodeDataDensity and printing out the info.
+ */
+public class ReportCommand extends Command {
+
+ private PrintStream out;
+
+ public ReportCommand(Configuration conf, final PrintStream out) {
+ super(conf);
+ this.out = out;
+
+ addValidCommandParameters(DiskBalancer.REPORT,
+ "Report volume information of nodes.");
+
+ String desc = String.format(
+ "Top number of nodes to be processed. Default: %d", getDefaultTop());
+ addValidCommandParameters(DiskBalancer.TOP, desc);
+
+ desc = String.format("Print out volume information for a DataNode.");
+ addValidCommandParameters(DiskBalancer.NODE, desc);
+ }
+
+ @Override
+ public void execute(CommandLine cmd) throws Exception {
+ StrBuilder result = new StrBuilder();
+ String outputLine = "Processing report command";
+ recordOutput(result, outputLine);
+
+ Preconditions.checkState(cmd.hasOption(DiskBalancer.REPORT));
+ verifyCommandOptions(DiskBalancer.REPORT, cmd);
+ readClusterInfo(cmd);
+
+ final String nodeFormat =
+ "%d/%d %s[%s:%d] - <%s>: %d volumes with node data density %.2f.";
+ final String nodeFormatWithoutSequence =
+ "%s[%s:%d] - <%s>: %d volumes with node data density %.2f.";
+ final String volumeFormat =
+ "[%s: volume-%s] - %.2f used: %d/%d, %.2f free: %d/%d, "
+ + "isFailed: %s, isReadOnly: %s, isSkip: %s, isTransient: %s.";
+
+ if (cmd.hasOption(DiskBalancer.NODE)) {
+ /*
+ * Reporting volume information for a specific DataNode
+ */
+ handleNodeReport(cmd, result, nodeFormatWithoutSequence, volumeFormat);
+
+ } else { // handle TOP
+ /*
+ * Reporting volume information for top X DataNode(s)
+ */
+ handleTopReport(cmd, result, nodeFormat);
+ }
+
+ out.println(result.toString());
+ }
+
+ private void handleTopReport(final CommandLine cmd, final StrBuilder result,
+ final String nodeFormat) {
+ Collections.sort(getCluster().getNodes(), Collections.reverseOrder());
+
+ /* extract value that identifies top X DataNode(s) */
+ setTopNodes(parseTopNodes(cmd, result));
+
+ /*
+ * Reporting volume information of top X DataNode(s) in summary
+ */
+ final String outputLine = String.format(
+ "Reporting top %d DataNode(s) benefiting from running DiskBalancer.",
+ getTopNodes());
+ recordOutput(result, outputLine);
+
+ ListIterator<DiskBalancerDataNode> li = getCluster().getNodes()
+ .listIterator();
+
+ for (int i = 0; i < getTopNodes() && li.hasNext(); i++) {
+ DiskBalancerDataNode dbdn = li.next();
+ result.appendln(String.format(nodeFormat,
+ i+1,
+ getTopNodes(),
+ dbdn.getDataNodeName(),
+ dbdn.getDataNodeIP(),
+ dbdn.getDataNodePort(),
+ dbdn.getDataNodeUUID(),
+ dbdn.getVolumeCount(),
+ dbdn.getNodeDataDensity()));
+ }
+ }
+
+ private void handleNodeReport(final CommandLine cmd, StrBuilder result,
+ final String nodeFormat, final String volumeFormat) {
+ String outputLine = "";
+ /*
+ * get value that identifies a DataNode from command line, it could be UUID,
+ * IP address or host name.
+ */
+ final String nodeVal = cmd.getOptionValue(DiskBalancer.NODE);
+
+ if (StringUtils.isBlank(nodeVal)) {
+ outputLine = "The value for '-node' is neither specified or empty.";
+ recordOutput(result, outputLine);
+ } else {
+ /*
+ * Reporting volume information for a specific DataNode
+ */
+ outputLine = String.format(
+ "Reporting volume information for DataNode '%s'.", nodeVal);
+ recordOutput(result, outputLine);
+
+ final String trueStr = "True";
+ final String falseStr = "False";
+ DiskBalancerDataNode dbdn = getNode(nodeVal);
+
+ if (dbdn == null) {
+ outputLine = String.format(
+ "Can't find a DataNode that matches '%s'.", nodeVal);
+ recordOutput(result, outputLine);
+ } else {
+ result.appendln(String.format(nodeFormat,
+ dbdn.getDataNodeName(),
+ dbdn.getDataNodeIP(),
+ dbdn.getDataNodePort(),
+ dbdn.getDataNodeUUID(),
+ dbdn.getVolumeCount(),
+ dbdn.getNodeDataDensity()));
+
+ for (DiskBalancerVolumeSet vset : dbdn.getVolumeSets().values()) {
+ for (DiskBalancerVolume vol : vset.getVolumes()) {
+ result.appendln(String.format(volumeFormat,
+ vol.getStorageType(),
+ vol.getPath(),
+ vol.getUsedRatio(),
+ vol.getUsed(),
+ vol.getCapacity(),
+ vol.getFreeRatio(),
+ vol.getFreeSpace(),
+ vol.getCapacity(),
+ vol.isFailed() ? trueStr : falseStr,
+ vol.isReadOnly() ? trueStr : falseStr,
+ vol.isSkip() ? trueStr : falseStr,
+ vol.isTransient() ? trueStr : falseStr));
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ protected String getHelp() {
+ return "Report volume information for a specific DataNode or top X "
+ + "one(s) benefiting from running DiskBalancer, "
+ + "top defaults to " + getDefaultTop() + ". E.g.:\n"
+ + "hdfs diskbalancer -uri http://namenode.uri -report\n"
+ + "hdfs diskbalancer -uri http://namenode.uri -report -top 5\n"
+ + "hdfs diskbalancer -uri http://namenode.uri -report "
+ + "-node {DataNodeID | IP | Hostname}";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b502102b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
index 2a39609..a6a8bdc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/datamodel/DiskBalancerVolume.java
@@ -140,6 +140,36 @@ public class DiskBalancerVolume {
}
/**
+ * Get free space of the volume.
+ *
+ * @return long
+ */
+ @JsonIgnore
+ public long getFreeSpace() {
+ return getCapacity() - getUsed();
+ }
+
+ /**
+ * Get ratio between used space and capacity.
+ *
+ * @return double
+ */
+ @JsonIgnore
+ public double getUsedRatio() {
+ return (1.0 * getUsed()) / getCapacity();
+ }
+
+ /**
+ * Get ratio between free space and capacity.
+ *
+ * @return double
+ */
+ @JsonIgnore
+ public double getFreeRatio() {
+ return (1.0 * getFreeSpace()) / getCapacity();
+ }
+
+ /**
* Sets the capacity of this volume.
*
* @param totalCapacity long
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b502102b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
index dde2ce4..1251e96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DiskBalancer.java
@@ -29,12 +29,14 @@ import org.apache.hadoop.hdfs.server.diskbalancer.command.Command;
import org.apache.hadoop.hdfs.server.diskbalancer.command.ExecuteCommand;
import org.apache.hadoop.hdfs.server.diskbalancer.command.PlanCommand;
import org.apache.hadoop.hdfs.server.diskbalancer.command.QueryCommand;
+import org.apache.hadoop.hdfs.server.diskbalancer.command.ReportCommand;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.io.PrintStream;
import java.net.URISyntaxException;
/**
@@ -94,6 +96,22 @@ public class DiskBalancer extends Configured implements Tool {
*/
public static final String EXECUTE = "execute";
/**
+ * The report command prints out a disk fragmentation report about the data
+ * cluster. By default it prints the DEFAULT_TOP machines names with high
+ * nodeDataDensity {DiskBalancerDataNode#getNodeDataDensity} values. This
+ * means that these are the nodes that deviates from the ideal data
+ * distribution.
+ */
+ public static final String REPORT = "report";
+ /**
+ * specify top number of nodes to be processed.
+ */
+ public static final String TOP = "top";
+ /**
+ * specify default top number of nodes to be processed.
+ */
+ public static final int DEFAULT_TOP = 100;
+ /**
* Name or address of the node to execute against.
*/
public static final String NODE = "node";
@@ -157,9 +175,21 @@ public class DiskBalancer extends Configured implements Tool {
*/
@Override
public int run(String[] args) throws Exception {
+ return run(args, System.out);
+ }
+
+ /**
+ * Execute the command with the given arguments.
+ *
+ * @param args command specific arguments.
+ * @param out the output stream used for printing
+ * @return exit code.
+ * @throws Exception
+ */
+ public int run(String[] args, final PrintStream out) throws Exception {
Options opts = getOpts();
CommandLine cmd = parseArgs(args, opts);
- return dispatch(cmd, opts);
+ return dispatch(cmd, opts, out);
}
/**
@@ -173,6 +203,7 @@ public class DiskBalancer extends Configured implements Tool {
addExecuteCommands(opts);
addQueryCommands(opts);
addCancelCommands(opts);
+ addReportCommands(opts);
return opts;
}
@@ -256,6 +287,26 @@ public class DiskBalancer extends Configured implements Tool {
}
/**
+ * Adds report command options.
+ * @param opt Options
+ */
+ private void addReportCommands(Options opt) {
+ Option report = new Option(REPORT, false,
+ "Report volume information of DataNode(s)"
+ + " benefiting from running DiskBalancer. "
+ + "-report [top -X] | [-node {DataNodeID | IP | Hostname}].");
+ opt.addOption(report);
+
+ Option top = new Option(TOP, true,
+ "specify the top number of nodes to be processed.");
+ opt.addOption(top);
+
+ Option node = new Option(NODE, true,
+ "Name of the datanode in the format of DataNodeID, IP or hostname.");
+ opt.addOption(node);
+ }
+
+ /**
* This function parses all command line arguments and returns the appropriate
* values.
*
@@ -272,10 +323,12 @@ public class DiskBalancer extends Configured implements Tool {
* Dispatches calls to the right command Handler classes.
*
* @param cmd - CommandLine
+ * @param opts options of command line
+ * @param out the output stream used for printing
* @throws IOException
* @throws URISyntaxException
*/
- private int dispatch(CommandLine cmd, Options opts)
+ private int dispatch(CommandLine cmd, Options opts, final PrintStream out)
throws IOException, URISyntaxException {
Command currentCommand = null;
@@ -297,6 +350,10 @@ public class DiskBalancer extends Configured implements Tool {
currentCommand = new CancelCommand(getConf());
}
+ if (cmd.hasOption(DiskBalancer.REPORT)) {
+ currentCommand = new ReportCommand(getConf(), out);
+ }
+
if(currentCommand == null) {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp(80, "hdfs diskbalancer -uri [args]",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b502102b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
new file mode 100644
index 0000000..57e59f6
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/diskbalancer/command/TestDiskBalancerCommand.java
@@ -0,0 +1,299 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.hdfs.server.diskbalancer.command;
+
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.List;
+import java.util.Scanner;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ClusterConnector;
+import org.apache.hadoop.hdfs.server.diskbalancer.connectors.ConnectorFactory;
+import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Tests various CLI commands of DiskBalancer.
+ */
+public class TestDiskBalancerCommand {
+ private MiniDFSCluster cluster;
+ private URI clusterJson;
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, true);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3)
+ .storagesPerDatanode(2).build();
+ cluster.waitActive();
+
+ clusterJson = getClass().getResource(
+ "/diskBalancer/data-cluster-64node-3disk.json").toURI();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ // Just make sure we can shutdown datanodes.
+ cluster.getDataNodes().get(0).shutdown();
+ cluster.shutdown();
+ }
+ }
+
+ private void testReportSimple() throws Exception {
+ final String cmdLine = String.format("hdfs diskbalancer -uri %s -report",
+ clusterJson.toString());
+ final List<String> outputs = runCommand(cmdLine);
+
+ assertThat(
+ outputs.get(0),
+ containsString("Processing report command"));
+ assertThat(
+ outputs.get(1),
+ is(allOf(containsString("No top limit specified"),
+ containsString("using default top value"), containsString("100"))));
+ assertThat(
+ outputs.get(2),
+ is(allOf(
+ containsString("Reporting top"),
+ containsString("64"),
+ containsString(
+ "DataNode(s) benefiting from running DiskBalancer"))));
+ assertThat(
+ outputs.get(32),
+ is(allOf(containsString("30/64 null[null:0]"),
+ containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"),
+ containsString("9 volumes with node data density 1.97"))));
+
+ }
+
+ private void testReportLessThanTotal() throws Exception {
+ final String cmdLine = String.format(
+ "hdfs diskbalancer -uri %s -report -top 32", clusterJson.toString());
+ final List<String> outputs = runCommand(cmdLine);
+
+ assertThat(
+ outputs.get(0),
+ containsString("Processing report command"));
+ assertThat(
+ outputs.get(1),
+ is(allOf(
+ containsString("Reporting top"),
+ containsString("32"),
+ containsString(
+ "DataNode(s) benefiting from running DiskBalancer"))));
+ assertThat(
+ outputs.get(31),
+ is(allOf(containsString("30/32 null[null:0]"),
+ containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"),
+ containsString("9 volumes with node data density 1.97"))));
+ }
+
+ private void testReportMoreThanTotal() throws Exception {
+ final String cmdLine = String.format(
+ "hdfs diskbalancer -uri %s -report -top 128", clusterJson.toString());
+ final List<String> outputs = runCommand(cmdLine);
+
+ assertThat(
+ outputs.get(0),
+ containsString("Processing report command"));
+ assertThat(
+ outputs.get(1),
+ is(allOf(
+ containsString("Reporting top"),
+ containsString("64"),
+ containsString(
+ "DataNode(s) benefiting from running DiskBalancer"))));
+ assertThat(
+ outputs.get(31),
+ is(allOf(containsString("30/64 null[null:0]"),
+ containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"),
+ containsString("9 volumes with node data density 1.97"))));
+
+ }
+
+ private void testReportInvalidTopLimit() throws Exception {
+ final String cmdLine = String.format(
+ "hdfs diskbalancer -uri %s -report -top xx", clusterJson.toString());
+ final List<String> outputs = runCommand(cmdLine);
+
+ assertThat(
+ outputs.get(0),
+ containsString("Processing report command"));
+ assertThat(
+ outputs.get(1),
+ is(allOf(containsString("Top limit input is not numeric"),
+ containsString("using default top value"), containsString("100"))));
+ assertThat(
+ outputs.get(2),
+ is(allOf(
+ containsString("Reporting top"),
+ containsString("64"),
+ containsString(
+ "DataNode(s) benefiting from running DiskBalancer"))));
+ assertThat(
+ outputs.get(32),
+ is(allOf(containsString("30/64 null[null:0]"),
+ containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"),
+ containsString("9 volumes with node data density 1.97"))));
+ }
+
+ private void testReportNode() throws Exception {
+ final String cmdLine = String
+ .format(
+ "hdfs diskbalancer -uri %s -report -node "
+ + "a87654a9-54c7-4693-8dd9-c9c7021dc340",
+ clusterJson.toString());
+ final List<String> outputs = runCommand(cmdLine);
+
+ assertThat(
+ outputs.get(0),
+ containsString("Processing report command"));
+ assertThat(
+ outputs.get(1),
+ is(allOf(containsString("Reporting volume information for DataNode"),
+ containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"))));
+ assertThat(
+ outputs.get(2),
+ is(allOf(containsString("null[null:0]"),
+ containsString("a87654a9-54c7-4693-8dd9-c9c7021dc340"),
+ containsString("9 volumes with node data density 1.97"))));
+ assertThat(
+ outputs.get(3),
+ is(allOf(containsString("DISK"),
+ containsString("/tmp/disk/xx3j3ph3zd"),
+ containsString("0.72 used: 289544224916/400000000000"),
+ containsString("0.28 free: 110455775084/400000000000"))));
+ assertThat(
+ outputs.get(4),
+ is(allOf(containsString("DISK"),
+ containsString("/tmp/disk/Mxfcfmb24Y"),
+ containsString("0.92 used: 733099315216/800000000000"),
+ containsString("0.08 free: 66900684784/800000000000"))));
+ assertThat(
+ outputs.get(5),
+ is(allOf(containsString("DISK"),
+ containsString("DISK"),
+ containsString("/tmp/disk/KmHefYNURo"),
+ containsString("0.20 used: 39160240782/200000000000"),
+ containsString("0.80 free: 160839759218/200000000000"))));
+ assertThat(
+ outputs.get(6),
+ is(allOf(containsString("RAM_DISK"),
+ containsString("/tmp/disk/MXRyYsCz3U"),
+ containsString("0.55 used: 438102096853/800000000000"),
+ containsString("0.45 free: 361897903147/800000000000"))));
+ assertThat(
+ outputs.get(7),
+ is(allOf(containsString("RAM_DISK"),
+ containsString("/tmp/disk/DtmAygEU6f"),
+ containsString("0.34 used: 134602910470/400000000000"),
+ containsString("0.66 free: 265397089530/400000000000"))));
+ assertThat(
+ outputs.get(8),
+ is(allOf(containsString("RAM_DISK"),
+ containsString("/tmp/disk/BoBlQFxhfw"),
+ containsString("0.60 used: 477590453390/800000000000"),
+ containsString("0.40 free: 322409546610/800000000000"))));
+ assertThat(
+ outputs.get(9),
+ is(allOf(containsString("SSD"),
+ containsString("/tmp/disk/BGe09Y77dI"),
+ containsString("0.89 used: 890446265501/1000000000000"),
+ containsString("0.11 free: 109553734499/1000000000000"))));
+ assertThat(
+ outputs.get(10),
+ is(allOf(containsString("SSD"),
+ containsString("/tmp/disk/JX3H8iHggM"),
+ containsString("0.31 used: 2782614512957/9000000000000"),
+ containsString("0.69 free: 6217385487043/9000000000000"))));
+ assertThat(
+ outputs.get(11),
+ is(allOf(containsString("SSD"),
+ containsString("/tmp/disk/uLOYmVZfWV"),
+ containsString("0.75 used: 1509592146007/2000000000000"),
+ containsString("0.25 free: 490407853993/2000000000000"))));
+ }
+
+ @Test(timeout=60000)
+ public void testReportCommmand() throws Exception {
+
+ /* test basic report */
+ testReportSimple();
+
+ /* test less than 64 DataNode(s) as total, e.g., -report -top 32 */
+ testReportLessThanTotal();
+
+ /* test more than 64 DataNode(s) as total, e.g., -report -top 128 */
+ testReportMoreThanTotal();
+
+ /* test invalid top limit, e.g., -report -top xx */
+ testReportInvalidTopLimit();
+
+ /* test -report -node DataNodeID */
+ testReportNode();
+ }
+
+ @Test
+ public void testReadClusterFromJson() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setBoolean(DFSConfigKeys.DFS_DISK_BALANCER_ENABLED, true);
+
+ ClusterConnector jsonConnector = ConnectorFactory.getCluster(clusterJson,
+ conf);
+ DiskBalancerCluster diskBalancerCluster = new DiskBalancerCluster(
+ jsonConnector);
+ diskBalancerCluster.readClusterInfo();
+ assertEquals(64, diskBalancerCluster.getNodes().size());
+ }
+
+ private List<String> runCommand(final String cmdLine) throws Exception {
+
+ String[] cmds = StringUtils.split(cmdLine, ' ');
+ Configuration conf = new HdfsConfiguration();
+ org.apache.hadoop.hdfs.tools.DiskBalancer db =
+ new org.apache.hadoop.hdfs.tools.DiskBalancer(conf);
+
+ ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
+ PrintStream out = new PrintStream(bufOut);
+ db.run(cmds, out);
+
+ Scanner scanner = new Scanner(bufOut.toString());
+ List<String> outputs = Lists.newArrayList();
+ while (scanner.hasNextLine()) {
+ outputs.add(scanner.nextLine());
+ }
+ return outputs;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org