You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/12/18 10:41:45 UTC
hadoop git commit: HDDS-539. Ozone datanode command ignores the
invalid options. Contributed by Vinicius Higa Murakami.
Repository: hadoop
Updated Branches:
refs/heads/trunk 94b368f29 -> ee10ba26d
HDDS-539. Ozone datanode command ignores the invalid options. Contributed by Vinicius Higa Murakami.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee10ba26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee10ba26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee10ba26
Branch: refs/heads/trunk
Commit: ee10ba26dd1e1cbddef7f8be19da595e70e3108e
Parents: 94b368f
Author: Shashikant Banerjee <sh...@apache.org>
Authored: Tue Dec 18 16:10:27 2018 +0530
Committer: Shashikant Banerjee <sh...@apache.org>
Committed: Tue Dec 18 16:10:27 2018 +0530
----------------------------------------------------------------------
.../hadoop/ozone/HddsDatanodeService.java | 118 +++++----
.../hadoop/ozone/TestHddsDatanodeService.java | 3 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 6 +-
.../ozone/ozShell/TestOzoneDatanodeShell.java | 250 +++++++++++++++++++
4 files changed, 325 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee10ba26/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
index 348196c..fb2eba3 100644
--- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java
@@ -1,19 +1,19 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
package org.apache.hadoop.ozone;
@@ -21,20 +21,20 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.HddsUtils;
+import org.apache.hadoop.hdds.cli.GenericCli;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine
- .DatanodeStateMachine;
-import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.util.ServicePlugin;
import org.apache.hadoop.util.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import picocli.CommandLine.Command;
import java.io.File;
import java.io.IOException;
@@ -48,12 +48,16 @@ import static org.apache.hadoop.util.ExitUtil.terminate;
/**
* Datanode service plugin to start the HDDS container services.
*/
-public class HddsDatanodeService implements ServicePlugin {
+
+@Command(name = "ozone datanode",
+ hidden = true, description = "Start the datanode for ozone",
+ versionProvider = HddsVersionProvider.class,
+ mixinStandardHelpOptions = true)
+public class HddsDatanodeService extends GenericCli implements ServicePlugin {
private static final Logger LOG = LoggerFactory.getLogger(
HddsDatanodeService.class);
-
private OzoneConfiguration conf;
private DatanodeDetails datanodeDetails;
private DatanodeStateMachine datanodeStateMachine;
@@ -80,6 +84,50 @@ public class HddsDatanodeService implements ServicePlugin {
}
}
+ @VisibleForTesting
+ public static HddsDatanodeService createHddsDatanodeService(
+ String[] args, Configuration conf) {
+ return createHddsDatanodeService(args, conf, true);
+ }
+
+ /**
+ * Create an Datanode instance based on the supplied command-line arguments.
+ * <p>
+ * This method is intended for unit tests only. It suppresses the
+ * startup/shutdown message and skips registering Unix signal handlers.
+ *
+ * @param args command line arguments.
+ * @param conf HDDS configuration
+ * @param printBanner if true, then log a verbose startup message.
+ * @return Datanode instance
+ */
+ private static HddsDatanodeService createHddsDatanodeService(
+ String[] args, Configuration conf, boolean printBanner) {
+ if (args.length == 0 && printBanner) {
+ StringUtils
+ .startupShutdownMessage(HddsDatanodeService.class, args, LOG);
+ return new HddsDatanodeService(conf);
+ } else {
+ new HddsDatanodeService().run(args);
+ return null;
+ }
+ }
+
+ public static void main(String[] args) {
+ try {
+ Configuration conf = new OzoneConfiguration();
+ HddsDatanodeService hddsDatanodeService =
+ createHddsDatanodeService(args, conf, true);
+ if (hddsDatanodeService != null) {
+ hddsDatanodeService.start(null);
+ hddsDatanodeService.join();
+ }
+ } catch (Throwable e) {
+ LOG.error("Exception in HddsDatanodeService.", e);
+ terminate(1, e);
+ }
+ }
+
/**
* Starts HddsDatanode services.
*
@@ -87,6 +135,7 @@ public class HddsDatanodeService implements ServicePlugin {
*/
@Override
public void start(Object service) {
+ DefaultMetricsSystem.initialize("HddsDatanode");
OzoneConfiguration.activate();
if (service instanceof Configurable) {
conf = new OzoneConfiguration(((Configurable) service).getConf());
@@ -148,7 +197,7 @@ public class HddsDatanodeService implements ServicePlugin {
} catch (RuntimeException e) {
String pluginsValue = conf.get(HDDS_DATANODE_PLUGINS_KEY);
LOG.error("Unable to load HDDS DataNode plugins. " +
- "Specified list of plugins: {}",
+ "Specified list of plugins: {}",
pluginsValue, e);
throw e;
}
@@ -170,8 +219,8 @@ public class HddsDatanodeService implements ServicePlugin {
public OzoneConfiguration getConf() {
return conf;
}
+
/**
- *
* Return DatanodeDetails if set, return null otherwise.
*
* @return DatanodeDetails
@@ -213,7 +262,7 @@ public class HddsDatanodeService implements ServicePlugin {
}
@Override
- public void close() throws IOException {
+ public void close() {
if (plugins != null) {
for (ServicePlugin plugin : plugins) {
try {
@@ -224,33 +273,4 @@ public class HddsDatanodeService implements ServicePlugin {
}
}
}
-
- public static HddsDatanodeService createHddsDatanodeService(
- Configuration conf) {
- return new HddsDatanodeService(conf);
- }
-
- public static void main(String[] args) {
- try {
- if (DFSUtil.parseHelpArgument(
- args, "Starts HDDS Datanode", System.out, false)) {
- System.exit(0);
- }
- Configuration conf = new OzoneConfiguration();
- GenericOptionsParser hParser = new GenericOptionsParser(conf, args);
- if (!hParser.isParseSuccessful()) {
- GenericOptionsParser.printGenericCommandUsage(System.err);
- System.exit(1);
- }
- StringUtils.startupShutdownMessage(HddsDatanodeService.class, args, LOG);
- DefaultMetricsSystem.initialize("HddsDatanode");
- HddsDatanodeService hddsDatanodeService =
- createHddsDatanodeService(conf);
- hddsDatanodeService.start(null);
- hddsDatanodeService.join();
- } catch (Throwable e) {
- LOG.error("Exception in HddsDatanodeService.", e);
- terminate(1, e);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee10ba26/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
index 18cd00c..b1ac4b6 100644
--- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
+++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/TestHddsDatanodeService.java
@@ -42,6 +42,7 @@ public class TestHddsDatanodeService {
private File testDir;
private Configuration conf;
private HddsDatanodeService service;
+ private String[] args = new String[] {};
@Before
public void setUp() {
@@ -63,7 +64,7 @@ public class TestHddsDatanodeService {
@Test
public void testStartup() throws IOException {
- service = HddsDatanodeService.createHddsDatanodeService(conf);
+ service = HddsDatanodeService.createHddsDatanodeService(args, conf);
service.start(null);
service.join();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee10ba26/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index ceea3ee..ee47b44 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -268,8 +268,9 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
// wait for node to be removed from SCM healthy node list.
waitForClusterToBeReady();
}
+ String[] args = new String[]{};
HddsDatanodeService service =
- HddsDatanodeService.createHddsDatanodeService(conf);
+ HddsDatanodeService.createHddsDatanodeService(args, conf);
hddsDatanodes.add(i, service);
service.start(null);
if (waitForDatanode) {
@@ -478,6 +479,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
configureHddsDatanodes();
String scmAddress = scm.getDatanodeRpcAddress().getHostString() +
":" + scm.getDatanodeRpcAddress().getPort();
+ String[] args = new String[] {};
conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, scmAddress);
List<HddsDatanodeService> hddsDatanodes = new ArrayList<>();
for (int i = 0; i < numOfDatanodes; i++) {
@@ -495,7 +497,7 @@ public final class MiniOzoneClusterImpl implements MiniOzoneCluster {
ratisDir.toString());
hddsDatanodes.add(
- HddsDatanodeService.createHddsDatanodeService(dnConf));
+ HddsDatanodeService.createHddsDatanodeService(args, dnConf));
}
return hddsDatanodes;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee10ba26/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
new file mode 100644
index 0000000..5fefc53
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ozShell/TestOzoneDatanodeShell.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.ozShell;
+
+import com.google.common.base.Strings;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdds.cli.MissingSubcommandException;
+import org.apache.hadoop.hdds.client.ReplicationFactor;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.HddsDatanodeService;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.client.rest.RestClient;
+import org.apache.hadoop.ozone.client.rpc.RpcClient;
+import org.apache.hadoop.ozone.om.helpers.ServiceInfo;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+import picocli.CommandLine.ExecutionException;
+import picocli.CommandLine.IExceptionHandler2;
+import picocli.CommandLine.ParameterException;
+import picocli.CommandLine.ParseResult;
+import picocli.CommandLine.RunLast;
+
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
+import static org.junit.Assert.fail;
+
+/**
+ * This test class specified for testing Ozone datanode shell command.
+ */
+@RunWith(value = Parameterized.class)
+
+public class TestOzoneDatanodeShell {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestOzoneDatanodeShell.class);
+
+ /**
+ * Set the timeout for every test.
+ */
+ @Rule
+ public Timeout testTimeout = new Timeout(300000);
+
+ private static String url;
+ private static File baseDir;
+ private static OzoneConfiguration conf = null;
+ private static MiniOzoneCluster cluster = null;
+ private static ClientProtocol client = null;
+ private static HddsDatanodeService datanode = null;
+
+ private final ByteArrayOutputStream out = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream err = new ByteArrayOutputStream();
+ private static final PrintStream OLD_OUT = System.out;
+ private static final PrintStream OLD_ERR = System.err;
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> clientProtocol() {
+ Object[][] params = new Object[][]{
+ {RpcClient.class},
+ {RestClient.class}};
+ return Arrays.asList(params);
+ }
+
+ @Parameterized.Parameter
+ public Class clientProtocol;
+ /**
+ * Create a MiniDFSCluster for testing with using distributed Ozone
+ * handler type.
+ *
+ * @throws Exception
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+
+ String path = GenericTestUtils.getTempPath(
+ TestOzoneShell.class.getSimpleName());
+ baseDir = new File(path);
+ baseDir.mkdirs();
+
+ datanode = new HddsDatanodeService();
+
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(3)
+ .build();
+ conf.setInt(OZONE_REPLICATION, ReplicationFactor.THREE.getValue());
+ conf.setQuietMode(false);
+ client = new RpcClient(conf);
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+
+ if (baseDir != null) {
+ FileUtil.fullyDelete(baseDir, true);
+ }
+ }
+
+ @Before
+ public void setup() {
+ System.setOut(new PrintStream(out));
+ System.setErr(new PrintStream(err));
+ if(clientProtocol.equals(RestClient.class)) {
+ String hostName = cluster.getOzoneManager().getHttpServer()
+ .getHttpAddress().getHostName();
+ int port = cluster
+ .getOzoneManager().getHttpServer().getHttpAddress().getPort();
+ url = String.format("http://" + hostName + ":" + port);
+ } else {
+ List<ServiceInfo> services = null;
+ try {
+ services = cluster.getOzoneManager().getServiceList();
+ } catch (IOException e) {
+ LOG.error("Could not get service list from OM");
+ }
+ String hostName = services.stream().filter(
+ a -> a.getNodeType().equals(HddsProtos.NodeType.OM))
+ .collect(Collectors.toList()).get(0).getHostname();
+
+ String port = cluster.getOzoneManager().getRpcPort();
+ url = String.format("o3://" + hostName + ":" + port);
+ }
+ }
+
+ @After
+ public void reset() {
+ // reset stream after each unit test
+ out.reset();
+ err.reset();
+
+ // restore system streams
+ System.setOut(OLD_OUT);
+ System.setErr(OLD_ERR);
+ }
+
+
+ private void executeDatanode(HddsDatanodeService hdds, String[] args) {
+ List<String> arguments = new ArrayList(Arrays.asList(args));
+ LOG.info("Executing ozone datanode command with args {}", arguments);
+ CommandLine cmd = datanode.getCmd();
+
+ IExceptionHandler2<List<Object>> exceptionHandler =
+ new IExceptionHandler2<List<Object>>() {
+ @Override
+ public List<Object> handleParseException(ParameterException ex,
+ String[] args) {
+ throw ex;
+ }
+
+ @Override
+ public List<Object> handleExecutionException(ExecutionException ex,
+ ParseResult parseResult) {
+ throw ex;
+ }
+ };
+ cmd.parseWithHandlers(new RunLast(),
+ exceptionHandler, args);
+ }
+
+ /**
+ * Execute command, assert exception message and returns true if error
+ * was thrown and contains the specified usage string.
+ */
+ private void executeDatanodeWithError(HddsDatanodeService hdds, String[] args,
+ String expectedError, String usage) {
+ if (Strings.isNullOrEmpty(expectedError)) {
+ executeDatanode(hdds, args);
+ } else {
+ try {
+ executeDatanode(hdds, args);
+ fail("Exception is expected from command execution " + Arrays
+ .asList(args));
+ } catch (Exception ex) {
+ if (!Strings.isNullOrEmpty(expectedError)) {
+ Throwable exceptionToCheck = ex;
+ if (exceptionToCheck.getCause() != null) {
+ exceptionToCheck = exceptionToCheck.getCause();
+ }
+ Assert.assertTrue(
+ String.format(
+ "Error of shell code doesn't contain the " +
+ "exception [%s] in [%s]",
+ expectedError, exceptionToCheck.getMessage()),
+ exceptionToCheck.getMessage().contains(expectedError));
+ Assert.assertTrue(
+ exceptionToCheck instanceof MissingSubcommandException);
+ Assert.assertTrue(
+ ((MissingSubcommandException) exceptionToCheck)
+ .getUsage().contains(usage));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testDatanodeIncompleteCommand() {
+ LOG.info("Running testDatanodeIncompleteCommand");
+ String expectedError = "Incomplete command";
+ String[] args = new String[]{}; //executing 'ozone datanode'
+
+ executeDatanodeWithError(datanode, args, expectedError,
+ "Usage: ozone datanode [-hV] [--verbose] [-D=<String=String>]...");
+
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org