You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/05/30 21:12:03 UTC
[13/50] [abbrv] hadoop git commit: HDDS-96. Add an option in ozone
script to generate a site file with minimally required ozone configs.
Contributed by Dinesh Chitlangia.
HDDS-96. Add an option in ozone script to generate a site file with minimally required ozone configs.
Contributed by Dinesh Chitlangia.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8733012a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8733012a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8733012a
Branch: refs/heads/HDDS-48
Commit: 8733012ae35f2762d704f94975a762885d116795
Parents: 1e0d4b1
Author: Anu Engineer <ae...@apache.org>
Authored: Fri May 25 13:06:14 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Fri May 25 13:06:14 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdds/conf/OzoneConfiguration.java | 6 +-
hadoop-ozone/common/src/main/bin/ozone | 4 +
...TestGenerateOzoneRequiredConfigurations.java | 100 +++++++++++
.../GenerateOzoneRequiredConfigurations.java | 174 +++++++++++++++++++
.../hadoop/ozone/genconf/package-info.java | 24 +++
5 files changed, 305 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
index f07718c..36d953c 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java
@@ -137,7 +137,7 @@ public class OzoneConfiguration extends Configuration {
@Override
public String toString() {
- return this.getName() + " " + this.getValue() + this.getTag();
+ return this.getName() + " " + this.getValue() + " " + this.getTag();
}
@Override
@@ -152,11 +152,11 @@ public class OzoneConfiguration extends Configuration {
}
}
- public static void activate(){
+ public static void activate() {
// adds the default resources
Configuration.addDefaultResource("hdfs-default.xml");
Configuration.addDefaultResource("hdfs-site.xml");
Configuration.addDefaultResource("ozone-default.xml");
Configuration.addDefaultResource("ozone-site.xml");
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/common/src/main/bin/ozone
----------------------------------------------------------------------
diff --git a/hadoop-ozone/common/src/main/bin/ozone b/hadoop-ozone/common/src/main/bin/ozone
index 00261c7..6843bdd 100755
--- a/hadoop-ozone/common/src/main/bin/ozone
+++ b/hadoop-ozone/common/src/main/bin/ozone
@@ -47,6 +47,7 @@ function hadoop_usage
hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager "
hadoop_add_subcommand "version" client "print the version"
+ hadoop_add_subcommand "genconf" client "generate minimally required ozone configs and output to ozone-site.xml in specified path"
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
}
@@ -118,6 +119,9 @@ function ozonecmd_case
version)
HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
;;
+ genconf)
+ HADOOP_CLASSNAME=org.apache.hadoop.ozone.genconf.GenerateOzoneRequiredConfigurations
+ ;;
*)
HADOOP_CLASSNAME="${subcmd}"
if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
new file mode 100644
index 0000000..82582a6
--- /dev/null
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/genconf/TestGenerateOzoneRequiredConfigurations.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.genconf;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+
+/**
+ * Tests GenerateOzoneRequiredConfigurations.
+ */
+public class TestGenerateOzoneRequiredConfigurations {
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration conf;
+
+ /**
+ * Create a MiniDFSCluster for testing.
+ * <p>
+ * Ozone is made active by setting OZONE_ENABLED = true and
+ * OZONE_HANDLER_TYPE_KEY = "distributed"
+ *
+ * @throws IOException
+ */
+ @BeforeClass
+ public static void init() throws Exception {
+ conf = new OzoneConfiguration();
+ cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(1).build();
+ cluster.waitForClusterToBeReady();
+ }
+
+ /**
+ * Shutdown MiniDFSCluster.
+ */
+ @AfterClass
+ public static void shutdown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ /**
+ * Tests a valid path and generates ozone-site.xml.
+ * @throws Exception
+ */
+ @Test
+ public void generateConfigurationsSuccess() throws Exception {
+ String[] args = new String[]{"-output", "."};
+ GenerateOzoneRequiredConfigurations.main(args);
+
+ Assert.assertEquals("Path is valid",
+ true, GenerateOzoneRequiredConfigurations.isValidPath(args[1]));
+
+ Assert.assertEquals("Permission is valid",
+ true, GenerateOzoneRequiredConfigurations.canWrite(args[1]));
+
+ Assert.assertEquals("Config file generated",
+ 0, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1]));
+ }
+
+ /**
+ * Test to avoid generating ozone-site.xml when invalid permission.
+ * @throws Exception
+ */
+ @Test
+ public void generateConfigurationsFailure() throws Exception {
+ String[] args = new String[]{"-output", "/"};
+ GenerateOzoneRequiredConfigurations.main(args);
+
+ Assert.assertEquals("Path is valid",
+ true, GenerateOzoneRequiredConfigurations.isValidPath(args[1]));
+
+ Assert.assertEquals("Invalid permission",
+ false, GenerateOzoneRequiredConfigurations.canWrite(args[1]));
+
+ Assert.assertEquals("Config file not generated",
+ 1, GenerateOzoneRequiredConfigurations.generateConfigurations(args[1]));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java
new file mode 100644
index 0000000..6296c9d
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/GenerateOzoneRequiredConfigurations.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.genconf;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Marshaller;
+import java.io.File;
+import java.net.URL;
+import java.nio.file.InvalidPathException;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * GenerateOzoneRequiredConfigurations - A tool to generate ozone-site.xml<br>
+ * This tool generates an ozone-site.xml with minimally required configs.
+ * This tool can be invoked as follows:<br>
+ * <ul>
+ * <li>ozone genconf -output <Path to output file></li>
+ * <li>ozone genconf -help</li>
+ * </ul>
+ */
+public final class GenerateOzoneRequiredConfigurations {
+
+ private static final String OUTPUT = "-output";
+ private static final String HELP = "-help";
+ private static final String USAGE = "Usage: \nozone genconf "
+ + OUTPUT + " <Path to output file> \n"
+ + "ozone genconf "
+ + HELP;
+ private static final int SUCCESS = 0;
+ private static final int FAILURE = 1;
+
+ private GenerateOzoneRequiredConfigurations() {
+
+ }
+ /**
+ * Entry point for using genconf tool.
+ *
+ * @param args
+ * @throws JAXBException
+ */
+ public static void main(String[] args) {
+
+ try {
+ if (args.length == 0) {
+ System.out.println(USAGE);
+ System.exit(1);
+ }
+
+ switch (args[0]) {
+ case OUTPUT:
+ if (args.length > 1) {
+ int result = generateConfigurations(args[1]);
+ } else {
+ System.out.println("Path to output file is mandatory");
+ System.out.println(USAGE);
+ System.exit(1);
+ }
+ break;
+
+ case HELP:
+ System.out.println(USAGE);
+ System.exit(0);
+ break;
+
+ default:
+ System.out.println(USAGE);
+ System.exit(1);
+ }
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
+ * Check if the path is valid.
+ *
+ * @param path
+ * @return true, if path is valid, else return false
+ */
+ public static boolean isValidPath(String path) {
+ try {
+ Paths.get(path);
+ } catch (InvalidPathException | NullPointerException ex) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Check if user has permission to write in the specified path.
+ *
+ * @param path
+ * @return true, if the user has permission to write, else returns false
+ */
+ public static boolean canWrite(String path) {
+ File file = new File(path);
+ return file.canWrite();
+ }
+
+ /**
+ * Generate ozone-site.xml at specified path.
+ *
+ * @param path
+ * @return SUCCESS(0) if file can be generated, else returns FAILURE(1)
+ * @throws JAXBException
+ */
+ public static int generateConfigurations(String path) throws JAXBException {
+
+ if (!isValidPath(path)) {
+ System.out.println("Invalid path or insufficient permission");
+ return FAILURE;
+ }
+
+ if (!canWrite(path)) {
+ System.out.println("Invalid path or insufficient permission");
+ return FAILURE;
+ }
+
+ OzoneConfiguration oc = new OzoneConfiguration();
+
+ ClassLoader cL = Thread.currentThread().getContextClassLoader();
+ if (cL == null) {
+ cL = OzoneConfiguration.class.getClassLoader();
+ }
+ URL url = cL.getResource("ozone-default.xml");
+
+ List<OzoneConfiguration.Property> allProperties =
+ oc.readPropertyFromXml(url);
+
+ List<OzoneConfiguration.Property> requiredProperties = new ArrayList<>();
+
+ for (OzoneConfiguration.Property p : allProperties) {
+ if (p.getTag() != null && p.getTag().contains("REQUIRED")) {
+ requiredProperties.add(p);
+ }
+ }
+
+ OzoneConfiguration.XMLConfiguration requiredConfig =
+ new OzoneConfiguration.XMLConfiguration();
+ requiredConfig.setProperties(requiredProperties);
+
+ JAXBContext context =
+ JAXBContext.newInstance(OzoneConfiguration.XMLConfiguration.class);
+ Marshaller m = context.createMarshaller();
+ m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE);
+ m.marshal(requiredConfig, new File(path, "ozone-site.xml"));
+
+ System.out.println("ozone-site.xml has been generated at " + path);
+
+ return SUCCESS;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8733012a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java
new file mode 100644
index 0000000..4817d39
--- /dev/null
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/genconf/package-info.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.genconf;
+
+/**
+ * Command line tool to generate required Ozone configs to an ozone-site.xml.
+ */
+
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org