You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/12/05 19:31:49 UTC
[hadoop] branch trunk updated: YARN-9781. SchedConfCli to get
current stored scheduler configuration. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new c71befa YARN-9781. SchedConfCli to get current stored scheduler configuration. Contributed by Prabhu Joseph
c71befa is described below
commit c71befaf8fec1e5a962359c9cc131fb47adfd9d1
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Thu Dec 5 20:31:24 2019 +0100
YARN-9781. SchedConfCli to get current stored scheduler configuration. Contributed by Prabhu Joseph
---
.../hadoop/yarn/client/cli/SchedConfCLI.java | 135 ++++++++++++++++-----
.../hadoop/yarn/client/cli/TestSchedConfCLI.java | 44 +++++--
.../apache/hadoop/yarn}/webapp/dao/ConfInfo.java | 10 +-
.../resourcemanager/webapp/RMWebServices.java | 2 +-
4 files changed, 149 insertions(+), 42 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
index 146db9e..2f2b601 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/SchedConfCLI.java
@@ -38,13 +38,24 @@ import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.webapp.dao.QueueConfigInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Marshaller;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
+import java.io.StringReader;
+import java.io.StringWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
@@ -65,12 +76,16 @@ public class SchedConfCLI extends Configured implements Tool {
private static final String REMOVE_QUEUES_OPTION = "removeQueues";
private static final String UPDATE_QUEUES_OPTION = "updateQueues";
private static final String GLOBAL_OPTIONS = "globalUpdates";
+ private static final String GET_SCHEDULER_CONF = "getConf";
private static final String FORMAT_CONF = "formatConfig";
private static final String HELP_CMD = "help";
private static final String CONF_ERR_MSG = "Specify configuration key " +
"value as confKey=confVal.";
+ private SSLFactory sslFactory;
+ private Client client;
+
public SchedConfCLI() {
super(new YarnConfiguration());
}
@@ -93,6 +108,8 @@ public class SchedConfCLI extends Configured implements Tool {
"Update queue configurations");
opts.addOption("global", GLOBAL_OPTIONS, true,
"Update global scheduler configurations");
+ opts.addOption("getconf", GET_SCHEDULER_CONF, false,
+ "Get current scheduler configurations");
opts.addOption("format", FORMAT_CONF, false,
"Format Scheduler Configuration and reload from" +
" capacity-scheduler.xml");
@@ -115,6 +132,7 @@ public class SchedConfCLI extends Configured implements Tool {
boolean hasOption = false;
boolean format = false;
+ boolean getConf = false;
SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
try {
if (parsedCli.hasOption(ADD_QUEUES_OPTION)) {
@@ -139,6 +157,10 @@ public class SchedConfCLI extends Configured implements Tool {
hasOption = true;
format = true;
}
+ if (parsedCli.hasOption(GET_SCHEDULER_CONF)) {
+ hasOption = true;
+ getConf = true;
+ }
} catch (IllegalArgumentException e) {
System.err.println(e.getMessage());
@@ -154,27 +176,98 @@ public class SchedConfCLI extends Configured implements Tool {
Configuration conf = getConf();
if (format) {
return WebAppUtils.execOnActiveRM(conf, this::formatSchedulerConf, null);
+ } else if (getConf) {
+ return WebAppUtils.execOnActiveRM(conf, this::getSchedulerConf, null);
} else {
return WebAppUtils.execOnActiveRM(conf,
this::updateSchedulerConfOnRMNode, updateInfo);
}
}
- @VisibleForTesting
- int formatSchedulerConf(String webAppAddress, WebResource resource)
+ private static void prettyFormatWithIndent(String input, int indent)
throws Exception {
+ Source xmlInput = new StreamSource(new StringReader(input));
+ StringWriter sw = new StringWriter();
+ StreamResult xmlOutput = new StreamResult(sw);
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ transformerFactory.setAttribute("indent-number", indent);
+ Transformer transformer = transformerFactory.newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.transform(xmlInput, xmlOutput);
+ System.out.println(xmlOutput.getWriter().toString());
+ }
+
+ private WebResource initializeWebResource(String webAppAddress) {
Configuration conf = getConf();
- SSLFactory clientSslFactory = null;
if (YarnConfiguration.useHttps(conf)) {
- clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
+ sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
}
- Client webServiceClient = createWebServiceClient(clientSslFactory);
+ client = createWebServiceClient(sslFactory);
+ return client.resource(webAppAddress);
+ }
+
+ private void destroyClient() {
+ if (client != null) {
+ client.destroy();
+ }
+ if (sslFactory != null) {
+ sslFactory.destroy();
+ }
+ }
+
+ @VisibleForTesting
+ int getSchedulerConf(String webAppAddress, WebResource resource)
+ throws Exception {
ClientResponse response = null;
resource = (resource != null) ? resource :
- webServiceClient.resource(webAppAddress);
+ initializeWebResource(webAppAddress);
+ try {
+ Builder builder;
+ if (UserGroupInformation.isSecurityEnabled()) {
+ builder = resource
+ .path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").accept(MediaType.APPLICATION_XML);
+ } else {
+ builder = resource
+ .path("ws").path("v1").path("cluster").path("scheduler-conf")
+ .queryParam("user.name", UserGroupInformation.getCurrentUser()
+ .getShortUserName()).accept(MediaType.APPLICATION_XML);
+ }
+ response = builder.get(ClientResponse.class);
+ if (response != null) {
+ if (response.getStatus() == Status.OK.getStatusCode()) {
+ ConfInfo schedulerConf = response.getEntity(ConfInfo.class);
+ JAXBContext jaxbContext = JAXBContext.newInstance(ConfInfo.class);
+ Marshaller jaxbMarshaller = jaxbContext.createMarshaller();
+ StringWriter sw = new StringWriter();
+ jaxbMarshaller.marshal(schedulerConf, sw);
+ prettyFormatWithIndent(sw.toString(), 2);
+ return 0;
+ } else {
+ System.err.println("Failed to get scheduler configuration: "
+ + response.getEntity(String.class));
+ }
+ } else {
+ System.err.println("Failed to get scheduler configuration: " +
+ "null response");
+ }
+ return -1;
+ } finally {
+ if (response != null) {
+ response.close();
+ }
+ destroyClient();
+ }
+ }
+ @VisibleForTesting
+ int formatSchedulerConf(String webAppAddress, WebResource resource)
+ throws Exception {
+ ClientResponse response = null;
+ resource = (resource != null) ? resource :
+ initializeWebResource(webAppAddress);
try {
- Builder builder = null;
+ Builder builder;
if (UserGroupInformation.isSecurityEnabled()) {
builder = resource
.path("ws").path("v1").path("cluster")
@@ -206,27 +299,15 @@ public class SchedConfCLI extends Configured implements Tool {
if (response != null) {
response.close();
}
- if (webServiceClient != null) {
- webServiceClient.destroy();
- }
- if (clientSslFactory != null) {
- clientSslFactory.destroy();
- }
+ destroyClient();
}
}
@VisibleForTesting
int updateSchedulerConfOnRMNode(String webAppAddress,
SchedConfUpdateInfo updateInfo) throws Exception {
- Configuration conf = getConf();
- SSLFactory clientSslFactory = null;
- if (YarnConfiguration.useHttps(conf)) {
- clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
- }
- Client webServiceClient = createWebServiceClient(clientSslFactory);
ClientResponse response = null;
- WebResource resource = webServiceClient.resource(webAppAddress);
-
+ WebResource resource = initializeWebResource(webAppAddress);
try {
Builder builder = null;
if (UserGroupInformation.isSecurityEnabled()) {
@@ -258,12 +339,7 @@ public class SchedConfCLI extends Configured implements Tool {
if (response != null) {
response.close();
}
- if (webServiceClient != null) {
- webServiceClient.destroy();
- }
- if (clientSslFactory != null) {
- clientSslFactory.destroy();
- }
+ destroyClient();
}
}
@@ -380,7 +456,8 @@ public class SchedConfCLI extends Configured implements Tool {
+ "[-update \"queueUpdatePath1:confKey1=confVal1\"] "
+ "[-global globalConfKey1=globalConfVal1,"
+ "globalConfKey2=globalConfVal2] "
- + "[-format]\n"
+ + "[-format] "
+ + "[-getconf]\n"
+ "Example (adding queues): yarn schedulerconf -add "
+ "\"root.a.a1:capacity=100,maximum-capacity=100;root.a.a2:capacity=0,"
+ "maximum-capacity=0\"\n"
@@ -393,6 +470,8 @@ public class SchedConfCLI extends Configured implements Tool {
+ "-global yarn.scheduler.capacity.maximum-applications=10000\n"
+ "Example (format scheduler configuration): yarn schedulerconf "
+ "-format\n"
+ + "Example (get scheduler configuration): yarn schedulerconf "
+ + "-getconf\n"
+ "Note: This is an alpha feature, the syntax/options are subject to "
+ "change, please run at your own risk.");
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
index 8a6c9c2..4233b4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestSchedConfCLI.java
@@ -200,6 +200,38 @@ public class TestSchedConfCLI extends JerseyTestBase {
config.setMaximumCapacity(a, 100f);
}
+ private void cleanUp() throws Exception {
+ if (rm != null) {
+ rm.stop();
+ }
+ CONF_FILE.delete();
+ if (CONF_FILE.exists()) {
+ throw new RuntimeException("Failed to delete configuration file");
+ }
+ if (OLD_CONF_FILE.exists()) {
+ if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
+ throw new RuntimeException("Failed to re-copy old" +
+ " configuration file");
+ }
+ }
+ super.tearDown();
+ }
+
+ @Test(timeout = 10000)
+ public void testGetSchedulerConf() throws Exception {
+ try {
+ super.setUp();
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
+ int exitCode = cli.getSchedulerConf("", resource());
+ assertEquals("SchedConfCLI failed to run", 0, exitCode);
+ assertTrue("Failed to get scheduler configuration",
+ sysOutStream.toString().contains("testqueue"));
+ } finally {
+ cleanUp();
+ }
+ }
+
@Test(timeout = 10000)
public void testFormatSchedulerConf() throws Exception {
try {
@@ -229,17 +261,7 @@ public class TestSchedConfCLI extends JerseyTestBase {
schedulerConf = provider.getConfiguration();
assertNull(schedulerConf.get("schedKey1"));
} finally {
- if (rm != null) {
- rm.stop();
- }
- CONF_FILE.delete();
- if (OLD_CONF_FILE.exists()) {
- if (!OLD_CONF_FILE.renameTo(CONF_FILE)) {
- throw new RuntimeException("Failed to re-copy old" +
- " configuration file");
- }
- }
- super.tearDown();
+ cleanUp();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java
similarity index 89%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java
index bfc450b..1971efa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ConfInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/dao/ConfInfo.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+package org.apache.hadoop.yarn.webapp.dao;
import org.apache.hadoop.conf.Configuration;
@@ -24,11 +24,14 @@ import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
+/**
+ * This class stores the Scheduler Configuration.
+ */
@XmlRootElement(name = "configuration")
@XmlAccessorType(XmlAccessType.FIELD)
public class ConfInfo {
- protected ArrayList<ConfItem> property = new ArrayList<>();
+ private ArrayList<ConfItem> property = new ArrayList<>();
public ConfInfo() {
} // JAXB needs this
@@ -46,6 +49,9 @@ public class ConfInfo {
return property;
}
+ /**
+ * This class stores the Configuration Property.
+ */
@XmlAccessorType(XmlAccessType.FIELD)
public static class ConfItem {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index f2f03a6..67d09da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -197,7 +197,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfigVersionInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webapp.WebServices;
@@ -210,6 +209,7 @@ import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.apache.hadoop.yarn.webapp.dao.ConfInfo;
import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org