You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ey...@apache.org on 2020/01/06 18:26:22 UTC
[hadoop] branch trunk updated: YARN-9956. Improved connection error
message for YARN ApiServerClient. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
eyang pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new d81d45f YARN-9956. Improved connection error message for YARN ApiServerClient. Contributed by Prabhu Joseph
d81d45f is described below
commit d81d45ff2fc9a1c424222e021f9306bf64c916b2
Author: Eric Yang <ey...@apache.org>
AuthorDate: Mon Jan 6 13:24:16 2020 -0500
YARN-9956. Improved connection error message for YARN ApiServerClient.
Contributed by Prabhu Joseph
---
.../yarn/service/client/ApiServiceClient.java | 78 ++++++++++++----------
.../yarn/service/client/TestApiServiceClient.java | 22 ++++++
.../service/client/TestSecureApiServiceClient.java | 2 +
.../hadoop/yarn/client/util/YarnClientUtils.java | 2 +-
4 files changed, 68 insertions(+), 36 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 834bb03..3c2c3c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.util.YarnClientUtils;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -94,7 +95,7 @@ public class ApiServiceClient extends AppAdminClient {
/**
* Calculate Resource Manager address base on working REST API.
*/
- String getRMWebAddress() {
+ String getRMWebAddress() throws IOException {
Configuration conf = getConfig();
String scheme = "http://";
String path = "/app/v1/services/version";
@@ -105,43 +106,50 @@ public class ApiServiceClient extends AppAdminClient {
rmAddress = conf
.get("yarn.resourcemanager.webapp.https.address");
}
- boolean useKerberos = UserGroupInformation.isSecurityEnabled();
- List<String> rmServers = getRMHAWebAddresses(conf);
- for (String host : rmServers) {
- try {
- Client client = Client.create();
- client.setFollowRedirects(false);
- StringBuilder sb = new StringBuilder();
- sb.append(scheme)
- .append(host)
- .append(path);
- if (!useKerberos) {
- try {
- String username = UserGroupInformation.getCurrentUser().getShortUserName();
- sb.append("?user.name=")
- .append(username);
- } catch (IOException e) {
- LOG.debug("Fail to resolve username: {}", e);
+
+ if (HAUtil.isHAEnabled(conf)) {
+ boolean useKerberos = UserGroupInformation.isSecurityEnabled();
+ List<String> rmServers = getRMHAWebAddresses(conf);
+ StringBuilder diagnosticsMsg = new StringBuilder();
+ for (String host : rmServers) {
+ try {
+ Client client = Client.create();
+ client.setFollowRedirects(false);
+ StringBuilder sb = new StringBuilder();
+ sb.append(scheme)
+ .append(host)
+ .append(path);
+ if (!useKerberos) {
+ try {
+ String username = UserGroupInformation.getCurrentUser()
+ .getShortUserName();
+ sb.append("?user.name=")
+ .append(username);
+ } catch (IOException e) {
+ LOG.debug("Fail to resolve username: {}", e);
+ }
}
+ Builder builder = client
+ .resource(sb.toString()).type(MediaType.APPLICATION_JSON);
+ if (useKerberos) {
+ String[] server = host.split(":");
+ String challenge = YarnClientUtils.generateToken(server[0]);
+ builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
+ challenge);
+ LOG.debug("Authorization: Negotiate {}", challenge);
+ }
+ ClientResponse test = builder.get(ClientResponse.class);
+ if (test.getStatus() == 200) {
+ return scheme + host;
+ }
+ } catch (Exception e) {
+ LOG.info("Fail to connect to: " + host);
+ LOG.debug("Root cause: ", e);
+ diagnosticsMsg.append("Error connecting to " + host
+ + " due to " + e.getMessage() + "\n");
}
- Builder builder = client
- .resource(sb.toString()).type(MediaType.APPLICATION_JSON);
- if (useKerberos) {
- String[] server = host.split(":");
- String challenge = YarnClientUtils.generateToken(server[0]);
- builder.header(HttpHeaders.AUTHORIZATION, "Negotiate " +
- challenge);
- LOG.debug("Authorization: Negotiate {}", challenge);
- }
- ClientResponse test = builder.get(ClientResponse.class);
- if (test.getStatus() == 200) {
- rmAddress = host;
- break;
- }
- } catch (Exception e) {
- LOG.info("Fail to connect to: "+host);
- LOG.debug("Root cause: {}", e);
}
+ throw new IOException(diagnosticsMsg.toString());
}
return scheme+rmAddress;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
index afdf16e..65849a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestApiServiceClient.java
@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletResponse;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
@@ -118,6 +119,27 @@ public class TestApiServiceClient {
}
@Test
+ public void testGetRMWebAddress() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1");
+ conf.set(YarnConfiguration.RM_HA_ID, "rm1");
+ conf.set("yarn.resourcemanager.webapp.address.rm1", "localhost:0");
+ ApiServiceClient asc1 = new ApiServiceClient(conf);
+ boolean exceptionCaught = false;
+ String diagnosticsMsg = null;
+ try {
+ String rmWebAddress = asc1.getRMWebAddress();
+ } catch (IOException e){
+ exceptionCaught = true;
+ diagnosticsMsg = e.getMessage();
+ }
+ assertTrue("ApiServiceClient failed to throw exception", exceptionCaught);
+ assertTrue("Exception Message does not match",
+ diagnosticsMsg.contains("Error connecting to localhost:0"));
+ }
+
+ @Test
public void testLaunch() {
String fileName = "target/test-classes/example-app.json";
String appName = "example-app";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
index 8c7da62..e2d613a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSecureApiServiceClient.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.SaslRpcServer.QualityOfProtection;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.log4j.Logger;
import org.eclipse.jetty.server.Server;
@@ -152,6 +153,7 @@ public class TestSecureApiServiceClient extends KerberosSecurityTestcase {
rmServers.add("localhost:8088");
testConf.set("yarn.resourcemanager.webapp.address",
"localhost:8088");
+ testConf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
asc = new ApiServiceClient() {
@Override
List<String> getRMHAWebAddresses(Configuration conf) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
index abed6c6..94b13a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/util/YarnClientUtils.java
@@ -247,7 +247,7 @@ public abstract class YarnClientUtils {
StandardCharsets.US_ASCII);
} catch (GSSException | IllegalAccessException
| NoSuchFieldException | ClassNotFoundException e) {
- LOG.error("Error: {}", e);
+ LOG.error("Error: ", e);
throw new AuthenticationException(e);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org