You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/10/09 04:48:35 UTC
git commit: AMBARI-7706. Slider View: view should check HDFS access
and stack-version compatibility along with using YARN user (srimanth)
Repository: ambari
Updated Branches:
refs/heads/trunk ac31c6b7f -> e98948acd
AMBARI-7706. Slider View: view should check HDFS access and stack-version compatibility along with using YARN user (srimanth)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e98948ac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e98948ac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e98948ac
Branch: refs/heads/trunk
Commit: e98948acda3f4900e28073e1f764928fbf5f2aa2
Parents: ac31c6b
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Wed Oct 8 18:44:03 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Wed Oct 8 19:48:28 2014 -0700
----------------------------------------------------------------------
.../slider/SliderAppsViewControllerImpl.java | 141 ++++++++++++++++++-
1 file changed, 138 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e98948ac/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
----------------------------------------------------------------------
diff --git a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
index 068ce9c..069a188 100644
--- a/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
+++ b/contrib/views/slider/src/main/java/org/apache/ambari/view/slider/SliderAppsViewControllerImpl.java
@@ -24,6 +24,7 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.lang.reflect.UndeclaredThrowableException;
+import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
@@ -36,6 +37,7 @@ import java.util.Set;
import java.util.zip.ZipException;
import org.apache.ambari.view.ViewContext;
+import org.apache.ambari.view.slider.ViewStatus.Validation;
import org.apache.ambari.view.slider.clients.AmbariCluster;
import org.apache.ambari.view.slider.clients.AmbariClusterInfo;
import org.apache.ambari.view.slider.clients.AmbariHostComponent;
@@ -48,6 +50,8 @@ import org.apache.ambari.view.slider.rest.client.SliderAppMasterClient.SliderApp
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -133,6 +137,11 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
AmbariCluster cluster = ambariClient.getCluster(clusterInfo);
AmbariServiceInfo hdfsServiceInfo = null;
AmbariServiceInfo yarnServiceInfo = null;
+ // Validate stack-version
+ Validation validateStackVersion = validateStackVersion(clusterInfo.getVersion());
+ if (validateStackVersion != null) {
+ status.getValidations().add(validateStackVersion);
+ }
for (AmbariServiceInfo svc : cluster.getServices()) {
if ("HDFS".equals(svc.getId())) {
hdfsServiceInfo = svc;
@@ -174,6 +183,12 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
newHadoopConfigs.put("security_enabled",
clusterEnvConfigs.get("security_enabled"));
}
+ if (cluster.getDesiredConfigs().containsKey("hdfs-site")) {
+ Map<String, String> hdfsSiteConfigs = ambariClient
+ .getConfiguration(cluster, "hdfs-site", cluster
+ .getDesiredConfigs().get("hdfs-site"));
+ newHadoopConfigs.putAll(hdfsSiteConfigs);
+ }
if (cluster.getDesiredConfigs().containsKey("yarn-site")) {
Map<String, String> yarnSiteConfigs = ambariClient
.getConfiguration(cluster, "yarn-site", cluster
@@ -182,6 +197,15 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
status.getParameters().put(PROPERTY_YARN_RM_WEBAPP_URL,
newHadoopConfigs.get("yarn.resourcemanager.webapp.address"));
}
+ if (cluster.getDesiredConfigs().containsKey("yarn-env")) {
+ Map<String, String> yarnEnvConfigs = ambariClient.getConfiguration(cluster, "yarn-env", cluster
+ .getDesiredConfigs().get("yarn-env"));
+ String yarnUser = yarnEnvConfigs.get("yarn_user");
+ if (yarnUser == null || yarnUser.trim().length() < 1) {
+ yarnUser = "yarn";
+ }
+ newHadoopConfigs.put("yarn_user", yarnUser);
+ }
if (cluster.getDesiredConfigs().containsKey("zookeeper-env")) {
Map<String, String> zkEnvConfigs = ambariClient.getConfiguration(
cluster, "zookeeper-env",
@@ -231,6 +255,10 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
newHadoopConfigs.put(PROPERTY_GANGLIA_CUSTOM_CLUSTERS, clustersCsv);
status.getParameters().put(PROPERTY_GANGLIA_CUSTOM_CLUSTERS, clustersCsv);
}
+ Validation validateHDFSAccess = validateHDFSAccess(newHadoopConfigs, hdfsServiceInfo);
+ if (validateHDFSAccess != null) {
+ status.getValidations().add(validateHDFSAccess);
+ }
} else {
status.getValidations().add(
new ViewStatus.Validation("Ambari cluster with ID ["
@@ -259,6 +287,72 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
return status;
}
+ /**
+ * Slider-view supports only some stack-versions. This method validates that the targeted cluster is supported.
+ *
+ * @param clusterVersion
+ * @return
+ */
+ private Validation validateStackVersion(String clusterVersion) {
+ // Assuming cluster versions are of the format "X-a.b.c.d"
+ String stackName = clusterVersion;
+ String stackVersion = clusterVersion;
+ int dashIndex = clusterVersion.indexOf('-');
+ if (dashIndex > -1 && dashIndex < clusterVersion.length() - 1) {
+ stackName = stackName.substring(0, dashIndex);
+ stackVersion = stackVersion.substring(dashIndex + 1);
+ }
+ String[] versionSplits = stackVersion.split("\\.");
+ if (!"HDP".equals(stackName) || versionSplits.length < 2) {
+ return new Validation("Stack version (" + clusterVersion + ") used by cluster is not supported");
+ }
+ try {
+ int majorVersion = Integer.parseInt(versionSplits[0]);
+ int minorVersion = Integer.parseInt(versionSplits[1]);
+ if (!(majorVersion >= 2 && minorVersion >= 2)) {
+ return new Validation("Stack version (" + clusterVersion + ") used by cluster is not supported");
+ }
+ } catch (NumberFormatException e) {
+ return new Validation("Stack version (" + clusterVersion + ") used by cluster is not supported");
+ }
+ return null;
+ }
+
+ private Validation validateHDFSAccess(final Map<String, String> hadoopConfigs, AmbariServiceInfo hdfsServiceInfo) {
+ if (hdfsServiceInfo != null && hdfsServiceInfo.isStarted()) {
+ if (hadoopConfigs.containsKey("fs.defaultFS")) {
+ final String fsPath = hadoopConfigs.get("fs.defaultFS");
+ try {
+ invokeHDFSClientRunnable(new HDFSClientRunnable<Boolean>() {
+ @Override
+ public Boolean run() throws IOException, InterruptedException {
+ HdfsConfiguration hdfsConfiguration = new HdfsConfiguration();
+ for (Entry<String, String> entry : hadoopConfigs.entrySet()) {
+ hdfsConfiguration.set(entry.getKey(), entry.getValue());
+ }
+ FileSystem fs = FileSystem.get(URI.create(fsPath), hdfsConfiguration, getUserToRunAs(hadoopConfigs));
+ Path homePath = fs.getHomeDirectory();
+ fs.listFiles(homePath, false);
+ fs.close();
+ return Boolean.TRUE;
+ }
+ }, hadoopConfigs);
+ } catch (IOException e) {
+ String message = "Error validating access to user home directory: " + e.getMessage();
+ logger.warn(message, e);
+ return new Validation(message);
+ } catch (InterruptedException e) {
+ String message = "Error validating access to user home directory: " + e.getMessage();
+ logger.warn(message, e);
+ return new Validation(message);
+ }
+ } else {
+ return new Validation("Unknown filesystem location for verification");
+ }
+ }
+ return null;
+ }
+
private String getApplicationIdString(ApplicationId appId) {
return Long.toString(appId.getClusterTimestamp()) + "_"
+ Integer.toString(appId.getId());
@@ -281,9 +375,20 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
public T run(SliderClient sliderClient) throws YarnException, IOException, InterruptedException;
}
+ private static interface HDFSClientRunnable<T> {
+ public T run() throws IOException, InterruptedException;
+ }
+
private String getUserToRunAs() {
+ return getUserToRunAs(getHadoopConfigs());
+ }
+
+ private String getUserToRunAs(Map<String, String> hadoopConfigs) {
String user = getViewParameterValue(PARAM_SLIDER_USER);
if (user == null || user.trim().length() < 1) {
+ if (hadoopConfigs.containsKey("yarn_user")) {
+ return hadoopConfigs.get("yarn_user");
+ }
return "yarn";
} else if ("${username}".equals(user)) {
return viewContext.getUsername();
@@ -292,11 +397,42 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
}
}
+ private <T> T invokeHDFSClientRunnable(final HDFSClientRunnable<T> runnable, Map<String, String> hadoopConfigs)
+ throws IOException, InterruptedException {
+ ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ try {
+ boolean securityEnabled = Boolean.valueOf(hadoopConfigs.get("security_enabled"));
+ UserGroupInformation sliderUser;
+ if (securityEnabled) {
+ String viewPrincipal = getViewParameterValue(PARAM_VIEW_PRINCIPAL);
+ String viewPrincipalKeytab = getViewParameterValue(PARAM_VIEW_PRINCIPAL_KEYTAB);
+ UserGroupInformation ambariUser = UserGroupInformation.loginUserFromKeytabAndReturnUGI(viewPrincipal, viewPrincipalKeytab);
+ sliderUser = UserGroupInformation.createProxyUser(getUserToRunAs(), ambariUser);
+ } else {
+ sliderUser = UserGroupInformation.getBestUGI(null, getUserToRunAs());
+ }
+ try {
+ T value = sliderUser.doAs(new PrivilegedExceptionAction<T>() {
+ @Override
+ public T run() throws Exception {
+ return runnable.run();
+ }
+ });
+ return value;
+ } catch (UndeclaredThrowableException e) {
+ throw e;
+ }
+ } finally {
+ Thread.currentThread().setContextClassLoader(currentClassLoader);
+ }
+ }
+
private <T> T invokeSliderClientRunnable(final SliderClientContextRunnable<T> runnable) throws IOException, InterruptedException, YarnException {
ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
try {
- boolean securityEnabled = Boolean.valueOf(getViewParameterValue(PROPERTY_SLIDER_SECURITY_ENABLED));
+ boolean securityEnabled = Boolean.valueOf(getHadoopConfigs().get(PROPERTY_SLIDER_SECURITY_ENABLED));
UserGroupInformation sliderUser;
if (securityEnabled) {
String viewPrincipal = getViewParameterValue(PARAM_VIEW_PRINCIPAL);
@@ -637,8 +773,7 @@ public class SliderAppsViewControllerImpl implements SliderAppsViewController {
for(Entry<String, String> entry: hadoopConfigs.entrySet()) {
yarnConfig.set(entry.getKey(), entry.getValue());
}
- yarnConfig.set("slider.security.enabled", hadoopConfigs.get("security_enabled"));
- yarnConfig.set("slider.yarn.queue", "default"); //TODO Remove?
+ yarnConfig.set(PROPERTY_SLIDER_SECURITY_ENABLED, hadoopConfigs.get("security_enabled"));
if (hadoopConfigs.containsKey(PROPERTY_SLIDER_ZK_QUORUM)) {
yarnConfig.set(PROPERTY_SLIDER_ZK_QUORUM, hadoopConfigs.get(PROPERTY_SLIDER_ZK_QUORUM));
}