You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2015/12/03 00:38:57 UTC
incubator-ranger git commit: RANGER-754:Ranger YARN Plugin lookup and
test connection should support SPENGO enabled HTTP Authentication
Repository: incubator-ranger
Updated Branches:
refs/heads/ranger-0.5 f9fc17f14 -> eb966b30a
RANGER-754:Ranger YARN Plugin lookup and test connection should support SPENGO enabled HTTP Authentication
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/eb966b30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/eb966b30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/eb966b30
Branch: refs/heads/ranger-0.5
Commit: eb966b30a7154064c5a83ada48346a2074785210
Parents: f9fc17f
Author: rmani <rm...@hortonworks.com>
Authored: Wed Dec 2 15:31:37 2015 -0800
Committer: rmani <rm...@hortonworks.com>
Committed: Wed Dec 2 15:38:31 2015 -0800
----------------------------------------------------------------------
.../plugin/client/HadoopConfigHolder.java | 62 +++--
.../service-defs/ranger-servicedef-yarn.json | 35 ++-
.../ranger/services/yarn/client/YarnClient.java | 228 ++++++++++---------
.../services/yarn/client/YarnConnectionMgr.java | 18 +-
.../services/yarn/client/YarnResourceMgr.java | 14 +-
5 files changed, 217 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb966b30/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java
index f95e10e..9d14ae6 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/client/HadoopConfigHolder.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
+import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -96,6 +97,7 @@ public class HadoopConfigHolder {
dataSource2HadoopConfigHolder.put(aDatasourceName, ret) ;
}
}
+
return ret ;
}
@@ -265,14 +267,15 @@ public class HadoopConfigHolder {
userName = prop.getProperty(RANGER_LOGIN_USER_NAME_PROP) ;
keyTabFile = prop.getProperty(RANGER_LOGIN_KEYTAB_FILE_PROP) ;
password = prop.getProperty(RANGER_LOGIN_PASSWORD) ;
-
- if ( getHadoopSecurityAuthentication() != null) {
- isKerberosAuth = ( getHadoopSecurityAuthentication().equalsIgnoreCase(HADOOP_SECURITY_AUTHENTICATION_METHOD));
+
+ String hadoopSecurityAuthenticationn = getHadoopSecurityAuthentication();
+
+ if ( hadoopSecurityAuthenticationn != null) {
+ isKerberosAuth = ( hadoopSecurityAuthenticationn.equalsIgnoreCase(HADOOP_SECURITY_AUTHENTICATION_METHOD));
}
else {
isKerberosAuth = (userName != null) && (userName.indexOf("@") > -1) ;
}
-
}
}
@@ -342,21 +345,26 @@ public class HadoopConfigHolder {
}
public String getHadoopSecurityAuthentication() {
- Properties repoParam = null ;
String ret = null;
-
- HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(this.getDatasourceName()) ;
-
- if ( resourceName2PropertiesMap != null) {
- repoParam=resourceName2PropertiesMap.get(DEFAULT_RESOURCE_NAME);
+ String sectionName = RANGER_SECTION_NAME;
+
+ if ( defaultConfigFile != null) {
+ sectionName = defaultConfigFile;
+ }
+
+ if ( LOG.isDebugEnabled() ) {
+ LOG.debug("==> HadoopConfigHolder.getHadoopSecurityAuthentication( " + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + ")" );
}
+
+ ret = getProperties(sectionName,HADOOP_SECURITY_AUTHENTICATION);
- if ( repoParam != null ) {
- ret = (String)repoParam.get(HADOOP_SECURITY_AUTHENTICATION);
+ if ( LOG.isDebugEnabled() ) {
+ LOG.debug("<== HadoopConfigHolder.getHadoopSecurityAuthentication(" + " DataSource : " + sectionName + " Property : " + HADOOP_SECURITY_AUTHENTICATION + " Value : " + ret + ")" );
}
+
return ret;
- }
-
+ }
+
public String getUserName() {
return userName;
}
@@ -377,6 +385,32 @@ public class HadoopConfigHolder {
return rangerInternalPropertyKeys;
}
+
+ private String getProperties(String sectionName, String property) {
+
+ if ( LOG.isDebugEnabled() ) {
+ LOG.debug("==> HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + ")" );
+ }
+
+ Properties repoParam = null ;
+ String ret = null;
+
+ HashMap<String,Properties> resourceName2PropertiesMap = dataSource2ResourceListMap.get(this.getDatasourceName()) ;
+
+ if ( resourceName2PropertiesMap != null) {
+ repoParam=resourceName2PropertiesMap.get(sectionName);
+ }
+
+ if ( repoParam != null ) {
+ ret = (String)repoParam.get(property);
+ }
+
+ if ( LOG.isDebugEnabled() ) {
+ LOG.debug("<== HadoopConfigHolder.getProperties( " + " DataSource : " + sectionName + " Property : " + property + " Value : " + ret);
+ }
+
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb966b30/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json
----------------------------------------------------------------------
diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json
index ff1f39f..ff93dfe 100644
--- a/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json
+++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-yarn.json
@@ -82,6 +82,19 @@
{
"itemId": 4,
+ "name": "hadoop.security.authentication",
+ "type": "enum",
+ "subType": "authnType",
+ "mandatory": true,
+ "validationRegEx":"",
+ "validationMessage": "",
+ "uiHint":"",
+ "label": "Authentication Type",
+ "defaultValue": "simple"
+ },
+
+ {
+ "itemId": 5,
"name": "commonNameForCertificate",
"type": "string",
"mandatory": false,
@@ -90,11 +103,31 @@
"uiHint":"",
"label": "Common Name for Certificate"
}
+
],
"enums":
[
-
+ {
+ "itemId": 1,
+ "name": "authnType",
+ "elements":
+ [
+ {
+ "itemId": 1,
+ "name": "simple",
+ "label": "Simple"
+ },
+
+ {
+ "itemId": 2,
+ "name": "kerberos",
+ "label": "Kerberos"
+ }
+ ],
+
+ "defaultIndex": 0
+ }
],
"contextEnrichers":
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb966b30/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java
----------------------------------------------------------------------
diff --git a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java
index fc07760..0f3ed89 100644
--- a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java
+++ b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnClient.java
@@ -19,6 +19,9 @@
package org.apache.ranger.services.yarn.client;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -26,6 +29,7 @@ import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+import javax.security.auth.Subject;
import org.apache.commons.io.FilenameUtils;
import org.apache.log4j.Logger;
@@ -40,7 +44,7 @@ import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.WebResource;
-public class YarnClient {
+public class YarnClient extends BaseClient {
public static final Logger LOG = Logger.getLogger(YarnClient.class) ;
@@ -57,16 +61,27 @@ public class YarnClient {
String userName;
String password;
- public YarnClient(String yarnQueueUrl, String yarnUserName, String yarnPassWord) {
-
- this.yarnQUrl = yarnQueueUrl;
- this.userName = yarnUserName ;
- this.password = yarnPassWord;
+ public YarnClient(String serviceName, Map<String, String> configs) {
+
+ super(serviceName,configs,"yarn-client") ;
+
+ this.yarnQUrl = configs.get("yarn.url");
+ this.userName = configs.get("username");
+ this.password = configs.get("password");
+ if (this.yarnQUrl == null || this.yarnQUrl.isEmpty()) {
+ LOG.error("No value found for configuration 'yarn.url'. YARN resource lookup will fail");
+ }
+ if (this.userName == null || this.userName.isEmpty()) {
+ LOG.error("No value found for configuration 'usename'. YARN resource lookup will fail");
+ }
+ if (this.password == null || this.password.isEmpty()) {
+ LOG.error("No value found for configuration 'password'. YARN resource lookup will fail");
+ }
+
if (LOG.isDebugEnabled()) {
- LOG.debug("Yarn Client is build with url [" + yarnQueueUrl + "] user: [" + yarnPassWord + "], password: [" + "" + "]");
+ LOG.debug("Yarn Client is build with url [" + this.yarnQUrl + "] user: [" + this.userName + "], password: [" + "*********" + "]");
}
-
}
public List<String> getQueueList(final String queueNameMatching, final List<String> existingQueueList) {
@@ -74,107 +89,122 @@ public class YarnClient {
if (LOG.isDebugEnabled()) {
LOG.debug("Getting Yarn queue list for queueNameMatching : " + queueNameMatching);
}
- final String errMsg = errMessage;
+ final String errMsg = errMessage;
List<String> ret = null;
-
- Callable<List<String>> yarnQueueListGetter = new Callable<List<String>>() {
+
+ Callable<List<String>> callableYarnQListGetter = new Callable<List<String>>() {
+
@Override
public List<String> call() {
-
- List<String> lret = new ArrayList<String>();
-
- String url = yarnQUrl + YARN_LIST_API_ENDPOINT ;
-
- Client client = null ;
- ClientResponse response = null ;
-
- try {
- client = Client.create() ;
-
- WebResource webResource = client.resource(url);
-
- response = webResource.accept(EXPECTED_MIME_TYPE)
- .get(ClientResponse.class);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("getQueueList():calling " + url);
- }
-
- if (response != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("getQueueList():response.getStatus()= " + response.getStatus());
- }
- if (response.getStatus() == 200) {
- String jsonString = response.getEntity(String.class);
- Gson gson = new GsonBuilder().setPrettyPrinting().create();
- YarnSchedulerResponse yarnQResponse = gson.fromJson(jsonString, YarnSchedulerResponse.class);
- if (yarnQResponse != null) {
- List<String> yarnQueueList = yarnQResponse.getQueueNames();
- if (yarnQueueList != null) {
- for ( String yarnQueueName : yarnQueueList) {
- if ( existingQueueList != null && existingQueueList.contains(yarnQueueName)) {
- continue;
- }
- if (queueNameMatching == null || queueNameMatching.isEmpty()
- || yarnQueueName.startsWith(queueNameMatching)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("getQueueList():Adding yarnQueue " + yarnQueueName);
+ List<String> yarnQueueListGetter = null;
+
+ Subject subj = getLoginSubject();
+
+ if (subj != null) {
+ yarnQueueListGetter = Subject.doAs(subj, new PrivilegedAction<List<String>>() {
+
+ @Override
+ public List<String> run() {
+
+ List<String> lret = new ArrayList<String>();
+
+ String url = yarnQUrl + YARN_LIST_API_ENDPOINT ;
+
+ Client client = null ;
+
+ ClientResponse response = null ;
+
+ try {
+ client = Client.create() ;
+
+ WebResource webResource = client.resource(url);
+
+ response = webResource.accept(EXPECTED_MIME_TYPE)
+ .get(ClientResponse.class);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getQueueList():calling " + url);
+ }
+
+ if (response != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getQueueList():response.getStatus()= " + response.getStatus());
+ }
+ if (response.getStatus() == 200) {
+ String jsonString = response.getEntity(String.class);
+ Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ YarnSchedulerResponse yarnQResponse = gson.fromJson(jsonString, YarnSchedulerResponse.class);
+ if (yarnQResponse != null) {
+ List<String> yarnQueueList = yarnQResponse.getQueueNames();
+ if (yarnQueueList != null) {
+ for ( String yarnQueueName : yarnQueueList) {
+ if ( existingQueueList != null && existingQueueList.contains(yarnQueueName)) {
+ continue;
+ }
+ if (queueNameMatching == null || queueNameMatching.isEmpty()
+ || yarnQueueName.startsWith(queueNameMatching)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getQueueList():Adding yarnQueue " + yarnQueueName);
+ }
+ lret.add(yarnQueueName) ;
+ }
}
- lret.add(yarnQueueName) ;
}
}
- }
+ } else{
+ LOG.info("getQueueList():response.getStatus()= " + response.getStatus() + " for URL " + url + ", so returning null list");
+ String jsonString = response.getEntity(String.class);
+ LOG.info(jsonString);
+ lret = null;
}
- } else{
- LOG.info("getQueueList():response.getStatus()= " + response.getStatus() + " for URL " + url + ", so returning null list");
- String jsonString = response.getEntity(String.class);
- LOG.info(jsonString);
+ } else {
+ lret = null;
+ String msgDesc = "Unable to get a valid response for "
+ + "expected mime type : [" + EXPECTED_MIME_TYPE
+ + "] URL : " + url + " - got null response.";
+ LOG.error(msgDesc);
+ HadoopException hdpException = new HadoopException(msgDesc);
+ hdpException.generateResponseDataMap(false, msgDesc,
+ msgDesc + errMsg, null, null);
+ throw hdpException;
+ }
+ } catch (HadoopException he) {
lret = null;
+ throw he;
+ } catch (Throwable t) {
+ lret = null;
+ String msgDesc = "Exception while getting Yarn Queue List."
+ + " URL : " + url;
+ HadoopException hdpException = new HadoopException(msgDesc,
+ t);
+
+ LOG.error(msgDesc, t);
+
+ hdpException.generateResponseDataMap(false,
+ BaseClient.getMessage(t), msgDesc + errMsg, null,
+ null);
+ throw hdpException;
+
+ } finally {
+ if (response != null) {
+ response.close();
+ }
+
+ if (client != null) {
+ client.destroy();
+ }
}
- } else {
- lret = null;
- String msgDesc = "Unable to get a valid response for "
- + "expected mime type : [" + EXPECTED_MIME_TYPE
- + "] URL : " + url + " - got null response.";
- LOG.error(msgDesc);
- HadoopException hdpException = new HadoopException(msgDesc);
- hdpException.generateResponseDataMap(false, msgDesc,
- msgDesc + errMsg, null, null);
- throw hdpException;
- }
- } catch (HadoopException he) {
- lret = null;
- throw he;
- } catch (Throwable t) {
- lret = null;
- String msgDesc = "Exception while getting Yarn Queue List."
- + " URL : " + url;
- HadoopException hdpException = new HadoopException(msgDesc,
- t);
-
- LOG.error(msgDesc, t);
-
- hdpException.generateResponseDataMap(false,
- BaseClient.getMessage(t), msgDesc + errMsg, null,
- null);
- throw hdpException;
-
- } finally {
- if (response != null) {
- response.close();
- }
-
- if (client != null) {
- client.destroy();
+ return lret ;
}
+ } );
}
- return lret ;
- }
- } ;
+ return yarnQueueListGetter;
+ }
+ };
try {
- ret = timedTask(yarnQueueListGetter, 5, TimeUnit.SECONDS);
+ ret = timedTask(callableYarnQListGetter, 5, TimeUnit.SECONDS);
} catch ( Throwable t) {
LOG.error("Unable to get Yarn Queue list from [" + yarnQUrl + "]", t) ;
String msgDesc = "Unable to get a valid response for "
@@ -243,12 +273,7 @@ public class YarnClient {
+ errMsg, null, null);
throw hdpException;
} else {
- String yarnUrl = configs.get("yarn.url");
- String yarnUserName = configs.get("username");
- String yarnPassWord = configs.get("password");
- yarnClient = new YarnClient (yarnUrl, yarnUserName,
- yarnPassWord);
-
+ yarnClient = new YarnClient (serviceName, configs);
}
return yarnClient;
}
@@ -299,5 +324,4 @@ public class YarnClient {
TimeUnit timeUnit) throws Exception {
return callableObj.call();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb966b30/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java
----------------------------------------------------------------------
diff --git a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java
index e2cc2ef..1d39998 100644
--- a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java
+++ b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnConnectionMgr.java
@@ -19,25 +19,17 @@
package org.apache.ranger.services.yarn.client;
+import java.util.Map;
+
import org.apache.log4j.Logger;
public class YarnConnectionMgr {
public static final Logger LOG = Logger.getLogger(YarnConnectionMgr.class);
-
- public static YarnClient getYarnClient(final String yarnURL, String userName, String password) {
- YarnClient yarnClient = null;
- if (yarnURL == null || yarnURL.isEmpty()) {
- LOG.error("Can not create YarnClient: yarnURL is empty");
- } else if (userName == null || userName.isEmpty()) {
- LOG.error("Can not create YarnClient: YarnuserName is empty");
- } else if (password == null || password.isEmpty()) {
- LOG.error("Can not create YarnClient: YarnPassWord is empty");
- } else {
- yarnClient = new YarnClient(yarnURL, userName, password);
- }
- return yarnClient;
+
+ public static YarnClient getYarnClient(String serviceName, Map<String, String> configs) {
+ return new YarnClient(serviceName, configs);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb966b30/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java
----------------------------------------------------------------------
diff --git a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java
index 95d29c0..97fdf19 100644
--- a/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java
+++ b/plugin-yarn/src/main/java/org/apache/ranger/services/yarn/client/YarnResourceMgr.java
@@ -65,23 +65,17 @@ public class YarnResourceMgr {
} else {
yarnQueueName = userInput;
}
-
-
+
if (configs == null || configs.isEmpty()) {
LOG.error("Connection Config is empty");
-
} else {
-
- String url = configs.get("yarn.url");
- String username = configs.get("username");
- String password = configs.get("password");
- resultList = getYarnResource(url, username, password,yarnQueueName,yarnQueueList) ;
+ resultList = getYarnResource(serviceName, configs, yarnQueueName,yarnQueueList) ;
}
return resultList ;
}
- public static List<String> getYarnResource(String url, String username, String password,String yarnQueueName, List<String> yarnQueueList) {
- final YarnClient yarnClient = YarnConnectionMgr.getYarnClient(url, username, password);
+ public static List<String> getYarnResource(String serviceName, Map<String, String> configs, String yarnQueueName, List<String> yarnQueueList) {
+ final YarnClient yarnClient = YarnConnectionMgr.getYarnClient(serviceName, configs);
List<String> topologyList = null;
if (yarnClient != null) {
synchronized(yarnClient) {