You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/06/27 09:17:19 UTC
[05/10] incubator-ranger git commit: RANGER-561: YARN plugin updated
to fallback to YARN ACL, instead of creating Ranger policies
RANGER-561: YARN plugin updated to fallback to YARN ACL, instead of creating Ranger policies
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/a31d901c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/a31d901c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/a31d901c
Branch: refs/heads/tag-policy
Commit: a31d901cc0cbc608550e1c99c60c1fb6e7d13388
Parents: 088f3cf
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Jun 19 17:34:13 2015 -0400
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Jun 25 08:04:23 2015 -0700
----------------------------------------------------------------------
.../hadoop/constants/RangerHadoopConstants.java | 5 +
.../yarn/authorizer/RangerYarnAuthorizer.java | 208 +++++++++++++------
2 files changed, 148 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a31d901c/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java
index a800027..ac9e93d 100644
--- a/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java
+++ b/agents-common/src/main/java/org/apache/ranger/authorization/hadoop/constants/RangerHadoopConstants.java
@@ -41,17 +41,22 @@ public class RangerHadoopConstants {
public static final String STORM_ACCESS_VERIFIER_CLASS_NAME_PROP = "storm.authorization.verifier.classname" ;
public static final String STORM_ACCESS_VERIFIER_CLASS_NAME_DEFAULT_VALUE = "org.apache.ranger.pdp.storm.RangerAuthorizer" ;
+ public static final String RANGER_ADD_YARN_PERMISSION_PROP = "ranger.add-yarn-authorization" ;
+ public static final boolean RANGER_ADD_YARN_PERMISSION_DEFAULT = true ;
+
//
// Loging constants
//
public static final String AUDITLOG_FIELD_DELIMITER_PROP = "xasecure.auditlog.fieldDelimiterString";
public static final String AUDITLOG_RANGER_MODULE_ACL_NAME_PROP = "xasecure.auditlog.xasecureAcl.name" ;
public static final String AUDITLOG_HADOOP_MODULE_ACL_NAME_PROP = "xasecure.auditlog.hadoopAcl.name" ;
+ public static final String AUDITLOG_YARN_MODULE_ACL_NAME_PROP = "ranger.auditlog.yarnAcl.name" ;
public static final String DEFAULT_LOG_FIELD_DELIMITOR = "|" ;
public static final String DEFAULT_XASECURE_MODULE_ACL_NAME = "xasecure-acl" ;
public static final String DEFAULT_RANGER_MODULE_ACL_NAME = "ranger-acl" ;
public static final String DEFAULT_HADOOP_MODULE_ACL_NAME = "hadoop-acl" ;
+ public static final String DEFAULT_YARN_MODULE_ACL_NAME = "yarn-acl" ;
public static final String AUDITLOG_FIELDINFO_VISIBLE_PROP = "xasecure.auditlog.fieldInfoVisible" ;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a31d901c/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java
----------------------------------------------------------------------
diff --git a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java
index e322477..ab9b7a9 100644
--- a/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java
+++ b/plugin-yarn/src/main/java/org/apache/ranger/authorization/yarn/authorizer/RangerYarnAuthorizer.java
@@ -21,13 +21,9 @@
package org.apache.ranger.authorization.yarn.authorizer;
import java.net.InetAddress;
-import java.util.Collection;
-import java.util.HashSet;
+import java.util.HashMap;
import java.util.Map;
-import java.util.Set;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.MapUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -35,14 +31,16 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.security.*;
+import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
+import org.apache.ranger.authorization.hadoop.constants.RangerHadoopConstants;
import org.apache.ranger.authorization.utils.StringUtil;
import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler;
import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
import org.apache.ranger.plugin.policyengine.RangerAccessResult;
-import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
import org.apache.ranger.plugin.service.RangerBasePlugin;
-import org.apache.ranger.plugin.util.GrantRevokeRequest;
import com.google.common.collect.Sets;
@@ -51,11 +49,14 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider {
public static final String ACCESS_TYPE_SUBMIT_APP = "submit-app";
public static final String ACCESS_TYPE_ADMIN = "admin";
+ private static boolean yarnAuthEnabled = RangerHadoopConstants.RANGER_ADD_YARN_PERMISSION_DEFAULT;
+
private static final Log LOG = LogFactory.getLog(RangerYarnAuthorizer.class);
private static volatile RangerYarnPlugin yarnPlugin = null;
private AccessControlList admins = null;
+ private Map<PrivilegedEntity, Map<AccessType, AccessControlList>> yarnAcl = new HashMap<PrivilegedEntity, Map<AccessType, AccessControlList>>();
@Override
public void init(Configuration conf) {
@@ -78,6 +79,8 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider {
}
}
+ RangerYarnAuthorizer.yarnAuthEnabled = RangerConfiguration.getInstance().getBoolean(RangerHadoopConstants.RANGER_ADD_YARN_PERMISSION_PROP, RangerHadoopConstants.RANGER_ADD_YARN_PERMISSION_DEFAULT);
+
if(LOG.isDebugEnabled()) {
LOG.debug("<== RangerYarnAuthorizer.init()");
}
@@ -86,23 +89,34 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider {
@Override
public boolean checkPermission(AccessType accessType, PrivilegedEntity entity, UserGroupInformation ugi) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> RangerYarnAuthorizer.checkPermission(" + accessType + ", " + entity + ", " + ugi + ")");
+ LOG.debug("==> RangerYarnAuthorizer.checkPermission(" + accessType + ", " + toString(entity) + ", " + ugi + ")");
}
- boolean ret = false;
-
- RangerYarnPlugin plugin = yarnPlugin;
+ boolean ret = false;
+ RangerYarnPlugin plugin = yarnPlugin;
+ RangerYarnAuditHandler auditHandler = null;
+ RangerAccessResult result = null;
if(plugin != null) {
RangerYarnAccessRequest request = new RangerYarnAccessRequest(entity, getRangerAccessType(accessType), accessType.name(), ugi);
- RangerAccessResult result = plugin.isAccessAllowed(request);
+ auditHandler = new RangerYarnAuditHandler();
+ result = plugin.isAccessAllowed(request, auditHandler);
+ }
+
+ if(RangerYarnAuthorizer.yarnAuthEnabled && (result == null || !result.getIsAccessDetermined())) {
+ ret = isAllowedByYarnAcl(accessType, entity, ugi, auditHandler);
+ } else {
ret = result == null ? false : result.getIsAllowed();
}
+ if(auditHandler != null) {
+ auditHandler.flushAudit();
+ }
+
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerYarnAuthorizer.checkPermission(" + accessType + ", " + entity + ", " + ugi + "): " + ret);
+ LOG.debug("<== RangerYarnAuthorizer.checkPermission(" + accessType + ", " + toString(entity) + ", " + ugi + "): " + ret);
}
return ret;
@@ -116,6 +130,8 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider {
boolean ret = false;
+ AccessControlList admins = this.admins;
+
if(admins != null) {
ret = admins.isUserAllowed(ugi);
}
@@ -143,58 +159,51 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider {
@Override
public void setPermission(PrivilegedEntity entity, Map<AccessType, AccessControlList> permission, UserGroupInformation ugi) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> RangerYarnAuthorizer.setPermission(" + entity + ", " + permission + ", " + ugi + ")");
+ LOG.debug("==> RangerYarnAuthorizer.setPermission(" + toString(entity) + ", " + permission + ", " + ugi + ")");
}
- RangerYarnPlugin plugin = yarnPlugin;
+ yarnAcl.put(entity, permission);
- if(plugin != null && entity != null && !MapUtils.isEmpty(permission) && ugi != null) {
- RangerYarnResource resource = new RangerYarnResource(entity);
-
- GrantRevokeRequest request = new GrantRevokeRequest();
- request.setResource(resource.getAsMap());
- request.setGrantor(ugi.getShortUserName());
- request.setDelegateAdmin(Boolean.FALSE);
- request.setEnableAudit(Boolean.TRUE);
- request.setReplaceExistingPermissions(Boolean.FALSE);
- request.setIsRecursive(Boolean.TRUE);
-
- for(Map.Entry<AccessType, AccessControlList> e : permission.entrySet()) {
- AccessType accessType = e.getKey();
- AccessControlList acl = e.getValue();
-
- Set<String> accessTypes = new HashSet<String>();
- accessTypes.add(getRangerAccessType(accessType));
- request.setAccessTypes(accessTypes);
-
- if(acl.isAllAllowed()) {
- Set<String> publicGroup = new HashSet<String>();
- publicGroup.add(RangerPolicyEngine.GROUP_PUBLIC);
-
- request.setUsers(null);
- request.setGroups(publicGroup);
- } else if(CollectionUtils.isEmpty(acl.getUsers()) && CollectionUtils.isEmpty(acl.getGroups())) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("grantAccess(): empty users and groups - skipped");
- }
-
- continue;
- } else {
- request.setUsers(getSet(acl.getUsers()));
- request.setGroups(getSet(acl.getGroups()));
- }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerYarnAuthorizer.setPermission(" + toString(entity) + ", " + permission + ", " + ugi + ")");
+ }
+ }
- try {
- plugin.grantAccess(request, plugin.getResultProcessor());
- } catch(Exception excp) {
- LOG.error("grantAccess(" + request + ") failed", excp);
- }
+ public boolean isAllowedByYarnAcl(AccessType accessType, PrivilegedEntity entity, UserGroupInformation ugi, RangerYarnAuditHandler auditHandler) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerYarnAuthorizer.isAllowedByYarnAcl(" + accessType + ", " + toString(entity) + ", " + ugi + ")");
+ }
+
+ boolean ret = false;
+
+ for(Map.Entry<PrivilegedEntity, Map<AccessType, AccessControlList>> e : yarnAcl.entrySet()) {
+ PrivilegedEntity aclEntity = e.getKey();
+ Map<AccessType, AccessControlList> entityPermissions = e.getValue();
+
+ AccessControlList acl = entityPermissions == null ? null : entityPermissions.get(accessType);
+
+ if(acl == null || !acl.isUserAllowed(ugi)) {
+ continue;
}
+
+ if(! isSelfOrChildOf(entity, aclEntity)) {
+ continue;
+ }
+
+ ret = true;
+
+ break;
+ }
+
+ if(auditHandler != null) {
+ auditHandler.logYarnAclEvent(ret);
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== RangerYarnAuthorizer.setPermission(" + entity + ", " + permission + ", " + ugi + ")");
+ LOG.debug("<== RangerYarnAuthorizer.isAllowedByYarnAcl(" + accessType + ", " + toString(entity) + ", " + ugi + "): " + ret);
}
+
+ return ret;
}
private static String getRangerAccessType(AccessType accessType) {
@@ -213,22 +222,32 @@ public class RangerYarnAuthorizer extends YarnAuthorizationProvider {
return ret;
}
- private Set<String> getSet(Collection<String> strings) {
- Set<String> ret = null;
+ private boolean isSelfOrChildOf(PrivilegedEntity queue, PrivilegedEntity parentQueue) {
+ boolean ret = queue.equals(parentQueue);
+
+ if(!ret && queue.getType() == EntityType.QUEUE) {
+ String queueName = queue.getName();
+ String parentQueueName = parentQueue.getName();
- if(! CollectionUtils.isEmpty(strings)) {
- if(strings instanceof Set<?>) {
- ret = (Set<String>)strings;
- } else {
- ret = new HashSet<String>();
- for(String str : strings) {
- ret.add(str);
+ if(queueName.contains(".") && !StringUtil.isEmpty(parentQueueName)) {
+ if(parentQueueName.charAt(parentQueueName.length() - 1) != '.') {
+ parentQueueName += ".";
}
+
+ ret = queueName.startsWith(parentQueueName);
}
}
return ret;
}
+
+ private String toString(PrivilegedEntity entity) {
+ if(entity != null) {
+ return "{name=" + entity.getName() + "; type=" + entity.getType() + "}";
+ }
+
+ return "null";
+ }
}
class RangerYarnPlugin extends RangerBasePlugin {
@@ -273,4 +292,63 @@ class RangerYarnAccessRequest extends RangerAccessRequestImpl {
}
return ret ;
}
+}
+
+class RangerYarnAuditHandler extends RangerDefaultAuditHandler {
+ private static final Log LOG = LogFactory.getLog(RangerYarnAuditHandler.class);
+
+ private static final String YarnModuleName = RangerConfiguration.getInstance().get(RangerHadoopConstants.AUDITLOG_YARN_MODULE_ACL_NAME_PROP , RangerHadoopConstants.DEFAULT_YARN_MODULE_ACL_NAME) ;
+
+ private boolean isAuditEnabled = false;
+ private AuthzAuditEvent auditEvent = null;
+
+ public RangerYarnAuditHandler() {
+ }
+
+ @Override
+ public void processResult(RangerAccessResult result) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerYarnAuditHandler.logAudit(" + result + ")");
+ }
+
+ if(! isAuditEnabled && result.getIsAudited()) {
+ isAuditEnabled = true;
+ }
+
+ auditEvent = super.getAuthzEvents(result);
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerYarnAuditHandler.logAudit(" + result + "): " + auditEvent);
+ }
+ }
+
+ public void logYarnAclEvent(boolean accessGranted) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerYarnAuditHandler.logYarnAclEvent(" + accessGranted + ")");
+ }
+
+ if(auditEvent != null) {
+ auditEvent.setAccessResult((short) (accessGranted ? 1 : 0));
+ auditEvent.setAclEnforcer(YarnModuleName);
+ auditEvent.setPolicyId(-1);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerYarnAuditHandler.logYarnAclEvent(" + accessGranted + "): " + auditEvent);
+ }
+ }
+
+ public void flushAudit() {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerYarnAuditHandler.flushAudit(" + isAuditEnabled + ", " + auditEvent + ")");
+ }
+
+ if(isAuditEnabled) {
+ super.logAuthzAudit(auditEvent);
+ }
+
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerYarnAuditHandler.flushAudit(" + isAuditEnabled + ", " + auditEvent + ")");
+ }
+ }
}
\ No newline at end of file