You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2017/11/20 10:43:19 UTC
ranger git commit: RANGER-1883: Part 2 - All tag sources and sinks
should use the same kerberos identity and reuse it
Repository: ranger
Updated Branches:
refs/heads/master 6f4e28639 -> 10051777e
RANGER-1883: Part 2 - All tag sources and sinks should use the same kerberos identity and reuse it
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/10051777
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/10051777
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/10051777
Branch: refs/heads/master
Commit: 10051777edf21c5ab7914c0670e23040d105dc4c
Parents: 6f4e286
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Mon Nov 13 18:56:03 2017 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Nov 20 02:43:09 2017 -0800
----------------------------------------------------------------------
.../ranger/tagsync/process/TagSyncConfig.java | 6 ++
.../ranger/tagsync/process/TagSynchronizer.java | 94 ++++++++++++++++++--
.../tagsync/sink/tagadmin/TagAdminRESTSink.java | 52 +++--------
.../source/atlasrest/AtlasRESTTagSource.java | 30 ++-----
.../tagsync/source/atlasrest/AtlasRESTUtil.java | 39 ++++----
5 files changed, 132 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
index 3f35097..697c7cc 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSyncConfig.java
@@ -98,6 +98,8 @@ public class TagSyncConfig extends Configuration {
private static final String TAGSYNC_KERBEROS_PRICIPAL = "ranger.tagsync.kerberos.principal";
private static final String TAGSYNC_KERBEROS_KEYTAB = "ranger.tagsync.kerberos.keytab";
+ public static final String TAGSYNC_KERBEROS_IDENTITY = "tagsync.kerberos.identity";
+
private static String LOCAL_HOSTNAME = "unknown";
private Properties props;
@@ -399,6 +401,10 @@ public class TagSyncConfig extends Configuration {
return ret;
}
+ static public String getTagsyncKerberosIdentity(Properties prop) {
+ return prop.getProperty(TAGSYNC_KERBEROS_IDENTITY);
+ }
+
private TagSyncConfig() {
super(false);
init();
http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
index d36ecae..b07cd34 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
@@ -22,10 +22,14 @@ package org.apache.ranger.tagsync.process;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Logger;
import org.apache.ranger.tagsync.model.TagSink;
import org.apache.ranger.tagsync.model.TagSource;
+import javax.security.auth.Subject;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -36,6 +40,8 @@ public class TagSynchronizer {
private static final Logger LOG = Logger.getLogger(TagSynchronizer.class);
+ private static final String AUTH_TYPE_KERBEROS = "kerberos";
+
private static final String TAGSYNC_SOURCE_BASE = "ranger.tagsync.source.";
private static final String PROP_CLASS_NAME = "class";
@@ -97,15 +103,19 @@ public class TagSynchronizer {
printConfigurationProperties(properties);
- boolean ret = false;
+ boolean ret = initializeKerberosIdentity(properties);
- LOG.info("Initializing TAG source and sink");
+ if (ret) {
+ LOG.info("Initializing TAG source and sink");
- tagSink = initializeTagSink(properties);
+ tagSink = initializeTagSink(properties);
- if (tagSink != null) {
- initializeTagSources();
- ret = true;
+ if (tagSink != null) {
+ initializeTagSources();
+ ret = true;
+ }
+ } else {
+ LOG.error("Error initializing kerberos identity");
}
if (LOG.isDebugEnabled()) {
@@ -344,6 +354,78 @@ public class TagSynchronizer {
return tagSource;
}
+ private static boolean initializeKerberosIdentity(Properties props) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagSynchronizer.initializeKerberosIdentity()");
+ }
+
+ boolean ret = false;
+
+ String authenticationType = TagSyncConfig.getAuthenticationType(props);
+ String principal = TagSyncConfig.getKerberosPrincipal(props);
+ String keytab = TagSyncConfig.getKerberosKeytab(props);
+ String nameRules = TagSyncConfig.getNameRules(props);
+
+ if (LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("authenticationType=" + authenticationType);
+ LOG.debug("principal=" + principal);
+ LOG.debug("keytab" + keytab);
+ LOG.debug("nameRules=" + nameRules);
+ }
+ }
+ final boolean isKerberized = !StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab);
+
+ if (isKerberized) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to get kerberos identitiy");
+ }
+ Subject subject = null;
+ try {
+ subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules);
+ } catch(IOException exception) {
+ LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception);
+ }
+
+ UserGroupInformation kerberosIdentity;
+
+ if (subject != null) {
+ try {
+ UserGroupInformation.loginUserFromSubject(subject);
+ kerberosIdentity = UserGroupInformation.getLoginUser();
+ if (kerberosIdentity != null) {
+ props.put(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY, kerberosIdentity.getUserName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Got UGI, user:[" + kerberosIdentity.getUserName() + "]");
+ }
+ ret = true;
+ } else {
+ LOG.error("KerberosIdentity is null!");
+ }
+ } catch (IOException exception) {
+ LOG.error("Failed to get UGI from Subject:[" + subject + "]", exception);
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not configured for Kerberos Authentication");
+ }
+ props.remove(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY);
+
+ ret = true;
+ }
+
+ if (!ret) {
+ props.remove(TagSyncConfig.TAGSYNC_KERBEROS_IDENTITY);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagSynchronizer.initializeKerberosIdentity() : " + ret);
+ }
+
+ return ret;
+ }
+
private static String getStringProperty(Properties props, String propName) {
String ret = null;
http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java
index 4f6761f..c34b6ea 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/sink/tagadmin/TagAdminRESTSink.java
@@ -26,7 +26,6 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.security.SecureClientLogin;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ranger.admin.client.datatype.RESTResponse;
import org.apache.ranger.tagsync.model.TagSink;
@@ -34,7 +33,6 @@ import org.apache.ranger.plugin.util.RangerRESTClient;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServiceTags;
import org.apache.ranger.tagsync.process.TagSyncConfig;
-import javax.security.auth.Subject;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
@@ -55,18 +53,13 @@ public class TagAdminRESTSink implements TagSink, Runnable {
private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/";
- private static final String AUTH_TYPE_KERBEROS = "kerberos";
-
private long rangerAdminConnectionCheckInterval;
private RangerRESTClient tagRESTClient = null;
+ private boolean isKerberized;
+
private BlockingQueue<UploadWorkItem> uploadWorkItems;
-
- private String authenticationType;
- private String principal;
- private String keytab;
- private String nameRules;
private Thread myThread = null;
@@ -83,47 +76,28 @@ public class TagAdminRESTSink implements TagSink, Runnable {
String userName = TagSyncConfig.getTagAdminUserName(properties);
String password = TagSyncConfig.getTagAdminPassword(properties);
rangerAdminConnectionCheckInterval = TagSyncConfig.getTagAdminConnectionCheckInterval(properties);
- authenticationType = TagSyncConfig.getAuthenticationType(properties);
- nameRules = TagSyncConfig.getNameRules(properties);
- principal = TagSyncConfig.getKerberosPrincipal(properties);
- keytab = TagSyncConfig.getKerberosKeytab(properties);
+ isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null;
+
if (LOG.isDebugEnabled()) {
LOG.debug("restUrl=" + restUrl);
LOG.debug("sslConfigFile=" + sslConfigFile);
LOG.debug("userName=" + userName);
- LOG.debug("rangerAdminConnectionCheckInterval" + rangerAdminConnectionCheckInterval);
+ LOG.debug("rangerAdminConnectionCheckInterval=" + rangerAdminConnectionCheckInterval);
+ LOG.debug("isKerberized=" + isKerberized);
}
if (StringUtils.isNotBlank(restUrl)) {
tagRESTClient = new RangerRESTClient(restUrl, sslConfigFile);
- if(isKerberosEnabled()) {
- Subject subject = null;
- try {
- subject = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules);
- } catch(IOException exception) {
- LOG.error("Could not get Subject from principal:[" + principal + "], keytab:[" + keytab + "], nameRules:[" + nameRules + "]", exception);
- }
- if (subject != null) {
- try {
- UserGroupInformation.loginUserFromSubject(subject);
- ret = true;
- } catch (IOException exception) {
- LOG.error("Failed to get UGI from Subject:[" + subject + "]");
- }
- }
- } else {
+ if(!isKerberized) {
tagRESTClient.setBasicAuthInfo(userName, password);
- ret = true;
}
+ uploadWorkItems = new LinkedBlockingQueue<UploadWorkItem>();
+ ret = true;
} else {
LOG.error("No value specified for property 'ranger.tagsync.tagadmin.rest.url'!");
}
- if (ret) {
- uploadWorkItems = new LinkedBlockingQueue<UploadWorkItem>();
- }
-
if(LOG.isDebugEnabled()) {
LOG.debug("<== TagAdminRESTSink.initialize(), result=" + ret);
}
@@ -152,12 +126,8 @@ public class TagAdminRESTSink implements TagSink, Runnable {
return ret;
}
- private boolean isKerberosEnabled() {
- return !StringUtils.isEmpty(authenticationType) && authenticationType.trim().equalsIgnoreCase(AUTH_TYPE_KERBEROS) && SecureClientLogin.isKerberosCredentialExists(principal, keytab);
- }
-
private ServiceTags doUpload(ServiceTags serviceTags) throws Exception {
- if(isKerberosEnabled()) {
+ if(isKerberized) {
try{
UserGroupInformation userGroupInformation = UserGroupInformation.getLoginUser();
if (userGroupInformation != null) {
@@ -170,7 +140,7 @@ public class TagAdminRESTSink implements TagSink, Runnable {
}
if (userGroupInformation != null) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Using Principal = " + principal + ", keytab = " + keytab);
+ LOG.debug("Using Principal = " + userGroupInformation.getUserName());
}
final ServiceTags serviceTag = serviceTags;
ServiceTags ret = userGroupInformation.doAs(new PrivilegedAction<ServiceTags>() {
http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
index 7da4ed4..4e0ae90 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
@@ -29,7 +29,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.security.SecureClientLogin;
import org.apache.ranger.plugin.util.RangerRESTClient;
import org.apache.ranger.tagsync.model.AbstractTagSource;
import org.apache.ranger.plugin.util.ServiceTags;
@@ -47,16 +46,9 @@ import java.util.Properties;
public class AtlasRESTTagSource extends AbstractTagSource implements Runnable {
private static final Log LOG = LogFactory.getLog(AtlasRESTTagSource.class);
- static final String AUTH_TYPE_KERBEROS = "kerberos";
-
private long sleepTimeBetweenCycleInMillis;
- AtlasRESTUtil atlasRESTUtil = null;
-
- private String authenticationType;
- private String principal;
- private String keytab;
- private String nameRules;
+ private AtlasRESTUtil atlasRESTUtil = null;
private Thread myThread = null;
@@ -103,30 +95,18 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable {
boolean ret = AtlasResourceMapperUtil.initializeAtlasResourceMappers(properties);
sleepTimeBetweenCycleInMillis = TagSyncConfig.getTagSourceAtlasDownloadIntervalInMillis(properties);
+ final boolean isKerberized = TagSyncConfig.getTagsyncKerberosIdentity(properties) != null;
String restUrl = TagSyncConfig.getAtlasRESTEndpoint(properties);
String sslConfigFile = TagSyncConfig.getAtlasRESTSslConfigFile(properties);
String userName = TagSyncConfig.getAtlasRESTUserName(properties);
String password = TagSyncConfig.getAtlasRESTPassword(properties);
- authenticationType = TagSyncConfig.getAuthenticationType(properties);
- nameRules = TagSyncConfig.getNameRules(properties);
- principal = TagSyncConfig.getKerberosPrincipal(properties);
- keytab = TagSyncConfig.getKerberosKeytab(properties);
-
- final boolean kerberized = StringUtils.isNotEmpty(authenticationType)
- && authenticationType.trim().equalsIgnoreCase(AtlasRESTTagSource.AUTH_TYPE_KERBEROS)
- && SecureClientLogin.isKerberosCredentialExists(principal, keytab);
-
if (LOG.isDebugEnabled()) {
LOG.debug("restUrl=" + restUrl);
LOG.debug("sslConfigFile=" + sslConfigFile);
LOG.debug("userName=" + userName);
- LOG.debug("authenticationType=" + authenticationType);
- LOG.debug("principal=" + principal);
- LOG.debug("keytab=" + keytab);
- LOG.debug("nameRules=" + nameRules);
- LOG.debug("kerberized=" + kerberized);
+ LOG.debug("kerberized=" + isKerberized);
}
if (StringUtils.isNotEmpty(restUrl)) {
@@ -135,10 +115,10 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable {
}
RangerRESTClient atlasRESTClient = new RangerRESTClient(restUrl, sslConfigFile);
- if (!kerberized) {
+ if (!isKerberized) {
atlasRESTClient.setBasicAuthInfo(userName, password);
}
- atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, kerberized, authenticationType, principal, keytab, nameRules);
+ atlasRESTUtil = new AtlasRESTUtil(atlasRESTClient, isKerberized);
} else {
LOG.info("AtlasEndpoint not specified, Initial download of Atlas-entities cannot be done.");
ret = false;
http://git-wip-us.apache.org/repos/asf/ranger/blob/10051777/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
index 167fe68..00a101e 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -28,14 +28,14 @@ import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.security.SecureClientLogin;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Logger;
import org.apache.ranger.admin.client.datatype.RESTResponse;
import org.apache.ranger.plugin.util.RangerRESTClient;
import org.apache.ranger.tagsync.source.atlas.AtlasEntityWithTraits;
import org.apache.ranger.tagsync.source.atlas.AtlasResourceMapperUtil;
-import javax.security.auth.Subject;
+import java.io.IOException;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.HashMap;
@@ -66,22 +66,17 @@ public class AtlasRESTUtil {
private final Gson gson = new Gson();
private final RangerRESTClient atlasRESTClient;
- private final String principal;
- private final String keytab;
- private final String nameRules;
- private final boolean kerberized;
- public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean kerberized, String authenticationType, String principal, String keytab, String nameRules) {
+ private final boolean isKerberized;
+
+ public AtlasRESTUtil(RangerRESTClient atlasRESTClient, boolean isKerberized) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasRESTUtil()");
}
- this.kerberized = kerberized;
-
this.atlasRESTClient = atlasRESTClient;
- this.principal = principal;
- this.keytab = keytab;
- this.nameRules = nameRules;
+
+ this.isKerberized = isKerberized;
if (LOG.isDebugEnabled()) {
LOG.debug("<== AtlasRESTUtil()");
@@ -249,13 +244,23 @@ public class AtlasRESTUtil {
Map<String, Object> ret = new HashMap<String, Object>();
try {
- if (kerberized) {
+ UserGroupInformation userGroupInformation = null;
+ if (isKerberized) {
+ userGroupInformation = UserGroupInformation.getLoginUser();
+
+ try {
+ userGroupInformation.checkTGTAndReloginFromKeytab();
+ } catch (IOException ioe) {
+ LOG.error("Error renewing TGT and relogin", ioe);
+ userGroupInformation = null;
+ }
+ }
+ if (userGroupInformation != null) {
LOG.debug("Using kerberos authentication");
- Subject sub = SecureClientLogin.loginUserFromKeytab(principal, keytab, nameRules);
if(LOG.isDebugEnabled()) {
- LOG.debug("Using Principal = "+ principal + ", keytab = "+keytab);
+ LOG.debug("Using Principal = "+ userGroupInformation.getUserName());
}
- ret = Subject.doAs(sub, new PrivilegedAction<Map<String, Object>>() {
+ ret = userGroupInformation.doAs(new PrivilegedAction<Map<String, Object>>() {
@Override
public Map<String, Object> run() {
try{