You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by me...@apache.org on 2021/05/18 15:04:06 UTC
[ranger] branch master updated: RANGER-3264 : Add support/Fix Test
connection with Solr service
This is an automated email from the ASF dual-hosted git repository.
mehul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new c647cfd RANGER-3264 : Add support/Fix Test connection with Solr service
c647cfd is described below
commit c647cfdac8c44b3b7b30e4f02ad8052c935b67ac
Author: mateenmansoori <ma...@gmail.com>
AuthorDate: Tue May 18 14:14:09 2021 +0530
RANGER-3264 : Add support/Fix Test connection with Solr service
Signed-off-by: Mehul Parikh <me...@apache.org>
---
.../services/solr/client/ServiceSolrClient.java | 215 ++++++++++++++++++---
.../solr/client/ServiceSolrConnectionMgr.java | 23 +--
2 files changed, 195 insertions(+), 43 deletions(-)
diff --git a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
index d2f9055..4b69fb9 100644
--- a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
+++ b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
@@ -20,23 +20,38 @@
package org.apache.ranger.services.solr.client;
import java.io.IOException;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+import javax.security.auth.Subject;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.security.SecureClientLogin;
import org.apache.log4j.Logger;
import org.apache.ranger.plugin.client.BaseClient;
+import org.apache.ranger.plugin.client.HadoopConfigHolder;
+import org.apache.ranger.plugin.client.HadoopException;
import org.apache.ranger.plugin.service.ResourceLookupContext;
import org.apache.ranger.plugin.util.PasswordUtils;
import org.apache.ranger.plugin.util.TimedEventUtil;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder;
+import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
@@ -44,7 +59,6 @@ import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.util.SimpleOrderedMap;
-import org.apache.commons.collections.CollectionUtils;
public class ServiceSolrClient {
private static final Logger LOG = Logger.getLogger(ServiceSolrClient.class);
@@ -63,41 +77,58 @@ public class ServiceSolrClient {
private String username;
private String password;
+ private String serviceName;
private SolrClient solrClient = null;
private boolean isSolrCloud = true;
+ private boolean isKerberosAuth;
+ private String url;
+ private Subject loginSubject;
+ private String authType;
- public ServiceSolrClient(SolrClient solrClient,
- boolean isSolrCloud, Map<String, String> configs) {
- this.solrClient = solrClient;
- this.isSolrCloud = isSolrCloud;
+ public ServiceSolrClient(String serviceName, Map<String, String> configs, String url, boolean isSolrCloud) {
this.username = configs.get("username");
this.password = configs.get("password");
+ this.authType = configs.get(HadoopConfigHolder.RANGER_AUTH_TYPE);
+ this.isKerberosAuth = HadoopConfigHolder.HADOOP_SECURITY_AUTHENTICATION_METHOD.equalsIgnoreCase(this.authType);
+ this.url = url;
+ this.serviceName = serviceName;
+ this.isSolrCloud = isSolrCloud;
+ this.createSolrClientInstance();
+ this.login(configs);
}
public Map<String, Object> connectionTest() throws Exception {
- String errMsg = errMessage;
Map<String, Object> responseData = new HashMap<String, Object>();
+ if(this.isKerberosAuth) {
+ Subject.doAs(this.loginSubject, new PrivilegedAction<Void>() {
+ @Override
+ public Void run() {
+ testConnection(responseData);
+ return null;
+ }
+ });
+ } else {
+ testConnection(responseData);
+ }
+ return responseData;
+ }
+
+ private void testConnection(Map<String, Object> responseData) {
+ String successMsg = "ConnectionTest Successful";
try {
getCollectionList(null);
// If it doesn't throw exception, then assume the instance is
// reachable
- String successMsg = "ConnectionTest Successful";
- BaseClient.generateResponseDataMap(true, successMsg,
- successMsg, null, null, responseData);
- } catch (IOException e) {
+ BaseClient.generateResponseDataMap(true, successMsg, successMsg, null, null, responseData);
+ } catch (Exception e) {
LOG.error("Error connecting to Solr. solrClient=" + solrClient, e);
- String failureMsg = "Unable to connect to Solr instance."
- + e.getMessage();
- BaseClient.generateResponseDataMap(false, failureMsg,
- failureMsg + errMsg, null, null, responseData);
-
+ String failureMsg = "Unable to connect to Solr instance." + e.getMessage();
+ BaseClient.generateResponseDataMap(false, failureMsg, failureMsg + errMessage, null, null, responseData);
}
-
- return responseData;
}
- public List<String> getCollectionList(List<String> ignoreCollectionList)
+ private List<String> getCollectionList(List<String> ignoreCollectionList)
throws Exception {
if (!isSolrCloud) {
return getCoresList(ignoreCollectionList);
@@ -105,11 +136,10 @@ public class ServiceSolrClient {
CollectionAdminRequest<?> request = new CollectionAdminRequest.List();
String decPassword = getDecryptedPassword();
- if (username != null && decPassword != null) {
+ if (!this.isKerberosAuth && username != null && decPassword != null) {
request.setBasicAuthCredentials(username, decPassword);
}
SolrResponse response = request.process(solrClient);
-
List<String> list = new ArrayList<String>();
List<String> responseCollectionList = (ArrayList<String>)response.getResponse().get("collections");
if(CollectionUtils.isEmpty(responseCollectionList)) {
@@ -124,12 +154,12 @@ public class ServiceSolrClient {
return list;
}
- public List<String> getCoresList(List<String> ignoreCollectionList)
+ private List<String> getCoresList(List<String> ignoreCollectionList)
throws Exception {
CoreAdminRequest request = new CoreAdminRequest();
request.setAction(CoreAdminAction.STATUS);
String decPassword = getDecryptedPassword();
- if (username != null && decPassword != null) {
+ if (!this.isKerberosAuth && username != null && decPassword != null) {
request.setBasicAuthCredentials(username, decPassword);
}
CoreAdminResponse cores = request.process(solrClient);
@@ -145,7 +175,7 @@ public class ServiceSolrClient {
return coreList;
}
- public List<String> getFieldList(String collection,
+ private List<String> getFieldList(String collection,
List<String> ignoreFieldList) throws Exception {
// TODO: Best is to get the collections based on the collection value
// which could contain wild cards
@@ -158,7 +188,7 @@ public class ServiceSolrClient {
query.setRequestHandler(queryStr);
QueryRequest req = new QueryRequest(query);
String decPassword = getDecryptedPassword();
- if (username != null && decPassword != null) {
+ if (!this.isKerberosAuth && username != null && decPassword != null) {
req.setBasicAuthCredentials(username, decPassword);
}
QueryResponse response = req.process(solrClient);
@@ -182,7 +212,7 @@ public class ServiceSolrClient {
return fieldList;
}
- public List<String> getFieldList(List<String> collectionList,
+ private List<String> getFieldList(List<String> collectionList,
List<String> ignoreFieldList) throws Exception {
Set<String> fieldSet = new LinkedHashSet<String>();
@@ -232,7 +262,6 @@ public class ServiceSolrClient {
break;
}
}
-
if (userInput != null) {
try {
Callable<List<String>> callableObj = null;
@@ -248,7 +277,23 @@ public class ServiceSolrClient {
public List<String> call() {
List<String> retList = new ArrayList<String>();
try {
- List<String> list = getCollectionList(finalCollectionList);
+ List<String> list = null;
+ if (isKerberosAuth) {
+ list = Subject.doAs(loginSubject, new PrivilegedAction<List<String>>() {
+ @Override
+ public List<String> run() {
+ List<String> ret = null;
+ try {
+ ret = getCollectionList(finalCollectionList);
+ } catch (Exception e) {
+ LOG.error("Unable to get collections, Error : " + e.getMessage(), new Throwable(e));
+ }
+ return ret;
+ }
+ });
+ } else {
+ list = getCollectionList(finalCollectionList);
+ }
if (userInputFinal != null
&& !userInputFinal.isEmpty()) {
for (String value : list) {
@@ -271,8 +316,23 @@ public class ServiceSolrClient {
public List<String> call() {
List<String> retList = new ArrayList<String>();
try {
- List<String> list = getFieldList(
- finalCollectionList, finalFieldList);
+ List<String> list = null;
+ if (isKerberosAuth) {
+ list = Subject.doAs(loginSubject, new PrivilegedAction<List<String>>() {
+ @Override
+ public List<String> run() {
+ List<String> ret = new ArrayList<String>();
+ try {
+ ret = getFieldList(finalCollectionList, finalFieldList);
+ } catch (Exception e) {
+ LOG.error("Unable to get field list, Error : " + e.getMessage(), new Throwable(e));
+ }
+ return ret;
+ }
+ });
+ } else {
+ list = getFieldList(finalCollectionList, finalFieldList);
+ }
if (userInputFinal != null
&& !userInputFinal.isEmpty()) {
for (String value : list) {
@@ -298,7 +358,7 @@ public class ServiceSolrClient {
}
}
} catch (Exception e) {
- LOG.error("Unable to get hive resources.", e);
+ LOG.error("Unable to get solr resources.", e);
}
}
@@ -320,4 +380,99 @@ public class ServiceSolrClient {
return decryptedPwd;
}
+
+ private void createSolrClientInstance() {
+ this.setHttpClientBuilderForKrb();
+ if (this.isSolrCloud) {
+ List<String> zookeeperHosts = Arrays.asList(this.url);
+ this.solrClient = new CloudSolrClient.Builder(zookeeperHosts, Optional.empty()).build();
+ } else {
+ HttpSolrClient.Builder builder = new HttpSolrClient.Builder();
+ builder.withBaseSolrUrl(this.url);
+ this.solrClient = builder.build();
+ }
+ }
+
+ private void setHttpClientBuilderForKrb() {
+ if (this.isKerberosAuth) {
+ Krb5HttpClientBuilder krbBuild = new Krb5HttpClientBuilder();
+ SolrHttpClientBuilder kb = krbBuild.getBuilder();
+ HttpClientUtil.setHttpClientBuilder(kb);
+ }
+ }
+
+ private void login(Map<String, String> configs) {
+ try {
+ String adminPrincipal = configs.get(HadoopConfigHolder.RANGER_PRINCIPAL);
+ String adminKeytab = configs.get(HadoopConfigHolder.RANGER_KEYTAB);
+ String nameRules = configs.get(HadoopConfigHolder.RANGER_NAME_RULES);
+ if (StringUtils.isEmpty(nameRules)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Name Rule is empty. Setting Name Rule as 'DEFAULT'");
+ }
+ nameRules = "DEFAULT";
+ }
+ String userName = this.username;
+ if (StringUtils.isEmpty(adminPrincipal) || StringUtils.isEmpty(adminKeytab)) {
+ if (userName == null) {
+ throw createException("Unable to find login username for hadoop environment, [" + serviceName + "]", null);
+ }
+ String keyTabFile = configs.get(HadoopConfigHolder.RANGER_KEYTAB);
+ if (keyTabFile != null) {
+ if (this.isKerberosAuth) {
+ LOG.info("Init Login: security enabled, using username/keytab");
+ this.loginSubject = SecureClientLogin.loginUserFromKeytab(userName, keyTabFile, nameRules);
+ } else {
+ LOG.info("Init Login: using username");
+ this.loginSubject = SecureClientLogin.login(userName);
+ }
+ } else {
+ String encryptedPwd = this.password;
+ String password = null;
+ if (encryptedPwd != null) {
+ try {
+ password = PasswordUtils.decryptPassword(encryptedPwd);
+ } catch (Exception ex) {
+ LOG.info("Password decryption failed; trying connection with received password string");
+ password = null;
+ } finally {
+ if (password == null) {
+ password = encryptedPwd;
+ }
+ }
+ } else {
+ LOG.info("Password decryption failed: no password was configured");
+ }
+ if (this.isKerberosAuth) {
+ LOG.info("Init Login: using username/password");
+ this.loginSubject = SecureClientLogin.loginUserWithPassword(userName, password);
+ } else {
+ LOG.info("Init Login: security not enabled, using username");
+ this.loginSubject = SecureClientLogin.login(userName);
+ }
+ }
+ } else {
+ if (this.isKerberosAuth) {
+ LOG.info("Init Lookup Login: security enabled, using lookupPrincipal/lookupKeytab");
+ this.loginSubject = SecureClientLogin.loginUserFromKeytab(adminPrincipal, adminKeytab, nameRules);
+ } else {
+ LOG.info("Init Login: security not enabled, using username");
+ this.loginSubject = SecureClientLogin.login(userName);
+ }
+ }
+ } catch (IOException ioe) {
+ throw createException("Unable to login to Hadoop environment [" + serviceName + "]", ioe);
+ } catch (SecurityException se) {
+ throw createException("Unable to login to Hadoop environment [" + serviceName + "]", se);
+ }
+ }
+
+ private HadoopException createException(String msgDesc, Exception exp) {
+ HadoopException hdpException = new HadoopException(msgDesc, exp);
+ final String fullDescription = exp != null ? BaseClient.getMessage(exp) : msgDesc;
+ hdpException.generateResponseDataMap(false, fullDescription + errMessage,
+ msgDesc + errMessage, null, null);
+ return hdpException;
+ }
+
}
diff --git a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrConnectionMgr.java b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrConnectionMgr.java
index f56373b..f9aeca9 100644
--- a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrConnectionMgr.java
+++ b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrConnectionMgr.java
@@ -21,26 +21,23 @@ package org.apache.ranger.services.solr.client;
import java.util.Map;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.commons.lang.StringUtils;
public class ServiceSolrConnectionMgr {
+ private static final String SOLR_ZOOKEEPER_QUORUM = "solr.zookeeper.quorum";
+ private static final String SOLR_URL = "solr.url";
+
static public ServiceSolrClient getSolrClient(String serviceName,
Map<String, String> configs) throws Exception {
- String url = configs.get("solr.url");
- if (url != null) {
- //TODO: Determine whether the instance is SolrCloud
- boolean isSolrCloud = true;
- HttpSolrClient.Builder builder = new HttpSolrClient.Builder();
- builder.withBaseSolrUrl(url);
- SolrClient solrClient = builder.build();
- ServiceSolrClient serviceSolrClient = new ServiceSolrClient(
- solrClient, isSolrCloud, configs);
+ String solr_url = configs.get(SOLR_URL);
+ String zk_url = configs.get(SOLR_ZOOKEEPER_QUORUM);
+ if (solr_url != null || zk_url != null) {
+ final boolean isSolrCloud = StringUtils.isNotEmpty(zk_url);
+ final String url = isSolrCloud ? zk_url : solr_url;
+ ServiceSolrClient serviceSolrClient = new ServiceSolrClient(serviceName, configs, url, isSolrCloud);
return serviceSolrClient;
}
- // TODO: Need to add method to create SolrClient using ZooKeeper for
- // SolrCloud
throw new Exception("Required properties are not set for "
+ serviceName + ". URL or Zookeeper information not provided.");
}