You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by dh...@apache.org on 2021/09/09 09:42:40 UTC
[ranger] branch ranger-2.2 updated: RANGER-3339 : Make Ranger Solr
audit collection storage configurable
This is an automated email from the ASF dual-hosted git repository.
dhavalshah9131 pushed a commit to branch ranger-2.2
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.2 by this push:
new 8a39d80 RANGER-3339 : Make Ranger Solr audit collection storage configurable
8a39d80 is described below
commit 8a39d80347bb6119d9568a34d00113e4b83a7d46
Author: mateenmansoori <ma...@gmail.com>
AuthorDate: Tue Aug 17 19:54:38 2021 +0530
RANGER-3339 : Make Ranger Solr audit collection storage configurable
---
embeddedwebserver/scripts/ranger-admin-services.sh | 2 +-
.../server/tomcat/SolrCollectionBootstrapper.java | 75 ++++++++++++++--------
security-admin/scripts/install.properties | 1 +
security-admin/scripts/setup.sh | 8 +++
.../main/resources/conf.dist/ranger-admin-site.xml | 4 ++
5 files changed, 61 insertions(+), 29 deletions(-)
diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh
index d7cabbc..bced4ad 100755
--- a/embeddedwebserver/scripts/ranger-admin-services.sh
+++ b/embeddedwebserver/scripts/ranger-admin-services.sh
@@ -91,7 +91,7 @@ fi
SERVER_NAME=rangeradmin
start() {
SLEEP_TIME_AFTER_START=5
- nohup java -Dproc_rangeradmin ${JAVA_OPTS} -Dlog4j.configuration=file:${RANGER_ADMIN_LOG4J_CONF_FILE} -Duser=${USER} -Dhostname=${HOSTNAME} ${DB_SSL_PARAM} -Dservername=${SERVER_NAME} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.server.tomcat.Embedded [...]
+ nohup java -Dproc_rangeradmin ${JAVA_OPTS} -Dlog4j.configuration=file:${RANGER_ADMIN_LOG4J_CONF_FILE} -Duser=${USER} -Dhostname=${HOSTNAME} ${DB_SSL_PARAM} -Dservername=${SERVER_NAME} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPA [...]
VALUE_OF_PID=$!
echo "Starting Apache Ranger Admin Service"
sleep $SLEEP_TIME_AFTER_START
diff --git a/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java
index d04d1c8..1596beb 100644
--- a/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java
+++ b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java
@@ -35,6 +35,8 @@ import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.SecureClientLogin;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpPost;
@@ -65,6 +67,7 @@ public class SolrCollectionBootstrapper extends Thread {
final static String SOLR_ZK_HOSTS = "ranger.audit.solr.zookeepers";
final static String SOLR_COLLECTION_NAME = "ranger.audit.solr.collection.name";
final static String SOLR_CONFIG_NAME = "ranger.audit.solr.config.name";
+ final static String CONFIG_SET_LOCATION = "ranger.audit.solr.configset.location";
final static String SOLR_NO_SHARDS = "ranger.audit.solr.no.shards";
final static String SOLR_MAX_SHARD_PER_NODE = "ranger.audit.solr.max.shards.per.node";
final static String SOLR_NO_REPLICA = "ranger.audit.solr.no.replica";
@@ -94,6 +97,7 @@ public class SolrCollectionBootstrapper extends Thread {
String nameRules;
String solr_collection_name;
String solr_config_name;
+ private String customConfigSetLocation;
Path path_for_cloud_mode;
int no_of_replicas;
int max_node_per_shards;
@@ -149,29 +153,26 @@ public class SolrCollectionBootstrapper extends Thread {
String basedir = new File(".").getCanonicalPath();
String solrFileDir = new File(basedir).getParent();
- path_for_cloud_mode = Paths.get(solrFileDir, "contrib",
- "solr_for_audit_setup", "conf");
- configSetFolder = path_for_cloud_mode.toFile();
+ this.customConfigSetLocation = EmbeddedServerUtil.getConfig(CONFIG_SET_LOCATION);
+ logger.info("Provided custom configSet location : " + this.customConfigSetLocation);
+ if (StringUtils.isNotEmpty(this.customConfigSetLocation)) {
+ this.configSetFolder = new File(this.customConfigSetLocation);
+ } else {
+ path_for_cloud_mode = Paths.get(solrFileDir, "contrib", "solr_for_audit_setup", "conf");
+ configSetFolder = path_for_cloud_mode.toFile();
+ }
String sslEnabledProp = EmbeddedServerUtil.getConfig(SSL_ENABLED_PARAM);
isSSLEnabled = ("true".equalsIgnoreCase(sslEnabledProp));
}
public void run() {
logger.info("Started run method");
-
- String zkHosts = "";
- List<String> zookeeperHosts = null;
- if (EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS) != null
- && !StringUtil.isEmpty(EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS))) {
- zkHosts = EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS).trim();
- zookeeperHosts = new ArrayList<String>(Arrays.asList(zkHosts
- .split(",")));
- }
+ List<String> zookeeperHosts = getZkHosts();
if (zookeeperHosts != null
&& !zookeeperHosts.isEmpty()
&& zookeeperHosts.stream().noneMatch(
h -> h.equalsIgnoreCase("none"))) {
- logger.info("Solr zkHosts=" + zkHosts + ", collectionName="
+ logger.info("Solr zkHosts=" + zookeeperHosts + ", collectionName="
+ solr_collection_name);
while (!is_completed && (max_retry == TRY_UNTIL_SUCCESS || retry_counter < max_retry)) {
try {
@@ -215,6 +216,8 @@ public class SolrCollectionBootstrapper extends Thread {
solrCloudClient = new CloudSolrClient.Builder(zookeeperHosts,
Optional.empty()).build();
solrCloudClient.setDefaultCollection(solr_collection_name);
+ solrCloudClient.connect();
+ zkClient = solrCloudClient.getZkStateReader().getZkClient();
solrClient = solrCloudClient;
solr_cloud_mode = true;
@@ -263,9 +266,6 @@ public class SolrCollectionBootstrapper extends Thread {
private boolean uploadConfiguration() {
try {
- solrCloudClient.connect();
- zkClient = solrCloudClient.getZkStateReader().getZkClient();
-
if (zkClient != null) {
ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
@@ -274,12 +274,18 @@ public class SolrCollectionBootstrapper extends Thread {
try {
logger.info("Config does not exist with name " + solr_config_name);
String zipOfConfigs = null;
- String[] files = configSetFolder.list();
- for (String file : files) {
- if (file != null) {
- if (file.equals("solr_audit_conf.zip")) {
- zipOfConfigs = file;
- break;
+ if (this.configSetFolder.exists() && this.configSetFolder.isFile()) {
+ zipOfConfigs = this.configSetFolder.getAbsolutePath();
+ } else {
+ String[] files = this.configSetFolder.list();
+ if (files != null) {
+ for (String aFile : files) {
+ if (aFile != null) {
+ if (aFile.equals("solr_audit_conf.zip")) {
+ zipOfConfigs = this.configSetFolder + "/" + aFile;
+ break;
+ }
+ }
}
}
}
@@ -287,14 +293,10 @@ public class SolrCollectionBootstrapper extends Thread {
throw new FileNotFoundException(
"Could Not Find Configs Zip File : " + getConfigSetFolder());
}
- File file = new File(configSetFolder + "/" + zipOfConfigs);
+ File file = new File(zipOfConfigs);
byte[] arrByte = Files.readAllBytes(file.toPath());
ByteBuffer byteBuffer = ByteBuffer.wrap(arrByte);
- Set<String> nodes = solrCloudClient.getClusterStateProvider().getLiveNodes();
- String baseUrl = null;
- String[] nodeArr = nodes.toArray(new String[0]);
- /* getting nodes URL as 'solr_8983', so converting it to 'solr/9893' */
- baseUrl = nodeArr[0].replaceAll("_", "/");
+ String baseUrl = getBaseUrl();
String protocol = isSSLEnabled ? "https" : "http";
String uploadConfigsUrl = String.format("%s://%s/admin/configs?action=UPLOAD&name=%s", protocol,
baseUrl.toString(), solr_config_name);
@@ -447,4 +449,21 @@ public class SolrCollectionBootstrapper extends Thread {
return configSetFolder;
}
+ private static List<String> getZkHosts() {
+ String zkHosts = "";
+ List<String> zookeeperHosts = null;
+ if (!StringUtil.isEmpty(EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS))) {
+ zkHosts = EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS).trim();
+ zookeeperHosts = new ArrayList<String>(Arrays.asList(zkHosts.split(",")));
+ }
+ return zookeeperHosts;
+ }
+
+ private String getBaseUrl() {
+ Set<String> nodes = solrCloudClient.getClusterStateProvider().getLiveNodes();
+ String[] nodeArr = nodes.toArray(new String[0]);
+ // getting nodes URL as 'port_solr', so converting it to 'port/solr'
+ return nodeArr[0].replaceAll("_", "/");
+ }
+
}
diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties
index ae7a00d..5a8b00c 100644
--- a/security-admin/scripts/install.properties
+++ b/security-admin/scripts/install.properties
@@ -106,6 +106,7 @@ audit_solr_zookeepers=
audit_solr_collection_name=ranger_audits
#solr Properties for cloud mode
audit_solr_config_name=ranger_audits
+audit_solr_configset_location=
audit_solr_no_shards=1
audit_solr_no_replica=1
audit_solr_max_shards_per_node=1
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index f6f6f56..c3f51a0 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -156,6 +156,7 @@ lookup_keytab=$(get_prop 'lookup_keytab' $PROPFILE)
hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE)
audit_solr_collection_name=$(get_prop 'audit_solr_collection_name' $PROPFILE)
audit_solr_config_name=$(get_prop 'audit_solr_config_name' $PROPFILE)
+audit_solr_configset_location=$(get_prop 'audit_solr_configset_location' $PROPFILE)
audit_solr_no_shards=$(get_prop 'audit_solr_no_shards' $PROPFILE)
audit_solr_no_replica=$(get_prop 'audit_solr_no_replica' $PROPFILE)
audit_solr_max_shards_per_node=$(get_prop 'audit_solr_max_shards_per_node' $PROPFILE)
@@ -476,6 +477,13 @@ update_properties() {
updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
fi
+ if [ "${audit_solr_configset_location}" != "" ]
+ then
+ propertyName=ranger.audit.solr.configset.location
+ newPropertyValue="${audit_solr_configset_location}"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+ fi
+
if [ "${audit_solr_no_shards}" != "" ]
then
propertyName=ranger.audit.solr.no.shards
diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
index 793c479..d32a324 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
@@ -341,6 +341,10 @@
<value></value>
</property>
<property>
+ <name>ranger.audit.solr.configset.location</name>
+ <value></value>
+ </property>
+ <property>
<name>ranger.audit.solr.no.shards</name>
<value></value>
</property>