You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by dh...@apache.org on 2021/09/09 09:42:40 UTC

[ranger] branch ranger-2.2 updated: RANGER-3339 : Make Ranger Solr audit collection storage configurable

This is an automated email from the ASF dual-hosted git repository.

dhavalshah9131 pushed a commit to branch ranger-2.2
in repository https://gitbox.apache.org/repos/asf/ranger.git


The following commit(s) were added to refs/heads/ranger-2.2 by this push:
     new 8a39d80  RANGER-3339 : Make Ranger Solr audit collection storage configurable
8a39d80 is described below

commit 8a39d80347bb6119d9568a34d00113e4b83a7d46
Author: mateenmansoori <ma...@gmail.com>
AuthorDate: Tue Aug 17 19:54:38 2021 +0530

    RANGER-3339 : Make Ranger Solr audit collection storage configurable
---
 embeddedwebserver/scripts/ranger-admin-services.sh |  2 +-
 .../server/tomcat/SolrCollectionBootstrapper.java  | 75 ++++++++++++++--------
 security-admin/scripts/install.properties          |  1 +
 security-admin/scripts/setup.sh                    |  8 +++
 .../main/resources/conf.dist/ranger-admin-site.xml |  4 ++
 5 files changed, 61 insertions(+), 29 deletions(-)

diff --git a/embeddedwebserver/scripts/ranger-admin-services.sh b/embeddedwebserver/scripts/ranger-admin-services.sh
index d7cabbc..bced4ad 100755
--- a/embeddedwebserver/scripts/ranger-admin-services.sh
+++ b/embeddedwebserver/scripts/ranger-admin-services.sh
@@ -91,7 +91,7 @@ fi
 SERVER_NAME=rangeradmin
 start() {
 	SLEEP_TIME_AFTER_START=5
-	nohup  java -Dproc_rangeradmin ${JAVA_OPTS} -Dlog4j.configuration=file:${RANGER_ADMIN_LOG4J_CONF_FILE}  -Duser=${USER} -Dhostname=${HOSTNAME} ${DB_SSL_PARAM} -Dservername=${SERVER_NAME} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPATH" org.apache.ranger.server.tomcat.Embedded [...]
+	nohup  java -Dproc_rangeradmin ${JAVA_OPTS} -Dlog4j.configuration=file:${RANGER_ADMIN_LOG4J_CONF_FILE}  -Duser=${USER} -Dhostname=${HOSTNAME} ${DB_SSL_PARAM} -Dservername=${SERVER_NAME} -Dlogdir=${RANGER_ADMIN_LOG_DIR} -Dcatalina.base=${XAPOLICYMGR_EWS_DIR} -cp "${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/classes/conf:${XAPOLICYMGR_EWS_DIR}/lib/*:${XAPOLICYMGR_EWS_DIR}/webapp/WEB-INF/lib/*:${RANGER_JAAS_LIB_DIR}/*:${RANGER_JAAS_CONF_DIR}:${JAVA_HOME}/lib/*:${RANGER_HADOOP_CONF_DIR}/*:$CLASSPA [...]
 	VALUE_OF_PID=$!
 	echo "Starting Apache Ranger Admin Service"
 	sleep $SLEEP_TIME_AFTER_START
diff --git a/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java
index d04d1c8..1596beb 100644
--- a/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java
+++ b/embeddedwebserver/src/main/java/org/apache/ranger/server/tomcat/SolrCollectionBootstrapper.java
@@ -35,6 +35,8 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.security.SecureClientLogin;
 import org.apache.http.HttpEntity;
 import org.apache.http.client.methods.HttpPost;
@@ -65,6 +67,7 @@ public class SolrCollectionBootstrapper extends Thread {
 	final static String SOLR_ZK_HOSTS = "ranger.audit.solr.zookeepers";
 	final static String SOLR_COLLECTION_NAME = "ranger.audit.solr.collection.name";
 	final static String SOLR_CONFIG_NAME = "ranger.audit.solr.config.name";
+	final static String CONFIG_SET_LOCATION = "ranger.audit.solr.configset.location";
 	final static String SOLR_NO_SHARDS = "ranger.audit.solr.no.shards";
 	final static String SOLR_MAX_SHARD_PER_NODE = "ranger.audit.solr.max.shards.per.node";
 	final static String SOLR_NO_REPLICA = "ranger.audit.solr.no.replica";
@@ -94,6 +97,7 @@ public class SolrCollectionBootstrapper extends Thread {
 	String nameRules;
 	String solr_collection_name;
 	String solr_config_name;
+	private String customConfigSetLocation;
 	Path path_for_cloud_mode;
 	int no_of_replicas;
 	int max_node_per_shards;
@@ -149,29 +153,26 @@ public class SolrCollectionBootstrapper extends Thread {
 		String basedir = new File(".").getCanonicalPath();
 		String solrFileDir = new File(basedir).getParent();
 
-		path_for_cloud_mode = Paths.get(solrFileDir, "contrib",
-				"solr_for_audit_setup", "conf");
-		configSetFolder = path_for_cloud_mode.toFile();
+		this.customConfigSetLocation = EmbeddedServerUtil.getConfig(CONFIG_SET_LOCATION);
+		logger.info("Provided custom configSet location : " + this.customConfigSetLocation);
+		if (StringUtils.isNotEmpty(this.customConfigSetLocation)) {
+			this.configSetFolder = new File(this.customConfigSetLocation);
+		} else {
+			path_for_cloud_mode = Paths.get(solrFileDir, "contrib", "solr_for_audit_setup", "conf");
+			configSetFolder = path_for_cloud_mode.toFile();
+		}
                 String sslEnabledProp = EmbeddedServerUtil.getConfig(SSL_ENABLED_PARAM);
                 isSSLEnabled = ("true".equalsIgnoreCase(sslEnabledProp));
 	}
 
 	public void run() {
 		logger.info("Started run method");
-
-		String zkHosts = "";
-		List<String> zookeeperHosts = null;
-		if (EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS) != null
-				&& !StringUtil.isEmpty(EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS))) {
-			zkHosts = EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS).trim();
-			zookeeperHosts = new ArrayList<String>(Arrays.asList(zkHosts
-					.split(",")));
-		}
+		List<String> zookeeperHosts = getZkHosts();
 		if (zookeeperHosts != null
 				&& !zookeeperHosts.isEmpty()
 				&& zookeeperHosts.stream().noneMatch(
 						h -> h.equalsIgnoreCase("none"))) {
-			logger.info("Solr zkHosts=" + zkHosts + ", collectionName="
+			logger.info("Solr zkHosts=" + zookeeperHosts + ", collectionName="
 					+ solr_collection_name);
 			while (!is_completed && (max_retry == TRY_UNTIL_SUCCESS || retry_counter < max_retry)) {
 				try {
@@ -215,6 +216,8 @@ public class SolrCollectionBootstrapper extends Thread {
 			solrCloudClient = new CloudSolrClient.Builder(zookeeperHosts,
 					Optional.empty()).build();
 			solrCloudClient.setDefaultCollection(solr_collection_name);
+			solrCloudClient.connect();
+            zkClient = solrCloudClient.getZkStateReader().getZkClient();
 			solrClient = solrCloudClient;
 			solr_cloud_mode = true;
 
@@ -263,9 +266,6 @@ public class SolrCollectionBootstrapper extends Thread {
 
 	private boolean uploadConfiguration() {
 		try {
-			solrCloudClient.connect();
-			zkClient = solrCloudClient.getZkStateReader().getZkClient();
-
 			if (zkClient != null) {
 				ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
 
@@ -274,12 +274,18 @@ public class SolrCollectionBootstrapper extends Thread {
 					try {
 						logger.info("Config does not exist with name " + solr_config_name);
 						String zipOfConfigs = null;
-						String[] files = configSetFolder.list();
-						for (String file : files) {
-							if (file != null) {
-								if (file.equals("solr_audit_conf.zip")) {
-									zipOfConfigs = file;
-									break;
+						if (this.configSetFolder.exists() && this.configSetFolder.isFile()) {
+							zipOfConfigs = this.configSetFolder.getAbsolutePath();
+						} else {
+							String[] files = this.configSetFolder.list();
+							if (files != null) {
+								for (String aFile : files) {
+									if (aFile != null) {
+										if (aFile.equals("solr_audit_conf.zip")) {
+											zipOfConfigs = this.configSetFolder + "/" + aFile;
+											break;
+										}
+									}
 								}
 							}
 						}
@@ -287,14 +293,10 @@ public class SolrCollectionBootstrapper extends Thread {
 							throw new FileNotFoundException(
 									"Could Not Find Configs Zip File : " + getConfigSetFolder());
 						}
-						File file = new File(configSetFolder + "/" + zipOfConfigs);
+						File file = new File(zipOfConfigs);
 						byte[] arrByte = Files.readAllBytes(file.toPath());
 						ByteBuffer byteBuffer = ByteBuffer.wrap(arrByte);
-						Set<String> nodes = solrCloudClient.getClusterStateProvider().getLiveNodes();
-						String baseUrl = null;
-						String[] nodeArr = nodes.toArray(new String[0]);
-						/* getting nodes URL as 'solr_8983', so converting it to 'solr/9893' */
-						baseUrl = nodeArr[0].replaceAll("_", "/");
+						String baseUrl = getBaseUrl();
 						String protocol = isSSLEnabled ? "https" : "http";
 						String uploadConfigsUrl = String.format("%s://%s/admin/configs?action=UPLOAD&name=%s", protocol,
 								baseUrl.toString(), solr_config_name);
@@ -447,4 +449,21 @@ public class SolrCollectionBootstrapper extends Thread {
 		return configSetFolder;
 	}
 
+	private static List<String> getZkHosts() {
+		String zkHosts = "";
+		List<String> zookeeperHosts = null;
+		if (!StringUtil.isEmpty(EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS))) {
+			zkHosts = EmbeddedServerUtil.getConfig(SOLR_ZK_HOSTS).trim();
+			zookeeperHosts = new ArrayList<String>(Arrays.asList(zkHosts.split(",")));
+		}
+		return zookeeperHosts;
+	}
+
+	private String getBaseUrl() {
+		Set<String> nodes = solrCloudClient.getClusterStateProvider().getLiveNodes();
+		String[] nodeArr = nodes.toArray(new String[0]);
+		// getting nodes URL as 'port_solr', so converting it to 'port/solr'
+		return nodeArr[0].replaceAll("_", "/");
+	}
+
 }
diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties
index ae7a00d..5a8b00c 100644
--- a/security-admin/scripts/install.properties
+++ b/security-admin/scripts/install.properties
@@ -106,6 +106,7 @@ audit_solr_zookeepers=
 audit_solr_collection_name=ranger_audits
 #solr Properties for cloud mode
 audit_solr_config_name=ranger_audits
+audit_solr_configset_location=
 audit_solr_no_shards=1
 audit_solr_no_replica=1
 audit_solr_max_shards_per_node=1
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index f6f6f56..c3f51a0 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -156,6 +156,7 @@ lookup_keytab=$(get_prop 'lookup_keytab' $PROPFILE)
 hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE)
 audit_solr_collection_name=$(get_prop 'audit_solr_collection_name' $PROPFILE)
 audit_solr_config_name=$(get_prop 'audit_solr_config_name' $PROPFILE)
+audit_solr_configset_location=$(get_prop 'audit_solr_configset_location' $PROPFILE)
 audit_solr_no_shards=$(get_prop 'audit_solr_no_shards' $PROPFILE)
 audit_solr_no_replica=$(get_prop 'audit_solr_no_replica' $PROPFILE)
 audit_solr_max_shards_per_node=$(get_prop 'audit_solr_max_shards_per_node' $PROPFILE)
@@ -476,6 +477,13 @@ update_properties() {
                updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
         fi
 
+		if [ "${audit_solr_configset_location}" != "" ]
+        then
+               propertyName=ranger.audit.solr.configset.location
+               newPropertyValue="${audit_solr_configset_location}"
+               updatePropertyToFilePy $propertyName $newPropertyValue $to_file_ranger
+        fi
+
         if [ "${audit_solr_no_shards}" != "" ]
         then
                propertyName=ranger.audit.solr.no.shards
diff --git a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
index 793c479..d32a324 100644
--- a/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
+++ b/security-admin/src/main/resources/conf.dist/ranger-admin-site.xml
@@ -341,6 +341,10 @@
                 <value></value>
         </property>
         <property>
+                <name>ranger.audit.solr.configset.location</name>
+                <value></value>
+        </property>
+        <property>
                 <name>ranger.audit.solr.no.shards</name>
                 <value></value>
         </property>