You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by rm...@apache.org on 2016/06/05 18:07:40 UTC
incubator-ranger git commit: RANGER-994: Ranger support Audit to
Secure Solr
Repository: incubator-ranger
Updated Branches:
refs/heads/master f0067ecde -> a4140bc0b
RANGER-994: Ranger support Audit to Secure Solr
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/a4140bc0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/a4140bc0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/a4140bc0
Branch: refs/heads/master
Commit: a4140bc0b183e0d91a597be8d7394cd3a6452097
Parents: f0067ec
Author: rmani <rm...@hortonworks.com>
Authored: Sun Jun 5 10:22:47 2016 -0700
Committer: rmani <rm...@hortonworks.com>
Committed: Sun Jun 5 10:22:47 2016 -0700
----------------------------------------------------------------------
agents-audit/pom.xml | 11 +-
.../audit/destination/SolrAuditDestination.java | 26 +-
.../audit/utils/InMemoryJAASConfiguration.java | 353 +++++++++++++++++++
hbase-agent/pom.xml | 5 +
hdfs-agent/pom.xml | 5 +
hive-agent/pom.xml | 5 +
kms/pom.xml | 5 +
knox-agent/pom.xml | 5 +
plugin-kafka/pom.xml | 5 +
.../services/solr/client/ServiceSolrClient.java | 8 +-
plugin-yarn/pom.xml | 5 +
pom.xml | 2 +-
.../contrib/solr_for_audit_setup/setup.sh | 2 +
.../apache/ranger/common/PropertiesUtil.java | 8 +
.../patch/cliutil/DbToSolrMigrationUtil.java | 145 +++++++-
.../java/org/apache/ranger/solr/SolrMgr.java | 28 +-
src/main/assembly/hdfs-agent.xml | 2 +
src/main/assembly/hive-agent.xml | 2 +
src/main/assembly/knox-agent.xml | 4 +-
src/main/assembly/plugin-solr.xml | 2 -
src/main/assembly/plugin-yarn.xml | 2 +
storm-agent/pom.xml | 5 +
22 files changed, 611 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/agents-audit/pom.xml
----------------------------------------------------------------------
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index 35ef2b6..3e64ca3 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -67,11 +67,11 @@
<version>${kafka.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.ranger</groupId>
- <artifactId>ranger_solrj</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>${httpcomponents.httpclient.version}</version>
@@ -86,5 +86,6 @@
<artifactId>noggit</artifactId>
<version>${noggit.version}</version>
</dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index 43b8244..9fe7e27 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -29,13 +29,17 @@ import org.apache.commons.logging.LogFactory;
import org.apache.ranger.audit.model.AuditEventBase;
import org.apache.ranger.audit.model.AuthzAuditEvent;
import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.ranger.audit.utils.InMemoryJAASConfiguration;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
+
public class SolrAuditDestination extends AuditDestination {
private static final Log LOG = LogFactory
.getLog(SolrAuditDestination.class);
@@ -43,8 +47,9 @@ public class SolrAuditDestination extends AuditDestination {
public static final String PROP_SOLR_URLS = "urls";
public static final String PROP_SOLR_ZK = "zookeepers";
public static final String PROP_SOLR_COLLECTION = "collection";
-
public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
+ public static final String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
+
SolrClient solrClient = null;
public SolrAuditDestination() {
@@ -54,6 +59,7 @@ public class SolrAuditDestination extends AuditDestination {
public void init(Properties props, String propPrefix) {
LOG.info("init() called");
super.init(props, propPrefix);
+ init();
connect();
}
@@ -99,6 +105,7 @@ public class SolrAuditDestination extends AuditDestination {
+ zkHosts);
try {
// Instantiate
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
CloudSolrClient solrCloudClient = new CloudSolrClient(
zkHosts);
solrCloudClient.setDefaultCollection(collectionName);
@@ -110,6 +117,7 @@ public class SolrAuditDestination extends AuditDestination {
} else if (solrURLs != null && !solrURLs.isEmpty()) {
try {
LOG.info("Connecting to Solr using URLs=" + solrURLs);
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
LBHttpSolrClient lbSolrClient = new LBHttpSolrClient(
solrURLs.get(0));
lbSolrClient.setConnectionTimeout(1000);
@@ -211,4 +219,20 @@ public class SolrAuditDestination extends AuditDestination {
return true;
}
+ private void init() {
+ LOG.info("==>SolrAuditDestination.init()" );
+ try {
+ // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it
+ // Acutal solrclient JAAS configs are read from the ranger-<component>-audit.xml present in components conf folder and set by InMemoryJAASConfiguration
+ // Refer InMemoryJAASConfiguration doc for JAAS Configuration
+ if ( System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null ) {
+ System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null") ;
+ }
+ LOG.info("Loading SolrClient JAAS config from Ranger audit config if present...");
+ InMemoryJAASConfiguration.init(props);
+ } catch (Exception e) {
+ LOG.error("ERROR: Unable to load SolrClient JAAS config from Audit config file. Audit to Kerberized Solr will fail...", e);
+ }
+ LOG.info("<==SolrAuditDestination.init()" );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java b/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java
new file mode 100644
index 0000000..be9cdd3
--- /dev/null
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/utils/InMemoryJAASConfiguration.java
@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.audit.utils;
+
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.SortedSet;
+import java.util.StringTokenizer;
+import java.util.TreeSet;
+
+import javax.security.auth.login.AppConfigurationEntry;
+import javax.security.auth.login.Configuration;
+
+/**
+ * InMemoryJAASConfiguration
+ *
+ * An utility class - which has a static method init to load all JAAS configuration from Application properties file (eg: kafka.properties) and
+ * set it as part of the default lookup configuration for all JAAS configuration lookup.
+ *
+ * Example settings in application.properties:
+ *
+ * xasecure.audit.jaas.KafkaClient.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+ * xasecure.audit.jaas.KafkaClient.loginModuleControlFlag = required
+ * xasecure.audit.jaas.KafkaClient.option.useKeyTab = true
+ * xasecure.audit.jaas.KafkaClient.option.storeKey = true
+ * xasecure.audit.jaas.KafkaClient.option.serviceName = kafka
+ * xasecure.audit.jaas.KafkaClient.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+ * xasecure.audit.jaas.KafkaClient.option.principal = kafka-client-1@EXAMPLE.COM
+
+ * xasecure.audit.jaas.MyClient.0.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+ * xasecure.audit.jaas.MyClient.0.loginModuleControlFlag = required
+ * xasecure.audit.jaas.MyClient.0.option.useKeyTab = true
+ * xasecure.audit.jaas.MyClient.0.option.storeKey = true
+ * xasecure.audit.jaas.MyClient.0.option.serviceName = kafka
+ * xasecure.audit.jaas.MyClient.0.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+ * xasecure.audit.jaas.MyClient.0.option.principal = kafka-client-1@EXAMPLE.COM
+ *
+ * xasecure.audit.jaas.MyClient.1.loginModuleName = com.sun.security.auth.module.Krb5LoginModule
+ * xasecure.audit.jaas.MyClient.1.loginModuleControlFlag = optional
+ * xasecure.audit.jaas.MyClient.1.option.useKeyTab = true
+ * xasecure.audit.jaas.MyClient.1.option.storeKey = true
+ * xasecure.audit.jaas.MyClient.1.option.serviceName = kafka
+ * xasecure.audit.jaas.MyClient.1.option.keyTab = /etc/security/keytabs/kafka_client.keytab
+ * xasecure.audit.jaas.MyClient.1.option.principal = kafka-client-1@EXAMPLE.COM
+
+ * This will set the JAAS configuration - equivalent to the jaas.conf file entries:
+ * KafkaClient {
+ * com.sun.security.auth.module.Krb5LoginModule required
+ * useKeyTab=true
+ * storeKey=true
+ * serviceName=kafka
+ * keyTab="/etc/security/keytabs/kafka_client.keytab"
+ * principal="kafka-client-1@EXAMPLE.COM";
+ * };
+ * MyClient {
+ * com.sun.security.auth.module.Krb5LoginModule required
+ * useKeyTab=true
+ * storeKey=true
+ * serviceName=kafka keyTab="/etc/security/keytabs/kafka_client.keytab"
+ * principal="kafka-client-1@EXAMPLE.COM";
+ * };
+ * MyClient {
+ * com.sun.security.auth.module.Krb5LoginModule optional
+ * useKeyTab=true
+ * storeKey=true
+ * serviceName=kafka
+ * keyTab="/etc/security/keytabs/kafka_client.keytab"
+ * principal="kafka-client-1@EXAMPLE.COM";
+ * };
+ *
+ * Here is the syntax for atlas.properties to add JAAS configuration:
+ *
+ * The property name has to begin with 'xasecure.audit.jaas.' + clientId (in case of Kafka client,
+ * it expects the clientId to be KafkaClient).
+ * The following property must be there to specify the JAAS loginModule name
+ * 'xasecure.audit.jaas.' +' + clientId + '.loginModuleName'
+ * The following optional property should be set to specify the loginModuleControlFlag
+ * 'xasecure.audit.jaas.' +' + clientId + '.loginModuleControlFlag'
+ * Default value : required , Possible values: required, optional, sufficient, requisite
+ * Then you can add additional optional parameters as options for the configuration using the following
+ * syntax:
+ * 'xasecure.audit.jaas.' +' + clientId + '.option.' + <optionName> = <optionValue>
+ *
+ * The current setup will lookup JAAS configration from the atlas-application.properties first, if not available,
+ * it will delegate to the original configuration
+ *
+ */
+
+public final class InMemoryJAASConfiguration extends Configuration {
+
+ private static final Logger LOG = LoggerFactory.getLogger(InMemoryJAASConfiguration.class);
+
+ private static final String JAAS_CONFIG_PREFIX_PARAM = "xasecure.audit.jaas.";
+ private static final String JAAS_CONFIG_LOGIN_MODULE_NAME_PARAM = "loginModuleName";
+ private static final String JAAS_CONFIG_LOGIN_MODULE_CONTROL_FLAG_PARAM = "loginModuleControlFlag";
+ private static final String JAAS_CONFIG_LOGIN_OPTIONS_PREFIX = "option";
+ private static final String JAAS_PRINCIPAL_PROP = "principal";
+
+ private Configuration parent = null;
+ private Map<String, List<AppConfigurationEntry>> applicationConfigEntryMap = new HashMap<>();
+
+ public static void init(String propFile) throws Exception {
+ LOG.debug("==> InMemoryJAASConfiguration.init( {} ) ", propFile);
+
+ InputStream in = null;
+
+ try {
+ Properties properties = new Properties();
+ in = ClassLoader.getSystemResourceAsStream(propFile);
+ if (in == null) {
+ if (!propFile.startsWith("/")) {
+ in = ClassLoader.getSystemResourceAsStream("/" + propFile);
+ }
+ if (in == null) {
+ in = new FileInputStream(new File(propFile));
+ }
+ }
+ properties.load(in);
+ init(properties);
+ } catch (IOException e) {
+ if (in != null) {
+ try {
+ in.close();
+ } catch (Exception exception) {
+ // Ignore
+ }
+ }
+ throw new Exception("Failed to load JAAS application properties", e);
+ }
+ LOG.debug("<== InMemoryJAASConfiguration.init( {} ) ", propFile);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void init(org.apache.commons.configuration.Configuration configuration) throws Exception {
+ LOG.debug("==> InMemoryJAASConfiguration.init()");
+
+ if (configuration != null && !configuration.isEmpty()) {
+ Properties properties = new Properties();
+ Iterator<String> iterator = configuration.getKeys();
+ while (iterator.hasNext()) {
+ String key = iterator.next();
+ properties.put(key, configuration.getProperty(key));
+ }
+ init(properties);
+ } else {
+ throw new Exception("Failed to load JAAS application properties: configuration NULL or empty!");
+ }
+
+ LOG.debug("<== InMemoryJAASConfiguration.init()");
+ }
+
+ public static void init(Properties properties) throws Exception {
+ LOG.debug("==> InMemoryJAASConfiguration.init()");
+
+ if (properties != null && MapUtils.isNotEmpty(properties)) {
+ InMemoryJAASConfiguration conf = new InMemoryJAASConfiguration(properties);
+ Configuration.setConfiguration(conf);
+ } else {
+ throw new Exception("Failed to load JAAS application properties: properties NULL or empty!");
+ }
+
+ LOG.debug("<== InMemoryJAASConfiguration.init()");
+ }
+
+ @Override
+ public AppConfigurationEntry[] getAppConfigurationEntry(String name) {
+ LOG.trace("==> InMemoryJAASConfiguration.getAppConfigurationEntry( {} )", name);
+
+ AppConfigurationEntry[] ret = null;
+ List<AppConfigurationEntry> retList = applicationConfigEntryMap.get(name);
+ if (retList == null || retList.size() == 0) {
+ if (parent != null) {
+ ret = parent.getAppConfigurationEntry(name);
+ }
+ } else {
+ int sz = retList.size();
+ ret = new AppConfigurationEntry[sz];
+ ret = retList.toArray(ret);
+ }
+ LOG.trace("<== InMemoryJAASConfiguration.getAppConfigurationEntry( {} ) : {}", name, ArrayUtils.toString(ret));
+ return ret;
+ }
+
+ private InMemoryJAASConfiguration(Properties prop) {
+ parent = Configuration.getConfiguration();
+ initialize(prop);
+ }
+
+ private void initialize(Properties properties) {
+ LOG.debug("==> InMemoryJAASConfiguration.initialize()");
+
+ int prefixLen = JAAS_CONFIG_PREFIX_PARAM.length();
+
+ Map<String, SortedSet<Integer>> jaasClients = new HashMap<>();
+ for(String key : properties.stringPropertyNames()) {
+ if (key.startsWith(JAAS_CONFIG_PREFIX_PARAM)) {
+ String jaasKey = key.substring(prefixLen);
+ StringTokenizer tokenizer = new StringTokenizer(jaasKey, ".");
+ int tokenCount =tokenizer.countTokens();
+ if (tokenCount > 0) {
+ String clientId = tokenizer.nextToken();
+ SortedSet<Integer> indexList = jaasClients.get(clientId);
+ if (indexList == null) {
+ indexList = new TreeSet<Integer>();
+ jaasClients.put(clientId, indexList);
+ }
+ String indexStr = tokenizer.nextToken();
+
+ int indexId = isNumeric(indexStr) ? Integer.parseInt(indexStr) : -1;
+
+ Integer clientIdIndex = Integer.valueOf(indexId);
+
+ if (!indexList.contains(clientIdIndex)) {
+ indexList.add(clientIdIndex);
+ }
+
+ }
+ }
+ }
+ for(String jaasClient : jaasClients.keySet()) {
+
+ for(Integer index : jaasClients.get(jaasClient)) {
+
+ String keyPrefix = JAAS_CONFIG_PREFIX_PARAM + jaasClient + ".";
+
+ if (index > -1) {
+ keyPrefix = keyPrefix + String.valueOf(index) + ".";
+ }
+
+ String keyParam = keyPrefix + JAAS_CONFIG_LOGIN_MODULE_NAME_PARAM;
+ String loginModuleName = properties.getProperty(keyParam);
+
+ if (loginModuleName == null) {
+ LOG.error("Unable to add JAAS configuration for "
+ + "client [" + jaasClient + "] as it is missing param [" + keyParam + "]."
+ + " Skipping JAAS config for [" + jaasClient + "]");
+ continue;
+ } else {
+ loginModuleName = loginModuleName.trim();
+ }
+
+ keyParam = keyPrefix + JAAS_CONFIG_LOGIN_MODULE_CONTROL_FLAG_PARAM;
+ String controlFlag = properties.getProperty(keyParam);
+
+ AppConfigurationEntry.LoginModuleControlFlag loginControlFlag = null;
+ if (controlFlag != null) {
+ controlFlag = controlFlag.trim().toLowerCase();
+ if (controlFlag.equals("optional")) {
+ loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.OPTIONAL;
+ } else if (controlFlag.equals("requisite")) {
+ loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUISITE;
+ } else if (controlFlag.equals("sufficient")) {
+ loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.SUFFICIENT;
+ } else if (controlFlag.equals("required")) {
+ loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUIRED;
+ } else {
+ String validValues = "optional|requisite|sufficient|required";
+ LOG.warn("Unknown JAAS configuration value for (" + keyParam
+ + ") = [" + controlFlag + "], valid value are [" + validValues
+ + "] using the default value, REQUIRED");
+ loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUIRED;
+ }
+ } else {
+ LOG.warn("Unable to find JAAS configuration ("
+ + keyParam + "); using the default value, REQUIRED");
+ loginControlFlag = AppConfigurationEntry.LoginModuleControlFlag.REQUIRED;
+ }
+
+
+ Map<String, String> options = new HashMap<>();
+ String optionPrefix = keyPrefix + JAAS_CONFIG_LOGIN_OPTIONS_PREFIX + ".";
+ int optionPrefixLen = optionPrefix.length();
+ for(String key : properties.stringPropertyNames()) {
+ if (key.startsWith(optionPrefix)) {
+ String optionKey = key.substring(optionPrefixLen);
+ String optionVal = properties.getProperty(key);
+ if (optionVal != null) {
+ optionVal = optionVal.trim();
+
+ try {
+ if (optionKey.equalsIgnoreCase(JAAS_PRINCIPAL_PROP)) {
+ optionVal = SecurityUtil.getServerPrincipal(optionVal, (String) null);
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to build serverPrincipal. Using provided value:["
+ + optionVal + "]");
+ }
+ }
+ options.put(optionKey, optionVal);
+ }
+ }
+
+ AppConfigurationEntry entry = new AppConfigurationEntry(loginModuleName, loginControlFlag, options);
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Adding client: [").append(jaasClient).append("{").append(index).append("}]\n");
+ sb.append("\tloginModule: [").append(loginModuleName).append("]\n");
+ sb.append("\tcontrolFlag: [").append(loginControlFlag).append("]\n");
+ for (String key : options.keySet()) {
+ String val = options.get(key);
+ sb.append("\tOptions: [").append(key).append("] => [").append(val).append("]\n");
+ }
+ LOG.debug(sb.toString());
+ }
+
+ List<AppConfigurationEntry> retList = applicationConfigEntryMap.get(jaasClient);
+ if (retList == null) {
+ retList = new ArrayList<AppConfigurationEntry>();
+ applicationConfigEntryMap.put(jaasClient, retList);
+ }
+ retList.add(entry);
+
+
+ }
+ }
+ LOG.debug("<== InMemoryJAASConfiguration.initialize()");
+ }
+
+ private static boolean isNumeric(String str) {
+ return str.matches("-?\\d+(\\.\\d+)?"); //match a number with optional '-' and decimal.
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/hbase-agent/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/pom.xml b/hbase-agent/pom.xml
index 4a2a25f..b320bba 100644
--- a/hbase-agent/pom.xml
+++ b/hbase-agent/pom.xml
@@ -64,5 +64,10 @@
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/hdfs-agent/pom.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/pom.xml b/hdfs-agent/pom.xml
index 33ddd81..6e92a0e 100644
--- a/hdfs-agent/pom.xml
+++ b/hdfs-agent/pom.xml
@@ -120,6 +120,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/hive-agent/pom.xml
----------------------------------------------------------------------
diff --git a/hive-agent/pom.xml b/hive-agent/pom.xml
index a7ee08b..3161433 100644
--- a/hive-agent/pom.xml
+++ b/hive-agent/pom.xml
@@ -120,5 +120,10 @@
<artifactId>ranger-plugins-audit</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/kms/pom.xml
----------------------------------------------------------------------
diff --git a/kms/pom.xml b/kms/pom.xml
index a9f6c6c..9c9a606 100644
--- a/kms/pom.xml
+++ b/kms/pom.xml
@@ -441,6 +441,11 @@
<artifactId>ranger-plugins-common</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
</dependencies>
<build>
<pluginManagement>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/knox-agent/pom.xml
----------------------------------------------------------------------
diff --git a/knox-agent/pom.xml b/knox-agent/pom.xml
index 20d8237..3f92b25 100644
--- a/knox-agent/pom.xml
+++ b/knox-agent/pom.xml
@@ -100,5 +100,10 @@
<artifactId>htrace-core</artifactId>
<version>${htrace-core.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/plugin-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml
index e148539..7c6e82f 100644
--- a/plugin-kafka/pom.xml
+++ b/plugin-kafka/pom.xml
@@ -57,5 +57,10 @@
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
----------------------------------------------------------------------
diff --git a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
index fc82dfd..c0a9d3d 100644
--- a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
+++ b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
@@ -35,9 +35,9 @@ import org.apache.ranger.plugin.service.ResourceLookupContext;
import org.apache.ranger.plugin.util.TimedEventUtil;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
@@ -99,11 +99,11 @@ public class ServiceSolrClient {
return getCoresList(ignoreCollectionList);
}
- CollectionAdminRequest request = new CollectionAdminRequest.List();
- CollectionAdminResponse response = request.process(solrClient);
+ CollectionAdminRequest<?> request = new CollectionAdminRequest.List();
+ SolrResponse response = request.process(solrClient);
List<String> list = new ArrayList<String>();
- for (int i = 0; i < response.getCollectionStatus().size(); i++) {
+ for (int i = 0; i < response.getResponse().size(); i++) {
if (ignoreCollectionList == null
|| !ignoreCollectionList.contains(list.get(i))) {
list.add(list.get(i));
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/plugin-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/plugin-yarn/pom.xml b/plugin-yarn/pom.xml
index 92c7dfb..b87145d 100644
--- a/plugin-yarn/pom.xml
+++ b/plugin-yarn/pom.xml
@@ -56,5 +56,10 @@
<artifactId>hadoop-yarn-api</artifactId>
<version>${hadoop.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 29292a7..ef78fb5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -207,7 +207,7 @@
<servlet.api.version>2.5</servlet.api.version>
<slf4j-api.version>1.7.5</slf4j-api.version>
<snappy-java.version>1.0.4.1</snappy-java.version>
- <solr.version>5.2.0</solr.version>
+ <solr.version>5.5.1</solr.version>
<spring-ldap-core.version>1.3.1.RELEASE</spring-ldap-core.version>
<springframework.security.version>3.1.3.RELEASE</springframework.security.version>
<springframework.spring.version>2.5.6</springframework.spring.version>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/security-admin/contrib/solr_for_audit_setup/setup.sh
----------------------------------------------------------------------
diff --git a/security-admin/contrib/solr_for_audit_setup/setup.sh b/security-admin/contrib/solr_for_audit_setup/setup.sh
index 1c05762..c4a8817 100755
--- a/security-admin/contrib/solr_for_audit_setup/setup.sh
+++ b/security-admin/contrib/solr_for_audit_setup/setup.sh
@@ -451,6 +451,8 @@ echo "`date`|INFO|Configure Ranger to use the following URL http://`hostname -f`
if [ "$SOLR_DEPLOYMENT" = "solrcloud" ]; then
echo "`date`|INFO|Please refer to $SOLR_INSTALL_NOTES for instructions for setting up collections in SolrCloud"
fi
+echo "`date`|INFO| ** NOTE: If Solr is Secured then solrclient JAAS configuration has to be added to Ranger Admin and Ranger Plugin properties"
+echo "`date`|INFO| ** Refer documentation on how to configure Ranger for audit to Secure Solr"
echo "########## Done ###################"
echo "Created file $SOLR_INSTALL_NOTES with instructions to start and stop"
echo "###################################"
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java b/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java
index b9caa76..e2d8d39 100644
--- a/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/common/PropertiesUtil.java
@@ -30,6 +30,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Set;
+
import org.apache.log4j.Logger;
import org.apache.ranger.credentialapi.CredentialReader;
import org.springframework.beans.BeansException;
@@ -244,4 +245,11 @@ public class PropertiesUtil extends PropertyPlaceholderConfigurer {
public static Map<String, String> getPropertiesMap() {
return propertiesMap;
}
+ public static Properties getProps() {
+ Properties ret = new Properties();
+ if (propertiesMap != null) {
+ ret.putAll(propertiesMap);
+ }
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java
index 433f5c9..283f44f 100644
--- a/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/patch/cliutil/DbToSolrMigrationUtil.java
@@ -25,7 +25,9 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import java.util.Properties;
import java.util.UUID;
+
import org.apache.log4j.Logger;
import org.apache.ranger.db.RangerDaoManager;
import org.apache.ranger.entity.XXAccessAudit;
@@ -34,13 +36,19 @@ import org.apache.ranger.entity.XXAccessAuditV4;
import org.apache.ranger.entity.XXAccessAuditV5;
import org.apache.ranger.patch.BaseLoader;
import org.apache.ranger.solr.SolrAccessAuditsService;
+import org.apache.ranger.audit.utils.InMemoryJAASConfiguration;
import org.apache.ranger.authorization.utils.StringUtil;
import org.apache.ranger.biz.RangerBizUtil;
import org.apache.ranger.common.AppConstants;
import org.apache.ranger.common.DateUtil;
import org.apache.ranger.common.PropertiesUtil;
import org.apache.ranger.util.CLIUtil;
-import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
@@ -51,16 +59,24 @@ import org.springframework.util.CollectionUtils;
@Component
public class DbToSolrMigrationUtil extends BaseLoader {
private static Logger logger = Logger.getLogger(DbToSolrMigrationUtil.class);
- private HttpSolrServer solrServer=null;
private final static String CHECK_FILE_NAME = "migration_check_file.txt";
private final static Charset ENCODING = StandardCharsets.UTF_8;
+
+ public static SolrClient solrClient = null;
+ public final static String SOLR_URLS_PROP = "ranger.audit.solr.urls";
+ public final static String SOLR_ZK_HOSTS = "ranger.audit.solr.zookeepers";
+ public final static String SOLR_COLLECTION_NAME = "ranger.audit.solr.collection.name";
+ public final static String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
+ public final static String DEFAULT_COLLECTION_NAME = "ranger_audits";
+
@Autowired
RangerDaoManager daoManager;
@Autowired
SolrAccessAuditsService solrAccessAuditsService;
- public static void main(String[] args) {
+ public static void main(String[] args) throws Exception {
logger.info("main()");
+ logger.info("Note: If migrating to Secure Solr, make sure SolrClient JAAS Properites are configured in ranger-admin-site.xml");
try {
DbToSolrMigrationUtil loader = (DbToSolrMigrationUtil) CLIUtil
.getBean(DbToSolrMigrationUtil.class);
@@ -74,14 +90,18 @@ public class DbToSolrMigrationUtil extends BaseLoader {
} catch (Exception e) {
logger.error("Error loading", e);
System.exit(1);
+ } finally {
+ if (solrClient != null) {
+ solrClient.close();
+ }
}
}
@Override
public void init() throws Exception {
- String solrURL=PropertiesUtil.getProperty("ranger.audit.solr.urls");
- logger.info("solrURL:"+solrURL);
- solrServer = new HttpSolrServer(solrURL);
+ logger.info("==> DbToSolrMigrationUtil.init() Start.");
+ solrClient = createSolrClient();
+ logger.info("<== DbToSolrMigrationUtil.init() End.");
}
@Override
@@ -222,7 +242,7 @@ public class DbToSolrMigrationUtil extends BaseLoader {
public void send2solr(XXAccessAuditV4 xXAccessAudit) throws Throwable {
SolrInputDocument document = new SolrInputDocument();
toSolrDocument(xXAccessAudit,document);
- UpdateResponse response = solrServer.add(document);
+ UpdateResponse response = solrClient.add(document);
if (response.getStatus() != 0) {
logger.info("Response=" + response.toString() + ", status= "
+ response.getStatus() + ", event=" + xXAccessAudit.toString());
@@ -233,7 +253,7 @@ public class DbToSolrMigrationUtil extends BaseLoader {
public void send2solr(XXAccessAuditV5 xXAccessAudit) throws Throwable {
SolrInputDocument document = new SolrInputDocument();
toSolrDocument(xXAccessAudit,document);
- UpdateResponse response = solrServer.add(document);
+ UpdateResponse response = solrClient.add(document);
if (response.getStatus() != 0) {
logger.info("Response=" + response.toString() + ", status= "
+ response.getStatus() + ", event=" + xXAccessAudit.toString());
@@ -244,7 +264,7 @@ public class DbToSolrMigrationUtil extends BaseLoader {
public void send2solr(XXAccessAudit xXAccessAudit) throws Throwable {
SolrInputDocument document = new SolrInputDocument();
toSolrDocument(xXAccessAudit,document);
- UpdateResponse response = solrServer.add(document);
+ UpdateResponse response = solrClient.add(document);
if (response.getStatus() != 0) {
logger.info("Response=" + response.toString() + ", status= "
+ response.getStatus() + ", event=" + xXAccessAudit.toString());
@@ -332,4 +352,111 @@ public class DbToSolrMigrationUtil extends BaseLoader {
@Override
public void printStats() {
}
+
+ private SolrClient createSolrClient() throws Exception {
+ SolrClient solrClient = null;
+
+ registerSolrClientJAAS();
+ String zkHosts = PropertiesUtil
+ .getProperty(SOLR_ZK_HOSTS);
+ if (zkHosts == null) {
+ zkHosts = PropertiesUtil
+ .getProperty("ranger.audit.solr.zookeeper");
+ }
+ if (zkHosts == null) {
+ zkHosts = PropertiesUtil
+ .getProperty("ranger.solr.zookeeper");
+ }
+
+ String solrURL = PropertiesUtil
+ .getProperty(SOLR_URLS_PROP);
+ if (solrURL == null) {
+ // Try with url
+ solrURL = PropertiesUtil
+ .getProperty("ranger.audit.solr.url");
+ }
+ if (solrURL == null) {
+ // Let's try older property name
+ solrURL = PropertiesUtil
+ .getProperty("ranger.solr.url");
+ }
+
+ if (zkHosts != null && !zkHosts.trim().equals("")
+ && !zkHosts.trim().equalsIgnoreCase("none")) {
+ zkHosts = zkHosts.trim();
+ String collectionName = PropertiesUtil
+ .getProperty(SOLR_COLLECTION_NAME);
+ if (collectionName == null
+ || collectionName.equalsIgnoreCase("none")) {
+ collectionName = DEFAULT_COLLECTION_NAME;
+ }
+
+ logger.info("Solr zkHosts=" + zkHosts
+ + ", collectionName=" + collectionName);
+
+ try {
+ // Instantiate
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
+ CloudSolrClient solrCloudClient = new CloudSolrClient(
+ zkHosts);
+ solrCloudClient
+ .setDefaultCollection(collectionName);
+ solrClient = solrCloudClient;
+ solrCloudClient.close();
+ } catch (Exception e) {
+ logger.fatal(
+ "Can't connect to Solr server. ZooKeepers="
+ + zkHosts + ", collection="
+ + collectionName, e);
+ throw e;
+ }
+ } else {
+ if (solrURL == null || solrURL.isEmpty()
+ || solrURL.equalsIgnoreCase("none")) {
+ logger.fatal("Solr ZKHosts and URL for Audit are empty. Please set property "
+ + SOLR_ZK_HOSTS
+ + " or "
+ + SOLR_URLS_PROP);
+ } else {
+ try {
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
+ solrClient = new HttpSolrClient(solrURL);
+ if (solrClient instanceof HttpSolrClient) {
+ HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient;
+ httpSolrClient
+ .setAllowCompression(true);
+ httpSolrClient
+ .setConnectionTimeout(1000);
+ httpSolrClient.setMaxRetries(1);
+ httpSolrClient
+ .setRequestWriter(new BinaryRequestWriter());
+ }
+ } catch (Exception e) {
+ logger.fatal(
+ "Can't connect to Solr server. URL="
+ + solrURL, e);
+ throw e;
+ }
+ }
+ }
+ return solrClient;
+ }
+
+ private void registerSolrClientJAAS() {
+ logger.info("==> createSolrClient.registerSolrClientJAAS()" );
+ Properties props = PropertiesUtil.getProps();
+ try {
+ // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it
+ // Acutal solrclient JAAS configs are read from the ranger-admin-site.xml in ranger admin config folder and set by InMemoryJAASConfiguration
+ // Refer InMemoryJAASConfiguration doc for JAAS Configuration
+ if ( System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null ) {
+ System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null") ;
+ }
+ logger.info("Loading SolrClient JAAS config from Ranger audit config if present...");
+ InMemoryJAASConfiguration.init(props);
+ } catch (Exception e) {
+ logger.error("ERROR: Unable to load SolrClient JAAS config from ranger admin config file. Audit migration to Secure Solr will fail...",e);
+ }
+ logger.info("<==createSolrClient.registerSolrClientJAAS()" );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java
index 913b9f0..1b5793f 100644
--- a/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java
@@ -20,14 +20,18 @@
package org.apache.ranger.solr;
import java.util.Date;
+import java.util.Properties;
import org.apache.log4j.Logger;
+import org.apache.ranger.audit.utils.InMemoryJAASConfiguration;
import org.apache.ranger.biz.RangerBizUtil;
import org.apache.ranger.common.PropertiesUtil;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Krb5HttpClientConfigurer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
@@ -52,10 +56,12 @@ public class SolrMgr {
final static String SOLR_URLS_PROP = "ranger.audit.solr.urls";
final static String SOLR_ZK_HOSTS = "ranger.audit.solr.zookeepers";
final static String SOLR_COLLECTION_NAME = "ranger.audit.solr.collection.name";
+ final static String PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG = "java.security.auth.login.config";
+
public static final String DEFAULT_COLLECTION_NAME = "ranger_audits";
public SolrMgr() {
-
+ init();
}
void connect() {
@@ -103,6 +109,7 @@ public class SolrMgr {
try {
// Instantiate
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
CloudSolrClient solrCloudClient = new CloudSolrClient(
zkHosts);
solrCloudClient
@@ -124,6 +131,7 @@ public class SolrMgr {
+ SOLR_URLS_PROP);
} else {
try {
+ HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer());
solrClient = new HttpSolrClient(solrURL);
if (solrClient == null) {
logger.fatal("Can't connect to Solr. URL="
@@ -157,6 +165,24 @@ public class SolrMgr {
}
}
+ private void init() {
+ logger.info("==>SolrMgr.init()" );
+ Properties props = PropertiesUtil.getProps();
+ try {
+ // SolrJ requires "java.security.auth.login.config" property to be set to identify itself that it is kerberized. So using a dummy property for it
+ // Acutal solrclient JAAS configs are read from the ranger-admin-site.xml in ranger admin config folder and set by InMemoryJAASConfiguration
+ // Refer InMemoryJAASConfiguration doc for JAAS Configuration
+ if ( System.getProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG) == null ) {
+ System.setProperty(PROP_JAVA_SECURITY_AUTH_LOGIN_CONFIG, "/dev/null") ;
+ }
+ logger.info("Loading SolrClient JAAS config from Ranger audit config if present...");
+ InMemoryJAASConfiguration.init(props);
+ } catch (Exception e) {
+ logger.error("ERROR: Unable to load SolrClient JAAS config from ranger admin config file. Audit to Kerberized Solr will fail...", e);
+ }
+ logger.info("<==SolrMgr.init()" );
+ }
+
public SolrClient getSolrClient() {
if (solrClient == null) {
connect();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/src/main/assembly/hdfs-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hdfs-agent.xml b/src/main/assembly/hdfs-agent.xml
index f0fbe19..b099bbb 100644
--- a/src/main/assembly/hdfs-agent.xml
+++ b/src/main/assembly/hdfs-agent.xml
@@ -83,6 +83,8 @@
<include>org.eclipse.persistence:javax.persistence</include>
<include>org.eclipse.persistence:eclipselink</include>
<include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include>
<include>org.noggit:noggit:jar:${noggit.version}</include>
</includes>
<unpack>false</unpack>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/src/main/assembly/hive-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hive-agent.xml b/src/main/assembly/hive-agent.xml
index 92a312a..5d83b46 100644
--- a/src/main/assembly/hive-agent.xml
+++ b/src/main/assembly/hive-agent.xml
@@ -51,6 +51,8 @@
<include>org.eclipse.persistence:eclipselink</include>
<include>org.eclipse.persistence:javax.persistence</include>
<include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include>
<include>org.noggit:noggit:jar:${noggit.version}</include>
</includes>
<unpack>false</unpack>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/src/main/assembly/knox-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/knox-agent.xml b/src/main/assembly/knox-agent.xml
index 0657e87..3352153 100644
--- a/src/main/assembly/knox-agent.xml
+++ b/src/main/assembly/knox-agent.xml
@@ -51,7 +51,9 @@
<includes>
<include>commons-configuration:commons-configuration</include>
<include>com.google.code.gson:gson*</include>
- <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
+ <include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include>
<include>org.noggit:noggit:jar:${noggit.version}</include>
<include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include>
<include>org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version}</include>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/src/main/assembly/plugin-solr.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/plugin-solr.xml b/src/main/assembly/plugin-solr.xml
index c96e63f..f73f204 100644
--- a/src/main/assembly/plugin-solr.xml
+++ b/src/main/assembly/plugin-solr.xml
@@ -43,7 +43,6 @@
<outputDirectory>/lib/ranger-solr-plugin-impl</outputDirectory>
<unpack>false</unpack>
<includes>
- <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include>
<include>com.google.code.gson:gson</include>
<include>org.eclipse.persistence:eclipselink</include>
<include>org.eclipse.persistence:javax.persistence</include>
@@ -53,7 +52,6 @@
<include>org.codehaus.jackson:jackson-jaxrs</include>
<include>org.codehaus.jackson:jackson-mapper-asl</include>
<include>org.codehaus.jackson:jackson-xc</include>
-
</includes>
</dependencySet>
<dependencySet>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/src/main/assembly/plugin-yarn.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/plugin-yarn.xml b/src/main/assembly/plugin-yarn.xml
index 19e88d4..d28834b 100644
--- a/src/main/assembly/plugin-yarn.xml
+++ b/src/main/assembly/plugin-yarn.xml
@@ -52,6 +52,8 @@
<include>org.eclipse.persistence:eclipselink</include>
<include>org.eclipse.persistence:javax.persistence</include>
<include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponents.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponents.httpcore.version}</include>
<include>org.noggit:noggit:jar:${noggit.version}</include>
</includes>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/a4140bc0/storm-agent/pom.xml
----------------------------------------------------------------------
diff --git a/storm-agent/pom.xml b/storm-agent/pom.xml
index c816644..7e157a8 100644
--- a/storm-agent/pom.xml
+++ b/storm-agent/pom.xml
@@ -66,5 +66,10 @@
<artifactId>htrace-core</artifactId>
<version>${htrace-core.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.solr</groupId>
+ <artifactId>solr-solrj</artifactId>
+ <version>${solr.version}</version>
+ </dependency>
</dependencies>
</project>