You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by bo...@apache.org on 2015/03/18 00:33:38 UTC
[16/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Support for Solr as Audit Destination.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/40aa090d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/40aa090d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/40aa090d
Branch: refs/heads/master
Commit: 40aa090d7b23a524b9900f5b722d02b48c06b947
Parents: 8e6acd5
Author: Don Bosco Durai <bo...@apache.org>
Authored: Tue Mar 17 14:17:30 2015 -0700
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Tue Mar 17 14:17:30 2015 -0700
----------------------------------------------------------------------
agents-audit/pom.xml | 28 +-
.../audit/provider/AuditProviderFactory.java | 23 +-
.../audit/provider/BaseAuditProvider.java | 5 +-
.../provider/kafka/KafkaAuditProvider.java | 2 +-
.../audit/provider/solr/SolrAuditProvider.java | 275 ++++
hbase-agent/conf/ranger-hbase-audit-changes.cfg | 12 +
hbase-agent/conf/ranger-hbase-audit.xml | 48 +
hbase-agent/scripts/install.properties | 6 +
hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 11 +
hdfs-agent/conf/ranger-hdfs-audit.xml | 48 +
hdfs-agent/scripts/install.properties | 7 +
hive-agent/conf/ranger-hive-audit-changes.cfg | 12 +
hive-agent/conf/ranger-hive-audit.xml | 48 +
hive-agent/scripts/install.properties | 6 +
knox-agent/conf/ranger-knox-audit-changes.cfg | 12 +
knox-agent/conf/ranger-knox-audit.xml | 48 +
knox-agent/scripts/install.properties | 6 +
plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 12 +
plugin-yarn/conf/ranger-yarn-audit.xml | 48 +
plugin-yarn/scripts/install.properties | 6 +
pom.xml | 9 +
ranger_solrj/.gitignore | 1 +
ranger_solrj/pom.xml | 55 +
.../solr/client/solrj/ResponseParser.java | 53 +
.../apache/solr/client/solrj/SolrClient.java | 416 ++++++
.../org/apache/solr/client/solrj/SolrQuery.java | 1109 ++++++++++++++++
.../apache/solr/client/solrj/SolrRequest.java | 137 ++
.../apache/solr/client/solrj/SolrResponse.java | 65 +
.../apache/solr/client/solrj/SolrServer.java | 25 +
.../solr/client/solrj/SolrServerException.java | 54 +
.../client/solrj/StreamingResponseCallback.java | 37 +
.../client/solrj/beans/BindingException.java | 29 +
.../solrj/beans/DocumentObjectBinder.java | 470 +++++++
.../apache/solr/client/solrj/beans/Field.java | 38 +
.../solr/client/solrj/beans/package-info.java | 23 +
.../client/solrj/impl/BinaryRequestWriter.java | 120 ++
.../client/solrj/impl/BinaryResponseParser.java | 64 +
.../solr/client/solrj/impl/CloudSolrClient.java | 1232 ++++++++++++++++++
.../solr/client/solrj/impl/CloudSolrServer.java | 61 +
.../solrj/impl/ConcurrentUpdateSolrClient.java | 492 +++++++
.../solrj/impl/ConcurrentUpdateSolrServer.java | 46 +
.../client/solrj/impl/HttpClientConfigurer.java | 97 ++
.../solr/client/solrj/impl/HttpClientUtil.java | 365 ++++++
.../solr/client/solrj/impl/HttpSolrClient.java | 821 ++++++++++++
.../solr/client/solrj/impl/HttpSolrServer.java | 41 +
.../client/solrj/impl/LBHttpSolrClient.java | 730 +++++++++++
.../client/solrj/impl/LBHttpSolrServer.java | 43 +
.../client/solrj/impl/NoOpResponseParser.java | 83 ++
.../impl/StreamingBinaryResponseParser.java | 91 ++
.../client/solrj/impl/XMLResponseParser.java | 465 +++++++
.../solr/client/solrj/impl/package-info.java | 24 +
.../apache/solr/client/solrj/package-info.java | 23 +
.../solrj/request/AbstractUpdateRequest.java | 144 ++
.../solrj/request/CollectionAdminRequest.java | 860 ++++++++++++
.../request/ContentStreamUpdateRequest.java | 78 ++
.../client/solrj/request/CoreAdminRequest.java | 593 +++++++++
.../client/solrj/request/DirectXmlRequest.java | 66 +
.../solrj/request/DocumentAnalysisRequest.java | 199 +++
.../solrj/request/FieldAnalysisRequest.java | 270 ++++
.../client/solrj/request/IsUpdateRequest.java | 26 +
.../request/JavaBinUpdateRequestCodec.java | 251 ++++
.../solr/client/solrj/request/LukeRequest.java | 120 ++
.../solr/client/solrj/request/QueryRequest.java | 89 ++
.../client/solrj/request/RequestWriter.java | 146 +++
.../solr/client/solrj/request/SolrPing.java | 111 ++
.../client/solrj/request/UpdateRequest.java | 463 +++++++
.../solr/client/solrj/request/package-info.java | 23 +
.../solrj/response/AnalysisResponseBase.java | 252 ++++
.../solrj/response/CollectionAdminResponse.java | 79 ++
.../solrj/response/CoreAdminResponse.java | 58 +
.../response/DocumentAnalysisResponse.java | 258 ++++
.../solr/client/solrj/response/FacetField.java | 176 +++
.../solrj/response/FieldAnalysisResponse.java | 204 +++
.../client/solrj/response/FieldStatsInfo.java | 191 +++
.../solr/client/solrj/response/Group.java | 69 +
.../client/solrj/response/GroupCommand.java | 125 ++
.../client/solrj/response/GroupResponse.java | 56 +
.../client/solrj/response/IntervalFacet.java | 85 ++
.../client/solrj/response/LukeResponse.java | 270 ++++
.../solr/client/solrj/response/PivotField.java | 97 ++
.../client/solrj/response/QueryResponse.java | 586 +++++++++
.../solr/client/solrj/response/RangeFacet.java | 126 ++
.../client/solrj/response/SolrPingResponse.java | 28 +
.../client/solrj/response/SolrResponseBase.java | 91 ++
.../solrj/response/SpellCheckResponse.java | 273 ++++
.../client/solrj/response/TermsResponse.java | 89 ++
.../client/solrj/response/UpdateResponse.java | 30 +
.../client/solrj/response/package-info.java | 24 +
.../solr/client/solrj/util/ClientUtils.java | 251 ++++
.../solr/client/solrj/util/package-info.java | 23 +
.../org/apache/solr/common/EnumFieldValue.java | 116 ++
.../org/apache/solr/common/SolrDocument.java | 396 ++++++
.../apache/solr/common/SolrDocumentList.java | 68 +
.../org/apache/solr/common/SolrException.java | 208 +++
.../apache/solr/common/SolrInputDocument.java | 301 +++++
.../org/apache/solr/common/SolrInputField.java | 232 ++++
.../org/apache/solr/common/StringUtils.java | 26 +
.../org/apache/solr/common/cloud/Aliases.java | 63 +
.../solr/common/cloud/BeforeReconnect.java | 22 +
.../solr/common/cloud/ClosableThread.java | 27 +
.../apache/solr/common/cloud/ClusterState.java | 397 ++++++
.../solr/common/cloud/ClusterStateUtil.java | 230 ++++
.../solr/common/cloud/CompositeIdRouter.java | 327 +++++
.../solr/common/cloud/ConnectionManager.java | 237 ++++
.../common/cloud/DefaultConnectionStrategy.java | 75 ++
.../solr/common/cloud/DefaultZkACLProvider.java | 45 +
.../cloud/DefaultZkCredentialsProvider.java | 41 +
.../apache/solr/common/cloud/DocCollection.java | 201 +++
.../org/apache/solr/common/cloud/DocRouter.java | 227 ++++
.../solr/common/cloud/HashBasedRouter.java | 81 ++
.../solr/common/cloud/ImplicitDocRouter.java | 104 ++
.../apache/solr/common/cloud/OnReconnect.java | 22 +
.../apache/solr/common/cloud/PlainIdRouter.java | 23 +
.../org/apache/solr/common/cloud/Replica.java | 48 +
.../apache/solr/common/cloud/RoutingRule.java | 71 +
.../solr/common/cloud/SaslZkACLProvider.java | 49 +
.../org/apache/solr/common/cloud/Slice.java | 196 +++
.../apache/solr/common/cloud/SolrZkClient.java | 736 +++++++++++
.../apache/solr/common/cloud/SolrZooKeeper.java | 103 ++
...ParamsAllAndReadonlyDigestZkACLProvider.java | 89 ++
...tCredentialsDigestZkCredentialsProvider.java | 60 +
.../apache/solr/common/cloud/ZkACLProvider.java | 28 +
.../cloud/ZkClientConnectionStrategy.java | 113 ++
.../apache/solr/common/cloud/ZkCmdExecutor.java | 111 ++
.../solr/common/cloud/ZkConfigManager.java | 145 +++
.../solr/common/cloud/ZkCoreNodeProps.java | 74 ++
.../common/cloud/ZkCredentialsProvider.java | 45 +
.../apache/solr/common/cloud/ZkNodeProps.java | 154 +++
.../apache/solr/common/cloud/ZkOperation.java | 37 +
.../apache/solr/common/cloud/ZkStateReader.java | 925 +++++++++++++
.../solr/common/cloud/ZooKeeperException.java | 33 +
.../apache/solr/common/cloud/package-info.java | 23 +
.../org/apache/solr/common/luke/FieldFlag.java | 70 +
.../apache/solr/common/luke/package-info.java | 23 +
.../org/apache/solr/common/package-info.java | 23 +
.../solr/common/params/AnalysisParams.java | 60 +
.../solr/common/params/AppendedSolrParams.java | 55 +
.../solr/common/params/CollectionParams.java | 74 ++
.../apache/solr/common/params/CommonParams.java | 228 ++++
.../solr/common/params/CoreAdminParams.java | 151 +++
.../solr/common/params/CursorMarkParams.java | 48 +
.../solr/common/params/DefaultSolrParams.java | 68 +
.../apache/solr/common/params/DisMaxParams.java | 78 ++
.../apache/solr/common/params/EventParams.java | 29 +
.../apache/solr/common/params/ExpandParams.java | 32 +
.../apache/solr/common/params/FacetParams.java | 405 ++++++
.../apache/solr/common/params/GroupParams.java | 71 +
.../solr/common/params/HighlightParams.java | 82 ++
.../solr/common/params/MapSolrParams.java | 88 ++
.../common/params/ModifiableSolrParams.java | 210 +++
.../solr/common/params/MoreLikeThisParams.java | 74 ++
.../solr/common/params/MultiMapSolrParams.java | 92 ++
.../common/params/QueryElevationParams.java | 53 +
.../solr/common/params/RequiredSolrParams.java | 155 +++
.../apache/solr/common/params/ShardParams.java | 56 +
.../apache/solr/common/params/SimpleParams.java | 50 +
.../apache/solr/common/params/SolrParams.java | 363 ++++++
.../solr/common/params/SpatialParams.java | 41 +
.../solr/common/params/SpellingParams.java | 174 +++
.../apache/solr/common/params/StatsParams.java | 28 +
.../solr/common/params/TermVectorParams.java | 66 +
.../apache/solr/common/params/TermsParams.java | 120 ++
.../apache/solr/common/params/UpdateParams.java | 72 +
.../apache/solr/common/params/package-info.java | 22 +
.../org/apache/solr/common/util/Base64.java | 153 +++
.../org/apache/solr/common/util/ByteUtils.java | 126 ++
.../apache/solr/common/util/ContentStream.java | 81 ++
.../solr/common/util/ContentStreamBase.java | 260 ++++
.../solr/common/util/DataInputInputStream.java | 27 +
.../org/apache/solr/common/util/DateUtil.java | 260 ++++
.../apache/solr/common/util/ExecutorUtil.java | 64 +
.../solr/common/util/FastInputStream.java | 253 ++++
.../solr/common/util/FastOutputStream.java | 233 ++++
.../java/org/apache/solr/common/util/Hash.java | 422 ++++++
.../org/apache/solr/common/util/IOUtils.java | 37 +
.../apache/solr/common/util/IteratorChain.java | 87 ++
.../apache/solr/common/util/JavaBinCodec.java | 820 ++++++++++++
.../solr/common/util/JsonRecordReader.java | 586 +++++++++
.../org/apache/solr/common/util/NamedList.java | 708 ++++++++++
.../solr/common/util/ObjectReleaseTracker.java | 62 +
.../org/apache/solr/common/util/RetryUtil.java | 43 +
.../solr/common/util/SimpleOrderedMap.java | 67 +
.../common/util/SolrjNamedThreadFactory.java | 50 +
.../org/apache/solr/common/util/StrUtils.java | 309 +++++
.../org/apache/solr/common/util/URLUtil.java | 50 +
.../java/org/apache/solr/common/util/XML.java | 207 +++
.../apache/solr/common/util/XMLErrorLogger.java | 84 ++
.../apache/solr/common/util/package-info.java | 23 +
ranger_solrj/src/main/java/overview.html | 21 +
security-admin/pom.xml | 5 +
security-admin/scripts/install.properties | 10 +
security-admin/scripts/setup.sh | 48 +-
.../java/org/apache/ranger/biz/AssetMgr.java | 11 +-
.../org/apache/ranger/biz/RangerBizUtil.java | 434 +++---
.../java/org/apache/ranger/biz/XAuditMgr.java | 41 +-
.../apache/ranger/db/RangerDaoManagerBase.java | 3 +
.../ranger/solr/SolrAccessAuditsService.java | 253 ++++
.../java/org/apache/ranger/solr/SolrMgr.java | 99 ++
.../java/org/apache/ranger/solr/SolrUtil.java | 327 +++++
src/main/assembly/hbase-agent.xml | 6 +
src/main/assembly/hdfs-agent.xml | 6 +
src/main/assembly/hive-agent.xml | 6 +
src/main/assembly/knox-agent.xml | 6 +
src/main/assembly/storm-agent.xml | 6 +
storm-agent/conf/ranger-storm-audit-changes.cfg | 13 +
storm-agent/conf/ranger-storm-audit.xml | 48 +
storm-agent/scripts/install.properties | 6 +
207 files changed, 30674 insertions(+), 219 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/pom.xml
----------------------------------------------------------------------
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index e54ec36..6715575 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -63,9 +63,29 @@
<version>${log4j.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>0.8.2.0</version>
-</dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${kafka.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger_solrj</artifactId>
+ <version>${ranger.solrj.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpcomponent.httpmime.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ <version>${noggit.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index 9fbe29f..bb8fa6d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider;
import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider;
+import org.apache.ranger.audit.provider.solr.SolrAuditProvider;
/*
@@ -43,6 +44,7 @@ public class AuditProviderFactory {
private static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled";
private static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled" ;
private static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
+ private static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
private static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024;
private static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000;
@@ -99,8 +101,9 @@ public class AuditProviderFactory {
boolean isAuditToHdfsEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false);
boolean isAuditToLog4jEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
boolean isAuditToKafkaEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false);
+ boolean isAuditToSolrEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false);
- if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled)) {
+ if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled)) {
LOG.info("AuditProviderFactory: Audit not enabled..");
mProvider = getDefaultProvider();
@@ -111,6 +114,7 @@ public class AuditProviderFactory {
List<AuditProvider> providers = new ArrayList<AuditProvider>();
if(isAuditToDbEnabled) {
+ LOG.info("DbAuditProvider is enabled");
DbAuditProvider dbProvider = new DbAuditProvider();
boolean isAuditToDbAsync = BaseAuditProvider.getBooleanProperty(props, DbAuditProvider.AUDIT_DB_IS_ASYNC_PROP, false);
@@ -128,6 +132,8 @@ public class AuditProviderFactory {
}
if(isAuditToHdfsEnabled) {
+ LOG.info("HdfsAuditProvider is enabled");
+
HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
boolean isAuditToHdfsAsync = BaseAuditProvider.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false);
@@ -156,7 +162,20 @@ public class AuditProviderFactory {
providers.add(kafkaProvider);
}
}
-
+
+ if(isAuditToSolrEnabled) {
+ LOG.info("SolrAuditProvider is enabled");
+ SolrAuditProvider solrProvider = new SolrAuditProvider();
+ solrProvider.init(props);
+
+ if( solrProvider.isAsync()) {
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", solrProvider.getMaxQueueSize(), solrProvider.getMaxFlushInterval(), solrProvider);
+ providers.add(asyncProvider);
+ } else {
+ providers.add(solrProvider);
+ }
+ }
+
if(isAuditToLog4jEnabled) {
Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
index 14e6220..a068b8f 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
@@ -40,7 +40,7 @@ public abstract class BaseAuditProvider implements AuditProvider {
private int maxQueueSize = AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT;
private int maxFlushInterval = AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT;
-
+ protected Properties props = null;
public BaseAuditProvider() {
}
@@ -48,7 +48,8 @@ public abstract class BaseAuditProvider implements AuditProvider {
@Override
public void init(Properties props) {
LOG.info("BaseAuditProvider.init()");
-
+ this.props = props;
+
mLogFailureReportMinIntervalInMs = getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
index 54e73ea..0ec8790 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
@@ -117,7 +117,7 @@ public class KafkaAuditProvider extends BaseAuditProvider {
}
} catch (Throwable t) {
LOG.error("Error sending message to Kafka topic. topic=" + topic
- + ", message=" + message);
+ + ", message=" + message, t);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
new file mode 100644
index 0000000..1b463e6
--- /dev/null
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.provider.solr;
+
+import java.util.Date;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.BaseAuditProvider;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrInputDocument;
+
+public class SolrAuditProvider extends BaseAuditProvider {
+ private static final Log LOG = LogFactory.getLog(SolrAuditProvider.class);
+
+ public static final String AUDIT_MAX_QUEUE_SIZE_PROP = "xasecure.audit.solr.async.max.queue.size";
+ public static final String AUDIT_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.solr.async.max.flush.interval.ms";
+ public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms";
+
+ static final Object lock = new Object();
+ SolrClient solrClient = null;
+ Date lastConnectTime = null;
+ long lastFailTime = 0;
+
+ int retryWaitTime = 30000;
+
+ public SolrAuditProvider() {
+ }
+
+ @Override
+ public void init(Properties props) {
+ LOG.info("init() called");
+ super.init(props);
+
+ setMaxQueueSize(BaseAuditProvider.getIntProperty(props,
+ AUDIT_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT));
+ setMaxFlushInterval(BaseAuditProvider.getIntProperty(props,
+ AUDIT_MAX_QUEUE_SIZE_PROP,
+ AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT));
+ retryWaitTime = BaseAuditProvider.getIntProperty(props,
+ AUDIT_RETRY_WAIT_PROP, retryWaitTime);
+ }
+
+ void connect() {
+ if (solrClient == null) {
+ synchronized (lock) {
+ if (solrClient == null) {
+ String solrURL = BaseAuditProvider.getStringProperty(props,
+ "xasecure.audit.solr.solr_url");
+
+ if (lastConnectTime != null) {
+ // Let's wait for enough time before retrying
+ long diff = System.currentTimeMillis()
+ - lastConnectTime.getTime();
+ if (diff < retryWaitTime) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignore connecting to solr url="
+ + solrURL + ", lastConnect=" + diff
+ + "ms");
+ }
+ return;
+ }
+ }
+ lastConnectTime = new Date();
+
+ if (solrURL == null || solrURL.isEmpty()) {
+ LOG.fatal("Solr URL for Audit is empty");
+ return;
+ }
+
+ try {
+ // TODO: Need to support SolrCloud also
+ solrClient = new HttpSolrClient(solrURL);
+ if (solrClient instanceof HttpSolrClient) {
+ HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient;
+ httpSolrClient.setAllowCompression(true);
+ httpSolrClient.setConnectionTimeout(1000);
+ // solrClient.setSoTimeout(10000);
+ httpSolrClient.setMaxRetries(1);
+ }
+ } catch (Throwable t) {
+ LOG.fatal("Can't connect to Solr server. URL="
+ + solrURL, t);
+ }
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger.
+ * audit.model.AuditEventBase)
+ */
+ @Override
+ public void log(AuditEventBase event) {
+ if (!(event instanceof AuthzAuditEvent)) {
+ LOG.error(event.getClass().getName()
+ + " audit event class type is not supported");
+ return;
+ }
+ AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+ // TODO: This should be done at a higher level
+
+ if (authzEvent.getAgentHostname() == null) {
+ authzEvent.setAgentHostname(MiscUtil.getHostname());
+ }
+
+ if (authzEvent.getLogType() == null) {
+ authzEvent.setLogType("RangerAudit");
+ }
+
+ if (authzEvent.getEventId() == null) {
+ authzEvent.setEventId(MiscUtil.generateUniqueId());
+ }
+
+ try {
+ if (solrClient == null) {
+ connect();
+ if (solrClient == null) {
+ // Solr is still not initialized. So need to throw error
+ return;
+ }
+ }
+
+ if (lastFailTime > 0) {
+ long diff = System.currentTimeMillis() - lastFailTime;
+ if (diff < retryWaitTime) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignore sending audit. lastConnect=" + diff
+ + " ms");
+ }
+ return;
+ }
+ }
+ // Convert AuditEventBase to Solr document
+ SolrInputDocument document = toSolrDoc(authzEvent);
+ UpdateResponse response = solrClient.add(document);
+ if (response.getStatus() != 0) {
+ lastFailTime = System.currentTimeMillis();
+
+ // System.out.println("Response=" + response.toString()
+ // + ", status= " + response.getStatus() + ", event="
+ // + event);
+ // throw new Exception("Aborting. event=" + event +
+ // ", response="
+ // + response.toString());
+ } else {
+ lastFailTime = 0;
+ }
+
+ } catch (Throwable t) {
+ LOG.error("Error sending message to Solr", t);
+ }
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#start()
+ */
+ @Override
+ public void start() {
+ connect();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#stop()
+ */
+ @Override
+ public void stop() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete()
+ */
+ @Override
+ public void waitToComplete() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#isFlushPending()
+ */
+ @Override
+ public boolean isFlushPending() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#getLastFlushTime()
+ */
+ @Override
+ public long getLastFlushTime() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ // TODO Auto-generated method stub
+
+ }
+
+ SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", auditEvent.getEventId());
+ doc.addField("access", auditEvent.getAccessType());
+ doc.addField("enforcer", auditEvent.getAclEnforcer());
+ doc.addField("agent", auditEvent.getAgentId());
+ doc.addField("repo", auditEvent.getRepositoryName());
+ doc.addField("sess", auditEvent.getSessionId());
+ doc.addField("reqUser", auditEvent.getUser());
+ doc.addField("reqData", auditEvent.getRequestData());
+ doc.addField("resource", auditEvent.getResourcePath());
+ doc.addField("cliIP", auditEvent.getClientIP());
+ doc.addField("logType", auditEvent.getLogType());
+ doc.addField("result", auditEvent.getAccessResult());
+ doc.addField("policy", auditEvent.getPolicyId());
+ doc.addField("repoType", auditEvent.getRepositoryType());
+ doc.addField("resType", auditEvent.getResourceType());
+ doc.addField("reason", auditEvent.getResultReason());
+ doc.addField("action", auditEvent.getAction());
+ doc.addField("evtTime", auditEvent.getEventTime());
+ return doc;
+ }
+
+ public boolean isAsync() {
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
index e5c381a..221d20a 100644
--- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg
+++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/ranger-hbase-audit.xml b/hbase-agent/conf/ranger-hbase-audit.xml
index b39696b..e5bfb89 100644
--- a/hbase-agent/conf/ranger-hbase-audit.xml
+++ b/hbase-agent/conf/ranger-hbase-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index 5a81ad4..7ff29c9 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
index e5c381a..8d31016 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
+++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
@@ -31,3 +31,14 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml
index d26345d..09114ad 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit.xml
+++ b/hdfs-agent/conf/ranger-hdfs-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 93790e3..2e1b61a 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -89,6 +89,13 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg
index 9fa7608..2d6d414 100644
--- a/hive-agent/conf/ranger-hive-audit-changes.cfg
+++ b/hive-agent/conf/ranger-hive-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml
index d011b24..e753336 100644
--- a/hive-agent/conf/ranger-hive-audit.xml
+++ b/hive-agent/conf/ranger-hive-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index bbd1849..75b1b5d 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg
index 7ae334e..f97d10f 100644
--- a/knox-agent/conf/ranger-knox-audit-changes.cfg
+++ b/knox-agent/conf/ranger-knox-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/ranger-knox-audit.xml b/knox-agent/conf/ranger-knox-audit.xml
index 0fdcefc..6f0adb9 100644
--- a/knox-agent/conf/ranger-knox-audit.xml
+++ b/knox-agent/conf/ranger-knox-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index d821c5d..ecd9813 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -92,6 +92,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
index 4f2c5a2..e0dbea2 100644
--- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
+++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit.xml
----------------------------------------------------------------------
diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml
index c0096a4..f1e9687 100644
--- a/plugin-yarn/conf/ranger-yarn-audit.xml
+++ b/plugin-yarn/conf/ranger-yarn-audit.xml
@@ -184,4 +184,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/scripts/install.properties
----------------------------------------------------------------------
diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties
index d2d1ffe..bbe9f7f 100644
--- a/plugin-yarn/scripts/install.properties
+++ b/plugin-yarn/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ef39d37..00c8339 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
<module>knox-agent</module>
<module>storm-agent</module>
<module>plugin-yarn</module>
+ <module>ranger_solrj</module>
<module>security-admin</module>
<module>ugsync</module>
<module>unixauthclient</module>
@@ -129,6 +130,9 @@
<hamcrest.all.version>1.3</hamcrest.all.version>
<hbase.version>0.99.2</hbase.version>
<hive.version>1.2.0-SNAPSHOT</hive.version>
+ <httpcomponent.httpmime.version>4.2.5</httpcomponent.httpmime.version>
+ <httpcomponent.httpclient.version>4.2.5</httpcomponent.httpclient.version>
+ <httpcomponent.httpcore.version>4.2.5</httpcomponent.httpcore.version>
<calcite.version>0.9.2-incubating</calcite.version>
<tez.version>0.5.2</tez.version>
<javassist.version>3.12.1.GA</javassist.version>
@@ -138,16 +142,20 @@
<jersey-bundle.version>1.17.1</jersey-bundle.version>
<jersey-client.version>2.6</jersey-client.version>
<junit.version>4.11</junit.version>
+ <kafka.version>0.8.2.0</kafka.version>
<mockito.version>1.8.4</mockito.version>
<hamcrest-version>1.3</hamcrest-version>
<knox.gateway.version>0.5.0</knox.gateway.version>
<local.lib.dir>${project.basedir}/../lib/local</local.lib.dir>
<log4j.version>1.2.17</log4j.version>
<mysql-connector-java.version>5.1.31</mysql-connector-java.version>
+ <noggit.version>0.6</noggit.version>
<owasp-java-html-sanitizer.version>r239</owasp-java-html-sanitizer.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<security-agent-install-dir>hadoop-security/plugins</security-agent-install-dir>
<slf4j-api.version>1.7.5</slf4j-api.version>
+ <!--<solr.version>5.0.0</solr.version>-->
+ <ranger.solrj.version>0.4.0</ranger.solrj.version>
<springframework.spring.version>2.5.6</springframework.spring.version>
<!--
<springframework.spring.version>3.1.3.RELEASE</springframework.spring.version>
@@ -162,6 +170,7 @@
<tomcat.commons.el.version>5.5.23</tomcat.commons.el.version>
<tomcat.embed.version>7.0.55</tomcat.embed.version>
<velocity.version>1.7</velocity.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
<powermock.version>1.5.6</powermock.version>
<aspectj.version>1.8.2</aspectj.version>
<findbugs.plugin.version>3.0.0</findbugs.plugin.version>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/.gitignore
----------------------------------------------------------------------
diff --git a/ranger_solrj/.gitignore b/ranger_solrj/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/ranger_solrj/.gitignore
@@ -0,0 +1 @@
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/pom.xml
----------------------------------------------------------------------
diff --git a/ranger_solrj/pom.xml b/ranger_solrj/pom.xml
new file mode 100644
index 0000000..2b86140
--- /dev/null
+++ b/ranger_solrj/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>0.4.0</version>
+ </parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger_solrj</artifactId>
+ <version>0.4.0</version>
+ <name>ranger_solrj</name>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponent.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpcomponent.httpmime.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ <version>${noggit.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j-api.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
new file mode 100644
index 0000000..d5c3b38
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import java.io.Reader;
+import java.io.InputStream;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public abstract class ResponseParser
+{
+ public abstract String getWriterType(); // for example: wt=XML, JSON, etc
+
+ public abstract NamedList<Object> processResponse(InputStream body, String encoding);
+
+ public abstract NamedList<Object> processResponse(Reader reader);
+
+ /**
+ * A well behaved ResponseParser will return its content-type.
+ *
+ * @return the content-type this parser expects to parse
+ */
+ public String getContentType() {
+ return null;
+ }
+
+ /**
+ * @return the version param passed to solr
+ */
+ public String getVersion()
+ {
+ return "2.2";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
new file mode 100644
index 0000000..28b7f4f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import org.apache.solr.client.solrj.SolrRequest.METHOD;
+import org.apache.solr.client.solrj.beans.DocumentObjectBinder;
+import org.apache.solr.client.solrj.impl.StreamingBinaryResponseParser;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.SolrPing;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Abstraction through which all communication with a Solr server may be routed
+ *
+ * @since 5.0, replaced {@code SolrServer}
+ */
+public abstract class SolrClient implements Serializable, Closeable {
+
+ private static final long serialVersionUID = 1L;
+ private DocumentObjectBinder binder;
+
+ /**
+ * Adds a collection of documents
+ * @param docs the collection of documents
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException {
+ return add(docs, -1);
+ }
+
+ /**
+ * Adds a collection of documents, specifying max time before they become committed
+ * @param docs the collection of documents
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.add(docs);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Adds a collection of beans
+ * @param beans the collection of beans
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse addBeans(Collection<?> beans) throws SolrServerException, IOException {
+ return addBeans(beans, -1);
+ }
+
+ /**
+ * Adds a collection of beans specifying max time before they become committed
+ * @param beans the collection of beans
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException {
+ DocumentObjectBinder binder = this.getBinder();
+ ArrayList<SolrInputDocument> docs = new ArrayList<>(beans.size());
+ for (Object bean : beans) {
+ docs.add(binder.toSolrInputDocument(bean));
+ }
+ return add(docs, commitWithinMs);
+ }
+
+ /**
+ * Adds a single document
+ * @param doc the input document
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException {
+ return add(doc, -1);
+ }
+
+ /**
+ * Adds a single document specifying max time before it becomes committed
+ * @param doc the input document
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.add(doc);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Adds a single bean
+ * @param obj the input bean
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse addBean(Object obj) throws IOException, SolrServerException {
+ return addBean(obj, -1);
+ }
+
+ /**
+ * Adds a single bean specifying max time before it becomes committed
+ * @param obj the input bean
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException {
+ return add(getBinder().toSolrInputDocument(obj),commitWithinMs);
+ }
+
+ /**
+ * Performs an explicit commit, causing pending documents to be committed for indexing
+ * <p>
+ * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse commit() throws SolrServerException, IOException {
+ return commit(true, true);
+ }
+
+ /**
+ * Performs an explicit optimize, causing a merge of all segments to one.
+ * <p>
+ * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
+ * <p>
+ * Note: In most cases it is not required to do explicit optimize
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse optimize() throws SolrServerException, IOException {
+ return optimize(true, true, 1);
+ }
+
+ /**
+ * Performs an explicit commit, causing pending documents to be committed for indexing
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
+ return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher).process( this );
+ }
+
+ /**
+ * Performs an explicit commit, causing pending documents to be committed for indexing
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @param softCommit makes index changes visible while neither fsync-ing index files nor writing a new index descriptor
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException {
+ return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher, softCommit).process( this );
+ }
+
+ /**
+ * Performs an explicit optimize, causing a merge of all segments to one.
+ * <p>
+ * Note: In most cases it is not required to do explicit optimize
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
+ return optimize(waitFlush, waitSearcher, 1);
+ }
+
+ /**
+ * Performs an explicit optimize, causing a merge of all segments to one.
+ * <p>
+ * Note: In most cases it is not required to do explicit optimize
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @param maxSegments optimizes down to at most this number of segments
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException {
+ return new UpdateRequest().setAction(UpdateRequest.ACTION.OPTIMIZE, waitFlush, waitSearcher, maxSegments).process( this );
+ }
+
+ /**
+ * Performs a rollback of all non-committed documents pending.
+ * <p>
+ * Note that this is not a true rollback as in databases. Content you have previously
+ * added may have been committed due to autoCommit, buffer full, other client performing
+ * a commit etc.
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse rollback() throws SolrServerException, IOException {
+ return new UpdateRequest().rollback().process( this );
+ }
+
+ /**
+ * Deletes a single document by unique ID
+ * @param id the ID of the document to delete
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse deleteById(String id) throws SolrServerException, IOException {
+ return deleteById(id, -1);
+ }
+
+ /**
+ * Deletes a single document by unique ID, specifying max time before commit
+ * @param id the ID of the document to delete
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since 3.6
+ */
+ public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.deleteById(id);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Deletes a list of documents by unique ID
+ * @param ids the list of document IDs to delete
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException {
+ return deleteById(ids, -1);
+ }
+
+ /**
+ * Deletes a list of documents by unique ID, specifying max time before commit
+ * @param ids the list of document IDs to delete
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since 3.6
+ */
+ public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.deleteById(ids);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Deletes documents from the index based on a query
+ * @param query the query expressing what documents to delete
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException {
+ return deleteByQuery(query, -1);
+ }
+
+ /**
+ * Deletes documents from the index based on a query, specifying max time before commit
+ * @param query the query expressing what documents to delete
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since 3.6
+ */
+ public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.deleteByQuery(query);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Issues a ping request to check if the server is alive
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public SolrPingResponse ping() throws SolrServerException, IOException {
+ return new SolrPing().process(this);
+ }
+
+ /**
+ * Performs a query to the Solr server
+ * @param params an object holding all key/value parameters to send along the request
+ */
+ public QueryResponse query(SolrParams params) throws SolrServerException, IOException {
+ return new QueryRequest(params).process(this);
+ }
+
+ /**
+ * Performs a query to the Solr server
+ * @param params an object holding all key/value parameters to send along the request
+ * @param method specifies the HTTP method to use for the request, such as GET or POST
+ */
+ public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException, IOException {
+ return new QueryRequest(params, method).process(this);
+ }
+
+ /**
+ * Query solr, and stream the results. Unlike the standard query, this will
+ * send events for each Document rather then add them to the QueryResponse.
+ *
+ * Although this function returns a 'QueryResponse' it should be used with care
+ * since it excludes anything that was passed to callback. Also note that
+ * future version may pass even more info to the callback and may not return
+ * the results in the QueryResponse.
+ *
+ * @since solr 4.0
+ */
+ public QueryResponse queryAndStreamResponse(SolrParams params, StreamingResponseCallback callback) throws SolrServerException, IOException
+ {
+ ResponseParser parser = new StreamingBinaryResponseParser(callback);
+ QueryRequest req = new QueryRequest(params);
+ req.setStreamingResponseCallback(callback);
+ req.setResponseParser(parser);
+ return req.process(this);
+ }
+
+ /**
+ * Retrieves the SolrDocument associated with the given identifier.
+ *
+ * @return retrieved SolrDocument, null if no document is found.
+ */
+ public SolrDocument getById(String id) throws SolrServerException, IOException {
+ return getById(id, null);
+ }
+
+ /**
+ * Retrieves the SolrDocument associated with the given identifier and uses
+ * the SolrParams to execute the request.
+ *
+ * @return retrieved SolrDocument, null if no document is found.
+ */
+ public SolrDocument getById(String id, SolrParams params) throws SolrServerException, IOException {
+ SolrDocumentList docs = getById(Arrays.asList(id), params);
+ if (!docs.isEmpty()) {
+ return docs.get(0);
+ }
+ return null;
+ }
+
+ /**
+ * Retrieves the SolrDocuments associated with the given identifiers.
+ * If a document was not found, it will not be added to the SolrDocumentList.
+ */
+ public SolrDocumentList getById(Collection<String> ids) throws SolrServerException, IOException {
+ return getById(ids, null);
+ }
+
+ /**
+ * Retrieves the SolrDocuments associated with the given identifiers and uses
+ * the SolrParams to execute the request.
+ * If a document was not found, it will not be added to the SolrDocumentList.
+ */
+ public SolrDocumentList getById(Collection<String> ids, SolrParams params) throws SolrServerException, IOException {
+ if (ids == null || ids.isEmpty()) {
+ throw new IllegalArgumentException("Must provide an identifier of a document to retrieve.");
+ }
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams(params);
+ if (StringUtils.isEmpty(reqParams.get(CommonParams.QT))) {
+ reqParams.set(CommonParams.QT, "/get");
+ }
+ reqParams.set("ids", (String[]) ids.toArray());
+
+ return query(reqParams).getResults();
+ }
+
+ /**
+ * SolrServer implementations need to implement how a request is actually processed
+ */
+ public abstract NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException;
+
+ public DocumentObjectBinder getBinder() {
+ if(binder == null){
+ binder = new DocumentObjectBinder();
+ }
+ return binder;
+ }
+
+ /**
+ * Release allocated resources.
+ *
+ * @since solr 4.0
+ * @deprecated Use close() instead.
+ */
+ @Deprecated
+ public abstract void shutdown();
+
+ //@SuppressWarnings("deprecation")
+ public void close() throws IOException {
+ shutdown();
+ }
+}