You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by bo...@apache.org on 2015/03/18 00:33:23 UTC
[01/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Repository: incubator-ranger
Updated Branches:
refs/heads/master 8e6acd5d5 -> 82f714208
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/src/main/assembly/storm-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/storm-agent.xml b/src/main/assembly/storm-agent.xml
index 08eb0da..e76f50a 100644
--- a/src/main/assembly/storm-agent.xml
+++ b/src/main/assembly/storm-agent.xml
@@ -48,6 +48,12 @@
<include>org.codehaus.jackson:jackson-jaxrs:jar:${codehaus.jackson.version}</include>
<include>org.codehaus.jackson:jackson-core-asl:jar:${codehaus.jackson.version}</include>
<include>org.codehaus.jackson:jackson-mapper-asl:jar:${codehaus.jackson.version}</include>
+ <include>org.apache.ranger:ranger_solrj:jar:${ranger.solrj.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include>
+ <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include>
+ <include>org.noggit:noggit:jar:${noggit.version}</include>
+ <include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include>
</includes>
</dependencySet>
<dependencySet>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/storm-agent/conf/ranger-storm-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/storm-agent/conf/ranger-storm-audit-changes.cfg b/storm-agent/conf/ranger-storm-audit-changes.cfg
index 4f2c5a2..b7d2ed0 100644
--- a/storm-agent/conf/ranger-storm-audit-changes.cfg
+++ b/storm-agent/conf/ranger-storm-audit-changes.cfg
@@ -31,3 +31,16 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.is.async %XAAUDIT.SOLR.IS_ASYNC% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/storm-agent/conf/ranger-storm-audit.xml
----------------------------------------------------------------------
diff --git a/storm-agent/conf/ranger-storm-audit.xml b/storm-agent/conf/ranger-storm-audit.xml
index 9aa8a89..3e310d9 100644
--- a/storm-agent/conf/ranger-storm-audit.xml
+++ b/storm-agent/conf/ranger-storm-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/storm-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties
index 28a0fd7..0e476d6 100644
--- a/storm-agent/scripts/install.properties
+++ b/storm-agent/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
[03/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java
new file mode 100644
index 0000000..687525c
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/JavaBinCodec.java
@@ -0,0 +1,820 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.common.util;
+
+import org.apache.solr.common.EnumFieldValue;
+import org.noggit.CharArr;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.*;
+import java.util.Map.Entry;
+import java.nio.ByteBuffer;
+
+/**
+ * The class is designed to optimaly serialize/deserialize any supported types in Solr response. As we know there are only a limited type of
+ * items this class can do it with very minimal amount of payload and code. There are 15 known types and if there is an
+ * object in the object tree which does not fall into these types, It must be converted to one of these. Implement an
+ * ObjectResolver and pass it over It is expected that this class is used on both end of the pipes. The class has one
+ * read method and one write method for each of the datatypes
+ * <p>
+ * Note -- Never re-use an instance of this class for more than one marshal or unmarshall operation. Always create a new
+ * instance.
+ */
+public class JavaBinCodec {
+
+ public static final byte
+ NULL = 0,
+ BOOL_TRUE = 1,
+ BOOL_FALSE = 2,
+ BYTE = 3,
+ SHORT = 4,
+ DOUBLE = 5,
+ INT = 6,
+ LONG = 7,
+ FLOAT = 8,
+ DATE = 9,
+ MAP = 10,
+ SOLRDOC = 11,
+ SOLRDOCLST = 12,
+ BYTEARR = 13,
+ ITERATOR = 14,
+ /**
+ * this is a special tag signals an end. No value is associated with it
+ */
+ END = 15,
+
+ SOLRINPUTDOC = 16,
+ SOLRINPUTDOC_CHILDS = 17,
+ ENUM_FIELD_VALUE = 18,
+ MAP_ENTRY = 19,
+ // types that combine tag + length (or other info) in a single byte
+ TAG_AND_LEN = (byte) (1 << 5),
+ STR = (byte) (1 << 5),
+ SINT = (byte) (2 << 5),
+ SLONG = (byte) (3 << 5),
+ ARR = (byte) (4 << 5), //
+ ORDERED_MAP = (byte) (5 << 5), // SimpleOrderedMap (a NamedList subclass, and more common)
+ NAMED_LST = (byte) (6 << 5), // NamedList
+ EXTERN_STRING = (byte) (7 << 5);
+
+
+ private static byte VERSION = 2;
+ private ObjectResolver resolver;
+ protected FastOutputStream daos;
+
+ public JavaBinCodec() {
+ }
+
+ public JavaBinCodec(ObjectResolver resolver) {
+ this.resolver = resolver;
+ }
+
+ public void marshal(Object nl, OutputStream os) throws IOException {
+ init(FastOutputStream.wrap(os));
+ try {
+ daos.writeByte(VERSION);
+ writeVal(nl);
+ } finally {
+ daos.flushBuffer();
+ }
+ }
+
+ /** expert: sets a new output stream */
+ public void init(FastOutputStream os) {
+ daos = os;
+ }
+
+ byte version;
+
+ public Object unmarshal(InputStream is) throws IOException {
+ FastInputStream dis = FastInputStream.wrap(is);
+ version = dis.readByte();
+ if (version != VERSION) {
+ throw new RuntimeException("Invalid version (expected " + VERSION +
+ ", but " + version + ") or the data in not in 'javabin' format");
+ }
+ return readVal(dis);
+ }
+
+
+ public SimpleOrderedMap<Object> readOrderedMap(DataInputInputStream dis) throws IOException {
+ int sz = readSize(dis);
+ SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();
+ for (int i = 0; i < sz; i++) {
+ String name = (String) readVal(dis);
+ Object val = readVal(dis);
+ nl.add(name, val);
+ }
+ return nl;
+ }
+
+ public NamedList<Object> readNamedList(DataInputInputStream dis) throws IOException {
+ int sz = readSize(dis);
+ NamedList<Object> nl = new NamedList<>();
+ for (int i = 0; i < sz; i++) {
+ String name = (String) readVal(dis);
+ Object val = readVal(dis);
+ nl.add(name, val);
+ }
+ return nl;
+ }
+
+ public void writeNamedList(NamedList<?> nl) throws IOException {
+ writeTag(nl instanceof SimpleOrderedMap ? ORDERED_MAP : NAMED_LST, nl.size());
+ for (int i = 0; i < nl.size(); i++) {
+ String name = nl.getName(i);
+ writeExternString(name);
+ Object val = nl.getVal(i);
+ writeVal(val);
+ }
+ }
+
+ public void writeVal(Object val) throws IOException {
+ if (writeKnownType(val)) {
+ return;
+ } else {
+ Object tmpVal = val;
+ if (resolver != null) {
+ tmpVal = resolver.resolve(val, this);
+ if (tmpVal == null) return; // null means the resolver took care of it fully
+ if (writeKnownType(tmpVal)) return;
+ }
+ }
+
+ writeVal(val.getClass().getName() + ':' + val.toString());
+ }
+
+ protected static final Object END_OBJ = new Object();
+
+ protected byte tagByte;
+
+ public Object readVal(DataInputInputStream dis) throws IOException {
+ tagByte = dis.readByte();
+
+ // if ((tagByte & 0xe0) == 0) {
+ // if top 3 bits are clear, this is a normal tag
+
+ // OK, try type + size in single byte
+ switch (tagByte >>> 5) {
+ case STR >>> 5:
+ return readStr(dis);
+ case SINT >>> 5:
+ return readSmallInt(dis);
+ case SLONG >>> 5:
+ return readSmallLong(dis);
+ case ARR >>> 5:
+ return readArray(dis);
+ case ORDERED_MAP >>> 5:
+ return readOrderedMap(dis);
+ case NAMED_LST >>> 5:
+ return readNamedList(dis);
+ case EXTERN_STRING >>> 5:
+ return readExternString(dis);
+ }
+
+ switch (tagByte) {
+ case NULL:
+ return null;
+ case DATE:
+ return new Date(dis.readLong());
+ case INT:
+ return dis.readInt();
+ case BOOL_TRUE:
+ return Boolean.TRUE;
+ case BOOL_FALSE:
+ return Boolean.FALSE;
+ case FLOAT:
+ return dis.readFloat();
+ case DOUBLE:
+ return dis.readDouble();
+ case LONG:
+ return dis.readLong();
+ case BYTE:
+ return dis.readByte();
+ case SHORT:
+ return dis.readShort();
+ case MAP:
+ return readMap(dis);
+ case SOLRDOC:
+ return readSolrDocument(dis);
+ case SOLRDOCLST:
+ return readSolrDocumentList(dis);
+ case BYTEARR:
+ return readByteArray(dis);
+ case ITERATOR:
+ return readIterator(dis);
+ case END:
+ return END_OBJ;
+ case SOLRINPUTDOC:
+ return readSolrInputDocument(dis);
+ case ENUM_FIELD_VALUE:
+ return readEnumFieldValue(dis);
+ case MAP_ENTRY:
+ return readMapEntry(dis);
+ }
+
+ throw new RuntimeException("Unknown type " + tagByte);
+ }
+
+ public boolean writeKnownType(Object val) throws IOException {
+ if (writePrimitive(val)) return true;
+ if (val instanceof NamedList) {
+ writeNamedList((NamedList<?>) val);
+ return true;
+ }
+ if (val instanceof SolrDocumentList) { // SolrDocumentList is a List, so must come before List check
+ writeSolrDocumentList((SolrDocumentList) val);
+ return true;
+ }
+ if (val instanceof Collection) {
+ writeArray((Collection) val);
+ return true;
+ }
+ if (val instanceof Object[]) {
+ writeArray((Object[]) val);
+ return true;
+ }
+ if (val instanceof SolrDocument) {
+ //this needs special treatment to know which fields are to be written
+ if (resolver == null) {
+ writeSolrDocument((SolrDocument) val);
+ } else {
+ Object retVal = resolver.resolve(val, this);
+ if (retVal != null) {
+ if (retVal instanceof SolrDocument) {
+ writeSolrDocument((SolrDocument) retVal);
+ } else {
+ writeVal(retVal);
+ }
+ }
+ }
+ return true;
+ }
+ if (val instanceof SolrInputDocument) {
+ writeSolrInputDocument((SolrInputDocument)val);
+ return true;
+ }
+ if (val instanceof Map) {
+ writeMap((Map) val);
+ return true;
+ }
+ if (val instanceof Iterator) {
+ writeIterator((Iterator) val);
+ return true;
+ }
+ if (val instanceof Iterable) {
+ writeIterator(((Iterable) val).iterator());
+ return true;
+ }
+ if (val instanceof EnumFieldValue) {
+ writeEnumFieldValue((EnumFieldValue) val);
+ return true;
+ }
+ if (val instanceof Map.Entry) {
+ writeMapEntry((Map.Entry)val);
+ return true;
+ }
+ return false;
+ }
+
+ public void writeTag(byte tag) throws IOException {
+ daos.writeByte(tag);
+ }
+
+ public void writeTag(byte tag, int size) throws IOException {
+ if ((tag & 0xe0) != 0) {
+ if (size < 0x1f) {
+ daos.writeByte(tag | size);
+ } else {
+ daos.writeByte(tag | 0x1f);
+ writeVInt(size - 0x1f, daos);
+ }
+ } else {
+ daos.writeByte(tag);
+ writeVInt(size, daos);
+ }
+ }
+
+ public void writeByteArray(byte[] arr, int offset, int len) throws IOException {
+ writeTag(BYTEARR, len);
+ daos.write(arr, offset, len);
+ }
+
+ public byte[] readByteArray(DataInputInputStream dis) throws IOException {
+ byte[] arr = new byte[readVInt(dis)];
+ dis.readFully(arr);
+ return arr;
+ }
+
+ public void writeSolrDocument(SolrDocument doc) throws IOException {
+ List<SolrDocument> children = doc.getChildDocuments();
+ int sz = doc.size() + (children==null ? 0 : children.size());
+ writeTag(SOLRDOC);
+ writeTag(ORDERED_MAP, sz);
+ for (Map.Entry<String, Object> entry : doc) {
+ String name = entry.getKey();
+ writeExternString(name);
+ Object val = entry.getValue();
+ writeVal(val);
+ }
+ if (children != null) {
+ for (SolrDocument child : children) {
+ writeSolrDocument(child);
+ }
+ }
+ }
+
+ public SolrDocument readSolrDocument(DataInputInputStream dis) throws IOException {
+ tagByte = dis.readByte();
+ int size = readSize(dis);
+ SolrDocument doc = new SolrDocument();
+ for (int i = 0; i < size; i++) {
+ String fieldName;
+ Object obj = readVal(dis); // could be a field name, or a child document
+ if (obj instanceof SolrDocument) {
+ doc.addChildDocument((SolrDocument)obj);
+ continue;
+ } else {
+ fieldName = (String)obj;
+ }
+ Object fieldVal = readVal(dis);
+ doc.setField(fieldName, fieldVal);
+ }
+ return doc;
+ }
+
+ public SolrDocumentList readSolrDocumentList(DataInputInputStream dis) throws IOException {
+ SolrDocumentList solrDocs = new SolrDocumentList();
+ List list = (List) readVal(dis);
+ solrDocs.setNumFound((Long) list.get(0));
+ solrDocs.setStart((Long) list.get(1));
+ solrDocs.setMaxScore((Float) list.get(2));
+
+ @SuppressWarnings("unchecked")
+ List<SolrDocument> l = (List<SolrDocument>) readVal(dis);
+ solrDocs.addAll(l);
+ return solrDocs;
+ }
+
+ public void writeSolrDocumentList(SolrDocumentList docs)
+ throws IOException {
+ writeTag(SOLRDOCLST);
+ List<Number> l = new ArrayList<>(3);
+ l.add(docs.getNumFound());
+ l.add(docs.getStart());
+ l.add(docs.getMaxScore());
+ writeArray(l);
+ writeArray(docs);
+ }
+
+ public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) throws IOException {
+ int sz = readVInt(dis);
+ float docBoost = (Float)readVal(dis);
+ SolrInputDocument sdoc = new SolrInputDocument();
+ sdoc.setDocumentBoost(docBoost);
+ for (int i = 0; i < sz; i++) {
+ float boost = 1.0f;
+ String fieldName;
+ Object obj = readVal(dis); // could be a boost, a field name, or a child document
+ if (obj instanceof Float) {
+ boost = (Float)obj;
+ fieldName = (String)readVal(dis);
+ } else if (obj instanceof SolrInputDocument) {
+ sdoc.addChildDocument((SolrInputDocument)obj);
+ continue;
+ } else {
+ fieldName = (String)obj;
+ }
+ Object fieldVal = readVal(dis);
+ sdoc.setField(fieldName, fieldVal, boost);
+ }
+ return sdoc;
+ }
+
+ public void writeSolrInputDocument(SolrInputDocument sdoc) throws IOException {
+ List<SolrInputDocument> children = sdoc.getChildDocuments();
+ int sz = sdoc.size() + (children==null ? 0 : children.size());
+ writeTag(SOLRINPUTDOC, sz);
+ writeFloat(sdoc.getDocumentBoost());
+ for (SolrInputField inputField : sdoc.values()) {
+ if (inputField.getBoost() != 1.0f) {
+ writeFloat(inputField.getBoost());
+ }
+ writeExternString(inputField.getName());
+ writeVal(inputField.getValue());
+ }
+ if (children != null) {
+ for (SolrInputDocument child : children) {
+ writeSolrInputDocument(child);
+ }
+ }
+ }
+
+
+ public Map<Object,Object> readMap(DataInputInputStream dis)
+ throws IOException {
+ int sz = readVInt(dis);
+ Map<Object,Object> m = new LinkedHashMap<>();
+ for (int i = 0; i < sz; i++) {
+ Object key = readVal(dis);
+ Object val = readVal(dis);
+ m.put(key, val);
+
+ }
+ return m;
+ }
+
+ public void writeIterator(Iterator iter) throws IOException {
+ writeTag(ITERATOR);
+ while (iter.hasNext()) {
+ writeVal(iter.next());
+ }
+ writeVal(END_OBJ);
+ }
+
+ public List<Object> readIterator(DataInputInputStream fis) throws IOException {
+ ArrayList<Object> l = new ArrayList<>();
+ while (true) {
+ Object o = readVal(fis);
+ if (o == END_OBJ) break;
+ l.add(o);
+ }
+ return l;
+ }
+
+ public void writeArray(List l) throws IOException {
+ writeTag(ARR, l.size());
+ for (int i = 0; i < l.size(); i++) {
+ writeVal(l.get(i));
+ }
+ }
+
+ public void writeArray(Collection coll) throws IOException {
+ writeTag(ARR, coll.size());
+ for (Object o : coll) {
+ writeVal(o);
+ }
+
+ }
+
+ public void writeArray(Object[] arr) throws IOException {
+ writeTag(ARR, arr.length);
+ for (int i = 0; i < arr.length; i++) {
+ Object o = arr[i];
+ writeVal(o);
+ }
+ }
+
+ public List<Object> readArray(DataInputInputStream dis) throws IOException {
+ int sz = readSize(dis);
+ ArrayList<Object> l = new ArrayList<>(sz);
+ for (int i = 0; i < sz; i++) {
+ l.add(readVal(dis));
+ }
+ return l;
+ }
+
+ /**
+ * write {@link EnumFieldValue} as tag+int value+string value
+ * @param enumFieldValue to write
+ */
+ public void writeEnumFieldValue(EnumFieldValue enumFieldValue) throws IOException {
+ writeTag(ENUM_FIELD_VALUE);
+ writeInt(enumFieldValue.toInt());
+ writeStr(enumFieldValue.toString());
+ }
+
+ public void writeMapEntry(Entry<Object,Object> val) throws IOException {
+ writeTag(MAP_ENTRY);
+ writeVal(val.getKey());
+ writeVal(val.getValue());
+ }
+
+ /**
+ * read {@link EnumFieldValue} (int+string) from input stream
+ * @param dis data input stream
+ * @return {@link EnumFieldValue}
+ */
+ public EnumFieldValue readEnumFieldValue(DataInputInputStream dis) throws IOException {
+ Integer intValue = (Integer) readVal(dis);
+ String stringValue = (String) readVal(dis);
+ return new EnumFieldValue(intValue, stringValue);
+ }
+
+
+ public Map.Entry<Object,Object> readMapEntry(DataInputInputStream dis) throws IOException {
+ final Object key = readVal(dis);
+ final Object value = readVal(dis);
+ return new Map.Entry<Object,Object>() {
+
+ @Override
+ public Object getKey() {
+ return key;
+ }
+
+ @Override
+ public Object getValue() {
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ return "MapEntry[" + key.toString() + ":" + value.toString() + "]";
+ }
+
+ @Override
+ public Object setValue(Object value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int hashCode() {
+ int result = 31;
+ result *=31 + getKey().hashCode();
+ result *=31 + getValue().hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if(this == obj) {
+ return true;
+ }
+ if(!(obj instanceof Entry)) {
+ return false;
+ }
+ Map.Entry<Object, Object> entry = (Entry<Object, Object>) obj;
+ return (this.getKey().equals(entry.getKey()) && this.getValue().equals(entry.getValue()));
+ }
+ };
+ }
+
+ /**
+ * write the string as tag+length, with length being the number of UTF-8 bytes
+ */
+ public void writeStr(String s) throws IOException {
+ if (s == null) {
+ writeTag(NULL);
+ return;
+ }
+ int end = s.length();
+ int maxSize = end * 4;
+ if (bytes == null || bytes.length < maxSize) bytes = new byte[maxSize];
+ int sz = ByteUtils.UTF16toUTF8(s, 0, end, bytes, 0);
+
+ writeTag(STR, sz);
+ daos.write(bytes, 0, sz);
+ }
+
+ byte[] bytes;
+ CharArr arr = new CharArr();
+
+ public String readStr(DataInputInputStream dis) throws IOException {
+ int sz = readSize(dis);
+ if (bytes == null || bytes.length < sz) bytes = new byte[sz];
+ dis.readFully(bytes, 0, sz);
+
+ arr.reset();
+ ByteUtils.UTF8toUTF16(bytes, 0, sz, arr);
+ return arr.toString();
+ }
+
+ public void writeInt(int val) throws IOException {
+ if (val > 0) {
+ int b = SINT | (val & 0x0f);
+
+ if (val >= 0x0f) {
+ b |= 0x10;
+ daos.writeByte(b);
+ writeVInt(val >>> 4, daos);
+ } else {
+ daos.writeByte(b);
+ }
+
+ } else {
+ daos.writeByte(INT);
+ daos.writeInt(val);
+ }
+ }
+
+ public int readSmallInt(DataInputInputStream dis) throws IOException {
+ int v = tagByte & 0x0F;
+ if ((tagByte & 0x10) != 0)
+ v = (readVInt(dis) << 4) | v;
+ return v;
+ }
+
+
+ public void writeLong(long val) throws IOException {
+ if ((val & 0xff00000000000000L) == 0) {
+ int b = SLONG | ((int) val & 0x0f);
+ if (val >= 0x0f) {
+ b |= 0x10;
+ daos.writeByte(b);
+ writeVLong(val >>> 4, daos);
+ } else {
+ daos.writeByte(b);
+ }
+ } else {
+ daos.writeByte(LONG);
+ daos.writeLong(val);
+ }
+ }
+
+ public long readSmallLong(DataInputInputStream dis) throws IOException {
+ long v = tagByte & 0x0F;
+ if ((tagByte & 0x10) != 0)
+ v = (readVLong(dis) << 4) | v;
+ return v;
+ }
+
+ public void writeFloat(float val) throws IOException {
+ daos.writeByte(FLOAT);
+ daos.writeFloat(val);
+ }
+
+ public boolean writePrimitive(Object val) throws IOException {
+ if (val == null) {
+ daos.writeByte(NULL);
+ return true;
+ } else if (val instanceof String) {
+ writeStr((String) val);
+ return true;
+ } else if (val instanceof Number) {
+
+ if (val instanceof Integer) {
+ writeInt(((Integer) val).intValue());
+ return true;
+ } else if (val instanceof Long) {
+ writeLong(((Long) val).longValue());
+ return true;
+ } else if (val instanceof Float) {
+ writeFloat(((Float) val).floatValue());
+ return true;
+ } else if (val instanceof Double) {
+ daos.writeByte(DOUBLE);
+ daos.writeDouble(((Double) val).doubleValue());
+ return true;
+ } else if (val instanceof Byte) {
+ daos.writeByte(BYTE);
+ daos.writeByte(((Byte) val).intValue());
+ return true;
+ } else if (val instanceof Short) {
+ daos.writeByte(SHORT);
+ daos.writeShort(((Short) val).intValue());
+ return true;
+ }
+ return false;
+
+ } else if (val instanceof Date) {
+ daos.writeByte(DATE);
+ daos.writeLong(((Date) val).getTime());
+ return true;
+ } else if (val instanceof Boolean) {
+ if ((Boolean) val) daos.writeByte(BOOL_TRUE);
+ else daos.writeByte(BOOL_FALSE);
+ return true;
+ } else if (val instanceof byte[]) {
+ writeByteArray((byte[]) val, 0, ((byte[]) val).length);
+ return true;
+ } else if (val instanceof ByteBuffer) {
+ ByteBuffer buf = (ByteBuffer) val;
+ writeByteArray(buf.array(),buf.position(),buf.limit() - buf.position());
+ return true;
+ } else if (val == END_OBJ) {
+ writeTag(END);
+ return true;
+ }
+ return false;
+ }
+
+
+ public void writeMap(Map<?,?> val) throws IOException {
+ writeTag(MAP, val.size());
+ for (Map.Entry<?,?> entry : val.entrySet()) {
+ Object key = entry.getKey();
+ if (key instanceof String) {
+ writeExternString((String) key);
+ } else {
+ writeVal(key);
+ }
+ writeVal(entry.getValue());
+ }
+ }
+
+
+ public int readSize(DataInputInputStream in) throws IOException {
+ int sz = tagByte & 0x1f;
+ if (sz == 0x1f) sz += readVInt(in);
+ return sz;
+ }
+
+
+ /**
+ * Special method for variable length int (copied from lucene). Usually used for writing the length of a
+ * collection/array/map In most of the cases the length can be represented in one byte (length < 127) so it saves 3
+ * bytes/object
+ *
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public static void writeVInt(int i, FastOutputStream out) throws IOException {
+ while ((i & ~0x7F) != 0) {
+ out.writeByte((byte) ((i & 0x7f) | 0x80));
+ i >>>= 7;
+ }
+ out.writeByte((byte) i);
+ }
+
+ /**
+ * The counterpart for {@link #writeVInt(int, FastOutputStream)}
+ *
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public static int readVInt(DataInputInputStream in) throws IOException {
+ byte b = in.readByte();
+ int i = b & 0x7F;
+ for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ b = in.readByte();
+ i |= (b & 0x7F) << shift;
+ }
+ return i;
+ }
+
+
+ public static void writeVLong(long i, FastOutputStream out) throws IOException {
+ while ((i & ~0x7F) != 0) {
+ out.writeByte((byte) ((i & 0x7f) | 0x80));
+ i >>>= 7;
+ }
+ out.writeByte((byte) i);
+ }
+
+ public static long readVLong(DataInputInputStream in) throws IOException {
+ byte b = in.readByte();
+ long i = b & 0x7F;
+ for (int shift = 7; (b & 0x80) != 0; shift += 7) {
+ b = in.readByte();
+ i |= (long) (b & 0x7F) << shift;
+ }
+ return i;
+ }
+
+ private int stringsCount = 0;
+ private Map<String, Integer> stringsMap;
+ private List<String> stringsList;
+
+ public void writeExternString(String s) throws IOException {
+ if (s == null) {
+ writeTag(NULL);
+ return;
+ }
+ Integer idx = stringsMap == null ? null : stringsMap.get(s);
+ if (idx == null) idx = 0;
+ writeTag(EXTERN_STRING, idx);
+ if (idx == 0) {
+ writeStr(s);
+ if (stringsMap == null) stringsMap = new HashMap<>();
+ stringsMap.put(s, ++stringsCount);
+ }
+
+ }
+
+ public String readExternString(DataInputInputStream fis) throws IOException {
+ int idx = readSize(fis);
+ if (idx != 0) {// idx != 0 is the index of the extern string
+ return stringsList.get(idx - 1);
+ } else {// idx == 0 means it has a string value
+ String s = (String) readVal(fis);
+ if (stringsList == null) stringsList = new ArrayList<>();
+ stringsList.add(s);
+ return s;
+ }
+ }
+
+
+ public static interface ObjectResolver {
+ public Object resolve(Object o, JavaBinCodec codec) throws IOException;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java
new file mode 100644
index 0000000..c3afde7
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/JsonRecordReader.java
@@ -0,0 +1,586 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.common.util;
+
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.*;
+
+import org.noggit.JSONParser;
+
+import static org.noggit.JSONParser.*;
+
+/**
+ * A Streaming parser for json to emit one record at a time.
+ */
+
+public class JsonRecordReader {
+ public static final String DELIM = ".";
+
+ private Node rootNode = new Node("/", (Node) null);
+
+ public static JsonRecordReader getInst(String split, List<String> fieldMappings) {
+
+ JsonRecordReader jsonRecordReader = new JsonRecordReader(split);
+ for (String s : fieldMappings) {
+ String path = s;
+ int idx = s.indexOf(':');
+ String fieldName = null;
+ if (idx > 0) {
+ fieldName = s.substring(0, idx);
+ path = s.substring(idx + 1);
+ }
+ jsonRecordReader.addField(path, fieldName, true, false);
+ }
+ return jsonRecordReader;
+ }
+
+ /**
+ * A constructor called with a '|' separated list of path expressions
+ * which define sub sections of the JSON stream that are to be emitted as
+ * separate records.
+ *
+ * @param splitPath The PATH for which a record is emitted. Once the
+ * path tag is encountered, the Node.getInst method starts collecting wanted
+ * fields and at the close of the tag, a record is emitted containing all
+ * fields collected since the tag start. Once
+ * emitted the collected fields are cleared. Any fields collected in the
+ * parent tag or above will also be included in the record, but these are
+ * not cleared after emitting the record.
+ * <p>
+ * It uses the ' | ' syntax of PATH to pass in multiple paths.
+ */
+ private JsonRecordReader(String splitPath) {
+ String[] splits = splitPath.split("\\|");
+ for (String split : splits) {
+ split = split.trim();
+ if (split.startsWith("//"))
+ throw new RuntimeException("split cannot start with '//': " + split);
+ if (split.length() == 0)
+ continue;
+ // The created Node has a name set to the full split attribute path
+ addField(split, split, false, true);
+ }
+ }
+
+ /**
+ * Splits the path into a List of segments and calls build() to
+ * construct a tree of Nodes representing path segments. The resulting
+ * tree structure ends up describing all the paths we are interested in.
+ *
+ * @param path The path expression for this field
+ * @param fieldName The name for this field in the emitted record
+ * @param multiValued If 'true' then the emitted record will have values in
+ * a List<String>
+ * @param isRecord Flags that this PATH is from a forEach statement
+ */
+ private void addField(String path, String fieldName, boolean multiValued, boolean isRecord) {
+ if (!path.startsWith("/")) throw new RuntimeException("All paths must start with '/' " + path);
+ List<String> paths = splitEscapeQuote(path);
+ if (paths.size() == 0) {
+ if (isRecord) rootNode.isRecord = true;
+ return;//the patrh is "/"
+ }
+ // deal with how split behaves when seperator starts a string!
+ if ("".equals(paths.get(0).trim()))
+ paths.remove(0);
+ rootNode.build(paths, fieldName, multiValued, isRecord, path);
+ rootNode.buildOptimize();
+ }
+
+ /**
+ * Uses {@link #streamRecords streamRecords} to getInst the JSON source but with
+ * a handler that collects all the emitted records into a single List which
+ * is returned upon completion.
+ *
+ * @param r the stream reader
+ * @return results a List of emitted records
+ */
+ public List<Map<String, Object>> getAllRecords(Reader r) throws IOException {
+ final List<Map<String, Object>> results = new ArrayList<>();
+ streamRecords(r, new Handler() {
+ @Override
+ public void handle(Map<String, Object> record, String path) {
+ results.add(record);
+ }
+ });
+ return results;
+ }
+
+ /**
+ * Creates an JSONParser on top of whatever reader has been
+ * configured. Then calls getInst() with a handler which is
+ * invoked forEach record emitted.
+ *
+ * @param r the stream reader
+ * @param handler The callback instance
+ */
+ public void streamRecords(Reader r, Handler handler) throws IOException {
+ streamRecords(new JSONParser(r), handler);
+ }
+
+ public void streamRecords(JSONParser parser, Handler handler) throws IOException {
+ rootNode.parse(parser, handler,
+ new LinkedHashMap<String, Object>(),
+ new Stack<Set<String>>(), false);
+ }
+
+
+ /**
+ * For each node/leaf in the Node tree there is one object of this class.
+ * This tree of objects represents all the Paths we are interested in.
+ * For each path segment of interest we create a node. In most cases the
+ * node (branch) is rather basic , but for the final portion (leaf) of any
+ * path we add more information to the Node. When parsing the JSON document
+ * we step though this tree as we stream records from the reader. If the JSON
+ * document departs from this tree we skip start tags till we are back on
+ * the tree.
+ */
+ private static class Node {
+ String name; // generally: segment of the path represented by this Node
+ String fieldName; // the fieldname in the emitted record (key of the map)
+ String splitPath; // the full path from the forEach entity attribute
+ final LinkedHashMap<String, Node> childNodes = new LinkedHashMap<>(); // List of immediate child Nodes of this node
+ Node parent; // parent Node in the tree
+ boolean isLeaf = false; // flag: store/emit streamed text for this node
+ boolean isRecord = false; //flag: this Node starts a new record
+ Node wildCardChild;
+ Node recursiveWildCardChild;
+ private boolean useFqn = false;
+
+
+ public Node(String name, Node p) {
+ // Create a basic Node, suitable for the mid portions of any path.
+ // Node.pathName and Node.name are set to same value.
+ this.name = name;
+ parent = p;
+ }
+
+ public Node(String name, String fieldName) {
+ // This is only called from build() when describing an attribute.
+ this.name = name; // a segment from the path
+ this.fieldName = fieldName; // name to store collected values against
+ }
+
+
+ /**
+ * Walk the Node tree propagating any wildDescentant information to
+ * child nodes.
+ */
+ private void buildOptimize() {
+ if (parent != null && parent.recursiveWildCardChild != null && this.recursiveWildCardChild == null) {
+ this.recursiveWildCardChild = parent.recursiveWildCardChild;
+ }
+ for (Node n : childNodes.values()) n.buildOptimize();
+ }
+ static final String WILDCARD_PATH = "*";
+ static final String RECURSIVE_WILDCARD_PATH = "**";
+
+ /**
+ * Build a Node tree structure representing all paths of intrest to us.
+ * This must be done before parsing of the JSON stream starts. Each node
+ * holds one portion of an path. Taking each path segment in turn this
+ * method walks the Node tree and finds where the new segment should be
+ * inserted. It creates a Node representing a field's name, PATH and
+ * some flags and inserts the Node into the Node tree.
+ */
+ private void build(
+ List<String> paths, // a List of segments from the split paths
+ String fieldName, // the fieldName assoc with this path
+ boolean multiValued, // flag if this fieldName is multiValued or not
+ boolean record, // is this path a record or a field
+ String path) {
+ // recursively walk the paths Lists adding new Nodes as required
+ String segment = paths.remove(0); // shift out next path segment
+
+ if (segment.length() < 1) throw new RuntimeException("all pieces in path must be non empty " + path);
+
+ // does this "name" already exist as a child node.
+ Node n = getOrAddNode(segment, childNodes);
+ if (paths.isEmpty()) {
+ // We have emptied paths, we are for the moment a leaf of the tree.
+ // When parsing the actual input we have traversed to a position
+ // where we actutally have to do something. getOrAddNode() will
+ // have created and returned a new minimal Node with name and
+ // pathName already populated. We need to add more information.
+ if (record) {
+ //wild cards cannot be used in split
+ assert !WILDCARD_PATH.equals(n.name);
+ assert !RECURSIVE_WILDCARD_PATH.equals(n.name);
+ // split attribute
+ n.isRecord = true; // flag: split attribute, prepare to emit rec
+ n.splitPath = fieldName; // the full split attribute path
+ } else {
+ if (n.name.equals(WILDCARD_PATH)) {
+ wildCardChild = n;
+ }
+ if (n.name.equals(RECURSIVE_WILDCARD_PATH)) {
+ recursiveWildCardChild = n.recursiveWildCardChild = n;
+ }
+
+ // path with content we want to store and return
+ n.isLeaf = true; // we have to store text found here
+ n.fieldName = fieldName; // name to store collected text against
+ if ("$FQN".equals(n.fieldName)) {
+ n.fieldName = null;
+ n.useFqn = true;
+ }
+ }
+ } else {
+ //wildcards must only come at the end
+ if (WILDCARD_PATH.equals(name) || RECURSIVE_WILDCARD_PATH.equals(name))
+ throw new RuntimeException("wild cards are allowed only in the end " + path);
+ // recurse to handle next paths segment
+ n.build(paths, fieldName, multiValued, record, path);
+ }
+ }
+
+ private Node getOrAddNode(String pathName, Map<String, Node> children) {
+ Node n = children.get(pathName);
+ if (n != null) return n;
+ // new territory! add a new node for this path bitty
+ children.put(pathName, n = new Node(pathName, this));
+ return n;
+ }
+
+ /**
+ * Copies a supplied Map to a new Map which is returned. Used to copy a
+ * records values. If a fields value is a List then they have to be
+ * deep-copied for thread safety
+ */
+ private static Map<String, Object> getDeepCopy(Map<String, Object> values) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ for (Map.Entry<String, Object> entry : values.entrySet()) {
+ if (entry.getValue() instanceof List) {
+ result.put(entry.getKey(), new ArrayList((List) entry.getValue()));
+ } else {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ }
+ return result;
+ }
+
+ private void parse(JSONParser parser,
+ Handler handler,
+ Map<String, Object> values,
+ Stack<Set<String>> stack, // lists of values to purge
+ boolean recordStarted) throws IOException {
+
+ int event = -1;
+ for (; ; ) {
+ event = parser.nextEvent();
+ if (event == EOF) break;
+ if (event == OBJECT_START) {
+ handleObjectStart(parser, new HashSet<Node>(), handler, values, stack, recordStarted, null);
+ } else if (event == ARRAY_START) {
+ for (; ; ) {
+ event = parser.nextEvent();
+ if (event == ARRAY_END) break;
+ if (event == OBJECT_START) {
+ handleObjectStart(parser, new HashSet<Node>(), handler, values, stack, recordStarted, null);
+ }
+ }
+ }
+ }
+
+ }
+
+ /**
+ * If a new tag is encountered, check if it is of interest or not by seeing
+ * if it matches against our node tree. If we have deperted from the node
+ * tree then walk back though the tree's ancestor nodes checking to see if
+ * any // expressions exist for the node and compare them against the new
+ * tag. If matched then "jump" to that node, otherwise ignore the tag.
+ * <p>
+ * Note, the list of // expressions found while walking back up the tree
+ * is chached in the HashMap decends. Then if the new tag is to be skipped,
+ * any inner chil tags are compared against the cache and jumped to if
+ * matched.
+ */
+ private void handleObjectStart(final JSONParser parser, final Set<Node> childrenFound,
+ final Handler handler, final Map<String, Object> values,
+ final Stack<Set<String>> stack, boolean recordStarted,
+ MethodFrameWrapper frameWrapper)
+ throws IOException {
+
+ final boolean isRecordStarted = recordStarted || isRecord;
+ Set<String> valuesAddedinThisFrame = null;
+ if (isRecord) {
+ // This Node is a match for an PATH from a forEach attribute,
+ // prepare for the clean up that will occurr when the record
+ // is emitted after its END_ELEMENT is matched
+ valuesAddedinThisFrame = new HashSet<>();
+ stack.push(valuesAddedinThisFrame);
+ } else if (recordStarted) {
+ // This node is a child of some parent which matched against forEach
+ // attribute. Continue to add values to an existing record.
+ valuesAddedinThisFrame = stack.peek();
+ }
+
+ class Wrapper extends MethodFrameWrapper {
+ Wrapper(Node node, MethodFrameWrapper parent, String name) {
+ this.node = node;
+ this.parent = parent;
+ this.name = name;
+ }
+
+ @Override
+ public void walk(int event) throws IOException {
+ if (event == OBJECT_START) {
+ node.handleObjectStart(parser, childrenFound, handler, values, stack, isRecordStarted, this);
+ } else if (event == ARRAY_START) {
+ for (; ; ) {
+ event = parser.nextEvent();
+ if (event == ARRAY_END) break;
+ if (event == OBJECT_START) {
+ node.handleObjectStart(parser, childrenFound, handler, values, stack, isRecordStarted, this);
+ }
+ }
+ }
+
+ }
+ }
+
+ try {
+ for (; ; ) {
+ int event = parser.nextEvent();
+ if (event == OBJECT_END) {
+ if (isRecord()) {
+ handler.handle(getDeepCopy(values), splitPath);
+ }
+ return;
+ }
+ assert event == STRING;
+ assert parser.wasKey();
+ String name = parser.getString();
+
+ Node node = childNodes.get(name);
+ if (node == null) node = wildCardChild;
+ if (node == null) node = recursiveWildCardChild;
+
+ if (node != null) {
+ if (node.isLeaf) {//this is a leaf collect data here
+ event = parser.nextEvent();
+ String nameInRecord = node.fieldName == null ? getNameInRecord(name, frameWrapper, node) : node.fieldName;
+ MethodFrameWrapper runnable = null;
+ if (event == OBJECT_START || event == ARRAY_START) {
+ if (node.recursiveWildCardChild != null) runnable = new Wrapper(node, frameWrapper, name);
+ }
+ Object val = parseSingleFieldValue(event, parser, runnable);
+ if (val != null) {
+ putValue(values, nameInRecord, val);
+ if (isRecordStarted) valuesAddedinThisFrame.add(nameInRecord);
+ }
+
+ } else {
+ event = parser.nextEvent();
+ new Wrapper(node, frameWrapper, name).walk(event);
+ }
+ } else {
+ //this is not something we are interested in . skip it
+ event = parser.nextEvent();
+ if (event == STRING ||
+ event == LONG ||
+ event == NUMBER ||
+ event == BIGNUMBER ||
+ event == BOOLEAN ||
+ event == NULL) {
+ continue;
+ }
+ if (event == ARRAY_START) {
+ consumeTillMatchingEnd(parser, 0, 1);
+ continue;
+ }
+ if (event == OBJECT_START) {
+ consumeTillMatchingEnd(parser, 1, 0);
+ continue;
+ } else throw new RuntimeException("unexpected token " + event);
+
+ }
+ }
+ } finally {
+ if ((isRecord() || !isRecordStarted) && !stack.empty()) {
+ Set<String> cleanThis = stack.pop();
+ if (cleanThis != null) {
+ for (String fld : cleanThis) {
+ values.remove(fld);
+ }
+ }
+ }
+ }
+ }
+
+ private String getNameInRecord(String name, MethodFrameWrapper frameWrapper, Node n) {
+ if (frameWrapper == null || !n.useFqn) return name;
+ StringBuilder sb = new StringBuilder();
+ frameWrapper.prependName(sb);
+ return sb.append(DELIM).append(name).toString();
+ }
+
+ private boolean isRecord() {
+ return isRecord;
+ }
+
+
+ private void putValue(Map<String, Object> values, String fieldName, Object o) {
+ if (o == null) return;
+ Object val = values.get(fieldName);
+ if (val == null) {
+ values.put(fieldName, o);
+ return;
+ }
+ if (val instanceof List) {
+ List list = (List) val;
+ list.add(o);
+ return;
+ }
+ ArrayList l = new ArrayList();
+ l.add(val);
+ l.add(o);
+ values.put(fieldName, l);
+ }
+
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ } // end of class Node
+
+
+ /**
+ * The path is split into segments using the '/' as a seperator. However
+ * this method deals with special cases where there is a slash '/' character
+ * inside the attribute value e.g. x/@html='text/html'. We split by '/' but
+ * then reassemble things were the '/' appears within a quoted sub-string.
+ * <p>
+ * We have already enforced that the string must begin with a seperator. This
+ * method depends heavily on how split behaves if the string starts with the
+ * seperator or if a sequence of multiple seperator's appear.
+ */
+ private static List<String> splitEscapeQuote(String str) {
+ List<String> result = new LinkedList<>();
+ String[] ss = str.split("/");
+ for (int i = 0; i < ss.length; i++) { // i=1: skip seperator at start of string
+ StringBuilder sb = new StringBuilder();
+ int quoteCount = 0;
+ while (true) {
+ sb.append(ss[i]);
+ for (int j = 0; j < ss[i].length(); j++)
+ if (ss[i].charAt(j) == '\'') quoteCount++;
+ // have we got a split inside quoted sub-string?
+ if ((quoteCount % 2) == 0) break;
+ // yes!; replace the '/' and loop to concat next token
+ i++;
+ sb.append("/");
+ }
+ result.add(sb.toString());
+ }
+ return result;
+ }
+
+
+ /**
+ * Implement this interface to stream records as and when one is found.
+ */
+ public static interface Handler {
+ /**
+ * @param record The record map. The key is the field name as provided in
+ * the addField() methods. The value can be a single String (for single
+ * valued fields) or a List<String> (for multiValued).
+ * @param path The forEach path for which this record is being emitted
+ * If there is any change all parsing will be aborted and the Exception
+ * is propagated up
+ */
+ public void handle(Map<String, Object> record, String path);
+ }
+
+ public static Object parseSingleFieldValue(int ev, JSONParser parser, MethodFrameWrapper runnable) throws IOException {
+ switch (ev) {
+ case STRING:
+ return parser.getString();
+ case LONG:
+ return parser.getLong();
+ case NUMBER:
+ return parser.getDouble();
+ case BIGNUMBER:
+ return parser.getNumberChars().toString();
+ case BOOLEAN:
+ return parser.getBoolean();
+ case NULL:
+ parser.getNull();
+ return null;
+ case ARRAY_START:
+ return parseArrayFieldValue(ev, parser, runnable);
+ case OBJECT_START:
+ if (runnable != null) {
+ runnable.walk(OBJECT_START);
+ return null;
+ }
+ consumeTillMatchingEnd(parser, 1, 0);
+ return null;
+ default:
+ throw new RuntimeException("Error parsing JSON field value. Unexpected " + JSONParser.getEventString(ev));
+ }
+ }
+
+ static abstract class MethodFrameWrapper {
+ Node node;
+ MethodFrameWrapper parent;
+ String name;
+
+ void prependName(StringBuilder sb) {
+ if (parent != null) {
+ parent.prependName(sb);
+ sb.append(DELIM);
+ }
+ sb.append(name);
+ }
+
+ public abstract void walk(int event) throws IOException;
+ }
+
+ public static List<Object> parseArrayFieldValue(int ev, JSONParser parser, MethodFrameWrapper runnable) throws IOException {
+ assert ev == ARRAY_START;
+
+ ArrayList lst = new ArrayList(2);
+ for (; ; ) {
+ ev = parser.nextEvent();
+ if (ev == ARRAY_END) {
+ if (lst.isEmpty()) return null;
+ return lst;
+ }
+ Object val = parseSingleFieldValue(ev, parser, runnable);
+ if (val != null) lst.add(val);
+ }
+ }
+
+ public static void consumeTillMatchingEnd(JSONParser parser, int obj, int arr) throws IOException {
+ for (; ; ) {
+ int event = parser.nextEvent();
+ if (event == OBJECT_START) obj++;
+ if (event == OBJECT_END) obj--;
+ assert obj >= 0;
+ if (event == ARRAY_START) arr++;
+ if (event == ARRAY_END) arr--;
+ assert arr >= 0;
+ if (obj == 0 && arr == 0) break;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java
new file mode 100644
index 0000000..c539b2b
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/NamedList.java
@@ -0,0 +1,708 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+
+/**
+ * A simple container class for modeling an ordered list of name/value pairs.
+ *
+ * <p>
+ * Unlike Maps:
+ * </p>
+ * <ul>
+ * <li>Names may be repeated</li>
+ * <li>Order of elements is maintained</li>
+ * <li>Elements may be accessed by numeric index</li>
+ * <li>Names and Values can both be null</li>
+ * </ul>
+ *
+ * <p>
+ * A NamedList provides fast access by element number, but not by name.
+ * </p>
+ * <p>
+ * When a NamedList is serialized, order is considered more important than access
+ * by key, so ResponseWriters that output to a format such as JSON will normally
+ * choose a data structure that allows order to be easily preserved in various
+ * clients (i.e. not a straight map).
+ * If access by key is more important for serialization, see {@link SimpleOrderedMap},
+ * or simply use a regular {@link Map}
+ * </p>
+ *
+ */
+public class NamedList<T> implements Cloneable, Serializable, Iterable<Map.Entry<String,T>> {
+
+ private static final long serialVersionUID = 1957981902839867821L;
+ protected final List<Object> nvPairs;
+
+ /** Creates an empty instance */
+ public NamedList() {
+ nvPairs = new ArrayList<>();
+ }
+
+ /**
+ * Creates a NamedList instance containing the "name,value" pairs contained in the
+ * Entry[].
+ *
+ * <p>
+ * Modifying the contents of the Entry[] after calling this constructor may change
+ * the NamedList (in future versions of Solr), but this is not guaranteed and should
+ * not be relied upon. To modify the NamedList, refer to {@link #add(String, Object)}
+ * or {@link #remove(String)}.
+ * </p>
+ *
+ * @param nameValuePairs the name value pairs
+ */
+ public NamedList(Map.Entry<String, ? extends T>[] nameValuePairs) {
+ nvPairs = nameValueMapToList(nameValuePairs);
+ }
+
+ /**
+ * Creates a NamedList instance containing the "name,value" pairs contained in the
+ * Map.
+ *
+ * <p>
+ * Modifying the contents of the Map after calling this constructor may change
+ * the NamedList (in future versions of Solr), but this is not guaranteed and should
+ * not be relied upon. To modify the NamedList, refer to {@link #add(String, Object)}
+ * or {@link #remove(String)}.
+ * </p>
+ *
+ * @param nameValueMap the name value pairs
+ */
+ public NamedList(Map<String,? extends T> nameValueMap) {
+ if (null == nameValueMap) {
+ nvPairs = new ArrayList<>();
+ } else {
+ nvPairs = new ArrayList<>(nameValueMap.size());
+ for (Map.Entry<String,? extends T> ent : nameValueMap.entrySet()) {
+ nvPairs.add(ent.getKey());
+ nvPairs.add(ent.getValue());
+ }
+ }
+ }
+
+ /**
+ * Creates an instance backed by an explicitly specified list of
+ * pairwise names/values.
+ *
+ * <p>
+ * When using this constructor, runtime type safety is only guaranteed if
+ * all even numbered elements of the input list are of type "T".
+ * </p>
+ *
+ * @param nameValuePairs underlying List which should be used to implement a NamedList
+ * @deprecated Use {@link #NamedList(java.util.Map.Entry[])} for the NamedList instantiation
+ */
+ @Deprecated
+ public NamedList(List<Object> nameValuePairs) {
+ nvPairs=nameValuePairs;
+ }
+
+ /**
+ * Method to serialize Map.Entry<String, ?> to a List in which the even
+ * indexed elements (0,2,4. ..etc) are Strings and odd elements (1,3,5,) are of
+ * the type "T".
+ *
+ * @return Modified List as per the above description
+ * @deprecated This a temporary placeholder method until the guts of the class
+ * are actually replaced by List<String, ?>.
+ * @see <a href="https://issues.apache.org/jira/browse/SOLR-912">SOLR-912</a>
+ */
+ @Deprecated
+ private List<Object> nameValueMapToList(Map.Entry<String, ? extends T>[] nameValuePairs) {
+ List<Object> result = new ArrayList<>();
+ for (Map.Entry<String, ?> ent : nameValuePairs) {
+ result.add(ent.getKey());
+ result.add(ent.getValue());
+ }
+ return result;
+ }
+
+ /** The total number of name/value pairs */
+ public int size() {
+ return nvPairs.size() >> 1;
+ }
+
+ /**
+ * The name of the pair at the specified List index
+ *
+ * @return null if no name exists
+ */
+ public String getName(int idx) {
+ return (String)nvPairs.get(idx << 1);
+ }
+
+ /**
+ * The value of the pair at the specified List index
+ *
+ * @return may be null
+ */
+ @SuppressWarnings("unchecked")
+ public T getVal(int idx) {
+ return (T)nvPairs.get((idx << 1) + 1);
+ }
+
+ /**
+ * Adds a name/value pair to the end of the list.
+ */
+ public void add(String name, T val) {
+ nvPairs.add(name);
+ nvPairs.add(val);
+ }
+
+ /**
+ * Modifies the name of the pair at the specified index.
+ */
+ public void setName(int idx, String name) {
+ nvPairs.set(idx<<1, name);
+ }
+
+ /**
+ * Modifies the value of the pair at the specified index.
+ *
+ * @return the value that used to be at index
+ */
+ public T setVal(int idx, T val) {
+ int index = (idx<<1)+1;
+ @SuppressWarnings("unchecked")
+ T old = (T)nvPairs.get( index );
+ nvPairs.set(index, val);
+ return old;
+ }
+
+ /**
+ * Removes the name/value pair at the specified index.
+ *
+ * @return the value at the index removed
+ */
+ public T remove(int idx) {
+ int index = (idx<<1);
+ nvPairs.remove(index);
+ @SuppressWarnings("unchecked")
+ T result = (T)nvPairs.remove(index); // same index, as things shifted in previous remove
+ return result;
+ }
+
+ /**
+ * Scans the list sequentially beginning at the specified index and
+ * returns the index of the first pair with the specified name.
+ *
+ * @param name name to look for, may be null
+ * @param start index to begin searching from
+ * @return The index of the first matching pair, -1 if no match
+ */
+ public int indexOf(String name, int start) {
+ int sz = size();
+ for (int i=start; i<sz; i++) {
+ String n = getName(i);
+ if (name==null) {
+ if (n==null) return i; // matched null
+ } else if (name.equals(n)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ /**
+ * Gets the value for the first instance of the specified name
+ * found.
+ * <p>
+ * NOTE: this runs in linear time (it scans starting at the
+ * beginning of the list until it finds the first pair with
+ * the specified name).
+ *
+ * @return null if not found or if the value stored was null.
+ * @see #indexOf
+ * @see #get(String,int)
+ *
+ */
+ public T get(String name) {
+ return get(name,0);
+ }
+
+ /**
+ * Gets the value for the first instance of the specified name
+ * found starting at the specified index.
+ * <p>
+ * NOTE: this runs in linear time (it scans starting at the
+ * specified position until it finds the first pair with
+ * the specified name).
+ *
+ * @return null if not found or if the value stored was null.
+ * @see #indexOf
+ */
+ public T get(String name, int start) {
+ int sz = size();
+ for (int i=start; i<sz; i++) {
+ String n = getName(i);
+ if (name==null) {
+ if (n==null) return getVal(i);
+ } else if (name.equals(n)) {
+ return getVal(i);
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Gets the values for the the specified name
+ *
+ * @param name Name
+ * @return List of values
+ */
+ public List<T> getAll(String name) {
+ List<T> result = new ArrayList<>();
+ int sz = size();
+ for (int i = 0; i < sz; i++) {
+ String n = getName(i);
+ if (name==n || (name!=null && name.equals(n))) {
+ result.add(getVal(i));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Removes all values matching the specified name
+ *
+ * @param name Name
+ */
+ private void killAll(String name) {
+ int sz = size();
+ // Go through the list backwards, removing matches as found.
+ for (int i = sz - 1; i >= 0; i--) {
+ String n = getName(i);
+ if (name==n || (name!=null && name.equals(n))) {
+ remove(i);
+ }
+ }
+ }
+
+ /**
+ * Recursively parses the NamedList structure to arrive at a specific element.
+ * As you descend the NamedList tree, the last element can be any type,
+ * including NamedList, but the previous elements MUST be NamedList objects
+ * themselves. A null value is returned if the indicated hierarchy doesn't
+ * exist, but NamedList allows null values so that could be the actual value
+ * at the end of the path.
+ *
+ * This method is particularly useful for parsing the response from Solr's
+ * /admin/mbeans handler, but it also works for any complex structure.
+ *
+ * Explicitly casting the return value is recommended. An even safer option is
+ * to accept the return value as an object and then check its type.
+ *
+ * Usage examples:
+ *
+ * String coreName = (String) response.findRecursive
+ * ("solr-mbeans", "CORE", "core", "stats", "coreName");
+ * long numDoc = (long) response.findRecursive
+ * ("solr-mbeans", "CORE", "searcher", "stats", "numDocs");
+ *
+ * @param args
+ * One or more strings specifying the tree to navigate.
+ * @return the last entry in the given path hierarchy, null if not found.
+ */
+ public Object findRecursive(String... args) {
+ NamedList<?> currentList = null;
+ Object value = null;
+ for (int i = 0; i < args.length; i++) {
+ String key = args[i];
+ /*
+ * The first time through the loop, the current list is null, so we assign
+ * it to this list. Then we retrieve the first key from this list and
+ * assign it to value.
+ *
+ * On the next loop, we check whether the retrieved value is a NamedList.
+ * If it is, then we drop down to that NamedList, grab the value of the
+ * next key, and start the loop over. If it is not a NamedList, then we
+ * assign the value to null and break out of the loop.
+ *
+ * Assigning the value to null and then breaking out of the loop seems
+ * like the wrong thing to do, but there's a very simple reason that it
+ * works: If we have reached the last key, then the loop ends naturally
+ * after we retrieve the value, and that code is never executed.
+ */
+ if (currentList == null) {
+ currentList = this;
+ } else {
+ if (value instanceof NamedList) {
+ currentList = (NamedList<?>) value;
+ } else {
+ value = null;
+ break;
+ }
+ }
+ /*
+ * We do not need to do a null check on currentList for the following
+ * assignment. The instanceof check above will fail if the current list is
+ * null, and if that happens, the loop will end before this point.
+ */
+ value = currentList.get(key, 0);
+ }
+ return value;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('{');
+ int sz = size();
+ for (int i=0; i<sz; i++) {
+ if (i != 0) sb.append(',');
+ sb.append(getName(i));
+ sb.append('=');
+ sb.append(getVal(i));
+ }
+ sb.append('}');
+
+ return sb.toString();
+ }
+
+ public NamedList getImmutableCopy() {
+ NamedList copy = clone();
+ return new NamedList<>( Collections.unmodifiableList(copy.nvPairs));
+ }
+
+ public Map asMap(int maxDepth) {
+ LinkedHashMap result = new LinkedHashMap();
+ for(int i=0;i<size();i++){
+ Object val = getVal(i);
+ if (val instanceof NamedList && maxDepth> 0) {
+ //the maxDepth check is to avoid stack overflow due to infinite recursion
+ val = ((NamedList) val).asMap(maxDepth-1);
+ }
+ Object old = result.put(getName(i), val);
+ if(old!=null){
+ if (old instanceof List) {
+ List list = (List) old;
+ list.add(val);
+ result.put(getName(i),old);
+ } else {
+ ArrayList l = new ArrayList();
+ l.add(old);
+ l.add(val);
+ result.put(getName(i), l);
+ }
+ }
+ }
+ return result;
+ }
+
+ /**
+ *
+ * Helper class implementing Map.Entry<String, T> to store the key-value
+ * relationship in NamedList (the keys of which are String-s)
+ */
+ public static final class NamedListEntry<T> implements Map.Entry<String,T> {
+
+ public NamedListEntry() {
+
+ }
+
+ public NamedListEntry(String _key, T _value) {
+ key = _key;
+ value = _value;
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
+
+ @Override
+ public T getValue() {
+ return value;
+ }
+
+ @Override
+ public T setValue(T _value) {
+ T oldValue = value;
+ value = _value;
+ return oldValue;
+ }
+
+ private String key;
+
+ private T value;
+ }
+
+ /**
+ * Iterates over the Map and sequentially adds its key/value pairs
+ */
+ public boolean addAll(Map<String,T> args) {
+ for (Map.Entry<String, T> entry : args.entrySet() ) {
+ add(entry.getKey(), entry.getValue());
+ }
+ return args.size()>0;
+ }
+
+ /** Appends the elements of the given NamedList to this one. */
+ public boolean addAll(NamedList<T> nl) {
+ nvPairs.addAll(nl.nvPairs);
+ return nl.size()>0;
+ }
+
+ /**
+ * Makes a <i>shallow copy</i> of the named list.
+ */
+ @Override
+ public NamedList<T> clone() {
+ ArrayList<Object> newList = new ArrayList<>(nvPairs.size());
+ newList.addAll(nvPairs);
+ return new NamedList<>(newList);
+ }
+
+ //----------------------------------------------------------------------------
+ // Iterable interface
+ //----------------------------------------------------------------------------
+
+ /**
+ * Support the Iterable interface
+ */
+ @Override
+ public Iterator<Map.Entry<String,T>> iterator() {
+
+ final NamedList<T> list = this;
+
+ Iterator<Map.Entry<String,T>> iter = new Iterator<Map.Entry<String,T>>() {
+
+ int idx = 0;
+
+ @Override
+ public boolean hasNext() {
+ return idx < list.size();
+ }
+
+ @Override
+ public Map.Entry<String,T> next() {
+ final int index = idx++;
+ Map.Entry<String,T> nv = new Map.Entry<String,T>() {
+ @Override
+ public String getKey() {
+ return list.getName( index );
+ }
+
+ @Override
+ public T getValue() {
+ return list.getVal( index );
+ }
+
+ @Override
+ public String toString() {
+ return getKey()+"="+getValue();
+ }
+
+ @Override
+ public T setValue(T value) {
+ return list.setVal(index, value);
+ }
+ };
+ return nv;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ return iter;
+ }
+
+ /**
+ * NOTE: this runs in linear time (it scans starting at the
+ * beginning of the list until it finds the first pair with
+ * the specified name).
+ */
+ public T remove(String name) {
+ int idx = indexOf(name, 0);
+ if(idx != -1) return remove(idx);
+ return null;
+ }
+
+ /**
+ * Removes and returns all values for the specified name. Returns null if
+ * no matches found. This method will return all matching objects,
+ * regardless of data type. If you are parsing Solr config options, the
+ * {@link #removeConfigArgs(String)} or {@link #removeBooleanArg(String)}
+ * methods will probably work better.
+ *
+ * @param name Name
+ * @return List of values
+ */
+ public List<T> removeAll(String name) {
+ List<T> result = new ArrayList<>();
+ result = getAll(name);
+ if (result.size() > 0 ) {
+ killAll(name);
+ return result;
+ }
+ return null;
+ }
+
+ /**
+ * Used for getting a boolean argument from a NamedList object. If the name
+ * is not present, returns null. If there is more than one value with that
+ * name, or if the value found is not a Boolean or a String, throws an
+ * exception. If there is only one value present and it is a Boolean or a
+ * String, the value is removed and returned as a Boolean. If an exception
+ * is thrown, the NamedList is not modified. See {@link #removeAll(String)}
+ * and {@link #removeConfigArgs(String)} for additional ways of gathering
+ * configuration information from a NamedList.
+ *
+ * @param name
+ * The key to look up in the NamedList.
+ * @return The boolean value found.
+ * @throws SolrException
+ * If multiple values are found for the name or the value found is
+ * not a Boolean or a String.
+ */
+ public Boolean removeBooleanArg(final String name) {
+ Boolean bool = getBooleanArg(name);
+ if (null != bool) {
+ remove(name);
+ }
+ return bool;
+ }
+
+ /**
+ * Used for getting a boolean argument from a NamedList object. If the name
+ * is not present, returns null. If there is more than one value with that
+ * name, or if the value found is not a Boolean or a String, throws an
+ * exception. If there is only one value present and it is a Boolean or a
+ * String, the value is returned as a Boolean. The NamedList is not
+ * modified. See {@link #remove(String)}, {@link #removeAll(String)}
+ * and {@link #removeConfigArgs(String)} for additional ways of gathering
+ * configuration information from a NamedList.
+ *
+ * @param name The key to look up in the NamedList.
+ * @return The boolean value found.
+ * @throws SolrException
+ * If multiple values are found for the name or the value found is
+ * not a Boolean or a String.
+ */
+ public Boolean getBooleanArg(final String name) {
+ Boolean bool;
+ List<T> values = getAll(name);
+ if (0 == values.size()) {
+ return null;
+ }
+ if (values.size() > 1) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Only one '" + name + "' is allowed");
+ }
+ Object o = get(name);
+ if (o instanceof Boolean) {
+ bool = (Boolean)o;
+ } else if (o instanceof CharSequence) {
+ bool = Boolean.parseBoolean(o.toString());
+ } else {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "'" + name + "' must have type Boolean or CharSequence; found " + o.getClass());
+ }
+ return bool;
+ }
+
+ /**
+ * Used for getting one or many arguments from NamedList objects that hold
+ * configuration parameters. Finds all entries in the NamedList that match
+ * the given name. If they are all strings or arrays of strings, remove them
+ * from the NamedList and return the individual elements as a {@link Collection}.
+ * Parameter order will be preserved if the returned collection is handled as
+ * an {@link ArrayList}. Throws SolrException if any of the values associated
+ * with the name are not strings or arrays of strings. If exception is
+ * thrown, the NamedList is not modified. Returns an empty collection if no
+ * matches found. If you need to remove and retrieve all matching items from
+ * the NamedList regardless of data type, use {@link #removeAll(String)} instead.
+ * The {@link #removeBooleanArg(String)} method can be used for retrieving a
+ * boolean argument.
+ *
+ * @param name
+ * The key to look up in the NamedList.
+ * @return A collection of the values found.
+ * @throws SolrException
+ * If values are found for the input key that are not strings or
+ * arrays of strings.
+ */
+ @SuppressWarnings("rawtypes")
+ public Collection<String> removeConfigArgs(final String name)
+ throws SolrException {
+ List<T> objects = getAll(name);
+ List<String> collection = new ArrayList<>(size() / 2);
+ final String err = "init arg '" + name + "' must be a string "
+ + "(ie: 'str'), or an array (ie: 'arr') containing strings; found: ";
+
+ for (Object o : objects) {
+ if (o instanceof String) {
+ collection.add((String) o);
+ continue;
+ }
+
+ // If it's an array, convert to List (which is a Collection).
+ if (o instanceof Object[]) {
+ o = Arrays.asList((Object[]) o);
+ }
+
+ // If it's a Collection, collect each value.
+ if (o instanceof Collection) {
+ for (Object item : (Collection) o) {
+ if (!(item instanceof String)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, err + item.getClass());
+ }
+ collection.add((String) item);
+ }
+ continue;
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, err + o.getClass());
+ }
+
+ if (collection.size() > 0) {
+ killAll(name);
+ }
+
+ return collection;
+ }
+
+ public void clear() {
+ nvPairs.clear();
+ }
+
+ @Override
+ public int hashCode() {
+ return nvPairs.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof NamedList)) return false;
+ NamedList<?> nl = (NamedList<?>) obj;
+ return this.nvPairs.equals(nl.nvPairs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java
new file mode 100644
index 0000000..47ab21a
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/ObjectReleaseTracker.java
@@ -0,0 +1,62 @@
+package org.apache.solr.common.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ObjectReleaseTracker {
+ public static Map<Object,String> OBJECTS = new ConcurrentHashMap<>();
+
+ public static boolean track(Object object) {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ new ObjectTrackerException().printStackTrace(pw);
+ OBJECTS.put(object, sw.toString());
+ return true;
+ }
+
+ public static boolean release(Object object) {
+ OBJECTS.remove(object);
+ return true;
+ }
+
+ public static boolean clearObjectTrackerAndCheckEmpty() {
+ Set<Entry<Object,String>> entries = OBJECTS.entrySet();
+ boolean empty = entries.isEmpty();
+ if (entries.size() > 0) {
+ System.err.println("ObjectTracker found objects that were not released!!!");
+ }
+
+ for (Entry<Object,String> entry : entries) {
+ System.err.println(entry.getValue());
+ }
+
+ OBJECTS.clear();
+
+ return empty;
+ }
+
+ private static class ObjectTrackerException extends RuntimeException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java
new file mode 100644
index 0000000..83ee100
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/RetryUtil.java
@@ -0,0 +1,43 @@
+package org.apache.solr.common.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.concurrent.TimeUnit;
+
+public class RetryUtil {
+ public static interface RetryCmd {
+ public void execute() throws Throwable;
+ }
+
+ public static void retryOnThrowable(Class clazz, long timeoutms, long intervalms, RetryCmd cmd) throws Throwable {
+ long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutms, TimeUnit.MILLISECONDS);
+ while (true) {
+ try {
+ cmd.execute();
+ } catch (Throwable t) {
+ if (clazz.isInstance(t) && System.nanoTime() < timeout) {
+ Thread.sleep(intervalms);
+ continue;
+ }
+ throw t;
+ }
+ // success
+ break;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java
new file mode 100644
index 0000000..c9996d1
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/SimpleOrderedMap.java
@@ -0,0 +1,67 @@
+package org.apache.solr.common.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.*;
+
+
+/** <code>SimpleOrderedMap</code> is a {@link NamedList} where access by key is more
+ * important than maintaining order when it comes to representing the
+ * held data in other forms, as ResponseWriters normally do.
+ * It's normally not a good idea to repeat keys or use null keys, but this
+ * is not enforced. If key uniqueness enforcement is desired, use a regular {@link Map}.
+ * <p>
+ * For example, a JSON response writer may choose to write a SimpleOrderedMap
+ * as {"foo":10,"bar":20} and may choose to write a NamedList as
+ * ["foo",10,"bar",20]. An XML response writer may choose to render both
+ * the same way.
+ * </p>
+ * <p>
+ * This class does not provide efficient lookup by key, its main purpose is
+ * to hold data to be serialized. It aims to minimize overhead and to be
+ * efficient at adding new elements.
+ * </p>
+ */
+public class SimpleOrderedMap<T> extends NamedList<T> {
+ /** Creates an empty instance */
+ public SimpleOrderedMap() {
+ super();
+ }
+
+ /**
+ * Creates an instance backed by an explicitly specified list of
+ * pairwise names/values.
+ *
+ * @param nameValuePairs underlying List which should be used to implement a SimpleOrderedMap; modifying this List will affect the SimpleOrderedMap.
+ */
+ @Deprecated
+ public SimpleOrderedMap(List<Object> nameValuePairs) {
+ super(nameValuePairs);
+ }
+
+ public SimpleOrderedMap(Map.Entry<String, T>[] nameValuePairs) {
+ super(nameValuePairs);
+ }
+
+ @Override
+ public SimpleOrderedMap<T> clone() {
+ ArrayList<Object> newList = new ArrayList<>(nvPairs.size());
+ newList.addAll(nvPairs);
+ return new SimpleOrderedMap<>(newList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java
new file mode 100644
index 0000000..2a7c901
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/SolrjNamedThreadFactory.java
@@ -0,0 +1,50 @@
+package org.apache.solr.common.util;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class SolrjNamedThreadFactory implements ThreadFactory {
+ private static final AtomicInteger poolNumber = new AtomicInteger(1);
+ private final ThreadGroup group;
+ private final AtomicInteger threadNumber = new AtomicInteger(1);
+ private final String prefix;
+
+ public SolrjNamedThreadFactory(String namePrefix) {
+ SecurityManager s = System.getSecurityManager();
+ group = (s != null)? s.getThreadGroup() :
+ Thread.currentThread().getThreadGroup();
+ prefix = namePrefix + "-" +
+ poolNumber.getAndIncrement() +
+ "-thread-";
+ }
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = new Thread(group, r,
+ prefix + threadNumber.getAndIncrement(),
+ 0);
+
+ t.setDaemon(false);
+
+ if (t.getPriority() != Thread.NORM_PRIORITY)
+ t.setPriority(Thread.NORM_PRIORITY);
+ return t;
+ }
+}
\ No newline at end of file
[02/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/StrUtils.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/StrUtils.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/StrUtils.java
new file mode 100644
index 0000000..8ac87d7
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/StrUtils.java
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Locale;
+import java.io.IOException;
+
+import org.apache.solr.common.SolrException;
+
+/**
+ *
+ */
+public class StrUtils {
+ public static final char[] HEX_DIGITS = { '0', '1', '2', '3', '4', '5', '6',
+ '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' };
+
+ /**
+ * Split a string based on a separator, but don't split if it's inside
+ * a string. Assume '\' escapes the next char both inside and
+ * outside strings.
+ */
+ public static List<String> splitSmart(String s, char separator) {
+ ArrayList<String> lst = new ArrayList<>(4);
+ int pos=0, start=0, end=s.length();
+ char inString=0;
+ char ch=0;
+ while (pos < end) {
+ char prevChar=ch;
+ ch = s.charAt(pos++);
+ if (ch=='\\') { // skip escaped chars
+ pos++;
+ } else if (inString != 0 && ch==inString) {
+ inString=0;
+ } else if (ch=='\'' || ch=='"') {
+ // If char is directly preceeded by a number or letter
+ // then don't treat it as the start of a string.
+ // Examples: 50" TV, or can't
+ if (!Character.isLetterOrDigit(prevChar)) {
+ inString=ch;
+ }
+ } else if (ch==separator && inString==0) {
+ lst.add(s.substring(start,pos-1));
+ start=pos;
+ }
+ }
+ if (start < end) {
+ lst.add(s.substring(start,end));
+ }
+
+ /***
+ if (SolrCore.log.isLoggable(Level.FINEST)) {
+ SolrCore.log.trace("splitCommand=" + lst);
+ }
+ ***/
+
+ return lst;
+ }
+
+ /** Splits a backslash escaped string on the separator.
+ * <p>
+ * Current backslash escaping supported:
+ * <br> \n \t \r \b \f are escaped the same as a Java String
+ * <br> Other characters following a backslash are produced verbatim (\c => c)
+ *
+ * @param s the string to split
+ * @param separator the separator to split on
+ * @param decode decode backslash escaping
+ */
+ public static List<String> splitSmart(String s, String separator, boolean decode) {
+ ArrayList<String> lst = new ArrayList<>(2);
+ StringBuilder sb = new StringBuilder();
+ int pos=0, end=s.length();
+ while (pos < end) {
+ if (s.startsWith(separator,pos)) {
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ sb=new StringBuilder();
+ }
+ pos+=separator.length();
+ continue;
+ }
+
+ char ch = s.charAt(pos++);
+ if (ch=='\\') {
+ if (!decode) sb.append(ch);
+ if (pos>=end) break; // ERROR, or let it go?
+ ch = s.charAt(pos++);
+ if (decode) {
+ switch(ch) {
+ case 'n' : ch='\n'; break;
+ case 't' : ch='\t'; break;
+ case 'r' : ch='\r'; break;
+ case 'b' : ch='\b'; break;
+ case 'f' : ch='\f'; break;
+ }
+ }
+ }
+
+ sb.append(ch);
+ }
+
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ }
+
+ return lst;
+ }
+
+ /**
+ * Splits file names separated by comma character.
+ * File names can contain comma characters escaped by backslash '\'
+ *
+ * @param fileNames the string containing file names
+ * @return a list of file names with the escaping backslashed removed
+ */
+ public static List<String> splitFileNames(String fileNames) {
+ if (fileNames == null)
+ return Collections.<String>emptyList();
+
+ List<String> result = new ArrayList<>();
+ for (String file : fileNames.split("(?<!\\\\),")) {
+ result.add(file.replaceAll("\\\\(?=,)", ""));
+ }
+
+ return result;
+ }
+
+ /**
+ * Creates a backslash escaped string, joining all the items.
+ * @see #escapeTextWithSeparator
+ */
+ public static String join(List<?> items, char separator) {
+ StringBuilder sb = new StringBuilder(items.size() << 3);
+ boolean first=true;
+ for (Object o : items) {
+ String item = o.toString();
+ if (first) {
+ first = false;
+ } else {
+ sb.append(separator);
+ }
+ appendEscapedTextToBuilder(sb, item, separator);
+ }
+ return sb.toString();
+ }
+
+
+
+ public static List<String> splitWS(String s, boolean decode) {
+ ArrayList<String> lst = new ArrayList<>(2);
+ StringBuilder sb = new StringBuilder();
+ int pos=0, end=s.length();
+ while (pos < end) {
+ char ch = s.charAt(pos++);
+ if (Character.isWhitespace(ch)) {
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ sb=new StringBuilder();
+ }
+ continue;
+ }
+
+ if (ch=='\\') {
+ if (!decode) sb.append(ch);
+ if (pos>=end) break; // ERROR, or let it go?
+ ch = s.charAt(pos++);
+ if (decode) {
+ switch(ch) {
+ case 'n' : ch='\n'; break;
+ case 't' : ch='\t'; break;
+ case 'r' : ch='\r'; break;
+ case 'b' : ch='\b'; break;
+ case 'f' : ch='\f'; break;
+ }
+ }
+ }
+
+ sb.append(ch);
+ }
+
+ if (sb.length() > 0) {
+ lst.add(sb.toString());
+ }
+
+ return lst;
+ }
+
+ public static List<String> toLower(List<String> strings) {
+ ArrayList<String> ret = new ArrayList<>(strings.size());
+ for (String str : strings) {
+ ret.add(str.toLowerCase(Locale.ROOT));
+ }
+ return ret;
+ }
+
+
+
+ /** Return if a string starts with '1', 't', or 'T'
+ * and return false otherwise.
+ */
+ public static boolean parseBoolean(String s) {
+ char ch = s.length()>0 ? s.charAt(0) : 0;
+ return (ch=='1' || ch=='t' || ch=='T');
+ }
+
+ /** how to transform a String into a boolean... more flexible than
+ * Boolean.parseBoolean() to enable easier integration with html forms.
+ */
+ public static boolean parseBool(String s) {
+ if( s != null ) {
+ if( s.startsWith("true") || s.startsWith("on") || s.startsWith("yes") ) {
+ return true;
+ }
+ if( s.startsWith("false") || s.startsWith("off") || s.equals("no") ) {
+ return false;
+ }
+ }
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "invalid boolean value: "+s );
+ }
+
+ /**
+ * {@link NullPointerException} and {@link SolrException} free version of {@link #parseBool(String)}
+ * @return parsed boolean value (or def, if s is null or invalid)
+ */
+ public static boolean parseBool(String s, boolean def) {
+ if( s != null ) {
+ if( s.startsWith("true") || s.startsWith("on") || s.startsWith("yes") ) {
+ return true;
+ }
+ if( s.startsWith("false") || s.startsWith("off") || s.equals("no") ) {
+ return false;
+ }
+ }
+ return def;
+ }
+
+ /**
+ * URLEncodes a value, replacing only enough chars so that
+ * the URL may be unambiguously pasted back into a browser.
+ * <p>
+ * Characters with a numeric value less than 32 are encoded.
+ * &,=,%,+,space are encoded.
+ */
+ public static void partialURLEncodeVal(Appendable dest, String val) throws IOException {
+ for (int i=0; i<val.length(); i++) {
+ char ch = val.charAt(i);
+ if (ch < 32) {
+ dest.append('%');
+ if (ch < 0x10) dest.append('0');
+ dest.append(Integer.toHexString(ch));
+ } else {
+ switch (ch) {
+ case ' ': dest.append('+'); break;
+ case '&': dest.append("%26"); break;
+ case '%': dest.append("%25"); break;
+ case '=': dest.append("%3D"); break;
+ case '+': dest.append("%2B"); break;
+ default : dest.append(ch); break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates a new copy of the string with the separator backslash escaped.
+ * @see #join
+ */
+ public static String escapeTextWithSeparator(String item, char separator) {
+ StringBuilder sb = new StringBuilder(item.length() * 2);
+ appendEscapedTextToBuilder(sb, item, separator);
+ return sb.toString();
+ }
+
+ /**
+ * writes chars from item to out, backslash escaping as needed based on separator --
+ * but does not append the seperator itself
+ */
+ public static void appendEscapedTextToBuilder(StringBuilder out,
+ String item,
+ char separator) {
+ for (int i = 0; i < item.length(); i++) {
+ char ch = item.charAt(i);
+ if (ch == '\\' || ch == separator) {
+ out.append('\\');
+ }
+ out.append(ch);
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/URLUtil.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/URLUtil.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/URLUtil.java
new file mode 100644
index 0000000..6d273ec
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/URLUtil.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class URLUtil {
+
+ public final static Pattern URL_PREFIX = Pattern.compile("^([a-z]*?://).*");
+
+ public static String removeScheme(String url) {
+ Matcher matcher = URL_PREFIX.matcher(url);
+ if (matcher.matches()) {
+ return url.substring(matcher.group(1).length());
+ }
+
+ return url;
+ }
+
+ public static boolean hasScheme(String url) {
+ Matcher matcher = URL_PREFIX.matcher(url);
+ return matcher.matches();
+ }
+
+ public static String getScheme(String url) {
+ Matcher matcher = URL_PREFIX.matcher(url);
+ if (matcher.matches()) {
+ return matcher.group(1);
+ }
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/XML.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/XML.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/XML.java
new file mode 100644
index 0000000..50a6da2
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/XML.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.Writer;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ *
+ */
+public class XML {
+
+ //
+ // copied from some of my personal code... -YCS
+ // table created from python script.
+ // only have to escape quotes in attribute values, and don't really have to escape '>'
+ // many chars less than 0x20 are *not* valid XML, even when escaped!
+ // for example, <foo>�<foo> is invalid XML.
+ private static final String[] chardata_escapes=
+ {"#0;","#1;","#2;","#3;","#4;","#5;","#6;","#7;","#8;",null,null,"#11;","#12;",null,"#14;","#15;","#16;","#17;","#18;","#19;","#20;","#21;","#22;","#23;","#24;","#25;","#26;","#27;","#28;","#29;","#30;","#31;",null,null,null,null,null,null,"&",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"<",null,">"};
+
+ private static final String[] attribute_escapes=
+ {"#0;","#1;","#2;","#3;","#4;","#5;","#6;","#7;","#8;",null,null,"#11;","#12;",null,"#14;","#15;","#16;","#17;","#18;","#19;","#20;","#21;","#22;","#23;","#24;","#25;","#26;","#27;","#28;","#29;","#30;","#31;",null,null,""",null,null,null,"&",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"<"};
+
+
+
+ /*****************************************
+ #Simple python script used to generate the escape table above. -YCS
+ #
+ #use individual char arrays or one big char array for better efficiency
+ # or byte array?
+ #other={'&':'amp', '<':'lt', '>':'gt', "'":'apos', '"':'quot'}
+ #
+ other={'&':'amp', '<':'lt'}
+
+ maxi=ord(max(other.keys()))+1
+ table=[None] * maxi
+ #NOTE: invalid XML chars are "escaped" as #nn; *not* &#nn; because
+ #a real XML escape would cause many strict XML parsers to choke.
+ for i in range(0x20): table[i]='#%d;' % i
+ for i in '\n\r\t ': table[ord(i)]=None
+ for k,v in other.items():
+ table[ord(k)]='&%s;' % v
+
+ result=""
+ for i in range(maxi):
+ val=table[i]
+ if not val: val='null'
+ else: val='"%s"' % val
+ result += val + ','
+
+ print result
+ ****************************************/
+
+
+/*********
+ *
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public static void escapeCharData(String str, Writer out) throws IOException {
+ escape(str, out, chardata_escapes);
+ }
+
+ public static void escapeAttributeValue(String str, Writer out) throws IOException {
+ escape(str, out, attribute_escapes);
+ }
+
+ public static void escapeAttributeValue(char [] chars, int start, int length, Writer out) throws IOException {
+ escape(chars, start, length, out, attribute_escapes);
+ }
+
+
+ public final static void writeXML(Writer out, String tag, String val) throws IOException {
+ out.write('<');
+ out.write(tag);
+ if (val == null) {
+ out.write('/');
+ out.write('>');
+ } else {
+ out.write('>');
+ escapeCharData(val,out);
+ out.write('<');
+ out.write('/');
+ out.write(tag);
+ out.write('>');
+ }
+ }
+
+ /** does NOT escape character data in val, must already be valid XML */
+ public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ out.write('<');
+ out.write(tag);
+ for (int i=0; i<attrs.length; i++) {
+ out.write(' ');
+ out.write(attrs[i++].toString());
+ out.write('=');
+ out.write('"');
+ out.write(attrs[i].toString());
+ out.write('"');
+ }
+ if (val == null) {
+ out.write('/');
+ out.write('>');
+ } else {
+ out.write('>');
+ out.write(val);
+ out.write('<');
+ out.write('/');
+ out.write(tag);
+ out.write('>');
+ }
+ }
+
+ /** escapes character data in val */
+ public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ out.write('<');
+ out.write(tag);
+ for (int i=0; i<attrs.length; i++) {
+ out.write(' ');
+ out.write(attrs[i++].toString());
+ out.write('=');
+ out.write('"');
+ escapeAttributeValue(attrs[i].toString(), out);
+ out.write('"');
+ }
+ if (val == null) {
+ out.write('/');
+ out.write('>');
+ } else {
+ out.write('>');
+ escapeCharData(val,out);
+ out.write('<');
+ out.write('/');
+ out.write(tag);
+ out.write('>');
+ }
+ }
+
+ /** escapes character data in val */
+ public static void writeXML(Writer out, String tag, String val, Map<String, String> attrs) throws IOException {
+ out.write('<');
+ out.write(tag);
+ for (Map.Entry<String, String> entry : attrs.entrySet()) {
+ out.write(' ');
+ out.write(entry.getKey());
+ out.write('=');
+ out.write('"');
+ escapeAttributeValue(entry.getValue(), out);
+ out.write('"');
+ }
+ if (val == null) {
+ out.write('/');
+ out.write('>');
+ } else {
+ out.write('>');
+ escapeCharData(val,out);
+ out.write('<');
+ out.write('/');
+ out.write(tag);
+ out.write('>');
+ }
+ }
+
+ private static void escape(char [] chars, int offset, int length, Writer out, String [] escapes) throws IOException{
+ for (int i=offset; i<length; i++) {
+ char ch = chars[i];
+ if (ch<escapes.length) {
+ String replacement = escapes[ch];
+ if (replacement != null) {
+ out.write(replacement);
+ continue;
+ }
+ }
+ out.write(ch);
+ }
+ }
+
+ private static void escape(String str, Writer out, String[] escapes) throws IOException {
+ for (int i=0; i<str.length(); i++) {
+ char ch = str.charAt(i);
+ if (ch<escapes.length) {
+ String replacement = escapes[ch];
+ if (replacement != null) {
+ out.write(replacement);
+ continue;
+ }
+ }
+ out.write(ch);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/XMLErrorLogger.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/XMLErrorLogger.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/XMLErrorLogger.java
new file mode 100644
index 0000000..7f45ff9
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/XMLErrorLogger.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import org.slf4j.Logger;
+
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+import javax.xml.transform.ErrorListener;
+import javax.xml.transform.TransformerException;
+import javax.xml.stream.Location;
+import javax.xml.stream.XMLReporter;
+
+public final class XMLErrorLogger implements ErrorHandler,ErrorListener,XMLReporter {
+
+ private final Logger log;
+
+ public XMLErrorLogger(Logger log) {
+ this.log = log;
+ }
+
+ // ErrorHandler
+
+ @Override
+ public void warning(SAXParseException e) {
+ log.warn("XML parse warning in \""+e.getSystemId()+"\", line "+e.getLineNumber()+", column "+e.getColumnNumber()+": "+e.getMessage());
+ }
+
+ @Override
+ public void error(SAXParseException e) throws SAXException {
+ throw e;
+ }
+
+ @Override
+ public void fatalError(SAXParseException e) throws SAXException {
+ throw e;
+ }
+
+ // ErrorListener
+
+ @Override
+ public void warning(TransformerException e) {
+ log.warn(e.getMessageAndLocation());
+ }
+
+ @Override
+ public void error(TransformerException e) throws TransformerException {
+ throw e;
+ }
+
+ @Override
+ public void fatalError(TransformerException e) throws TransformerException {
+ throw e;
+ }
+
+ // XMLReporter
+
+ @Override
+ public void report(String message, String errorType, Object relatedInformation, Location loc) {
+ final StringBuilder sb = new StringBuilder("XML parser reported ").append(errorType);
+ if (loc != null) {
+ sb.append(" in \"").append(loc.getSystemId()).append("\", line ")
+ .append(loc.getLineNumber()).append(", column ").append(loc.getColumnNumber());
+ }
+ log.warn(sb.append(": ").append(message).toString());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/package-info.java
new file mode 100644
index 0000000..1da825f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common utility classes reused on both clients & server.
+ */
+package org.apache.solr.common.util;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/overview.html
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/overview.html b/ranger_solrj/src/main/java/overview.html
new file mode 100644
index 0000000..7a534ed
--- /dev/null
+++ b/ranger_solrj/src/main/java/overview.html
@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<body>
+Apache Solr Search Server: Solr-j
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/pom.xml
----------------------------------------------------------------------
diff --git a/security-admin/pom.xml b/security-admin/pom.xml
index 78b7e7f..e507afe 100644
--- a/security-admin/pom.xml
+++ b/security-admin/pom.xml
@@ -244,6 +244,11 @@
</exclusions>
</dependency>
<dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger_solrj</artifactId>
+ <version>${ranger.solrj.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/scripts/install.properties
----------------------------------------------------------------------
diff --git a/security-admin/scripts/install.properties b/security-admin/scripts/install.properties
index 03d09eb..ae66576 100644
--- a/security-admin/scripts/install.properties
+++ b/security-admin/scripts/install.properties
@@ -58,6 +58,14 @@ db_name=ranger
db_user=rangeradmin
db_password=
+#Source for Audit DB
+# * audit_db is solr or db
+audit_store=db
+
+# * audit_solr_url URL to Solr. E.g. http://<solr_host>:6083/solr/ranger_audits
+audit_solr_url=
+
+
#
# DB UserId for storing auditlog infromation
#
@@ -69,6 +77,8 @@ audit_db_name=ranger_audit
audit_db_user=rangerlogger
audit_db_password=
+
+
#------------------------- DB CONFIG - END ----------------------------------
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/scripts/setup.sh
----------------------------------------------------------------------
diff --git a/security-admin/scripts/setup.sh b/security-admin/scripts/setup.sh
index 6c992f7..04e22eb 100755
--- a/security-admin/scripts/setup.sh
+++ b/security-admin/scripts/setup.sh
@@ -129,8 +129,13 @@ init_variables(){
getPropertyFromFile 'db_root_password' $PROPFILE db_user
getPropertyFromFile 'db_user' $PROPFILE db_user
getPropertyFromFile 'db_password' $PROPFILE db_password
- getPropertyFromFile 'audit_db_user' $PROPFILE audit_db_user
- getPropertyFromFile 'audit_db_password' $PROPFILE audit_db_password
+ if [ "${audit_store}" == "solr" ]
+ then
+ getPropertyFromFile 'audit_solr_url' $PROPFILE audit_solr_url
+ else
+ getPropertyFromFile 'audit_db_user' $PROPFILE audit_db_user
+ getPropertyFromFile 'audit_db_password' $PROPFILE audit_db_password
+ fi
}
wait_for_tomcat_shutdown() {
@@ -820,6 +825,19 @@ update_properties() {
newPropertyValue="com.microsoft.sqlserver.jdbc.SQLServerDriver"
updatePropertyToFile $propertyName $newPropertyValue $to_file
fi
+
+ if [ "${audit_store}" == "solr" ]
+ then
+ propertyName=xa.audit.solr.url
+ newPropertyValue=${audit_solr_url}
+ updatePropertyToFile $propertyName $newPropertyValue $to_file
+ fi
+
+ propertyName=xa.audit.db.type
+ newPropertyValue=${audit_store}
+ updatePropertyToFile $propertyName $newPropertyValue $to_file
+
+
propertyName=xa.webapp.url.root
newPropertyValue="${policymgr_external_url}"
updatePropertyToFile $propertyName $newPropertyValue $to_file
@@ -878,41 +896,43 @@ update_properties() {
fi
###########
- audit_db_password_alias=auditDB.jdbc.password
+ if [ "${audit_store}" != "solr" ]
+ then
+ audit_db_password_alias=auditDB.jdbc.password
- echo "Starting configuration for Audit DB credentials:"
+ echo "Starting configuration for Audit DB credentials:"
- if [ "${keystore}" != "" ]
- then
+ if [ "${keystore}" != "" ]
+ then
$JAVA_HOME/bin/java -cp "cred/lib/*" org.apache.ranger.credentialapi.buildks create "$audit_db_password_alias" -value "$audit_db_password" -provider jceks://file$keystore
propertyName=auditDB.jdbc.credential.alias
newPropertyValue="${audit_db_password_alias}"
updatePropertyToFile $propertyName $newPropertyValue $to_file
-
+
propertyName=auditDB.jdbc.credential.provider.path
newPropertyValue="${keystore}"
updatePropertyToFile $propertyName $newPropertyValue $to_file
-
+
propertyName=auditDB.jdbc.password
newPropertyValue="_"
updatePropertyToFile $propertyName $newPropertyValue $to_file
- else
+ else
propertyName=auditDB.jdbc.password
newPropertyValue="${audit_db_password}"
updatePropertyToFile $propertyName $newPropertyValue $to_file
- fi
+ fi
- if test -f $keystore; then
+ if test -f $keystore; then
chown -R ${unix_user}:${unix_group} ${keystore}
#echo "$keystore found."
- else
+ else
#echo "$keystore not found. so use clear text password"
propertyName=auditDB.jdbc.password
newPropertyValue="${audit_db_password}"
updatePropertyToFile $propertyName $newPropertyValue $to_file
+ fi
fi
-
}
create_audit_db_user(){
@@ -1404,4 +1424,4 @@ execute_java_patches
else
exit 1
fi
-echo "Installation of Ranger PolicyManager Web Application is completed."
\ No newline at end of file
+echo "Installation of Ranger PolicyManager Web Application is completed."
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
index 3c3bd77..839f1fc 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java
@@ -66,6 +66,7 @@ import org.apache.ranger.service.XPermMapService;
import org.apache.ranger.service.XPolicyService;
import org.apache.ranger.service.XTrxLogService;
import org.apache.ranger.service.XUserService;
+import org.apache.ranger.solr.SolrAccessAuditsService;
import org.apache.ranger.util.RestUtil;
import org.apache.ranger.view.VXAccessAuditList;
import org.apache.ranger.view.VXAsset;
@@ -135,6 +136,9 @@ public class AssetMgr extends AssetMgrBase {
XUserMgr xUserMgr;
@Autowired
+ SolrAccessAuditsService solrAccessAuditsService;
+
+ @Autowired
@Qualifier(value = "transactionManager")
PlatformTransactionManager txManager;
@@ -1776,7 +1780,12 @@ public class AssetMgr extends AssetMgrBase {
}else if(!searchCriteria.getSortType().equalsIgnoreCase("asc")&& !searchCriteria.getSortType().equalsIgnoreCase("desc")){
searchCriteria.setSortType("desc");
}
- return xAccessAuditService.searchXAccessAudits(searchCriteria);
+ if (xaBizUtil.getAuditDBType().equalsIgnoreCase(RangerBizUtil.AUDIT_STORE_SOLR)) {
+ return solrAccessAuditsService.searchXAccessAudits(searchCriteria);
+ } else {
+ return xAccessAuditService.searchXAccessAudits(searchCriteria);
+ }
+ //return xAccessAuditService.searchXAccessAudits(searchCriteria);
}
public VXTrxLogList getTransactionReport(String transactionId) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
index b8659aa..37cc3d7 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/RangerBizUtil.java
@@ -17,7 +17,7 @@
* under the License.
*/
- package org.apache.ranger.biz;
+package org.apache.ranger.biz;
import java.io.File;
import java.util.ArrayList;
@@ -83,15 +83,17 @@ public class RangerBizUtil {
Map<String, Integer> classTypeMappings = new HashMap<String, Integer>();
private int maxFirstNameLength;
int maxDisplayNameLength = 150;
- boolean defaultAutoApprove = true;
- boolean showBlockedContent = true;
public final String EMPTY_CONTENT_DISPLAY_NAME = "...";
boolean enableResourceAccessControl;
private Random random;
private static final String PATH_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrst0123456789-_.";
private static char[] PATH_CHAR_SET = PATH_CHARS.toCharArray();
private static int PATH_CHAR_SET_LEN = PATH_CHAR_SET.length;
-
+ public static final String AUDIT_STORE_RDBMS = "DB";
+ public static final String AUDIT_STORE_SOLR = "solr";
+
+ String auditDBType = AUDIT_STORE_RDBMS;
+
static String fileSeparator = PropertiesUtil.getProperty(
"xa.file.separator", "/");
@@ -100,19 +102,20 @@ public class RangerBizUtil {
"xa.user.firstname.maxlength", "16"));
maxDisplayNameLength = PropertiesUtil.getIntProperty(
"xa.bookmark.name.maxlen", maxDisplayNameLength);
- showBlockedContent = PropertiesUtil.getBooleanProperty(
- "xa.content.show_blocked", showBlockedContent);
- defaultAutoApprove = PropertiesUtil.getBooleanProperty(
- "xa.mod.default", defaultAutoApprove);
groupEditableClasses = new HashSet<Class<?>>(
Arrays.asList(groupEditableClassesList));
enableResourceAccessControl = PropertiesUtil.getBooleanProperty(
"xa.resource.accessControl.enabled", true);
+ auditDBType = PropertiesUtil.getProperty("xa.audit.db.type",
+ auditDBType).toLowerCase();
+
+ logger.info("Audit datasource is " + auditDBType);
random = new Random();
}
- public <T extends XXDBBase> List<? extends XXDBBase> getParentObjects(T object) {
+ public <T extends XXDBBase> List<? extends XXDBBase> getParentObjects(
+ T object) {
List<XXDBBase> parentObjectList = null;
// if (checkParentAcess.contains(object.getMyClassType())) {
// parentObjectList = new ArrayList<MBase>();
@@ -216,8 +219,8 @@ public class RangerBizUtil {
}
public String getDisplayNameForClassName(XXDBBase obj) {
- String classTypeDisplayName = RangerConstants.getLabelFor_ClassTypes(obj
- .getMyClassType());
+ String classTypeDisplayName = RangerConstants
+ .getLabelFor_ClassTypes(obj.getMyClassType());
if (classTypeDisplayName == null) {
logger.error(
"Error get name for class type. obj=" + obj.toString(),
@@ -254,7 +257,8 @@ public class RangerBizUtil {
* @param userProfile
* @return
*/
- public String generatePublicName(VXPortalUser userProfile, XXPortalUser gjUser) {
+ public String generatePublicName(VXPortalUser userProfile,
+ XXPortalUser gjUser) {
return generatePublicName(userProfile.getFirstName(),
userProfile.getLastName());
}
@@ -298,7 +302,8 @@ public class RangerBizUtil {
public XXDBBase getMObject(VXDataObject vXDataObject) {
if (vXDataObject != null) {
- return getMObject(vXDataObject.getMyClassType(), vXDataObject.getId());
+ return getMObject(vXDataObject.getMyClassType(),
+ vXDataObject.getId());
}
return null;
}
@@ -308,8 +313,8 @@ public class RangerBizUtil {
return null;
}
if (objClassType == RangerConstants.CLASS_TYPE_USER_PROFILE) {
- return userMgr.mapXXPortalUserVXPortalUser(daoManager.getXXPortalUser().getById(
- objId));
+ return userMgr.mapXXPortalUserVXPortalUser(daoManager
+ .getXXPortalUser().getById(objId));
}
try {
AbstractBaseResourceService<?, ?> myService = AbstractBaseResourceService
@@ -426,9 +431,10 @@ public class RangerBizUtil {
xResourceList, xUserId, permission, reqTableType,
reqColumnType, false);
if (!matchFound) {
- vXResponse.setMsgDesc("You're not permitted to perform "
- + "the action for resource path : "
- + resourceName);
+ vXResponse
+ .setMsgDesc("You're not permitted to perform "
+ + "the action for resource path : "
+ + resourceName);
vXResponse.setStatusCode(VXResponse.STATUS_ERROR);
return vXResponse;
}
@@ -438,9 +444,10 @@ public class RangerBizUtil {
boolean matchFound = matchHivePolicy(resourceName,
xResourceList, xUserId, permission);
if (!matchFound) {
- vXResponse.setMsgDesc("You're not permitted to perform "
- + "the action for resource path : "
- + resourceName);
+ vXResponse
+ .setMsgDesc("You're not permitted to perform "
+ + "the action for resource path : "
+ + resourceName);
vXResponse.setStatusCode(VXResponse.STATUS_ERROR);
return vXResponse;
}
@@ -477,34 +484,34 @@ public class RangerBizUtil {
vXResponse.setStatusCode(VXResponse.STATUS_SUCCESS);
return vXResponse;
} else if (assetType == AppConstants.ASSET_KNOX) {
- String[] requestResNameList = resourceNames.split(",");
- for (String resourceName : requestResNameList) {
- boolean matchFound = matchKnoxPolicy(resourceName,
- xResourceList, vXResponse, xUserId, permission);
- if (!matchFound) {
- vXResponse.setMsgDesc("You're not permitted to perform "
- + "the action for resource path : " + resourceName);
- vXResponse.setStatusCode(VXResponse.STATUS_ERROR);
- return vXResponse;
- }
+ String[] requestResNameList = resourceNames.split(",");
+ for (String resourceName : requestResNameList) {
+ boolean matchFound = matchKnoxPolicy(resourceName,
+ xResourceList, vXResponse, xUserId, permission);
+ if (!matchFound) {
+ vXResponse.setMsgDesc("You're not permitted to perform "
+ + "the action for resource path : " + resourceName);
+ vXResponse.setStatusCode(VXResponse.STATUS_ERROR);
+ return vXResponse;
+ }
+ }
+ vXResponse.setStatusCode(VXResponse.STATUS_SUCCESS);
+ return vXResponse;
+ } else if (assetType == AppConstants.ASSET_STORM) {
+ String[] requestResNameList = resourceNames.split(",");
+ for (String resourceName : requestResNameList) {
+ boolean matchFound = matchStormPolicy(resourceName,
+ xResourceList, vXResponse, xUserId, permission);
+ if (!matchFound) {
+ vXResponse.setMsgDesc("You're not permitted to perform "
+ + "the action for resource path : " + resourceName);
+ vXResponse.setStatusCode(VXResponse.STATUS_ERROR);
+ return vXResponse;
}
- vXResponse.setStatusCode(VXResponse.STATUS_SUCCESS);
- return vXResponse;
- } else if (assetType == AppConstants.ASSET_STORM) {
- String[] requestResNameList = resourceNames.split(",");
- for (String resourceName : requestResNameList) {
- boolean matchFound = matchStormPolicy(resourceName,
- xResourceList, vXResponse, xUserId, permission);
- if (!matchFound) {
- vXResponse.setMsgDesc("You're not permitted to perform "
- + "the action for resource path : " + resourceName);
- vXResponse.setStatusCode(VXResponse.STATUS_ERROR);
- return vXResponse;
- }
- }
- vXResponse.setStatusCode(VXResponse.STATUS_SUCCESS);
- return vXResponse;
- }
+ }
+ vXResponse.setStatusCode(VXResponse.STATUS_SUCCESS);
+ return vXResponse;
+ }
return vXResponse;
}
@@ -619,21 +626,26 @@ public class RangerBizUtil {
* @return
*/
public boolean matchHbasePolicy(String resourceName,
- List<XXResource> xResourceList, VXResponse vXResponse, Long xUserId,
- int permission) {
- if(stringUtil.isEmpty(resourceName) || xResourceList==null || xUserId==null){
+ List<XXResource> xResourceList, VXResponse vXResponse,
+ Long xUserId, int permission) {
+ if (stringUtil.isEmpty(resourceName) || xResourceList == null
+ || xUserId == null) {
return false;
}
- String[] splittedResources = stringUtil.split(resourceName, fileSeparator);
+ String[] splittedResources = stringUtil.split(resourceName,
+ fileSeparator);
if (splittedResources.length < 1 || splittedResources.length > 3) {
logger.debug("Invalid resourceName name : " + resourceName);
return false;
}
- String tblName = splittedResources.length > 0 ? splittedResources[0] : StringUtil.WILDCARD_ASTERISK;
- String colFamName = splittedResources.length > 1 ? splittedResources[1] : StringUtil.WILDCARD_ASTERISK;
- String colName = splittedResources.length > 2 ? splittedResources[2] : StringUtil.WILDCARD_ASTERISK;
+ String tblName = splittedResources.length > 0 ? splittedResources[0]
+ : StringUtil.WILDCARD_ASTERISK;
+ String colFamName = splittedResources.length > 1 ? splittedResources[1]
+ : StringUtil.WILDCARD_ASTERISK;
+ String colName = splittedResources.length > 2 ? splittedResources[2]
+ : StringUtil.WILDCARD_ASTERISK;
boolean policyMatched = false;
// check all resources whether Hbase policy is enabled in any resource
@@ -643,29 +655,38 @@ public class RangerBizUtil {
continue;
}
Long resourceId = xResource.getId();
- boolean hasPermission = checkUsrPermForPolicy(xUserId, permission, resourceId);
+ boolean hasPermission = checkUsrPermForPolicy(xUserId, permission,
+ resourceId);
// if permission is enabled then load Tables,column family and
// columns list from resource
- if (! hasPermission) {
+ if (!hasPermission) {
continue;
}
// 1. does the policy match the table?
- String[] xTables = stringUtil.isEmpty(xResource.getTables()) ? null : stringUtil.split(xResource.getTables(), ",");
+ String[] xTables = stringUtil.isEmpty(xResource.getTables()) ? null
+ : stringUtil.split(xResource.getTables(), ",");
- boolean matchFound = (xTables == null || xTables.length == 0) ? true : matchPath(tblName, xTables);
+ boolean matchFound = (xTables == null || xTables.length == 0) ? true
+ : matchPath(tblName, xTables);
- if(matchFound) {
+ if (matchFound) {
// 2. does the policy match the column?
- String[] xColumnFamilies = stringUtil.isEmpty(xResource.getColumnFamilies()) ? null : stringUtil.split(xResource.getColumnFamilies(), ",");
+ String[] xColumnFamilies = stringUtil.isEmpty(xResource
+ .getColumnFamilies()) ? null : stringUtil.split(
+ xResource.getColumnFamilies(), ",");
+
+ matchFound = (xColumnFamilies == null || xColumnFamilies.length == 0) ? true
+ : matchPath(colFamName, xColumnFamilies);
- matchFound = (xColumnFamilies == null || xColumnFamilies.length == 0) ? true : matchPath(colFamName, xColumnFamilies);
-
- if(matchFound) {
+ if (matchFound) {
// 3. does the policy match the columnFamily?
- String[] xColumns = stringUtil.isEmpty(xResource.getColumns()) ? null : stringUtil.split(xResource.getColumns(), ",");
+ String[] xColumns = stringUtil.isEmpty(xResource
+ .getColumns()) ? null : stringUtil.split(
+ xResource.getColumns(), ",");
- matchFound = (xColumns == null || xColumns.length == 0) ? true : matchPath(colName, xColumns);
+ matchFound = (xColumns == null || xColumns.length == 0) ? true
+ : matchPath(colName, xColumns);
}
}
@@ -699,19 +720,24 @@ public class RangerBizUtil {
List<XXResource> xResourceList, Long xUserId, int permission,
int reqTableType, int reqColumnType, boolean isUdfPolicy) {
- if(stringUtil.isEmpty(resourceName) || xResourceList==null || xUserId==null){
+ if (stringUtil.isEmpty(resourceName) || xResourceList == null
+ || xUserId == null) {
return false;
}
- String[] splittedResources = stringUtil.split(resourceName, fileSeparator);// get list of resources
+ String[] splittedResources = stringUtil.split(resourceName,
+ fileSeparator);// get list of resources
if (splittedResources.length < 1 || splittedResources.length > 3) {
logger.debug("Invalid resource name : " + resourceName);
return false;
}
-
- String dbName = splittedResources.length > 0 ? splittedResources[0] : StringUtil.WILDCARD_ASTERISK;
- String tblName = splittedResources.length > 1 ? splittedResources[1] : StringUtil.WILDCARD_ASTERISK;
- String colName = splittedResources.length > 2 ? splittedResources[2] : StringUtil.WILDCARD_ASTERISK;
+
+ String dbName = splittedResources.length > 0 ? splittedResources[0]
+ : StringUtil.WILDCARD_ASTERISK;
+ String tblName = splittedResources.length > 1 ? splittedResources[1]
+ : StringUtil.WILDCARD_ASTERISK;
+ String colName = splittedResources.length > 2 ? splittedResources[2]
+ : StringUtil.WILDCARD_ASTERISK;
boolean policyMatched = false;
for (XXResource xResource : xResourceList) {
@@ -720,21 +746,24 @@ public class RangerBizUtil {
}
Long resourceId = xResource.getId();
- boolean hasPermission = checkUsrPermForPolicy(xUserId, permission, resourceId);
+ boolean hasPermission = checkUsrPermForPolicy(xUserId, permission,
+ resourceId);
- if (! hasPermission) {
+ if (!hasPermission) {
continue;
}
// 1. does the policy match the database?
- String[] xDatabases = stringUtil.isEmpty(xResource.getDatabases()) ? null : stringUtil.split(xResource.getDatabases(), ",");
+ String[] xDatabases = stringUtil.isEmpty(xResource.getDatabases()) ? null
+ : stringUtil.split(xResource.getDatabases(), ",");
- boolean matchFound = (xDatabases == null || xDatabases.length == 0) ? true : matchPath(dbName, xDatabases);
+ boolean matchFound = (xDatabases == null || xDatabases.length == 0) ? true
+ : matchPath(dbName, xDatabases);
- if (! matchFound) {
+ if (!matchFound) {
continue;
}
-
+
// Type(either UDFs policy or non-UDFs policy) of current policy
// should be of same as type of policy being iterated
if (!stringUtil.isEmpty(xResource.getUdfs()) && !isUdfPolicy) {
@@ -743,9 +772,10 @@ public class RangerBizUtil {
if (isUdfPolicy) {
// 2. does the policy match the UDF?
- String[] xUdfs = stringUtil.isEmpty(xResource.getUdfs()) ? null : stringUtil.split(xResource.getUdfs(), ",");
-
- if(! matchPath(tblName, xUdfs)) {
+ String[] xUdfs = stringUtil.isEmpty(xResource.getUdfs()) ? null
+ : stringUtil.split(xResource.getUdfs(), ",");
+
+ if (!matchPath(tblName, xUdfs)) {
continue;
} else {
policyMatched = true;
@@ -753,11 +783,13 @@ public class RangerBizUtil {
}
} else {
// 2. does the policy match the table?
- String[] xTables = stringUtil.isEmpty(xResource.getTables()) ? null : stringUtil.split(xResource.getTables(), ",");
+ String[] xTables = stringUtil.isEmpty(xResource.getTables()) ? null
+ : stringUtil.split(xResource.getTables(), ",");
- matchFound = (xTables == null || xTables.length == 0) ? true : matchPath(tblName, xTables);
+ matchFound = (xTables == null || xTables.length == 0) ? true
+ : matchPath(tblName, xTables);
- if(xResource.getTableType() == AppConstants.POLICY_EXCLUSION) {
+ if (xResource.getTableType() == AppConstants.POLICY_EXCLUSION) {
matchFound = !matchFound;
}
@@ -766,11 +798,13 @@ public class RangerBizUtil {
}
// 3. does current policy match the column?
- String[] xColumns = stringUtil.isEmpty(xResource.getColumns()) ? null : stringUtil.split(xResource.getColumns(), ",");
+ String[] xColumns = stringUtil.isEmpty(xResource.getColumns()) ? null
+ : stringUtil.split(xResource.getColumns(), ",");
- matchFound = (xColumns == null || xColumns.length == 0) ? true : matchPath(colName, xColumns);
+ matchFound = (xColumns == null || xColumns.length == 0) ? true
+ : matchPath(colName, xColumns);
- if(xResource.getColumnType() == AppConstants.POLICY_EXCLUSION) {
+ if (xResource.getColumnType() == AppConstants.POLICY_EXCLUSION) {
matchFound = !matchFound;
}
@@ -784,6 +818,7 @@ public class RangerBizUtil {
}
return policyMatched;
}
+
/**
* returns true if user is having required permission on given Hbase
* resource
@@ -796,8 +831,8 @@ public class RangerBizUtil {
* @return
*/
private boolean matchKnoxPolicy(String resourceName,
- List<XXResource> xResourceList, VXResponse vXResponse, Long xUserId,
- int permission) {
+ List<XXResource> xResourceList, VXResponse vXResponse,
+ Long xUserId, int permission) {
String[] splittedResources = stringUtil.split(resourceName,
fileSeparator);
@@ -817,11 +852,12 @@ public class RangerBizUtil {
Long resourceId = xResource.getId();
boolean hasPermission = checkUsrPermForPolicy(xUserId, permission,
resourceId);
- // if permission is enabled then load Topologies,services list from resource
+ // if permission is enabled then load Topologies,services list from
+ // resource
if (hasPermission) {
String[] xTopologies = (xResource.getTopologies() == null || xResource
- .getTopologies().equalsIgnoreCase("")) ? null : stringUtil
- .split(xResource.getTopologies(), ",");
+ .getTopologies().equalsIgnoreCase("")) ? null
+ : stringUtil.split(xResource.getTopologies(), ",");
String[] xServices = (xResource.getServices() == null || xResource
.getServices().equalsIgnoreCase("")) ? null
: stringUtil.split(xResource.getServices(), ",");
@@ -833,30 +869,31 @@ public class RangerBizUtil {
// check whether given table resource matches with any
// existing topology resource
if (index == 0) {
- if(xTopologies!=null){
- for (String xTopology : xTopologies) {
- if (matchPath(splittedResources[index], xTopology)) {
- matchFound = true;
- continue;
+ if (xTopologies != null) {
+ for (String xTopology : xTopologies) {
+ if (matchPath(splittedResources[index],
+ xTopology)) {
+ matchFound = true;
+ continue;
+ }
}
}
- }
- if(!matchFound) {
+ if (!matchFound) {
break;
}
} // check whether given service resource matches with
// any existing service resource
else if (index == 1) {
- if(xServices!=null){
- for (String xService : xServices) {
- if (matchPath(splittedResources[index],
- xService)) {
- matchFound = true;
- continue;
+ if (xServices != null) {
+ for (String xService : xServices) {
+ if (matchPath(splittedResources[index],
+ xService)) {
+ matchFound = true;
+ continue;
+ }
}
}
- }
- if(!matchFound) {
+ if (!matchFound) {
break;
}
}
@@ -870,85 +907,84 @@ public class RangerBizUtil {
return policyMatched;
}
- /**
- * returns true if user is having required permission on given STORM
- * resource
- *
- * @param resourceName
- * @param xResourceList
- * @param vXResponse
- * @param xUserId
- * @param permission
- * @return
- */
- private boolean matchStormPolicy(String resourceName,
- List<XXResource> xResourceList, VXResponse vXResponse, Long xUserId,
- int permission) {
-
- String[] splittedResources = stringUtil.split(resourceName,
- fileSeparator);
- int numberOfResources = splittedResources.length;
- if (numberOfResources < 1 || numberOfResources > 3) {
- logger.debug("Invalid policy name : " + resourceName);
- return false;
- }
-
- boolean policyMatched = false;
- // check all resources whether Knox policy is enabled in any resource
- // of provided resource list
- for (XXResource xResource : xResourceList) {
- if (xResource.getResourceStatus() != AppConstants.STATUS_ENABLED) {
- continue;
- }
- Long resourceId = xResource.getId();
- boolean hasPermission = checkUsrPermForPolicy(xUserId, permission,
- resourceId);
- // if permission is enabled then load Topologies,services list from resource
- if (hasPermission) {
- String[] xTopologies = (xResource.getTopologies() == null || xResource
- .getTopologies().equalsIgnoreCase("")) ? null : stringUtil
- .split(xResource.getTopologies(), ",");
- /*String[] xServices = (xResource.getServices() == null || xResource
- .getServices().equalsIgnoreCase("")) ? null
- : stringUtil.split(xResource.getServices(), ",");*/
-
- boolean matchFound = false;
-
- for (int index = 0; index < numberOfResources; index++) {
- matchFound = false;
- // check whether given table resource matches with any
- // existing topology resource
- if (index == 0) {
- if(xTopologies!=null){
- for (String xTopology : xTopologies) {
- if (matchPath(splittedResources[index], xTopology)) {
- matchFound = true;
- continue;
- }
- }
- }
- } // check whether given service resource matches with
- // any existing service resource
- /*else if (index == 1) {
- if(xServices!=null){
- for (String xService : xServices) {
- if (matchPath(splittedResources[index],
- xService)) {
- matchFound = true;
- continue;
- }
- }
- }
- }*/
- }
- if (matchFound) {
- policyMatched = true;
- break;
- }
- }
- }
- return policyMatched;
- }
+ /**
+ * returns true if user is having required permission on given STORM
+ * resource
+ *
+ * @param resourceName
+ * @param xResourceList
+ * @param vXResponse
+ * @param xUserId
+ * @param permission
+ * @return
+ */
+ private boolean matchStormPolicy(String resourceName,
+ List<XXResource> xResourceList, VXResponse vXResponse,
+ Long xUserId, int permission) {
+
+ String[] splittedResources = stringUtil.split(resourceName,
+ fileSeparator);
+ int numberOfResources = splittedResources.length;
+ if (numberOfResources < 1 || numberOfResources > 3) {
+ logger.debug("Invalid policy name : " + resourceName);
+ return false;
+ }
+
+ boolean policyMatched = false;
+ // check all resources whether Knox policy is enabled in any resource
+ // of provided resource list
+ for (XXResource xResource : xResourceList) {
+ if (xResource.getResourceStatus() != AppConstants.STATUS_ENABLED) {
+ continue;
+ }
+ Long resourceId = xResource.getId();
+ boolean hasPermission = checkUsrPermForPolicy(xUserId, permission,
+ resourceId);
+ // if permission is enabled then load Topologies,services list from
+ // resource
+ if (hasPermission) {
+ String[] xTopologies = (xResource.getTopologies() == null || xResource
+ .getTopologies().equalsIgnoreCase("")) ? null
+ : stringUtil.split(xResource.getTopologies(), ",");
+ /*
+ * String[] xServices = (xResource.getServices() == null ||
+ * xResource .getServices().equalsIgnoreCase("")) ? null :
+ * stringUtil.split(xResource.getServices(), ",");
+ */
+
+ boolean matchFound = false;
+
+ for (int index = 0; index < numberOfResources; index++) {
+ matchFound = false;
+ // check whether given table resource matches with any
+ // existing topology resource
+ if (index == 0) {
+ if (xTopologies != null) {
+ for (String xTopology : xTopologies) {
+ if (matchPath(splittedResources[index],
+ xTopology)) {
+ matchFound = true;
+ continue;
+ }
+ }
+ }
+ } // check whether given service resource matches with
+ // any existing service resource
+ /*
+ * else if (index == 1) { if(xServices!=null){ for (String
+ * xService : xServices) { if
+ * (matchPath(splittedResources[index], xService)) {
+ * matchFound = true; continue; } } } }
+ */
+ }
+ if (matchFound) {
+ policyMatched = true;
+ break;
+ }
+ }
+ }
+ return policyMatched;
+ }
/**
* returns path without meta characters
@@ -1027,8 +1063,11 @@ public class RangerBizUtil {
for (XXPermMap permMap : permMapList) {
if (permMap.getPermType() == permission) {
if (permMap.getPermFor() == AppConstants.XA_PERM_FOR_GROUP) {
- // check whether permission is enabled for public group or a group to which user belongs
- matchFound = (publicGroupId != null && publicGroupId == permMap.getGroupId()) || isGroupInList(permMap.getGroupId(), userGroups);
+ // check whether permission is enabled for public group or a
+ // group to which user belongs
+ matchFound = (publicGroupId != null && publicGroupId == permMap
+ .getGroupId())
+ || isGroupInList(permMap.getGroupId(), userGroups);
} else if (permMap.getPermFor() == AppConstants.XA_PERM_FOR_USER) {
// check whether permission is enabled to user
matchFound = permMap.getUserId().equals(xUserId);
@@ -1040,9 +1079,10 @@ public class RangerBizUtil {
}
return matchFound;
}
-
+
public Long getPublicGroupId() {
- XXGroup xXGroupPublic = daoManager.getXXGroup().findByGroupName(RangerConstants.GROUP_PUBLIC);
+ XXGroup xXGroupPublic = daoManager.getXXGroup().findByGroupName(
+ RangerConstants.GROUP_PUBLIC);
return xXGroupPublic != null ? xXGroupPublic.getId() : null;
}
@@ -1194,7 +1234,7 @@ public class RangerBizUtil {
*/
private boolean matchPath(String pathToCheckFragment,
String wildCardPathFragment) {
- if(pathToCheckFragment == null || wildCardPathFragment == null) {
+ if (pathToCheckFragment == null || wildCardPathFragment == null) {
return false;
}
@@ -1220,7 +1260,7 @@ public class RangerBizUtil {
}
}
}
-
+
private boolean matchPath(String pathToCheck, String[] wildCardPaths) {
if (pathToCheck != null && wildCardPaths != null) {
for (String wildCardPath : wildCardPaths) {
@@ -1229,7 +1269,7 @@ public class RangerBizUtil {
}
}
}
-
+
return false;
}
@@ -1308,4 +1348,12 @@ public class RangerBizUtil {
}
}
+ public String getAuditDBType() {
+ return auditDBType;
+ }
+
+ public void setAuditDBType(String auditDBType) {
+ this.auditDBType = auditDBType;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
index 9d69a95..d9812f9 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/XAuditMgr.java
@@ -17,9 +17,46 @@
* under the License.
*/
- package org.apache.ranger.biz;
+package org.apache.ranger.biz;
+
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.solr.SolrAccessAuditsService;
+import org.apache.ranger.view.VXAccessAudit;
+import org.apache.ranger.view.VXAccessAuditList;
+import org.apache.ranger.view.VXLong;
+import org.springframework.beans.factory.annotation.Autowired;
public class XAuditMgr extends XAuditMgrBase {
-}
+ @Autowired
+ SolrAccessAuditsService solrAccessAuditsService;
+
+ @Autowired
+ RangerBizUtil rangerBizUtil;
+
+ @Override
+ public VXAccessAudit getXAccessAudit(Long id) {
+ // TODO Auto-generated method stub
+ return super.getXAccessAudit(id);
+ }
+ @Override
+ public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
+ if (rangerBizUtil.getAuditDBType().equalsIgnoreCase("solr")) {
+ return solrAccessAuditsService.searchXAccessAudits(searchCriteria);
+ } else {
+ return super.searchXAccessAudits(searchCriteria);
+ }
+ }
+
+ @Override
+ public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) {
+ if (rangerBizUtil.getAuditDBType().equalsIgnoreCase("solr")) {
+ return solrAccessAuditsService
+ .getXAccessAuditSearchCount(searchCriteria);
+ } else {
+ return super.getXAccessAuditSearchCount(searchCriteria);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
index b1482b6..962eb02 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java
@@ -321,6 +321,9 @@ public abstract class RangerDaoManagerBase {
}
public XXAccessAuditDao getXXAccessAudit() {
+ //Load appropriate class based on audit store
+ //TODO: Need to fix this, currently hard coding Solr
+
return new XXAccessAuditDao(this);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
new file mode 100644
index 0000000..d5d68b1
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrAccessAuditsService.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.solr;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.common.MessageEnums;
+import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.common.SearchField;
+import org.apache.ranger.common.SortField;
+import org.apache.ranger.common.StringUtil;
+import org.apache.ranger.common.SearchField.DATA_TYPE;
+import org.apache.ranger.common.SearchField.SEARCH_TYPE;
+import org.apache.ranger.common.SortField.SORT_ORDER;
+import org.apache.ranger.view.VXAccessAudit;
+import org.apache.ranger.view.VXAccessAuditList;
+import org.apache.ranger.view.VXLong;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Service;
+
+@Service
+@Scope("singleton")
+public class SolrAccessAuditsService {
+ static Logger logger = Logger.getLogger(SolrAccessAuditsService.class);
+
+ @Autowired
+ SolrMgr solrMgr;
+
+ @Autowired
+ SolrUtil solrUtil;
+
+ @Autowired
+ RESTErrorUtil restErrorUtil;
+
+ @Autowired
+ StringUtil stringUtil;
+
+ public List<SortField> sortFields = new ArrayList<SortField>();
+ public List<SearchField> searchFields = new ArrayList<SearchField>();
+
+ public SolrAccessAuditsService() {
+
+ searchFields.add(new SearchField("id", "id",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("accessType", "access",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("aclEnforcer", "enforcer",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("agentId", "agent",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("repoName", "repo",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("sessionId", "sess",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("requestUser", "reqUser",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("requestData", "reqData",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL));
+ searchFields.add(new SearchField("resourcePath", "resource",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.PARTIAL));
+ searchFields.add(new SearchField("clientIP", "cliIP",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+
+ searchFields.add(new SearchField("auditType", "logType",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("accessResult", "result",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ // searchFields.add(new SearchField("assetId", "obj.assetId",
+ // SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("policyId", "policy",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("repoType", "repoType",
+ SearchField.DATA_TYPE.INTEGER, SearchField.SEARCH_TYPE.FULL));
+
+ searchFields.add(new SearchField("resourceType", "resType",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("reason", "reason",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+ searchFields.add(new SearchField("action", "action",
+ SearchField.DATA_TYPE.STRING, SearchField.SEARCH_TYPE.FULL));
+
+ searchFields.add(new SearchField("startDate", "evtTime",
+ DATA_TYPE.DATE, SEARCH_TYPE.GREATER_EQUAL_THAN));
+ searchFields.add(new SearchField("endDate", "evtTime", DATA_TYPE.DATE,
+ SEARCH_TYPE.LESS_EQUAL_THAN));
+
+ sortFields.add(new SortField("eventTime", "evtTime", true,
+ SORT_ORDER.DESC));
+ }
+
+ public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
+
+ // Make call to Solr
+ SolrClient solrClient = solrMgr.getSolrClient();
+
+ if (solrClient == null) {
+ logger.warn("Solr client is null, so not running the query.");
+ throw restErrorUtil.createRESTException(
+ "Error connecting to search engine",
+ MessageEnums.ERROR_SYSTEM);
+ }
+
+ VXAccessAuditList returnList = new VXAccessAuditList();
+ List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>();
+
+ QueryResponse response = solrUtil.searchResources(searchCriteria,
+ searchFields, sortFields, solrClient);
+ if (response == null) {
+ logger.warn("Error running search query. searchCriteria="
+ + searchCriteria.toString());
+ throw restErrorUtil.createRESTException(
+ "Error running search query", MessageEnums.ERROR_SYSTEM);
+ }
+ SolrDocumentList docs = response.getResults();
+ for (int i = 0; i < docs.size(); i++) {
+ SolrDocument doc = docs.get(i);
+ VXAccessAudit vXAccessAudit = populateViewBean(doc);
+ xAccessAuditList.add(vXAccessAudit);
+ }
+
+ returnList.setResultSize((int) docs.getNumFound());
+ returnList.setStartIndex((int) docs.getStart());
+ returnList.setVXAccessAudits(xAccessAuditList);
+
+ return returnList;
+ }
+
+ /**
+ * @param doc
+ * @return
+ */
+ private VXAccessAudit populateViewBean(SolrDocument doc) {
+ VXAccessAudit accessAudit = new VXAccessAudit();
+ Object value = null;
+ logger.info("doc=" + doc.toString());
+
+ value = doc.getFieldValue("id");
+ if (value != null) {
+ // TODO: Converting ID to hashcode for now
+ accessAudit.setId((long) value.hashCode());
+ }
+
+ value = doc.getFieldValue("access");
+ if (value != null) {
+ accessAudit.setAccessType(value.toString());
+ }
+
+ value = doc.getFieldValue("enforcer");
+ if (value != null) {
+ accessAudit.setAclEnforcer(value.toString());
+ }
+ value = doc.getFieldValue("agent");
+ if (value != null) {
+ accessAudit.setAgentId(value.toString());
+ }
+ value = doc.getFieldValue("repo");
+ if (value != null) {
+ accessAudit.setRepoName(value.toString());
+ }
+ value = doc.getFieldValue("sess");
+ if (value != null) {
+ accessAudit.setSessionId(value.toString());
+ }
+ value = doc.getFieldValue("reqUser");
+ if (value != null) {
+ accessAudit.setRequestUser(value.toString());
+ }
+ value = doc.getFieldValue("reqData");
+ if (value != null) {
+ accessAudit.setRequestData(value.toString());
+ }
+ value = doc.getFieldValue("resource");
+ if (value != null) {
+ accessAudit.setResourcePath(value.toString());
+ }
+ value = doc.getFieldValue("cliIP");
+ if (value != null) {
+ accessAudit.setClientIP(value.toString());
+ }
+ value = doc.getFieldValue("logType");
+ if (value != null) {
+ // TODO: Need to see what logType maps to in UI
+// accessAudit.setAuditType(solrUtil.toInt(value));
+ }
+ value = doc.getFieldValue("result");
+ if (value != null) {
+ accessAudit.setAccessResult(solrUtil.toInt(value));
+ }
+ value = doc.getFieldValue("policy");
+ if (value != null) {
+ accessAudit.setPolicyId(solrUtil.toLong(value));
+ }
+ value = doc.getFieldValue("repoType");
+ if (value != null) {
+ accessAudit.setRepoType(solrUtil.toInt(value));
+ }
+ value = doc.getFieldValue("resType");
+ if (value != null) {
+ accessAudit.setResourceType(value.toString());
+ }
+ value = doc.getFieldValue("reason");
+ if (value != null) {
+ accessAudit.setResultReason(value.toString());
+ }
+ value = doc.getFieldValue("action");
+ if (value != null) {
+ accessAudit.setAction(value.toString());
+ }
+ value = doc.getFieldValue("evtTime");
+ if (value != null) {
+ accessAudit.setEventTime(solrUtil.toDate(value));
+ }
+ return accessAudit;
+ }
+
+ /**
+ * @param searchCriteria
+ * @return
+ */
+ public VXLong getXAccessAuditSearchCount(SearchCriteria searchCriteria) {
+ long count = 100;
+
+ VXLong vXLong = new VXLong();
+ vXLong.setValue(count);
+ return vXLong;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java
new file mode 100644
index 0000000..60ef902
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrMgr.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.solr;
+
+import java.util.Date;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.biz.RangerBizUtil;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+/**
+ * This class initializes Solr
+ *
+ */
+@Component
+public class SolrMgr {
+
+ static final Logger logger = Logger.getLogger(SolrMgr.class);
+
+ @Autowired
+ RangerBizUtil rangerBizUtil;
+
+ static final Object lock = new Object();
+
+ SolrClient solrClient = null;
+ Date lastConnectTime = null;
+ boolean initDone = false;
+
+ public SolrMgr() {
+
+ }
+
+ void connect() {
+ if (!initDone) {
+ synchronized (lock) {
+ if (!initDone) {
+ if (rangerBizUtil.getAuditDBType().equalsIgnoreCase("solr")) {
+ String solrURL = PropertiesUtil
+ .getProperty("xa.audit.solr.url");
+ if (solrURL == null || solrURL.isEmpty()) {
+ logger.fatal("Solr URL for Audit is empty");
+ }
+ try {
+ solrClient = new HttpSolrClient(solrURL);
+ if (solrClient == null) {
+ logger.fatal("Can't connect to Solr. URL="
+ + solrURL);
+ } else {
+ initDone = true;
+ if (solrClient instanceof HttpSolrClient) {
+ HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient;
+ httpSolrClient.setAllowCompression(true);
+ httpSolrClient.setConnectionTimeout(1000);
+ // httpSolrClient.setSoTimeout(10000);
+ httpSolrClient.setMaxRetries(1);
+ httpSolrClient.setRequestWriter(new BinaryRequestWriter());
+ }
+ }
+
+ } catch (Throwable t) {
+ logger.fatal("Can't connect to Solr server. URL="
+ + solrURL, t);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public SolrClient getSolrClient() {
+ if (solrClient == null) {
+ connect();
+ }
+ return solrClient;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java
new file mode 100644
index 0000000..2ce63a7
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/solr/SolrUtil.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.solr;
+
+import java.text.SimpleDateFormat;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.apache.log4j.Logger;
+import org.apache.ranger.common.MessageEnums;
+import org.apache.ranger.common.PropertiesUtil;
+import org.apache.ranger.common.RESTErrorUtil;
+import org.apache.ranger.common.SearchCriteria;
+import org.apache.ranger.common.SearchField;
+import org.apache.ranger.common.SortField;
+import org.apache.ranger.common.StringUtil;
+import org.apache.ranger.common.SearchField.SEARCH_TYPE;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrQuery.ORDER;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class SolrUtil {
+ static final Logger logger = Logger.getLogger(SolrUtil.class);
+
+ @Autowired
+ RESTErrorUtil restErrorUtil;
+
+ @Autowired
+ StringUtil stringUtil;
+
+ SimpleDateFormat dateFormat = new SimpleDateFormat(
+ "yyyy-MM-dd'T'HH:mm:ss'Z'");
+
+ public SolrUtil() {
+ String timeZone = PropertiesUtil.getProperty("xa.solr.timezone");
+ if (timeZone != null) {
+ logger.info("Setting timezone to " + timeZone);
+ try {
+ dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
+ } catch (Throwable t) {
+ logger.error("Error setting timezone. timeZone=" + timeZone);
+ }
+ }
+ }
+
+ public QueryResponse runQuery(SolrClient solrClient, SolrQuery solrQuery) {
+ if (solrQuery != null) {
+ QueryResponse response;
+ try {
+ response = solrClient.query(solrQuery);
+ return response;
+ } catch (Throwable e) {
+ logger.error("Error from Solr server.", e);
+ }
+ }
+ return null;
+ }
+
+ public QueryResponse searchResources(SearchCriteria searchCriteria,
+ List<SearchField> searchFields, List<SortField> sortFieldList,
+ SolrClient solrClient) {
+ SolrQuery query = new SolrQuery();
+ query.setQuery("*:*");
+ if (searchCriteria.getParamList() != null) {
+ // For now assuming there is only date field where range query will
+ // be done. If we there are more than one, then we should create a
+ // hashmap for each field name
+ Date fromDate = null;
+ Date toDate = null;
+ String dateFieldName = null;
+
+ for (SearchField searchField : searchFields) {
+ Object paramValue = searchCriteria.getParamValue(searchField
+ .getClientFieldName());
+ if (paramValue == null || paramValue.toString().isEmpty()) {
+ continue;
+ }
+ String fieldName = searchField.getFieldName();
+ if (paramValue instanceof Collection) {
+ String fq = orList(fieldName, (Collection<?>) paramValue);
+ if (fq != null) {
+ query.addFilterQuery(fq);
+ }
+ } else if (searchField.getDataType() == SearchField.DATA_TYPE.DATE) {
+ if (!(paramValue instanceof Date)) {
+ logger.error("Search file is not of java object instanceof Date");
+ } else {
+ if (searchField.getSearchType() == SEARCH_TYPE.GREATER_EQUAL_THAN
+ || searchField.getSearchType() == SEARCH_TYPE.GREATER_THAN) {
+ fromDate = (Date) paramValue;
+ dateFieldName = fieldName;
+ } else if (searchField.getSearchType() == SEARCH_TYPE.LESS_EQUAL_THAN
+ || searchField.getSearchType() == SEARCH_TYPE.LESS_THAN) {
+ toDate = (Date) paramValue;
+ }
+ }
+ } else if (searchField.getSearchType() == SEARCH_TYPE.GREATER_EQUAL_THAN
+ || searchField.getSearchType() == SEARCH_TYPE.GREATER_THAN
+ || searchField.getSearchType() == SEARCH_TYPE.LESS_EQUAL_THAN
+ || searchField.getSearchType() == SEARCH_TYPE.LESS_THAN) {
+ // TODO: Need to handle range here
+ } else {
+ String fq = setField(fieldName, paramValue);
+ if (fq != null) {
+ query.addFilterQuery(fq);
+ }
+ }
+ }
+ if (fromDate != null || toDate != null) {
+ String fq = setDateRange(dateFieldName, fromDate, toDate);
+ if (fq != null) {
+ query.addFilterQuery(fq);
+ }
+ }
+ }
+
+ setSortClause(searchCriteria, sortFieldList, query);
+ query.setStart(searchCriteria.getStartIndex());
+ query.setRows(searchCriteria.getMaxRows());
+
+ // Fields to get
+ // query.setFields("myClassType", "id", "score", "globalId");
+ if (logger.isDebugEnabled()) {
+ logger.debug("SOLR QUERY=" + query.toString());
+ }
+ QueryResponse response = runQuery(solrClient, query);
+
+ if (response == null || response.getStatus() != 0) {
+ logger.error("Error running query. query=" + query.toString()
+ + ", response=" + response);
+ throw restErrorUtil.createRESTException("Error running query",
+ MessageEnums.ERROR_SYSTEM);
+ }
+ return response;
+ }
+
+ public String setField(String fieldName, Object value) {
+ if (value == null || value.toString().trim().length() == 0) {
+ return null;
+ }
+ return fieldName
+ + ":"
+ + ClientUtils.escapeQueryChars(value.toString().trim()
+ .toLowerCase());
+ }
+
+ public String setDateRange(String fieldName, Date fromDate, Date toDate) {
+ String fromStr = "*";
+ String toStr = "NOW";
+ if (fromDate != null) {
+ fromStr = dateFormat.format(fromDate);
+ }
+ if (toDate != null) {
+ toStr = dateFormat.format(toDate);
+ }
+ return fieldName + ":[" + fromStr + " TO " + toStr + "]";
+ }
+
+ public String orList(String fieldName, Collection<?> valueList) {
+ if (valueList == null || valueList.size() == 0) {
+ return null;
+ }
+ String expr = "";
+ int count = -1;
+ for (Object value : valueList) {
+ count++;
+ if (count > 0) {
+ expr += " OR ";
+ }
+ expr += fieldName
+ + ":"
+ + ClientUtils.escapeQueryChars(value.toString()
+ .toLowerCase());
+ }
+ if (valueList.size() == 0) {
+ return expr;
+ } else {
+ return "(" + expr + ")";
+ }
+
+ }
+
+ public String andList(String fieldName, Collection<?> valueList) {
+ if (valueList == null || valueList.size() == 0) {
+ return null;
+ }
+ String expr = "";
+ int count = -1;
+ for (Object value : valueList) {
+ count++;
+ if (count > 0) {
+ expr += " AND ";
+ }
+ expr += fieldName
+ + ":"
+ + ClientUtils.escapeQueryChars(value.toString()
+ .toLowerCase());
+ }
+ if (valueList.size() == 0) {
+ return expr;
+ } else {
+ return "(" + expr + ")";
+ }
+ }
+
+ public void setSortClause(SearchCriteria searchCriteria,
+ List<SortField> sortFields, SolrQuery query) {
+
+ // TODO: We are supporting single sort field only for now
+ String sortBy = searchCriteria.getSortBy();
+ String querySortBy = null;
+ if (!stringUtil.isEmpty(sortBy)) {
+ sortBy = sortBy.trim();
+ for (SortField sortField : sortFields) {
+ if (sortBy.equalsIgnoreCase(sortField.getParamName())) {
+ querySortBy = sortField.getFieldName();
+ // Override the sortBy using the normalized value
+ searchCriteria.setSortBy(sortField.getParamName());
+ break;
+ }
+ }
+ }
+
+ if (querySortBy == null) {
+ for (SortField sortField : sortFields) {
+ if (sortField.isDefault()) {
+ querySortBy = sortField.getFieldName();
+ // Override the sortBy using the default value
+ searchCriteria.setSortBy(sortField.getParamName());
+ searchCriteria.setSortType(sortField.getDefaultOrder()
+ .name());
+ break;
+ }
+ }
+ }
+
+ if (querySortBy != null) {
+ // Add sort type
+ String sortType = searchCriteria.getSortType();
+ ORDER order = ORDER.asc;
+ if (sortType != null && sortType.equalsIgnoreCase("desc")) {
+ order = ORDER.desc;
+
+ }
+ query.addSort(querySortBy, order);
+ }
+ }
+
+ // Utility methods
+ public int toInt(Object value) {
+ if (value == null) {
+ return 0;
+ }
+ if (value instanceof Integer) {
+ return (Integer) value;
+ }
+ if (value.toString().isEmpty()) {
+ return 0;
+ }
+ try {
+ return new Integer(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to integer. value=" + value, t);
+ }
+ return 0;
+ }
+
+ public long toLong(Object value) {
+ if (value == null) {
+ return 0;
+ }
+ if (value instanceof Long) {
+ return (Long) value;
+ }
+ if (value.toString().isEmpty()) {
+ return 0;
+ }
+ try {
+ return new Long(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to long. value=" + value, t);
+ }
+ return 0;
+ }
+
+ public Date toDate(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof Date) {
+ return (Date) value;
+ }
+ try {
+ // TODO: Do proper parsing based on Solr response value
+ return new Date(value.toString());
+ } catch (Throwable t) {
+ logger.error("Error converting value to date. value=" + value, t);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/src/main/assembly/hbase-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hbase-agent.xml b/src/main/assembly/hbase-agent.xml
index 6ab569f..8dfbd5e 100644
--- a/src/main/assembly/hbase-agent.xml
+++ b/src/main/assembly/hbase-agent.xml
@@ -36,6 +36,12 @@
<includes>
<include>com.google.code.gson:gson*</include>
<include>org.eclipse.persistence:eclipselink</include>
+ <include>org.apache.ranger:ranger_solrj:jar:${ranger.solrj.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include>
+ <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include>
+ <include>org.noggit:noggit:jar:${noggit.version}</include>
+ <include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include>
</includes>
<unpack>false</unpack>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/src/main/assembly/hdfs-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hdfs-agent.xml b/src/main/assembly/hdfs-agent.xml
index 7929231..27c2bf0 100644
--- a/src/main/assembly/hdfs-agent.xml
+++ b/src/main/assembly/hdfs-agent.xml
@@ -37,6 +37,12 @@
<include>com.google.code.gson:gson*</include>
<include>org.eclipse.persistence:javax.persistence</include>
<include>org.eclipse.persistence:eclipselink</include>
+ <include>org.apache.ranger:ranger_solrj:jar:${ranger.solrj.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include>
+ <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include>
+ <include>org.noggit:noggit:jar:${noggit.version}</include>
+ <include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include>
</includes>
<unpack>false</unpack>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/src/main/assembly/hive-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/hive-agent.xml b/src/main/assembly/hive-agent.xml
index e9035b4..f0bacb6 100644
--- a/src/main/assembly/hive-agent.xml
+++ b/src/main/assembly/hive-agent.xml
@@ -37,6 +37,12 @@
<include>com.google.code.gson:gson*</include>
<include>org.eclipse.persistence:eclipselink</include>
<include>org.eclipse.persistence:javax.persistence</include>
+ <include>org.apache.ranger:ranger_solrj:jar:${ranger.solrj.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include>
+ <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include>
+ <include>org.noggit:noggit:jar:${noggit.version}</include>
+ <include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include>
</includes>
<unpack>false</unpack>
</dependencySet>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/src/main/assembly/knox-agent.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/knox-agent.xml b/src/main/assembly/knox-agent.xml
index 25604b2..fe7000a 100644
--- a/src/main/assembly/knox-agent.xml
+++ b/src/main/assembly/knox-agent.xml
@@ -41,6 +41,12 @@
<include>com.google.code.gson:gson*</include>
<include>org.eclipse.persistence:eclipselink</include>
<include>org.eclipse.persistence:javax.persistence</include>
+ <include>org.apache.ranger:ranger_solrj:jar:${ranger.solrj.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include>
+ <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include>
+ <include>org.noggit:noggit:jar:${noggit.version}</include>
+ <include>org.apache.zookeeper:zookeeper:jar:${zookeeper.version}</include>
</includes>
<unpack>false</unpack>
</dependencySet>
[05/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/FacetParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/FacetParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/FacetParams.java
new file mode 100644
index 0000000..2dae65f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/FacetParams.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import java.util.EnumSet;
+import java.util.Locale;
+
+import org.apache.solr.common.SolrException;
+
+/**
+ * Facet parameters
+ */
+public interface FacetParams {
+
+ /**
+ * Should facet counts be calculated?
+ */
+ public static final String FACET = "facet";
+
+ /**
+ * Numeric option indicating the maximum number of threads to be used
+ * in counting facet field vales
+ */
+ public static final String FACET_THREADS = FACET + ".threads";
+
+ /** What method should be used to do the faceting */
+ public static final String FACET_METHOD = FACET + ".method";
+
+ /** Value for FACET_METHOD param to indicate that Solr should enumerate over terms
+ * in a field to calculate the facet counts.
+ */
+ public static final String FACET_METHOD_enum = "enum";
+
+ /** Value for FACET_METHOD param to indicate that Solr should enumerate over documents
+ * and count up terms by consulting an uninverted representation of the field values
+ * (such as the FieldCache used for sorting).
+ */
+ public static final String FACET_METHOD_fc = "fc";
+
+ /** Value for FACET_METHOD param, like FACET_METHOD_fc but counts per-segment.
+ */
+ public static final String FACET_METHOD_fcs = "fcs";
+
+ /**
+ * Any lucene formated queries the user would like to use for
+ * Facet Constraint Counts (multi-value)
+ */
+ public static final String FACET_QUERY = FACET + ".query";
+ /**
+ * Any field whose terms the user wants to enumerate over for
+ * Facet Constraint Counts (multi-value)
+ */
+ public static final String FACET_FIELD = FACET + ".field";
+
+ /**
+ * The offset into the list of facets.
+ * Can be overridden on a per field basis.
+ */
+ public static final String FACET_OFFSET = FACET + ".offset";
+
+ /**
+ * Numeric option indicating the maximum number of facet field counts
+ * be included in the response for each field - in descending order of count.
+ * Can be overridden on a per field basis.
+ */
+ public static final String FACET_LIMIT = FACET + ".limit";
+
+ /**
+ * Numeric option indicating the minimum number of hits before a facet should
+ * be included in the response. Can be overridden on a per field basis.
+ */
+ public static final String FACET_MINCOUNT = FACET + ".mincount";
+
+ /**
+ * Boolean option indicating whether facet field counts of "0" should
+ * be included in the response. Can be overridden on a per field basis.
+ */
+ public static final String FACET_ZEROS = FACET + ".zeros";
+
+ /**
+ * Boolean option indicating whether the response should include a
+ * facet field count for all records which have no value for the
+ * facet field. Can be overridden on a per field basis.
+ */
+ public static final String FACET_MISSING = FACET + ".missing";
+
+
+ static final String FACET_OVERREQUEST = FACET + ".overrequest";
+
+ /**
+ * The percentage to over-request by when performing initial distributed requests.
+ *
+ * default value is 1.5
+ */
+ public static final String FACET_OVERREQUEST_RATIO = FACET_OVERREQUEST + ".ratio";
+
+ /**
+ * An additional amount to over-request by when performing initial distributed requests. This
+ * value will be added after accounting for the over-request ratio.
+ *
+ * default value is 10
+ */
+ public static final String FACET_OVERREQUEST_COUNT = FACET_OVERREQUEST + ".count";
+
+
+ /**
+ * Comma separated list of fields to pivot
+ *
+ * example: author,type (for types by author / types within author)
+ */
+ public static final String FACET_PIVOT = FACET + ".pivot";
+
+ /**
+ * Minimum number of docs that need to match to be included in the sublist
+ *
+ * default value is 1
+ */
+ public static final String FACET_PIVOT_MINCOUNT = FACET_PIVOT + ".mincount";
+
+
+ /**
+ * String option: "count" causes facets to be sorted
+ * by the count, "index" results in index order.
+ */
+ public static final String FACET_SORT = FACET + ".sort";
+
+ public static final String FACET_SORT_COUNT = "count";
+ public static final String FACET_SORT_COUNT_LEGACY = "true";
+ public static final String FACET_SORT_INDEX = "index";
+ public static final String FACET_SORT_INDEX_LEGACY = "false";
+
+ /**
+ * Only return constraints of a facet field with the given prefix.
+ */
+ public static final String FACET_PREFIX = FACET + ".prefix";
+
+ /**
+ * When faceting by enumerating the terms in a field,
+ * only use the filterCache for terms with a df >= to this parameter.
+ */
+ public static final String FACET_ENUM_CACHE_MINDF = FACET + ".enum.cache.minDf";
+ /**
+ * Any field whose terms the user wants to enumerate over for
+ * Facet Contraint Counts (multi-value)
+ */
+ public static final String FACET_DATE = FACET + ".date";
+ /**
+ * Date string indicating the starting point for a date facet range.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_DATE_START = FACET_DATE + ".start";
+ /**
+ * Date string indicating the endinging point for a date facet range.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_DATE_END = FACET_DATE + ".end";
+ /**
+ * Date Math string indicating the interval of sub-ranges for a date
+ * facet range.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_DATE_GAP = FACET_DATE + ".gap";
+ /**
+ * Boolean indicating how counts should be computed if the range
+ * between 'start' and 'end' is not evenly divisible by 'gap'. If
+ * this value is true, then all counts of ranges involving the 'end'
+ * point will use the exact endpoint specified -- this includes the
+ * 'between' and 'after' counts as well as the last range computed
+ * using the 'gap'. If the value is false, then 'gap' is used to
+ * compute the effective endpoint closest to the 'end' param which
+ * results in the range between 'start' and 'end' being evenly
+ * divisible by 'gap'.
+ * The default is false.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_DATE_HARD_END = FACET_DATE + ".hardend";
+ /**
+ * String indicating what "other" ranges should be computed for a
+ * date facet range (multi-value).
+ * Can be overriden on a per field basis.
+ * @see FacetRangeOther
+ */
+ public static final String FACET_DATE_OTHER = FACET_DATE + ".other";
+
+ /**
+ * <p>
+ * Multivalued string indicating what rules should be applied to determine
+ * when the the ranges generated for date faceting should be inclusive or
+ * exclusive of their end points.
+ * </p>
+ * <p>
+ * The default value if none are specified is: [lower,upper,edge] <i>(NOTE: This is different then FACET_RANGE_INCLUDE)</i>
+ * </p>
+ * <p>
+ * Can be overriden on a per field basis.
+ * </p>
+ * @see FacetRangeInclude
+ * @see #FACET_RANGE_INCLUDE
+ */
+ public static final String FACET_DATE_INCLUDE = FACET_DATE + ".include";
+
+ /**
+ * Any numerical field whose terms the user wants to enumerate over
+ * Facet Contraint Counts for selected ranges.
+ */
+ public static final String FACET_RANGE = FACET + ".range";
+ /**
+ * Number indicating the starting point for a numerical range facet.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_RANGE_START = FACET_RANGE + ".start";
+ /**
+ * Number indicating the ending point for a numerical range facet.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_RANGE_END = FACET_RANGE + ".end";
+ /**
+ * Number indicating the interval of sub-ranges for a numerical
+ * facet range.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_RANGE_GAP = FACET_RANGE + ".gap";
+ /**
+ * Boolean indicating how counts should be computed if the range
+ * between 'start' and 'end' is not evenly divisible by 'gap'. If
+ * this value is true, then all counts of ranges involving the 'end'
+ * point will use the exact endpoint specified -- this includes the
+ * 'between' and 'after' counts as well as the last range computed
+ * using the 'gap'. If the value is false, then 'gap' is used to
+ * compute the effective endpoint closest to the 'end' param which
+ * results in the range between 'start' and 'end' being evenly
+ * divisible by 'gap'.
+ * The default is false.
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_RANGE_HARD_END = FACET_RANGE + ".hardend";
+ /**
+ * String indicating what "other" ranges should be computed for a
+ * numerical range facet (multi-value).
+ * Can be overriden on a per field basis.
+ */
+ public static final String FACET_RANGE_OTHER = FACET_RANGE + ".other";
+
+ /**
+ * <p>
+ * Multivalued string indicating what rules should be applied to determine
+ * when the the ranges generated for numeric faceting should be inclusive or
+ * exclusive of their end points.
+ * </p>
+ * <p>
+ * The default value if none are specified is: lower
+ * </p>
+ * <p>
+ * Can be overriden on a per field basis.
+ * </p>
+ * @see FacetRangeInclude
+ */
+ public static final String FACET_RANGE_INCLUDE = FACET_RANGE + ".include";
+
+ /**
+ * Any field whose values the user wants to enumerate as explicit intervals of terms.
+ */
+ public static final String FACET_INTERVAL = FACET + ".interval";
+
+ /**
+ * Set of terms for a single interval to facet on.
+ */
+ public static final String FACET_INTERVAL_SET = FACET_INTERVAL + ".set";
+
+ /** A spatial RPT field to generate a 2D "heatmap" (grid of facet counts) on. Just like the other faceting types,
+ * this may include a 'key' or local-params to facet multiple times. All parameters with this suffix can be
+ * overridden on a per-field basis. */
+ public static final String FACET_HEATMAP = "facet.heatmap";
+
+ /** The format of the heatmap: either png or ints2D (default). */
+ public static final String FACET_HEATMAP_FORMAT = FACET_HEATMAP + ".format";
+
+ /** The region the heatmap should minimally enclose. It defaults to the world if not set. The format can either be
+ * a minimum to maximum point range format: <pre>["-150 10" TO "-100 30"]</pre> (the first is bottom-left and second
+ * is bottom-right, both of which are parsed as points are parsed). OR, any WKT can be provided and it's bounding
+ * box will be taken. */
+ public static final String FACET_HEATMAP_GEOM = FACET_HEATMAP + ".geom";
+
+ /** Specify the heatmap grid level explicitly, instead of deriving it via distErr or distErrPct. */
+ public static final String FACET_HEATMAP_LEVEL = FACET_HEATMAP + ".gridLevel";
+
+ /** Used to determine the heatmap grid level to compute, defaulting to 0.15. It has the same interpretation of
+ * distErrPct when searching on RPT, but relative to the shape in 'bbox'. It's a fraction (not a %) of the radius of
+ * the shape that grid squares must fit into without exceeding. > 0 and <= 0.5.
+ * Mutually exclusive with distErr & gridLevel. */
+ public static final String FACET_HEATMAP_DIST_ERR_PCT = FACET_HEATMAP + ".distErrPct";
+
+ /** Used to determine the heatmap grid level to compute (optional). It has the same interpretation of maxDistErr or
+ * distErr with RPT. It's an absolute distance (in units of what's specified on the field type) that a grid square
+ * must maximally fit into (width & height). It can be used to to more explicitly specify the maximum grid square
+ * size without knowledge of what particular grid levels translate to. This can in turn be used with
+ * knowledge of the size of 'bbox' to get a target minimum number of grid cells.
+ * Mutually exclusive with distErrPct & gridLevel. */
+ public static final String FACET_HEATMAP_DIST_ERR = FACET_HEATMAP + ".distErr";
+
+ /** The maximum number of cells (grid squares) the client is willing to handle. If this limit would be exceeded, we
+ * throw an error instead. Defaults to 100k. */
+ public static final String FACET_HEATMAP_MAX_CELLS = FACET_HEATMAP + ".maxCells";
+
+ /**
+ * An enumeration of the legal values for {@link #FACET_RANGE_OTHER} and {@link #FACET_DATE_OTHER} ...
+ * <ul>
+ * <li>before = the count of matches before the start</li>
+ * <li>after = the count of matches after the end</li>
+ * <li>between = the count of all matches between start and end</li>
+ * <li>all = all of the above (default value)</li>
+ * <li>none = no additional info requested</li>
+ * </ul>
+ * @see #FACET_RANGE_OTHER
+ * @see #FACET_DATE_OTHER
+ */
+ public enum FacetRangeOther {
+ BEFORE, AFTER, BETWEEN, ALL, NONE;
+ @Override
+ public String toString() { return super.toString().toLowerCase(Locale.ROOT); }
+ public static FacetRangeOther get(String label) {
+ try {
+ return valueOf(label.toUpperCase(Locale.ROOT));
+ } catch (IllegalArgumentException e) {
+ throw new SolrException
+ (SolrException.ErrorCode.BAD_REQUEST,
+ label+" is not a valid type of 'other' range facet information",e);
+ }
+ }
+ }
+
+ /**
+ * An enumeration of the legal values for {@link #FACET_DATE_INCLUDE} and {@link #FACET_RANGE_INCLUDE}
+ * <br>
+ * <ul>
+ * <li>lower = all gap based ranges include their lower bound</li>
+ * <li>upper = all gap based ranges include their upper bound</li>
+ * <li>edge = the first and last gap ranges include their edge bounds (ie: lower
+ * for the first one, upper for the last one) even if the corresponding
+ * upper/lower option is not specified
+ * </li>
+ * <li>outer = the BEFORE and AFTER ranges
+ * should be inclusive of their bounds, even if the first or last ranges
+ * already include those boundaries.
+ * </li>
+ * <li>all = shorthand for lower, upper, edge, and outer</li>
+ * </ul>
+ * @see #FACET_DATE_INCLUDE
+ * @see #FACET_RANGE_INCLUDE
+ */
+ public enum FacetRangeInclude {
+ ALL, LOWER, UPPER, EDGE, OUTER;
+ @Override
+ public String toString() { return super.toString().toLowerCase(Locale.ROOT); }
+ public static FacetRangeInclude get(String label) {
+ try {
+ return valueOf(label.toUpperCase(Locale.ROOT));
+ } catch (IllegalArgumentException e) {
+ throw new SolrException
+ (SolrException.ErrorCode.BAD_REQUEST,
+ label+" is not a valid type of for range 'include' information",e);
+ }
+ }
+ /**
+ * Convinience method for parsing the param value according to the
+ * correct semantics and applying the default of "LOWER"
+ */
+ public static EnumSet<FacetRangeInclude> parseParam(final String[] param) {
+ // short circut for default behavior
+ if (null == param || 0 == param.length )
+ return EnumSet.of(LOWER);
+
+ // build up set containing whatever is specified
+ final EnumSet<FacetRangeInclude> include
+ = EnumSet.noneOf(FacetRangeInclude.class);
+ for (final String o : param) {
+ include.add(FacetRangeInclude.get(o));
+ }
+
+ // if set contains all, then we're back to short circuting
+ if (include.contains(FacetRangeInclude.ALL))
+ return EnumSet.allOf(FacetRangeInclude.class);
+
+ // use whatever we've got.
+ return include;
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/GroupParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/GroupParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/GroupParams.java
new file mode 100644
index 0000000..ca8e2c4
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/GroupParams.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * Group parameters
+ */
+public interface GroupParams {
+ public static final String GROUP = "group";
+
+ public static final String GROUP_QUERY = GROUP + ".query";
+ public static final String GROUP_FIELD = GROUP + ".field";
+ public static final String GROUP_FUNC = GROUP + ".func";
+ public static final String GROUP_SORT = GROUP + ".sort";
+
+ /** the limit for the number of documents in each group */
+ public static final String GROUP_LIMIT = GROUP + ".limit";
+ /** the offset for the doclist of each group */
+ public static final String GROUP_OFFSET = GROUP + ".offset";
+
+ /** treat the first group result as the main result. true/false */
+ public static final String GROUP_MAIN = GROUP + ".main";
+
+ /** treat the first group result as the main result. true/false */
+ public static final String GROUP_FORMAT = GROUP + ".format";
+
+ /**
+ * Whether to cache the first pass search (doc ids and score) for the second pass search.
+ * Also defines the maximum size of the group cache relative to maxdoc in a percentage.
+ * Values can be a positive integer, from 0 till 100. A value of 0 will disable the group cache.
+ * The default is 0.*/
+ public static final String GROUP_CACHE_PERCENTAGE = GROUP + ".cache.percent";
+
+ // Note: Since you can supply multiple fields to group on, but only have a facets for the whole result. It only makes
+ // sense to me to support these parameters for the first group.
+ /** Whether the docSet (for example for faceting) should be based on plain documents (a.k.a UNGROUPED) or on the groups (a.k.a GROUPED).
+ * The docSet will only the most relevant documents per group. It is if you query for everything with group.limit=1 */
+ public static final String GROUP_TRUNCATE = GROUP + ".truncate";
+
+ /** Whether the group count should be included in the response. */
+ public static final String GROUP_TOTAL_COUNT = GROUP + ".ngroups";
+
+ /** Whether to compute grouped facets based on the first specified group. */
+ public static final String GROUP_FACET = GROUP + ".facet";
+
+ /** Retrieve the top search groups (top group values) from the shards being queried. */
+ public static final String GROUP_DISTRIBUTED_FIRST = GROUP + ".distributed.first";
+
+ /** Retrieve the top groups from the shards being queries based on the specified search groups in
+ * the {@link #GROUP_DISTRIBUTED_TOPGROUPS_PREFIX} parameters.
+ */
+ public static final String GROUP_DISTRIBUTED_SECOND = GROUP + ".distributed.second";
+
+ public static final String GROUP_DISTRIBUTED_TOPGROUPS_PREFIX = GROUP + ".topgroups.";
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/HighlightParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/HighlightParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/HighlightParams.java
new file mode 100644
index 0000000..ceccd74
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/HighlightParams.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ *
+ * @since solr 1.3
+ */
+public interface HighlightParams {
+ public static final String HIGHLIGHT = "hl";
+ public static final String Q = HIGHLIGHT+".q";
+ public static final String QPARSER = HIGHLIGHT+".qparser";
+ public static final String FIELDS = HIGHLIGHT+".fl";
+ public static final String SNIPPETS = HIGHLIGHT+".snippets";
+ public static final String FRAGSIZE = HIGHLIGHT+".fragsize";
+ public static final String INCREMENT = HIGHLIGHT+".increment";
+ public static final String MAX_CHARS = HIGHLIGHT+".maxAnalyzedChars";
+ public static final String FORMATTER = HIGHLIGHT+".formatter";
+ public static final String ENCODER = HIGHLIGHT+".encoder";
+ public static final String FRAGMENTER = HIGHLIGHT+".fragmenter";
+ public static final String PRESERVE_MULTI = HIGHLIGHT+".preserveMulti";
+ public static final String FRAG_LIST_BUILDER = HIGHLIGHT+".fragListBuilder";
+ public static final String FRAGMENTS_BUILDER = HIGHLIGHT+".fragmentsBuilder";
+ public static final String BOUNDARY_SCANNER = HIGHLIGHT+".boundaryScanner";
+ public static final String BS_MAX_SCAN = HIGHLIGHT+".bs.maxScan";
+ public static final String BS_CHARS = HIGHLIGHT+".bs.chars";
+ public static final String BS_TYPE = HIGHLIGHT+".bs.type";
+ public static final String BS_LANGUAGE = HIGHLIGHT+".bs.language";
+ public static final String BS_COUNTRY = HIGHLIGHT+".bs.country";
+ public static final String BS_VARIANT = HIGHLIGHT+".bs.variant";
+ public static final String FIELD_MATCH = HIGHLIGHT+".requireFieldMatch";
+ public static final String DEFAULT_SUMMARY = HIGHLIGHT + ".defaultSummary";
+ public static final String ALTERNATE_FIELD = HIGHLIGHT+".alternateField";
+ public static final String ALTERNATE_FIELD_LENGTH = HIGHLIGHT+".maxAlternateFieldLength";
+ public static final String MAX_MULTIVALUED_TO_EXAMINE = HIGHLIGHT + ".maxMultiValuedToExamine";
+ public static final String MAX_MULTIVALUED_TO_MATCH = HIGHLIGHT + ".maxMultiValuedToMatch";
+
+ public static final String USE_PHRASE_HIGHLIGHTER = HIGHLIGHT+".usePhraseHighlighter";
+ public static final String HIGHLIGHT_MULTI_TERM = HIGHLIGHT+".highlightMultiTerm";
+ public static final String PAYLOADS = HIGHLIGHT+".payloads";
+
+ public static final String MERGE_CONTIGUOUS_FRAGMENTS = HIGHLIGHT + ".mergeContiguous";
+
+ public static final String USE_FVH = HIGHLIGHT + ".useFastVectorHighlighter";
+ public static final String TAG_PRE = HIGHLIGHT + ".tag.pre";
+ public static final String TAG_POST = HIGHLIGHT + ".tag.post";
+ public static final String TAG_ELLIPSIS = HIGHLIGHT + ".tag.ellipsis";
+ public static final String PHRASE_LIMIT = HIGHLIGHT + ".phraseLimit";
+ public static final String MULTI_VALUED_SEPARATOR = HIGHLIGHT + ".multiValuedSeparatorChar";
+
+ // Formatter
+ public static final String SIMPLE = "simple";
+ public static final String SIMPLE_PRE = HIGHLIGHT+"."+SIMPLE+".pre";
+ public static final String SIMPLE_POST = HIGHLIGHT+"."+SIMPLE+".post";
+
+ // Regex fragmenter
+ public static final String REGEX = "regex";
+ public static final String SLOP = HIGHLIGHT+"."+REGEX+".slop";
+ public static final String PATTERN = HIGHLIGHT+"."+REGEX+".pattern";
+ public static final String MAX_RE_CHARS = HIGHLIGHT+"."+REGEX+".maxAnalyzedChars";
+
+ // Scoring parameters
+ public static final String SCORE = "score";
+ public static final String SCORE_K1 = HIGHLIGHT +"."+SCORE+".k1";
+ public static final String SCORE_B = HIGHLIGHT +"."+SCORE+".b";
+ public static final String SCORE_PIVOT = HIGHLIGHT +"."+SCORE+".pivot";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/MapSolrParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/MapSolrParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/MapSolrParams.java
new file mode 100644
index 0000000..f2a1c37
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/MapSolrParams.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import org.apache.solr.common.util.StrUtils;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map;
+import java.io.IOException;
+
+/**
+ *
+ */
+public class MapSolrParams extends SolrParams {
+ protected final Map<String,String> map;
+
+ public MapSolrParams(Map<String,String> map) {
+ this.map = map;
+ }
+
+ @Override
+ public String get(String name) {
+ Object o = map.get(name);
+ if(o == null) return null;
+ if (o instanceof String) return (String) o;
+ if (o instanceof String[]) {
+ String[] strings = (String[]) o;
+ if(strings.length == 0) return null;
+ return strings[0];
+ }
+ return String.valueOf(o);
+ }
+
+ @Override
+ public String[] getParams(String name) {
+ Object val = map.get(name);
+ if (val instanceof String[]) return (String[]) val;
+ return val==null ? null : new String[]{String.valueOf(val)};
+ }
+
+ @Override
+ public Iterator<String> getParameterNamesIterator() {
+ return map.keySet().iterator();
+ }
+
+ public Map<String,String> getMap() { return map; }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(128);
+ try {
+ boolean first=true;
+
+ for (Map.Entry<String,String> entry : map.entrySet()) {
+ String key = entry.getKey();
+ Object val = entry.getValue();
+ if (val instanceof String[]) {
+ String[] strings = (String[]) val;
+ val = StrUtils.join(Arrays.asList(strings),',');
+ }
+ if (!first) sb.append('&');
+ first=false;
+ sb.append(key);
+ sb.append('=');
+ StrUtils.partialURLEncodeVal(sb, val==null ? "" : String.valueOf(val));
+ }
+ }
+ catch (IOException e) {throw new RuntimeException(e);} // can't happen
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/ModifiableSolrParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/ModifiableSolrParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/ModifiableSolrParams.java
new file mode 100644
index 0000000..b84f4aa
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/ModifiableSolrParams.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.LinkedHashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+
+/**
+ * This class is similar to MultiMapSolrParams except you can edit the
+ * parameters after it is initialized. It has helper functions to set/add
+ * integer and boolean param values.
+ *
+ * @since solr 1.3
+ */
+public class ModifiableSolrParams extends SolrParams
+{
+ private Map<String,String[]> vals;
+
+ public ModifiableSolrParams()
+ {
+ // LinkedHashMap so params show up in CGI in the same order as they are entered
+ vals = new LinkedHashMap<>();
+ }
+
+ /** Constructs a new ModifiableSolrParams directly using the provided Map<String,String[]> */
+ public ModifiableSolrParams( Map<String,String[]> v )
+ {
+ vals = v;
+ }
+
+ /** Constructs a new ModifiableSolrParams, copying values from an existing SolrParams */
+ public ModifiableSolrParams(SolrParams params)
+ {
+ vals = new LinkedHashMap<>();
+ if( params != null ) {
+ this.add( params );
+ }
+ }
+
+ //----------------------------------------------------------------
+ //----------------------------------------------------------------
+
+ /**
+ * Replace any existing parameter with the given name. if val==null remove key from params completely.
+ */
+ public ModifiableSolrParams set( String name, String ... val ) {
+ if (val==null || (val.length==1 && val[0]==null)) {
+ vals.remove(name);
+ } else {
+ vals.put( name, val );
+ }
+ return this;
+ }
+
+ public ModifiableSolrParams set( String name, int val ) {
+ set( name, String.valueOf(val) );
+ return this;
+ }
+
+ public ModifiableSolrParams set( String name, boolean val ) {
+ set( name, String.valueOf(val) );
+ return this;
+ }
+
+ /**
+ * Add the given values to any existing name
+ * @param name Key
+ * @param val Array of value(s) added to the name. NOTE: If val is null
+ * or a member of val is null, then a corresponding null reference
+ * will be included when a get method is called on the key later.
+ * @return this
+ */
+ public ModifiableSolrParams add( String name, String ... val ) {
+ String[] old = vals.put(name, val);
+ if( old != null ) {
+ if( val == null || val.length < 1 ) {
+ String[] both = new String[old.length+1];
+ System.arraycopy(old, 0, both, 0, old.length);
+ both[old.length] = null;
+ vals.put( name, both );
+ }
+ else {
+ String[] both = new String[old.length+val.length];
+ System.arraycopy(old, 0, both, 0, old.length);
+ System.arraycopy(val, 0, both, old.length, val.length);
+ vals.put( name, both );
+ }
+ }
+ return this;
+ }
+
+ public void add(SolrParams params)
+ {
+ Iterator<String> names = params.getParameterNamesIterator();
+ while (names.hasNext()) {
+ String name = names.next();
+ set(name, params.getParams(name));
+ }
+ }
+
+ /**
+ * remove a field at the given name
+ */
+ public String[] remove( String name )
+ {
+ return vals.remove( name );
+ }
+
+ /** clear all parameters */
+ public void clear()
+ {
+ vals.clear();
+ }
+
+ /**
+ * remove the given value for the given name
+ *
+ * @return true if the item was removed, false if null or not present
+ */
+ public boolean remove(String name, String value) {
+ String[] tmp = vals.get(name);
+ if (tmp==null) return false;
+ for (int i=0; i<tmp.length; i++) {
+ if (tmp[i].equals(value)) {
+ String[] tmp2 = new String[tmp.length-1];
+ if (tmp2.length==0) {
+ tmp2 = null;
+ remove(name);
+ } else {
+ System.arraycopy(tmp, 0, tmp2, 0, i);
+ System.arraycopy(tmp, i+1, tmp2, i, tmp.length-i-1);
+ set(name, tmp2);
+ }
+ return true;
+ }
+ }
+ return false;
+ }
+
+ //----------------------------------------------------------------
+ //----------------------------------------------------------------
+
+ @Override
+ public String get(String param) {
+ String[] v = vals.get( param );
+ if( v!= null && v.length > 0 ) {
+ return v[0];
+ }
+ return null;
+ }
+
+ @Override
+ public Iterator<String> getParameterNamesIterator() {
+ return vals.keySet().iterator();
+ }
+
+ public Set<String> getParameterNames() {
+ return vals.keySet();
+ }
+
+ @Override
+ public String[] getParams(String param) {
+ return vals.get( param );
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(128);
+ try {
+ boolean first=true;
+
+ for (Map.Entry<String,String[]> entry : vals.entrySet()) {
+ String key = entry.getKey();
+ String[] valarr = entry.getValue();
+ for (String val : valarr) {
+ if (!first) sb.append('&');
+ first=false;
+ sb.append(key);
+ sb.append('=');
+ if( val != null ) {
+ sb.append( URLEncoder.encode( val, "UTF-8" ) );
+ }
+ }
+ }
+ }
+ catch (IOException e) {throw new RuntimeException(e);} // can't happen
+
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/MoreLikeThisParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/MoreLikeThisParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/MoreLikeThisParams.java
new file mode 100644
index 0000000..0d808c2
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/MoreLikeThisParams.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import java.util.Locale;
+
+/**
+ * @since solr 1.3
+ */
+public interface MoreLikeThisParams
+{
+ // enable more like this -- this only applies to 'StandardRequestHandler' maybe DismaxRequestHandler
+ public final static String MLT = "mlt";
+
+ public final static String PREFIX = "mlt.";
+
+ public final static String SIMILARITY_FIELDS = PREFIX + "fl";
+ public final static String MIN_TERM_FREQ = PREFIX + "mintf";
+ public final static String MAX_DOC_FREQ = PREFIX + "maxdf";
+ public final static String MIN_DOC_FREQ = PREFIX + "mindf";
+ public final static String MIN_WORD_LEN = PREFIX + "minwl";
+ public final static String MAX_WORD_LEN = PREFIX + "maxwl";
+ public final static String MAX_QUERY_TERMS = PREFIX + "maxqt";
+ public final static String MAX_NUM_TOKENS_PARSED = PREFIX + "maxntp";
+ public final static String BOOST = PREFIX + "boost"; // boost or not?
+ public final static String QF = PREFIX + "qf"; //boosting applied to mlt fields
+
+ // the /mlt request handler uses 'rows'
+ public final static String DOC_COUNT = PREFIX + "count";
+
+ // Do you want to include the original document in the results or not
+ public final static String MATCH_INCLUDE = PREFIX + "match.include";
+
+ // If multiple docs are matched in the query, what offset do you want?
+ public final static String MATCH_OFFSET = PREFIX + "match.offset";
+
+ // Do you want to include the original document in the results or not
+ public final static String INTERESTING_TERMS = PREFIX + "interestingTerms"; // false,details,(list or true)
+
+ public enum TermStyle {
+ NONE,
+ LIST,
+ DETAILS;
+
+ public static TermStyle get( String p )
+ {
+ if( p != null ) {
+ p = p.toUpperCase(Locale.ROOT);
+ if( p.equals( "DETAILS" ) ) {
+ return DETAILS;
+ }
+ else if( p.equals( "LIST" ) ) {
+ return LIST;
+ }
+ }
+ return NONE;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/MultiMapSolrParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/MultiMapSolrParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/MultiMapSolrParams.java
new file mode 100644
index 0000000..684396b
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/MultiMapSolrParams.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import org.apache.solr.common.util.StrUtils;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.io.IOException;
+
+/**
+ *
+ */
+public class MultiMapSolrParams extends SolrParams {
+ protected final Map<String,String[]> map;
+
+ public static void addParam(String name, String val, Map<String,String[]> map) {
+ String[] arr = map.get(name);
+ if (arr ==null) {
+ arr =new String[]{val};
+ } else {
+ String[] newarr = new String[arr.length+1];
+ System.arraycopy(arr,0,newarr,0,arr.length);
+ newarr[arr.length]=val;
+ arr =newarr;
+ }
+ map.put(name, arr);
+ }
+
+ public MultiMapSolrParams(Map<String,String[]> map) {
+ this.map = map;
+ }
+
+ @Override
+ public String get(String name) {
+ String[] arr = map.get(name);
+ return arr==null ? null : arr[0];
+ }
+
+ @Override
+ public String[] getParams(String name) {
+ return map.get(name);
+ }
+
+ @Override
+ public Iterator<String> getParameterNamesIterator() {
+ return map.keySet().iterator();
+ }
+
+ public Map<String,String[]> getMap() { return map; }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(128);
+ try {
+ boolean first=true;
+
+ for (Map.Entry<String,String[]> entry : map.entrySet()) {
+ String key = entry.getKey();
+ String[] valarr = entry.getValue();
+
+ for (String val : valarr) {
+ if (!first) sb.append('&');
+ first=false;
+ sb.append(key);
+ sb.append('=');
+ StrUtils.partialURLEncodeVal(sb, val==null ? "" : val);
+ }
+ }
+ }
+ catch (IOException e) {throw new RuntimeException(e);} // can't happen
+
+ return sb.toString();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/QueryElevationParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/QueryElevationParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/QueryElevationParams.java
new file mode 100644
index 0000000..d5a0b41
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/QueryElevationParams.java
@@ -0,0 +1,53 @@
+package org.apache.solr.common.params;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Parameters used with the QueryElevationComponent
+ *
+ **/
+public interface QueryElevationParams {
+
+ String ENABLE = "enableElevation";
+ String EXCLUSIVE = "exclusive";
+ String FORCE_ELEVATION = "forceElevation";
+ String IDS = "elevateIds";
+ String EXCLUDE = "excludeIds";
+ /**
+ * The name of the field that editorial results will be written out as when using the QueryElevationComponent, which
+ * automatically configures the EditorialMarkerFactory. The default name is "elevated"
+ * <br>
+ * See http://wiki.apache.org/solr/DocTransformers
+ */
+ String EDITORIAL_MARKER_FIELD_NAME = "editorialMarkerFieldName";
+ /**
+ * The name of the field that excluded editorial results will be written out as when using the QueryElevationComponent, which
+ * automatically configures the EditorialMarkerFactory. The default name is "excluded". This is only used
+ * when {@link #MARK_EXCLUDES} is set to true at query time.
+ * <br>
+ * See http://wiki.apache.org/solr/DocTransformers
+ */
+ String EXCLUDE_MARKER_FIELD_NAME = "excludeMarkerFieldName";
+
+ /**
+ * Instead of removing excluded items from the results, passing in this parameter allows you to get back the excluded items, but to mark them
+ * as excluded.
+ */
+ String MARK_EXCLUDES = "markExcludes";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/RequiredSolrParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/RequiredSolrParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/RequiredSolrParams.java
new file mode 100644
index 0000000..0b78d41
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/RequiredSolrParams.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import org.apache.solr.common.SolrException;
+
+import java.util.Iterator;
+
+/**
+ * This is a simple wrapper to SolrParams that will throw a 400
+ * exception if you ask for a parameter that does not exist. Fields
+ * specified with
+ *
+ * In short, any value you for from a <code>RequiredSolrParams</code>
+ * will return a valid non-null value or throw a 400 exception.
+ * (If you pass in <code>null</code> as the default value, you can
+ * get a null return value)
+ *
+ *
+ * @since solr 1.2
+ */
+public class RequiredSolrParams extends SolrParams {
+ protected final SolrParams params;
+
+ public RequiredSolrParams(SolrParams params) {
+ this.params = params;
+ }
+
+ /** get the param from params, fail if not found **/
+ @Override
+ public String get(String param) {
+ String val = params.get(param);
+ if( val == null ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Missing required parameter: "+param );
+ }
+ return val;
+ }
+
+ @Override
+ public String getFieldParam(final String field, final String param) {
+ final String fpname = fpname(field,param);
+ String val = params.get(fpname);
+ if (null == val) {
+ // don't call this.get, we want a specified exception message
+ val = params.get(param);
+ if (null == val) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "Missing required parameter: "+fpname+
+ " (or default: "+param+")" );
+ }
+ }
+ return val;
+ }
+
+ @Override
+ public String[] getFieldParams(final String field, final String param) {
+ final String fpname = fpname(field,param);
+ String[] val = params.getParams(fpname);
+ if (null == val) {
+ // don't call this.getParams, we want a specified exception message
+ val = params.getParams(param);
+ if (null == val) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "Missing required parameter: "+fpname+
+ " (or default: "+param+")" );
+ }
+ }
+ return val;
+ }
+
+
+ @Override
+ public String[] getParams(String param) {
+ String[] vals = params.getParams(param);
+ if( vals == null || vals.length == 0 ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "Missing required parameter: "+param );
+ }
+ return vals;
+ }
+
+ /** returns an Iterator over the parameter names */
+ @Override
+ public Iterator<String> getParameterNamesIterator() {
+ return params.getParameterNamesIterator();
+ }
+
+ @Override
+ public String toString() {
+ return "{required("+params+")}";
+ }
+
+ //----------------------------------------------------------
+ // Functions with a default value - pass directly to the
+ // wrapped SolrParams (they won't return null - unless it's the default)
+ //----------------------------------------------------------
+
+ @Override
+ public String get(String param, String def) {
+ return params.get(param, def);
+ }
+
+ @Override
+ public int getInt(String param, int def) {
+ return params.getInt(param, def);
+ }
+
+ @Override
+ public float getFloat(String param, float def) {
+ return params.getFloat(param, def);
+ }
+
+ @Override
+ public boolean getBool(String param, boolean def) {
+ return params.getBool(param, def);
+ }
+
+ @Override
+ public int getFieldInt(String field, String param, int def) {
+ return params.getFieldInt(field, param, def);
+ }
+
+ @Override
+ public boolean getFieldBool(String field, String param, boolean def) {
+ return params.getFieldBool(field, param, def);
+ }
+
+ @Override
+ public float getFieldFloat(String field, String param, float def) {
+ return params.getFieldFloat(field, param, def);
+ }
+
+ @Override
+ public String getFieldParam(String field, String param, String def) {
+ return params.getFieldParam(field, param, def);
+ }
+
+ public void check(String... params){
+ for (String param : params) get(param);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/ShardParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/ShardParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/ShardParams.java
new file mode 100644
index 0000000..46a9fc0
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/ShardParams.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * Parameters used for distributed search.
+ */
+public interface ShardParams {
+ /** the shards to use (distributed configuration) */
+ public static final String SHARDS = "shards";
+
+ /** per-shard start and rows */
+ public static final String SHARDS_ROWS = "shards.rows";
+ public static final String SHARDS_START = "shards.start";
+
+ /** IDs of the shard documents */
+ public static final String IDS = "ids";
+
+ /** whether the request goes to a shard */
+ public static final String IS_SHARD = "isShard";
+
+ /** The requested URL for this shard */
+ public static final String SHARD_URL = "shard.url";
+
+ /** The Request Handler for shard requests */
+ public static final String SHARDS_QT = "shards.qt";
+
+ /** Request detailed match info for each shard (true/false) */
+ public static final String SHARDS_INFO = "shards.info";
+
+ /** Should things fail if there is an error? (true/false) */
+ public static final String SHARDS_TOLERANT = "shards.tolerant";
+
+ /** query purpose for shard requests */
+ public static final String SHARDS_PURPOSE = "shards.purpose";
+
+ public static final String _ROUTE_ = "_route_";
+
+ /** Force a single-pass distributed query? (true/false) */
+ public static final String DISTRIB_SINGLE_PASS = "distrib.singlePass";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/SimpleParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/SimpleParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/SimpleParams.java
new file mode 100644
index 0000000..e770402
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/SimpleParams.java
@@ -0,0 +1,50 @@
+package org.apache.solr.common.params;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Parameters used by the SimpleQParser.
+ */
+public interface SimpleParams {
+ /** Query fields and boosts. */
+ public static String QF = "qf";
+
+ /** Override the currently enabled/disabled query operators. */
+ public static String QO = "q.operators";
+
+ /** Enables {@code AND} operator (+) */
+ public static final String AND_OPERATOR = "AND";
+ /** Enables {@code NOT} operator (-) */
+ public static final String NOT_OPERATOR = "NOT";
+ /** Enables {@code OR} operator (|) */
+ public static final String OR_OPERATOR = "OR";
+ /** Enables {@code PREFIX} operator (*) */
+ public static final String PREFIX_OPERATOR = "PREFIX";
+ /** Enables {@code PHRASE} operator (") */
+ public static final String PHRASE_OPERATOR = "PHRASE";
+ /** Enables {@code PRECEDENCE} operators: {@code (} and {@code )} */
+ public static final String PRECEDENCE_OPERATORS = "PRECEDENCE";
+ /** Enables {@code ESCAPE} operator (\) */
+ public static final String ESCAPE_OPERATOR = "ESCAPE";
+ /** Enables {@code WHITESPACE} operators: ' ' '\n' '\r' '\t' */
+ public static final String WHITESPACE_OPERATOR = "WHITESPACE";
+ /** Enables {@code FUZZY} operator (~) */
+ public static final String FUZZY_OPERATOR = "FUZZY";
+ /** Enables {@code NEAR} operator (~) */
+ public static final String NEAR_OPERATOR = "NEAR";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/SolrParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/SolrParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/SolrParams.java
new file mode 100644
index 0000000..36d0df1
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/SolrParams.java
@@ -0,0 +1,363 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.StrUtils;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/** SolrParams hold request parameters.
+ *
+ *
+ */
+public abstract class SolrParams implements Serializable {
+
+ /** returns the String value of a param, or null if not set */
+ public abstract String get(String param);
+
+ /** returns an array of the String values of a param, or null if none */
+ public abstract String[] getParams(String param);
+
+ /** returns an Iterator over the parameter names */
+ public abstract Iterator<String> getParameterNamesIterator();
+
+ /** returns the value of the param, or def if not set */
+ public String get(String param, String def) {
+ String val = get(param);
+ return val==null ? def : val;
+ }
+
+ /** returns a RequiredSolrParams wrapping this */
+ public RequiredSolrParams required()
+ {
+ // TODO? should we want to stash a reference?
+ return new RequiredSolrParams(this);
+ }
+
+ protected String fpname(String field, String param) {
+ return "f."+field+'.'+param;
+ }
+
+ /** returns the String value of the field parameter, "f.field.param", or
+ * the value for "param" if that is not set.
+ */
+ public String getFieldParam(String field, String param) {
+ String val = get(fpname(field,param));
+ return val!=null ? val : get(param);
+ }
+
+ /** returns the String value of the field parameter, "f.field.param", or
+ * the value for "param" if that is not set. If that is not set, def
+ */
+ public String getFieldParam(String field, String param, String def) {
+ String val = get(fpname(field,param));
+ return val!=null ? val : get(param, def);
+ }
+
+ /** returns the String values of the field parameter, "f.field.param", or
+ * the values for "param" if that is not set.
+ */
+ public String[] getFieldParams(String field, String param) {
+ String[] val = getParams(fpname(field,param));
+ return val!=null ? val : getParams(param);
+ }
+
+ /** Returns the Boolean value of the param, or null if not set */
+ public Boolean getBool(String param) {
+ String val = get(param);
+ return val==null ? null : StrUtils.parseBool(val);
+ }
+
+ /** Returns the boolean value of the param, or def if not set */
+ public boolean getBool(String param, boolean def) {
+ String val = get(param);
+ return val==null ? def : StrUtils.parseBool(val);
+ }
+
+ /** Returns the Boolean value of the field param,
+ or the value for param, or null if neither is set. */
+ public Boolean getFieldBool(String field, String param) {
+ String val = getFieldParam(field, param);
+ return val==null ? null : StrUtils.parseBool(val);
+ }
+
+ /** Returns the boolean value of the field param,
+ or the value for param, or def if neither is set. */
+ public boolean getFieldBool(String field, String param, boolean def) {
+ String val = getFieldParam(field, param);
+ return val==null ? def : StrUtils.parseBool(val);
+ }
+
+ /** Returns the Integer value of the param, or null if not set */
+ public Integer getInt(String param) {
+ String val = get(param);
+ try {
+ return val==null ? null : Integer.valueOf(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the Long value of the param, or null if not set */
+ public Long getLong(String param, Long def) {
+ String val = get(param);
+ try {
+ return val== null ? def : Long.parseLong(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the int value of the param, or def if not set */
+ public int getInt(String param, int def) {
+ String val = get(param);
+ try {
+ return val==null ? def : Integer.parseInt(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the Long value of the param, or null if not set */
+ public Long getLong(String param) {
+ String val = get(param);
+ try {
+ return val == null ? null : Long.valueOf(val);
+ } catch (Exception ex) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex);
+ }
+ }
+
+ /** Returns the long value of the param, or def if not set */
+ public long getLong(String param, long def) {
+ String val = get(param);
+ try {
+ return val == null ? def : Long.parseLong(val);
+ } catch (Exception ex) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex);
+ }
+ }
+
+
+ /**
+ * @return The int value of the field param, or the value for param
+ * or <code>null</code> if neither is set.
+ **/
+ public Integer getFieldInt(String field, String param) {
+ String val = getFieldParam(field, param);
+ try {
+ return val==null ? null : Integer.valueOf(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the int value of the field param,
+ or the value for param, or def if neither is set. */
+ public int getFieldInt(String field, String param, int def) {
+ String val = getFieldParam(field, param);
+ try {
+ return val==null ? def : Integer.parseInt(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+
+ /** Returns the Float value of the param, or null if not set */
+ public Float getFloat(String param) {
+ String val = get(param);
+ try {
+ return val==null ? null : Float.valueOf(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the float value of the param, or def if not set */
+ public float getFloat(String param, float def) {
+ String val = get(param);
+ try {
+ return val==null ? def : Float.parseFloat(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the Float value of the param, or null if not set */
+ public Double getDouble(String param) {
+ String val = get(param);
+ try {
+ return val==null ? null : Double.valueOf(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the float value of the param, or def if not set */
+ public double getDouble(String param, double def) {
+ String val = get(param);
+ try {
+ return val==null ? def : Double.parseDouble(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+
+ /** Returns the float value of the field param. */
+ public Float getFieldFloat(String field, String param) {
+ String val = getFieldParam(field, param);
+ try {
+ return val==null ? null : Float.valueOf(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the float value of the field param,
+ or the value for param, or def if neither is set. */
+ public float getFieldFloat(String field, String param, float def) {
+ String val = getFieldParam(field, param);
+ try {
+ return val==null ? def : Float.parseFloat(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the float value of the field param. */
+ public Double getFieldDouble(String field, String param) {
+ String val = getFieldParam(field, param);
+ try {
+ return val==null ? null : Double.valueOf(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ /** Returns the float value of the field param,
+ or the value for param, or def if neither is set. */
+ public double getFieldDouble(String field, String param, double def) {
+ String val = getFieldParam(field, param);
+ try {
+ return val==null ? def : Double.parseDouble(val);
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, ex.getMessage(), ex );
+ }
+ }
+
+ public static SolrParams wrapDefaults(SolrParams params, SolrParams defaults) {
+ if (params == null)
+ return defaults;
+ if (defaults == null)
+ return params;
+ return new DefaultSolrParams(params,defaults);
+ }
+
+ public static SolrParams wrapAppended(SolrParams params, SolrParams defaults) {
+ if (params == null)
+ return defaults;
+ if (defaults == null)
+ return params;
+ return AppendedSolrParams.wrapAppended(params,defaults);
+ }
+
+ /** Create a Map<String,String> from a NamedList given no keys are repeated */
+ public static Map<String,String> toMap(NamedList params) {
+ HashMap<String,String> map = new HashMap<>();
+ for (int i=0; i<params.size(); i++) {
+ map.put(params.getName(i), params.getVal(i).toString());
+ }
+ return map;
+ }
+
+ /** Create a Map<String,String[]> from a NamedList */
+ public static Map<String,String[]> toMultiMap(NamedList params) {
+ HashMap<String,String[]> map = new HashMap<>();
+ for (int i=0; i<params.size(); i++) {
+ String name = params.getName(i);
+ String val = params.getVal(i).toString();
+ MultiMapSolrParams.addParam(name,val,map);
+ }
+ return map;
+ }
+
+ /** Create SolrParams from NamedList. */
+ public static SolrParams toSolrParams(NamedList params) {
+ // if no keys are repeated use the faster MapSolrParams
+ HashMap<String,String> map = new HashMap<>();
+ for (int i=0; i<params.size(); i++) {
+ String prev = map.put(params.getName(i), params.getVal(i).toString());
+ if (prev!=null) return new MultiMapSolrParams(toMultiMap(params));
+ }
+ return new MapSolrParams(map);
+ }
+
+ /** Create filtered SolrParams. */
+ public SolrParams toFilteredSolrParams(List<String> names) {
+ NamedList<String> nl = new NamedList<>();
+ for (Iterator<String> it = getParameterNamesIterator(); it.hasNext();) {
+ final String name = it.next();
+ if (names.contains(name)) {
+ final String[] values = getParams(name);
+ for (String value : values) {
+ nl.add(name, value);
+ }
+ }
+ }
+ return toSolrParams(nl);
+ }
+
+ /** Convert this to a NamedList */
+ public NamedList<Object> toNamedList() {
+ final SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
+
+ for(Iterator<String> it=getParameterNamesIterator(); it.hasNext(); ) {
+ final String name = it.next();
+ final String [] values = getParams(name);
+ if(values.length==1) {
+ result.add(name,values[0]);
+ } else {
+ // currently no reason not to use the same array
+ result.add(name,values);
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/SpatialParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/SpatialParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/SpatialParams.java
new file mode 100644
index 0000000..043f216
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/SpatialParams.java
@@ -0,0 +1,41 @@
+package org.apache.solr.common.params;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ *
+ *
+ **/
+public interface SpatialParams {
+ public static final String POINT = "pt";
+ public static final String DISTANCE = "d";
+ public static final String FIELD = "sfield"; // the field that contains the points we are measuring from "pt"
+ /**
+ * km - kilometers
+ * mi - miles
+ */
+ public static final String UNITS = "units";
+ /**
+ * The distance measure to use.
+ */
+ public static final String MEASURE = "meas";
+ /**
+ * The radius of the sphere to use to in calculating spherical distances like Haversine
+ */
+ public static final String SPHERE_RADIUS = "sphere_radius";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/SpellingParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/SpellingParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/SpellingParams.java
new file mode 100644
index 0000000..918592d
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/SpellingParams.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * Parameters used for spellchecking
+ *
+ * @since solr 1.3
+ */
+public interface SpellingParams {
+
+ public static final String SPELLCHECK_PREFIX = "spellcheck.";
+
+ /**
+ * The name of the dictionary to be used for giving the suggestion for a
+ * request. The value for this parameter is configured in solrconfig.xml
+ */
+ public static final String SPELLCHECK_DICT = SPELLCHECK_PREFIX + "dictionary";
+
+ /**
+ * The count of suggestions to return for each query term not in the index and/or dictionary.
+ * <p>
+ * If this parameter is absent in the request then only one suggestion is
+ * returned. If it is more than one then a maximum of given suggestions are
+ * returned for each token in the query.
+ */
+ public static final String SPELLCHECK_COUNT = SPELLCHECK_PREFIX + "count";
+
+ /**
+ * The count of suggestions to return for each query term existing in the index and/or dictionary.
+ * <p>
+ * If this parameter is absent in the request then no suggestions are generated. This parameter allows
+ * for receiving alternative terms to use in context-sensitive spelling corrections.
+ */
+ public static final String SPELLCHECK_ALTERNATIVE_TERM_COUNT = SPELLCHECK_PREFIX + "alternativeTermCount";
+
+ /**
+ * <p>
+ * The maximum number of hits the request can return in order to both
+ * generate spelling suggestions and set the "correctlySpelled" element to "false".
+ * Note that this parameter is typically of use only in conjunction with "spellcheck.alternativeTermCount".
+ * </p>
+ * <p>
+ * If left unspecified, the default behavior will prevail. That is, "correctlySpelled" will be false and suggestions
+ * will be returned only if one or more of the query terms are absent from the dictionary and/or index. If set to zero,
+ * the "correctlySpelled" flag will be false only if the response returns zero hits. If set to a value greater than zero,
+ * suggestions will be returned even if hits are returned (up to the specified number). This number also will serve as
+ * the threshold in determining the value of "correctlySpelled". Specifying a value greater than zero is useful
+ * for creating "did-you-mean" suggestions for queries that return a low number of hits.
+ * </p>
+ */
+ public static final String SPELLCHECK_MAX_RESULTS_FOR_SUGGEST = SPELLCHECK_PREFIX + "maxResultsForSuggest";
+
+ /**
+ * When this parameter is set to true and the misspelled word exists in the
+ * user field, only words that occur more frequently in the Solr field than
+ * the one given will be returned. The default value is false.
+ * <p>
+ * <b>This is applicable only for dictionaries built from Solr fields.</b>
+ */
+ public static final String SPELLCHECK_ONLY_MORE_POPULAR = SPELLCHECK_PREFIX + "onlyMorePopular";
+
+ /**
+ * Whether to use the extended response format, which is more complicated but
+ * richer. Returns the document frequency for each suggestion and returns one
+ * suggestion block for each term in the query string. Default is false.
+ * <p>
+ * <b>This is applicable only for dictionaries built from Solr fields.</b>
+ */
+ public static final String SPELLCHECK_EXTENDED_RESULTS = SPELLCHECK_PREFIX + "extendedResults";
+
+ /**
+ * Use the value for this parameter as the query to spell check.
+ * <p>
+ * This parameter is <b>optional</b>. If absent, then the q parameter is
+ * used.
+ */
+ public static final String SPELLCHECK_Q = SPELLCHECK_PREFIX + "q";
+
+ /**
+ * Whether to build the index or not. Optional and false by default.
+ */
+ public static final String SPELLCHECK_BUILD = SPELLCHECK_PREFIX + "build";
+
+ /**
+ * Whether to reload the index. Optional and false by default.
+ */
+ public static final String SPELLCHECK_RELOAD = SPELLCHECK_PREFIX + "reload";
+
+ /**
+ * Take the top suggestion for each token and create a new query from it
+ */
+ public static final String SPELLCHECK_COLLATE = SPELLCHECK_PREFIX + "collate";
+ /**
+ * <p>
+ * The maximum number of collations to return. Default=1. Ignored if "spellcheck.collate" is false.
+ * </p>
+ */
+ public static final String SPELLCHECK_MAX_COLLATIONS = SPELLCHECK_PREFIX + "maxCollations";
+ /**
+ * <p>
+ * The maximum number of collations to test by querying against the index.
+ * When testing, the collation is substituted for the original query's "q" param. Any "qf"s are retained.
+ * If this is set to zero, does not test for hits before returning collations (returned collations may result in zero hits).
+ * Default=0. Ignored of "spellcheck.collate" is false.
+ * </p>
+ */
+ public static final String SPELLCHECK_MAX_COLLATION_TRIES = SPELLCHECK_PREFIX + "maxCollationTries";
+ /**
+ * <p>
+ * The maximum number of word correction combinations to rank and evaluate prior to deciding which collation
+ * candidates to test against the index. This is a performance safety-net in cases a user enters a query with
+ * many misspelled words. The default is 10,000 combinations.
+ * </p>
+ */
+ public static final String SPELLCHECK_MAX_COLLATION_EVALUATIONS = SPELLCHECK_PREFIX + "maxCollationEvaluations";
+ /**
+ * <p>
+ * For use with {@link SpellingParams#SPELLCHECK_MAX_COLLATION_TRIES} and
+ * {@link SpellingParams#SPELLCHECK_COLLATE_EXTENDED_RESULTS}.
+ * A performance optimization in cases where the exact number of hits a collation would return is not needed.
+ * Specify "0" to return the exact # of hits, otherwise give the maximum documents Lucene should collect
+ * with which to base an estimate. The higher the value the more likely the estimates will be accurate
+ * (at expense of performance).
+ * </p>
+ *
+ * <p>
+ * The default is 0 (report exact hit-counts) when {@link SpellingParams#SPELLCHECK_COLLATE_EXTENDED_RESULTS} is TRUE.
+ * When {@link SpellingParams#SPELLCHECK_COLLATE_EXTENDED_RESULTS} is FALSE, this optimization is always performed.
+ * </p>
+ */
+ public static final String SPELLCHECK_COLLATE_MAX_COLLECT_DOCS = SPELLCHECK_PREFIX + "collateMaxCollectDocs";
+ /**
+ * <p>
+ * Whether to use the Extended Results Format for collations.
+ * Includes "before>after" pairs to easily allow clients to generate messages like "no results for PORK. did you mean POLK?"
+ * Also indicates the # of hits each collation will return on re-query. Default=false, which retains 1.4-compatible output.
+ * </p>
+ * <p>
+ * Note: that if {@link SpellingParams#SPELLCHECK_COLLATE_MAX_COLLECT_DOCS} is set to a value greater than 0,
+ * then the hit counts returned by this will be estimated.
+ * </p>
+ */
+ public static final String SPELLCHECK_COLLATE_EXTENDED_RESULTS = SPELLCHECK_PREFIX + "collateExtendedResults";
+
+ /**
+ * <p>
+ * For use with {@link SpellingParams#SPELLCHECK_MAX_COLLATION_TRIES}, use this to override any original query parameters
+ * when issuing test queries. For instance, if the original query has "mm=1" but it is preferred to test collations
+ * with "mm=100%", then use "spellcheck.collateParam.mm=100%".
+ * </p>
+ */
+ public static final String SPELLCHECK_COLLATE_PARAM_OVERRIDE = SPELLCHECK_PREFIX + "collateParam.";
+ /**
+ * Certain spelling implementations may allow for an accuracy setting.
+ */
+ public static final String SPELLCHECK_ACCURACY = SPELLCHECK_PREFIX + "accuracy";
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/StatsParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/StatsParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/StatsParams.java
new file mode 100644
index 0000000..6fb7935
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/StatsParams.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * Stats Parameters
+ */
+public interface StatsParams {
+ public static final String STATS = "stats";
+ public static final String STATS_FIELD = STATS + ".field";
+ public static final String STATS_FACET = STATS + ".facet";
+ public static final String STATS_CALC_DISTINCT = STATS + ".calcdistinct";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/TermVectorParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/TermVectorParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/TermVectorParams.java
new file mode 100644
index 0000000..cfa1afd
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/TermVectorParams.java
@@ -0,0 +1,66 @@
+package org.apache.solr.common.params;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ *
+ *
+ **/
+public interface TermVectorParams {
+
+ public static final String TV_PREFIX = "tv.";
+
+ /**
+ * Return Term Frequency info
+ * */
+ public static final String TF = TV_PREFIX + "tf";
+ /**
+ * Return Term Vector position information
+ *
+ * */
+ public static final String POSITIONS = TV_PREFIX + "positions";
+ /**
+ * Return offset information, if available
+ * */
+ public static final String OFFSETS = TV_PREFIX + "offsets";
+ /**
+ * Return IDF information. May be expensive
+ * */
+ public static final String DF = TV_PREFIX + "df";
+
+ /**
+ * Return TF-IDF calculation, i.e. (tf / idf). May be expensive.
+ */
+ public static final String TF_IDF = TV_PREFIX + "tf_idf";
+
+
+ /**
+ * Return all the options: TF, positions, offsets, idf
+ */
+ public static final String ALL = TV_PREFIX + "all";
+
+ /**
+ * The fields to get term vectors for
+ */
+ public static final String FIELDS = TV_PREFIX + "fl";
+
+ /**
+ * The Doc Ids (Lucene internal ids) of the docs to get the term vectors for
+ */
+ public static final String DOC_IDS = TV_PREFIX + "docIds";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/TermsParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/TermsParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/TermsParams.java
new file mode 100644
index 0000000..9558efc
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/TermsParams.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import java.util.regex.Pattern;
+
+/**
+ *
+ *
+ **/
+public interface TermsParams {
+ /**
+ * The component name. Set to true to turn on the TermsComponent
+ */
+ public static final String TERMS = "terms";
+
+ /**
+ * Used for building up the other terms
+ */
+ public static final String TERMS_PREFIX = TERMS + ".";
+
+ /**
+ * Required. Specify the field to look up terms in.
+ */
+ public static final String TERMS_FIELD = TERMS_PREFIX + "fl";
+
+ /**
+ * Optional. The lower bound term to start at. The TermEnum will start at the next term after this term in the dictionary.
+ *
+ * If not specified, the empty string is used
+ */
+ public static final String TERMS_LOWER = TERMS_PREFIX + "lower";
+
+ /**
+ * Optional. The term to stop at.
+ *
+ * @see #TERMS_UPPER_INCLUSIVE
+ */
+ public static final String TERMS_UPPER = TERMS_PREFIX + "upper";
+ /**
+ * Optional. If true, include the upper bound term in the results. False by default.
+ */
+ public static final String TERMS_UPPER_INCLUSIVE = TERMS_PREFIX + "upper.incl";
+
+ /**
+ * Optional. If true, include the lower bound term in the results, otherwise skip to the next one. True by default.
+ */
+ public static final String TERMS_LOWER_INCLUSIVE = TERMS_PREFIX + "lower.incl";
+
+ /**
+ * Optional. The number of results to return. If not specified, looks for {@link org.apache.solr.common.params.CommonParams#ROWS}. If that's not specified, uses 10.
+ */
+ public static final String TERMS_LIMIT = TERMS_PREFIX + "limit";
+
+ public static final String TERMS_PREFIX_STR = TERMS_PREFIX + "prefix";
+
+ public static final String TERMS_REGEXP_STR = TERMS_PREFIX + "regex";
+
+ public static final String TERMS_REGEXP_FLAG = TERMS_REGEXP_STR + ".flag";
+
+ public static enum TermsRegexpFlag {
+ UNIX_LINES(Pattern.UNIX_LINES),
+ CASE_INSENSITIVE(Pattern.CASE_INSENSITIVE),
+ COMMENTS(Pattern.COMMENTS),
+ MULTILINE(Pattern.MULTILINE),
+ LITERAL(Pattern.LITERAL),
+ DOTALL(Pattern.DOTALL),
+ UNICODE_CASE(Pattern.UNICODE_CASE),
+ CANON_EQ(Pattern.CANON_EQ);
+
+ int value;
+
+ TermsRegexpFlag(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return value;
+ }
+ }
+
+ /**
+ * Optional. The minimum value of docFreq to be returned. 1 by default
+ */
+ public static final String TERMS_MINCOUNT = TERMS_PREFIX + "mincount";
+ /**
+ * Optional. The maximum value of docFreq to be returned. -1 by default means no boundary
+ */
+ public static final String TERMS_MAXCOUNT = TERMS_PREFIX + "maxcount";
+
+ /**
+ * Optional. If true, return the raw characters of the indexed term, regardless of if it is readable.
+ * For instance, the index form of numeric numbers is not human readable. The default is false.
+ */
+ public static final String TERMS_RAW = TERMS_PREFIX + "raw";
+
+ /**
+ * Optional. If sorting by frequency is enabled. Defaults to sorting by count.
+ */
+ public static final String TERMS_SORT = TERMS_PREFIX + "sort";
+
+ public static final String TERMS_SORT_COUNT = "count";
+ public static final String TERMS_SORT_INDEX = "index";
+}
+
[13/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
new file mode 100644
index 0000000..96ed211
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientUtil.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.InflaterInputStream;
+
+import org.apache.http.Header;
+import org.apache.http.HeaderElement;
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpException;
+import org.apache.http.HttpRequest;
+import org.apache.http.HttpRequestInterceptor;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpResponseInterceptor;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.params.ClientParamBean;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.conn.scheme.Scheme;
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.conn.ssl.X509HostnameVerifier;
+import org.apache.http.entity.HttpEntityWrapper;
+//import org.apache.http.impl.client.CloseableHttpClient; //RANGER_UPDATE - to use SystemDefaultHttpClient
+import org.apache.http.impl.client.AbstractHttpClient; //RANGER_UPDATE
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.SystemDefaultHttpClient;
+import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; // jdoc
+import org.apache.http.params.HttpConnectionParams;
+import org.apache.http.protocol.HttpContext;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for creating/configuring httpclient instances.
+ */
+public class HttpClientUtil {
+ // socket timeout measured in ms, closes a socket if read
+ // takes longer than x ms to complete. throws
+ // java.net.SocketTimeoutException: Read timed out exception
+ public static final String PROP_SO_TIMEOUT = "socketTimeout";
+ // connection timeout measures in ms, closes a socket if connection
+ // cannot be established within x ms. with a
+ // java.net.SocketTimeoutException: Connection timed out
+ public static final String PROP_CONNECTION_TIMEOUT = "connTimeout";
+ // Maximum connections allowed per host
+ public static final String PROP_MAX_CONNECTIONS_PER_HOST = "maxConnectionsPerHost";
+ // Maximum total connections allowed
+ public static final String PROP_MAX_CONNECTIONS = "maxConnections";
+ // Retry http requests on error
+ public static final String PROP_USE_RETRY = "retry";
+ // Allow compression (deflate,gzip) if server supports it
+ public static final String PROP_ALLOW_COMPRESSION = "allowCompression";
+ // Follow redirects
+ public static final String PROP_FOLLOW_REDIRECTS = "followRedirects";
+ // Basic auth username
+ public static final String PROP_BASIC_AUTH_USER = "httpBasicAuthUser";
+ // Basic auth password
+ public static final String PROP_BASIC_AUTH_PASS = "httpBasicAuthPassword";
+
+ public static final String SYS_PROP_CHECK_PEER_NAME = "solr.ssl.checkPeerName";
+
+ private static final Logger logger = LoggerFactory
+ .getLogger(HttpClientUtil.class);
+
+ static final DefaultHttpRequestRetryHandler NO_RETRY = new DefaultHttpRequestRetryHandler(
+ 0, false);
+
+ private static HttpClientConfigurer configurer = new HttpClientConfigurer();
+
+ private HttpClientUtil(){}
+
+ /**
+ * Replace the {@link HttpClientConfigurer} class used in configuring the http
+ * clients with a custom implementation.
+ */
+ public static void setConfigurer(HttpClientConfigurer newConfigurer) {
+ configurer = newConfigurer;
+ }
+
+ public static HttpClientConfigurer getConfigurer() {
+ return configurer;
+ }
+
+ /**
+ * Creates new http client by using the provided configuration.
+ *
+ * @param params
+ * http client configuration, if null a client with default
+ * configuration (no additional configuration) is created.
+ */
+ //public static CloseableHttpClient createClient(final SolrParams params) { //RANGER_UPDATE
+ public static AbstractHttpClient createClient(final SolrParams params) {
+ final ModifiableSolrParams config = new ModifiableSolrParams(params);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Creating new http client, config:" + config);
+ }
+ final DefaultHttpClient httpClient = new SystemDefaultHttpClient();
+ configureClient(httpClient, config);
+ return httpClient;
+ }
+
+ /**
+ * Creates new http client by using the provided configuration.
+ *
+ */
+ //public static CloseableHttpClient createClient(final SolrParams params, ClientConnectionManager cm) {
+ public static AbstractHttpClient createClient(final SolrParams params, ClientConnectionManager cm) {
+ final ModifiableSolrParams config = new ModifiableSolrParams(params);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Creating new http client, config:" + config);
+ }
+ final DefaultHttpClient httpClient = new DefaultHttpClient(cm);
+ configureClient(httpClient, config);
+ return httpClient;
+ }
+
+ /**
+ * Configures {@link DefaultHttpClient}, only sets parameters if they are
+ * present in config.
+ */
+ public static void configureClient(final DefaultHttpClient httpClient,
+ SolrParams config) {
+ configurer.configure(httpClient, config);
+ }
+
+ public static void close(HttpClient httpClient) {
+// if (httpClient instanceof CloseableHttpClient) { //RANGER_UPDATE
+// org.apache.solr.common.util.IOUtils.closeQuietly((CloseableHttpClient) httpClient); //RANGER_UPDATE
+// } else { //RANGER_UPDATE
+ httpClient.getConnectionManager().shutdown();
+// } //RANGER_UPDATE
+ }
+
+ /**
+ * Control HTTP payload compression.
+ *
+ * @param allowCompression
+ * true will enable compression (needs support from server), false
+ * will disable compression.
+ */
+ public static void setAllowCompression(DefaultHttpClient httpClient,
+ boolean allowCompression) {
+ httpClient
+ .removeRequestInterceptorByClass(UseCompressionRequestInterceptor.class);
+ httpClient
+ .removeResponseInterceptorByClass(UseCompressionResponseInterceptor.class);
+ if (allowCompression) {
+ httpClient.addRequestInterceptor(new UseCompressionRequestInterceptor());
+ httpClient
+ .addResponseInterceptor(new UseCompressionResponseInterceptor());
+ }
+ }
+
+ /**
+ * Set http basic auth information. If basicAuthUser or basicAuthPass is null
+ * the basic auth configuration is cleared. Currently this is not preemtive
+ * authentication. So it is not currently possible to do a post request while
+ * using this setting.
+ */
+ public static void setBasicAuth(DefaultHttpClient httpClient,
+ String basicAuthUser, String basicAuthPass) {
+ if (basicAuthUser != null && basicAuthPass != null) {
+ httpClient.getCredentialsProvider().setCredentials(AuthScope.ANY,
+ new UsernamePasswordCredentials(basicAuthUser, basicAuthPass));
+ } else {
+ httpClient.getCredentialsProvider().clear();
+ }
+ }
+
+ /**
+ * Set max connections allowed per host. This call will only work when
+ * {@link ThreadSafeClientConnManager} or
+ * {@link PoolingClientConnectionManager} is used.
+ */
+ public static void setMaxConnectionsPerHost(HttpClient httpClient,
+ int max) {
+ // would have been nice if there was a common interface
+ if (httpClient.getConnectionManager() instanceof ThreadSafeClientConnManager) {
+ ThreadSafeClientConnManager mgr = (ThreadSafeClientConnManager)httpClient.getConnectionManager();
+ mgr.setDefaultMaxPerRoute(max);
+ } else if (httpClient.getConnectionManager() instanceof PoolingClientConnectionManager) {
+ PoolingClientConnectionManager mgr = (PoolingClientConnectionManager)httpClient.getConnectionManager();
+ mgr.setDefaultMaxPerRoute(max);
+ }
+ }
+
+ /**
+ * Set max total connections allowed. This call will only work when
+ * {@link ThreadSafeClientConnManager} or
+ * {@link PoolingClientConnectionManager} is used.
+ */
+ public static void setMaxConnections(final HttpClient httpClient,
+ int max) {
+ // would have been nice if there was a common interface
+ if (httpClient.getConnectionManager() instanceof ThreadSafeClientConnManager) {
+ ThreadSafeClientConnManager mgr = (ThreadSafeClientConnManager)httpClient.getConnectionManager();
+ mgr.setMaxTotal(max);
+ } else if (httpClient.getConnectionManager() instanceof PoolingClientConnectionManager) {
+ PoolingClientConnectionManager mgr = (PoolingClientConnectionManager)httpClient.getConnectionManager();
+ mgr.setMaxTotal(max);
+ }
+ }
+
+
+ /**
+ * Defines the socket timeout (SO_TIMEOUT) in milliseconds. A timeout value of
+ * zero is interpreted as an infinite timeout.
+ *
+ * @param timeout timeout in milliseconds
+ */
+ public static void setSoTimeout(HttpClient httpClient, int timeout) {
+ HttpConnectionParams.setSoTimeout(httpClient.getParams(),
+ timeout);
+ }
+
+ /**
+ * Control retry handler
+ * @param useRetry when false the client will not try to retry failed requests.
+ */
+ public static void setUseRetry(final DefaultHttpClient httpClient,
+ boolean useRetry) {
+ if (!useRetry) {
+ httpClient.setHttpRequestRetryHandler(NO_RETRY);
+ } else {
+ // if the request is not fully sent, we retry
+ // streaming updates are not a problem, because they are not retryable
+ httpClient.setHttpRequestRetryHandler(new DefaultHttpRequestRetryHandler(){
+ @Override
+ protected boolean handleAsIdempotent(final HttpRequest request) {
+ return false; // we can't tell if a Solr request is idempotent
+ }
+ });
+ }
+ }
+
+ /**
+ * Set connection timeout. A timeout value of zero is interpreted as an
+ * infinite timeout.
+ *
+ * @param timeout
+ * connection Timeout in milliseconds
+ */
+ public static void setConnectionTimeout(final HttpClient httpClient,
+ int timeout) {
+ HttpConnectionParams.setConnectionTimeout(httpClient.getParams(),
+ timeout);
+ }
+
+ /**
+ * Set follow redirects.
+ *
+ * @param followRedirects When true the client will follow redirects.
+ */
+ public static void setFollowRedirects(HttpClient httpClient,
+ boolean followRedirects) {
+ new ClientParamBean(httpClient.getParams()).setHandleRedirects(followRedirects);
+ }
+
+ public static void setHostNameVerifier(DefaultHttpClient httpClient,
+ X509HostnameVerifier hostNameVerifier) {
+ Scheme httpsScheme = httpClient.getConnectionManager().getSchemeRegistry().get("https");
+ if (httpsScheme != null) {
+ SSLSocketFactory sslSocketFactory = (SSLSocketFactory) httpsScheme.getSchemeSocketFactory();
+ sslSocketFactory.setHostnameVerifier(hostNameVerifier);
+ }
+ }
+
+ public static void setStaleCheckingEnabled(final HttpClient httpClient, boolean enabled) {
+ HttpConnectionParams.setStaleCheckingEnabled(httpClient.getParams(), enabled);
+ }
+
+ public static void setTcpNoDelay(final HttpClient httpClient, boolean tcpNoDelay) {
+ HttpConnectionParams.setTcpNoDelay(httpClient.getParams(), tcpNoDelay);
+ }
+
+ private static class UseCompressionRequestInterceptor implements
+ HttpRequestInterceptor {
+
+ @Override
+ public void process(HttpRequest request, HttpContext context)
+ throws HttpException, IOException {
+ if (!request.containsHeader("Accept-Encoding")) {
+ request.addHeader("Accept-Encoding", "gzip, deflate");
+ }
+ }
+ }
+
+ private static class UseCompressionResponseInterceptor implements
+ HttpResponseInterceptor {
+
+ @Override
+ public void process(final HttpResponse response, final HttpContext context)
+ throws HttpException, IOException {
+
+ HttpEntity entity = response.getEntity();
+ Header ceheader = entity.getContentEncoding();
+ if (ceheader != null) {
+ HeaderElement[] codecs = ceheader.getElements();
+ for (int i = 0; i < codecs.length; i++) {
+ if (codecs[i].getName().equalsIgnoreCase("gzip")) {
+ response
+ .setEntity(new GzipDecompressingEntity(response.getEntity()));
+ return;
+ }
+ if (codecs[i].getName().equalsIgnoreCase("deflate")) {
+ response.setEntity(new DeflateDecompressingEntity(response
+ .getEntity()));
+ return;
+ }
+ }
+ }
+ }
+ }
+
+ private static class GzipDecompressingEntity extends HttpEntityWrapper {
+ public GzipDecompressingEntity(final HttpEntity entity) {
+ super(entity);
+ }
+
+ @Override
+ public InputStream getContent() throws IOException, IllegalStateException {
+ return new GZIPInputStream(wrappedEntity.getContent());
+ }
+
+ @Override
+ public long getContentLength() {
+ return -1;
+ }
+ }
+
+ private static class DeflateDecompressingEntity extends
+ GzipDecompressingEntity {
+ public DeflateDecompressingEntity(final HttpEntity entity) {
+ super(entity);
+ }
+
+ @Override
+ public InputStream getContent() throws IOException, IllegalStateException {
+ return new InflaterInputStream(wrappedEntity.getContent());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
new file mode 100644
index 0000000..a073265
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrClient.java
@@ -0,0 +1,821 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpEntityEnclosingRequestBase;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.methods.HttpUriRequest;
+import org.apache.http.conn.ClientConnectionManager;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.entity.mime.FormBodyPart;
+import org.apache.http.entity.mime.HttpMultipartMode;
+import org.apache.http.entity.mime.MultipartEntity;
+import org.apache.http.entity.mime.content.InputStreamBody;
+import org.apache.http.entity.mime.content.StringBody;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.ConnectException;
+import java.net.SocketTimeoutException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+public class HttpSolrClient extends SolrClient {
+
+ private static final String UTF_8 = StandardCharsets.UTF_8.name();
+ private static final String DEFAULT_PATH = "/select";
+ private static final long serialVersionUID = -946812319974801896L;
+
+ /**
+ * User-Agent String.
+ */
+ public static final String AGENT = "Solr[" + HttpSolrClient.class.getName() + "] 1.0";
+
+ private static Logger log = LoggerFactory.getLogger(HttpSolrClient.class);
+
+ /**
+ * The URL of the Solr server.
+ */
+ protected volatile String baseUrl;
+
+ /**
+ * Default value: null / empty.
+ * <p>
+ * Parameters that are added to every request regardless. This may be a place
+ * to add something like an authentication token.
+ */
+ protected ModifiableSolrParams invariantParams;
+
+ /**
+ * Default response parser is BinaryResponseParser
+ * <p>
+ * This parser represents the default Response Parser chosen to parse the
+ * response if the parser were not specified as part of the request.
+ *
+ * @see org.apache.solr.client.solrj.impl.BinaryResponseParser
+ */
+ protected volatile ResponseParser parser;
+
+ /**
+ * The RequestWriter used to write all requests to Solr
+ *
+ * @see org.apache.solr.client.solrj.request.RequestWriter
+ */
+ protected volatile RequestWriter requestWriter = new RequestWriter();
+
+ private final HttpClient httpClient;
+
+ private volatile boolean followRedirects = false;
+
+ private volatile int maxRetries = 0;
+
+ private volatile boolean useMultiPartPost;
+ private final boolean internalClient;
+
+ private volatile Set<String> queryParams = Collections.emptySet();
+
+ /**
+ * @param baseURL
+ * The URL of the Solr server. For example, "
+ * <code>http://localhost:8983/solr/</code>" if you are using the
+ * standard distribution Solr webapp on your local machine.
+ */
+ public HttpSolrClient(String baseURL) {
+ this(baseURL, null, new BinaryResponseParser());
+ }
+
+ public HttpSolrClient(String baseURL, HttpClient client) {
+ this(baseURL, client, new BinaryResponseParser());
+ }
+
+ public HttpSolrClient(String baseURL, HttpClient client, ResponseParser parser) {
+ this.baseUrl = baseURL;
+ if (baseUrl.endsWith("/")) {
+ baseUrl = baseUrl.substring(0, baseUrl.length() - 1);
+ }
+ if (baseUrl.indexOf('?') >= 0) {
+ throw new RuntimeException(
+ "Invalid base url for solrj. The base URL must not contain parameters: "
+ + baseUrl);
+ }
+
+ if (client != null) {
+ httpClient = client;
+ internalClient = false;
+ } else {
+ internalClient = true;
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 128);
+ params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 32);
+ params.set(HttpClientUtil.PROP_FOLLOW_REDIRECTS, followRedirects);
+ httpClient = HttpClientUtil.createClient(params);
+ }
+
+ this.parser = parser;
+ }
+
+ public Set<String> getQueryParams() {
+ return queryParams;
+ }
+
+ /**
+ * Expert Method
+ * @param queryParams set of param keys to only send via the query string
+ * Note that the param will be sent as a query string if the key is part
+ * of this Set or the SolrRequest's query params.
+ * @see org.apache.solr.client.solrj.SolrRequest#getQueryParams
+ */
+ public void setQueryParams(Set<String> queryParams) {
+ this.queryParams = queryParams;
+ }
+
+ /**
+ * Process the request. If
+ * {@link org.apache.solr.client.solrj.SolrRequest#getResponseParser()} is
+ * null, then use {@link #getParser()}
+ *
+ * @param request
+ * The {@link org.apache.solr.client.solrj.SolrRequest} to process
+ * @return The {@link org.apache.solr.common.util.NamedList} result
+ * @throws IOException If there is a low-level I/O error.
+ *
+ * @see #request(org.apache.solr.client.solrj.SolrRequest,
+ * org.apache.solr.client.solrj.ResponseParser)
+ */
+ @Override
+ public NamedList<Object> request(final SolrRequest request)
+ throws SolrServerException, IOException {
+ ResponseParser responseParser = request.getResponseParser();
+ if (responseParser == null) {
+ responseParser = parser;
+ }
+ return request(request, responseParser);
+ }
+
+ public NamedList<Object> request(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException {
+ return executeMethod(createMethod(request),processor);
+ }
+
+ /**
+ * @lucene.experimental
+ */
+ public static class HttpUriRequestResponse {
+ public HttpUriRequest httpUriRequest;
+ public Future<NamedList<Object>> future;
+ }
+
+ /**
+ * @lucene.experimental
+ */
+ public HttpUriRequestResponse httpUriRequest(final SolrRequest request)
+ throws SolrServerException, IOException {
+ ResponseParser responseParser = request.getResponseParser();
+ if (responseParser == null) {
+ responseParser = parser;
+ }
+ return httpUriRequest(request, responseParser);
+ }
+
+ /**
+ * @lucene.experimental
+ */
+ public HttpUriRequestResponse httpUriRequest(final SolrRequest request, final ResponseParser processor) throws SolrServerException, IOException {
+ HttpUriRequestResponse mrr = new HttpUriRequestResponse();
+ final HttpRequestBase method = createMethod(request);
+ ExecutorService pool = Executors.newFixedThreadPool(1, new SolrjNamedThreadFactory("httpUriRequest"));
+ try {
+ mrr.future = pool.submit(new Callable<NamedList<Object>>(){
+
+ @Override
+ public NamedList<Object> call() throws Exception {
+ return executeMethod(method, processor);
+ }});
+
+ } finally {
+ pool.shutdown();
+ }
+ assert method != null;
+ mrr.httpUriRequest = method;
+ return mrr;
+ }
+
+ protected ModifiableSolrParams calculateQueryParams(Set<String> queryParamNames,
+ ModifiableSolrParams wparams) {
+ ModifiableSolrParams queryModParams = new ModifiableSolrParams();
+ if (queryParamNames != null) {
+ for (String param : queryParamNames) {
+ String[] value = wparams.getParams(param) ;
+ if (value != null) {
+ for (String v : value) {
+ queryModParams.add(param, v);
+ }
+ wparams.remove(param);
+ }
+ }
+ }
+ return queryModParams;
+ }
+
+ protected HttpRequestBase createMethod(final SolrRequest request) throws IOException, SolrServerException {
+ HttpRequestBase method = null;
+ InputStream is = null;
+ SolrParams params = request.getParams();
+ Collection<ContentStream> streams = requestWriter.getContentStreams(request);
+ String path = requestWriter.getPath(request);
+ if (path == null || !path.startsWith("/")) {
+ path = DEFAULT_PATH;
+ }
+
+ ResponseParser parser = request.getResponseParser();
+ if (parser == null) {
+ parser = this.parser;
+ }
+
+ // The parser 'wt=' and 'version=' params are used instead of the original
+ // params
+ ModifiableSolrParams wparams = new ModifiableSolrParams(params);
+ if (parser != null) {
+ wparams.set(CommonParams.WT, parser.getWriterType());
+ wparams.set(CommonParams.VERSION, parser.getVersion());
+ }
+ if (invariantParams != null) {
+ wparams.add(invariantParams);
+ }
+
+ int tries = maxRetries + 1;
+ try {
+ while( tries-- > 0 ) {
+ // Note: since we aren't do intermittent time keeping
+ // ourselves, the potential non-timeout latency could be as
+ // much as tries-times (plus scheduling effects) the given
+ // timeAllowed.
+ try {
+ if( SolrRequest.METHOD.GET == request.getMethod() ) {
+ if( streams != null ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!" );
+ }
+ method = new HttpGet( baseUrl + path + ClientUtils.toQueryString( wparams, false ) );
+ }
+ else if( SolrRequest.METHOD.POST == request.getMethod() || SolrRequest.METHOD.PUT == request.getMethod() ) {
+
+ String url = baseUrl + path;
+ boolean hasNullStreamName = false;
+ if (streams != null) {
+ for (ContentStream cs : streams) {
+ if (cs.getName() == null) {
+ hasNullStreamName = true;
+ break;
+ }
+ }
+ }
+ boolean isMultipart = ((this.useMultiPartPost && SolrRequest.METHOD.POST == request.getMethod())
+ || ( streams != null && streams.size() > 1 )) && !hasNullStreamName;
+
+ LinkedList<NameValuePair> postOrPutParams = new LinkedList<>();
+ if (streams == null || isMultipart) {
+ // send server list and request list as query string params
+ ModifiableSolrParams queryParams = calculateQueryParams(this.queryParams, wparams);
+ queryParams.add(calculateQueryParams(request.getQueryParams(), wparams));
+ String fullQueryUrl = url + ClientUtils.toQueryString( queryParams, false );
+ HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ?
+ new HttpPost(fullQueryUrl) : new HttpPut(fullQueryUrl);
+ if (!isMultipart) {
+ postOrPut.addHeader("Content-Type",
+ "application/x-www-form-urlencoded; charset=UTF-8");
+ }
+
+ List<FormBodyPart> parts = new LinkedList<>();
+ Iterator<String> iter = wparams.getParameterNamesIterator();
+ while (iter.hasNext()) {
+ String p = iter.next();
+ String[] vals = wparams.getParams(p);
+ if (vals != null) {
+ for (String v : vals) {
+ if (isMultipart) {
+ parts.add(new FormBodyPart(p, new StringBody(v, StandardCharsets.UTF_8)));
+ } else {
+ postOrPutParams.add(new BasicNameValuePair(p, v));
+ }
+ }
+ }
+ }
+
+ if (isMultipart && streams != null) {
+ for (ContentStream content : streams) {
+ String contentType = content.getContentType();
+ if(contentType==null) {
+ contentType = BinaryResponseParser.BINARY_CONTENT_TYPE; // default
+ }
+ String name = content.getName();
+ if(name==null) {
+ name = "";
+ }
+ parts.add(new FormBodyPart(name,
+ new InputStreamBody(
+ content.getStream(),
+ contentType,
+ content.getName())));
+ }
+ }
+
+ if (parts.size() > 0) {
+ MultipartEntity entity = new MultipartEntity(HttpMultipartMode.STRICT);
+ for(FormBodyPart p: parts) {
+ entity.addPart(p);
+ }
+ postOrPut.setEntity(entity);
+ } else {
+ //not using multipart
+ postOrPut.setEntity(new UrlEncodedFormEntity(postOrPutParams, StandardCharsets.UTF_8));
+ }
+
+ method = postOrPut;
+ }
+ // It is has one stream, it is the post body, put the params in the URL
+ else {
+ String pstr = ClientUtils.toQueryString(wparams, false);
+ HttpEntityEnclosingRequestBase postOrPut = SolrRequest.METHOD.POST == request.getMethod() ?
+ new HttpPost(url + pstr) : new HttpPut(url + pstr);
+
+ // Single stream as body
+ // Using a loop just to get the first one
+ final ContentStream[] contentStream = new ContentStream[1];
+ for (ContentStream content : streams) {
+ contentStream[0] = content;
+ break;
+ }
+ if (contentStream[0] instanceof RequestWriter.LazyContentStream) {
+ postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) {
+ @Override
+ public Header getContentType() {
+ return new BasicHeader("Content-Type", contentStream[0].getContentType());
+ }
+
+ @Override
+ public boolean isRepeatable() {
+ return false;
+ }
+
+ });
+ } else {
+ postOrPut.setEntity(new InputStreamEntity(contentStream[0].getStream(), -1) {
+ @Override
+ public Header getContentType() {
+ return new BasicHeader("Content-Type", contentStream[0].getContentType());
+ }
+
+ @Override
+ public boolean isRepeatable() {
+ return false;
+ }
+ });
+ }
+ method = postOrPut;
+ }
+ }
+ else {
+ throw new SolrServerException("Unsupported method: "+request.getMethod() );
+ }
+ }
+ catch( NoHttpResponseException r ) {
+ method = null;
+ if(is != null) {
+ is.close();
+ }
+ // If out of tries then just rethrow (as normal error).
+ if (tries < 1) {
+ throw r;
+ }
+ }
+ }
+ } catch (IOException ex) {
+ throw new SolrServerException("error reading streams", ex);
+ }
+
+ return method;
+ }
+
+ protected NamedList<Object> executeMethod(HttpRequestBase method, final ResponseParser processor) throws SolrServerException {
+ method.addHeader("User-Agent", AGENT);
+
+ InputStream respBody = null;
+ boolean shouldClose = true;
+ boolean success = false;
+ try {
+ // Execute the method.
+ final HttpResponse response = httpClient.execute(method);
+ int httpStatus = response.getStatusLine().getStatusCode();
+
+ // Read the contents
+ respBody = response.getEntity().getContent();
+ Header ctHeader = response.getLastHeader("content-type");
+ String contentType;
+ if (ctHeader != null) {
+ contentType = ctHeader.getValue();
+ } else {
+ contentType = "";
+ }
+
+ // handle some http level checks before trying to parse the response
+ switch (httpStatus) {
+ case HttpStatus.SC_OK:
+ case HttpStatus.SC_BAD_REQUEST:
+ case HttpStatus.SC_CONFLICT: // 409
+ break;
+ case HttpStatus.SC_MOVED_PERMANENTLY:
+ case HttpStatus.SC_MOVED_TEMPORARILY:
+ if (!followRedirects) {
+ throw new SolrServerException("Server at " + getBaseURL()
+ + " sent back a redirect (" + httpStatus + ").");
+ }
+ break;
+ default:
+ if (processor == null) {
+ throw new RemoteSolrException(baseUrl, httpStatus, "non ok status: " + httpStatus
+ + ", message:" + response.getStatusLine().getReasonPhrase(),
+ null);
+ }
+ }
+ if (processor == null) {
+
+ // no processor specified, return raw stream
+ NamedList<Object> rsp = new NamedList<>();
+ rsp.add("stream", respBody);
+ // Only case where stream should not be closed
+ shouldClose = false;
+ success = true;
+ return rsp;
+ }
+
+ String procCt = processor.getContentType();
+ if (procCt != null) {
+ String procMimeType = ContentType.parse(procCt).getMimeType().trim().toLowerCase(Locale.ROOT);
+ String mimeType = ContentType.parse(contentType).getMimeType().trim().toLowerCase(Locale.ROOT);
+ if (!procMimeType.equals(mimeType)) {
+ // unexpected mime type
+ String msg = "Expected mime type " + procMimeType + " but got " + mimeType + ".";
+ Header encodingHeader = response.getEntity().getContentEncoding();
+ String encoding;
+ if (encodingHeader != null) {
+ encoding = encodingHeader.getValue();
+ } else {
+ encoding = "UTF-8"; // try UTF-8
+ }
+ try {
+ msg = msg + " " + IOUtils.toString(respBody, encoding);
+ } catch (IOException e) {
+ throw new RemoteSolrException(baseUrl, httpStatus, "Could not parse response with encoding " + encoding, e);
+ }
+ throw new RemoteSolrException(baseUrl, httpStatus, msg, null);
+ }
+ }
+
+ NamedList<Object> rsp = null;
+ String charset = EntityUtils.getContentCharSet(response.getEntity());
+ try {
+ rsp = processor.processResponse(respBody, charset);
+ } catch (Exception e) {
+ throw new RemoteSolrException(baseUrl, httpStatus, e.getMessage(), e);
+ }
+ if (httpStatus != HttpStatus.SC_OK) {
+ NamedList<String> metadata = null;
+ String reason = null;
+ try {
+ NamedList err = (NamedList) rsp.get("error");
+ if (err != null) {
+ reason = (String) err.get("msg");
+ if(reason == null) {
+ reason = (String) err.get("trace");
+ }
+ metadata = (NamedList<String>)err.get("metadata");
+ }
+ } catch (Exception ex) {}
+ if (reason == null) {
+ StringBuilder msg = new StringBuilder();
+ msg.append(response.getStatusLine().getReasonPhrase());
+ msg.append("\n\n");
+ msg.append("request: " + method.getURI());
+ reason = java.net.URLDecoder.decode(msg.toString(), UTF_8);
+ }
+ RemoteSolrException rss = new RemoteSolrException(baseUrl, httpStatus, reason, null);
+ if (metadata != null) rss.setMetadata(metadata);
+ throw rss;
+ }
+ success = true;
+ return rsp;
+ } catch (ConnectException e) {
+ throw new SolrServerException("Server refused connection at: "
+ + getBaseURL(), e);
+ } catch (SocketTimeoutException e) {
+ throw new SolrServerException(
+ "Timeout occured while waiting response from server at: "
+ + getBaseURL(), e);
+ } catch (IOException e) {
+ throw new SolrServerException(
+ "IOException occured when talking to server at: " + getBaseURL(), e);
+ } finally {
+ if (respBody != null && shouldClose) {
+ try {
+ respBody.close();
+ } catch (IOException e) {
+ log.error("", e);
+ } finally {
+ if (!success) {
+ method.abort();
+ }
+ }
+ }
+ }
+ }
+
+ // -------------------------------------------------------------------
+ // -------------------------------------------------------------------
+
+ /**
+ * Retrieve the default list of parameters are added to every request
+ * regardless.
+ *
+ * @see #invariantParams
+ */
+ public ModifiableSolrParams getInvariantParams() {
+ return invariantParams;
+ }
+
+ public String getBaseURL() {
+ return baseUrl;
+ }
+
+ public void setBaseURL(String baseURL) {
+ this.baseUrl = baseURL;
+ }
+
+ public ResponseParser getParser() {
+ return parser;
+ }
+
+ /**
+ * Note: This setter method is <b>not thread-safe</b>.
+ *
+ * @param processor
+ * Default Response Parser chosen to parse the response if the parser
+ * were not specified as part of the request.
+ * @see org.apache.solr.client.solrj.SolrRequest#getResponseParser()
+ */
+ public void setParser(ResponseParser processor) {
+ parser = processor;
+ }
+
+ /**
+ * Return the HttpClient this instance uses.
+ */
+ public HttpClient getHttpClient() {
+ return httpClient;
+ }
+
+ /**
+ * HttpConnectionParams.setConnectionTimeout
+ *
+ * @param timeout
+ * Timeout in milliseconds
+ **/
+ public void setConnectionTimeout(int timeout) {
+ HttpClientUtil.setConnectionTimeout(httpClient, timeout);
+ }
+
+ /**
+ * Set SoTimeout (read timeout). This is desirable
+ * for queries, but probably not for indexing.
+ *
+ * @param timeout
+ * Timeout in milliseconds
+ **/
+ public void setSoTimeout(int timeout) {
+ HttpClientUtil.setSoTimeout(httpClient, timeout);
+ }
+
+ /**
+ * Configure whether the client should follow redirects or not.
+ * <p>
+ * This defaults to false under the assumption that if you are following a
+ * redirect to get to a Solr installation, something is misconfigured
+ * somewhere.
+ * </p>
+ */
+ public void setFollowRedirects(boolean followRedirects) {
+ this.followRedirects = followRedirects;
+ HttpClientUtil.setFollowRedirects(httpClient, followRedirects);
+ }
+
+ /**
+ * Allow server->client communication to be compressed. Currently gzip and
+ * deflate are supported. If the server supports compression the response will
+ * be compressed. This method is only allowed if the http client is of type
+ * DefatulHttpClient.
+ */
+ public void setAllowCompression(boolean allowCompression) {
+ if (httpClient instanceof DefaultHttpClient) {
+ HttpClientUtil.setAllowCompression((DefaultHttpClient) httpClient, allowCompression);
+ } else {
+ throw new UnsupportedOperationException(
+ "HttpClient instance was not of type DefaultHttpClient");
+ }
+ }
+
+ /**
+ * Set maximum number of retries to attempt in the event of transient errors.
+ * <p>
+ * Maximum number of retries to attempt in the event of transient errors.
+ * Default: 0 (no) retries. No more than 1 recommended.
+ * </p>
+ * @param maxRetries
+ * No more than 1 recommended
+ */
+ public void setMaxRetries(int maxRetries) {
+ if (maxRetries > 1) {
+ log.warn("HttpSolrServer: maximum Retries " + maxRetries
+ + " > 1. Maximum recommended retries is 1.");
+ }
+ this.maxRetries = maxRetries;
+ }
+
+ public void setRequestWriter(RequestWriter requestWriter) {
+ this.requestWriter = requestWriter;
+ }
+
+ /**
+ * Adds the documents supplied by the given iterator.
+ *
+ * @param docIterator
+ * the iterator which returns SolrInputDocument instances
+ *
+ * @return the response from the SolrServer
+ */
+ public UpdateResponse add(Iterator<SolrInputDocument> docIterator)
+ throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.setDocIterator(docIterator);
+ return req.process(this);
+ }
+
+ /**
+ * Adds the beans supplied by the given iterator.
+ *
+ * @param beanIterator
+ * the iterator which returns Beans
+ *
+ * @return the response from the SolrServer
+ */
+ public UpdateResponse addBeans(final Iterator<?> beanIterator)
+ throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.setDocIterator(new Iterator<SolrInputDocument>() {
+
+ @Override
+ public boolean hasNext() {
+ return beanIterator.hasNext();
+ }
+
+ @Override
+ public SolrInputDocument next() {
+ Object o = beanIterator.next();
+ if (o == null) return null;
+ return getBinder().toSolrInputDocument(o);
+ }
+
+ @Override
+ public void remove() {
+ beanIterator.remove();
+ }
+ });
+ return req.process(this);
+ }
+
+ /**
+ * Close the {@link ClientConnectionManager} from the internal client.
+ */
+ @Override
+ public void close() throws IOException {
+ shutdown();
+ }
+
+ @Override
+ @Deprecated
+ public void shutdown() {
+ if (httpClient != null && internalClient) {
+ HttpClientUtil.close(httpClient);
+ }
+ }
+
+ /**
+ * Set the maximum number of connections that can be open to a single host at
+ * any given time. If http client was created outside the operation is not
+ * allowed.
+ */
+ public void setDefaultMaxConnectionsPerHost(int max) {
+ if (internalClient) {
+ HttpClientUtil.setMaxConnectionsPerHost(httpClient, max);
+ } else {
+ throw new UnsupportedOperationException(
+ "Client was created outside of HttpSolrServer");
+ }
+ }
+
+ /**
+ * Set the maximum number of connections that can be open at any given time.
+ * If http client was created outside the operation is not allowed.
+ */
+ public void setMaxTotalConnections(int max) {
+ if (internalClient) {
+ HttpClientUtil.setMaxConnections(httpClient, max);
+ } else {
+ throw new UnsupportedOperationException(
+ "Client was created outside of HttpSolrServer");
+ }
+ }
+
+ public boolean isUseMultiPartPost() {
+ return useMultiPartPost;
+ }
+
+ /**
+ * Set the multipart connection properties
+ */
+ public void setUseMultiPartPost(boolean useMultiPartPost) {
+ this.useMultiPartPost = useMultiPartPost;
+ }
+
+ /**
+ * Subclass of SolrException that allows us to capture an arbitrary HTTP
+ * status code that may have been returned by the remote server or a
+ * proxy along the way.
+ */
+ public static class RemoteSolrException extends SolrException {
+ /**
+ * @param remoteHost the host the error was received from
+ * @param code Arbitrary HTTP status code
+ * @param msg Exception Message
+ * @param th Throwable to wrap with this Exception
+ */
+ public RemoteSolrException(String remoteHost, int code, String msg, Throwable th) {
+ super(code, "Error from server at " + remoteHost + ": " + msg, th);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
new file mode 100644
index 0000000..ef5d439
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpSolrServer.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.ResponseParser;
+
+/**
+ * @deprecated Use {@link org.apache.solr.client.solrj.impl.HttpSolrClient}
+ */
+@Deprecated
+public class HttpSolrServer extends HttpSolrClient {
+
+ public HttpSolrServer(String baseURL) {
+ super(baseURL);
+ }
+
+ public HttpSolrServer(String baseURL, HttpClient client) {
+ super(baseURL, client);
+ }
+
+ public HttpSolrServer(String baseURL, HttpClient client, ResponseParser parser) {
+ super(baseURL, client, parser);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
new file mode 100644
index 0000000..c524b1f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrClient.java
@@ -0,0 +1,730 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.*;
+import org.apache.solr.client.solrj.request.IsUpdateRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.apache.solr.common.SolrException;
+
+import java.io.IOException;
+import java.lang.ref.WeakReference;
+import java.net.ConnectException;
+import java.net.MalformedURLException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.URL;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.*;
+
+/**
+ * LBHttpSolrClient or "LoadBalanced HttpSolrClient" is a load balancing wrapper around
+ * {@link HttpSolrClient}. This is useful when you
+ * have multiple Solr servers and the requests need to be Load Balanced among them.
+ *
+ * Do <b>NOT</b> use this class for indexing in master/slave scenarios since documents must be sent to the
+ * correct master; no inter-node routing is done.
+ *
+ * In SolrCloud (leader/replica) scenarios, it is usually better to use
+ * {@link CloudSolrClient}, but this class may be used
+ * for updates because the server will forward them to the appropriate leader.
+ *
+ * <p>
+ * It offers automatic failover when a server goes down and it detects when the server comes back up.
+ * <p>
+ * Load balancing is done using a simple round-robin on the list of servers.
+ * <p>
+ * If a request to a server fails by an IOException due to a connection timeout or read timeout then the host is taken
+ * off the list of live servers and moved to a 'dead server list' and the request is resent to the next live server.
+ * This process is continued till it tries all the live servers. If at least one server is alive, the request succeeds,
+ * and if not it fails.
+ * <blockquote><pre>
+ * SolrClient lbHttpSolrClient = new LBHttpSolrClient("http://host1:8080/solr/", "http://host2:8080/solr", "http://host2:8080/solr");
+ * //or if you wish to pass the HttpClient do as follows
+ * httpClient httpClient = new HttpClient();
+ * SolrClient lbHttpSolrClient = new LBHttpSolrClient(httpClient, "http://host1:8080/solr/", "http://host2:8080/solr", "http://host2:8080/solr");
+ * </pre></blockquote>
+ * This detects if a dead server comes alive automatically. The check is done in fixed intervals in a dedicated thread.
+ * This interval can be set using {@link #setAliveCheckInterval} , the default is set to one minute.
+ * <p>
+ * <b>When to use this?</b><br> This can be used as a software load balancer when you do not wish to setup an external
+ * load balancer. Alternatives to this code are to use
+ * a dedicated hardware load balancer or using Apache httpd with mod_proxy_balancer as a load balancer. See <a
+ * href="http://en.wikipedia.org/wiki/Load_balancing_(computing)">Load balancing on Wikipedia</a>
+ *
+ * @since solr 1.4
+ */
+public class LBHttpSolrClient extends SolrClient {
+ private static Set<Integer> RETRY_CODES = new HashSet<>(4);
+
+ static {
+ RETRY_CODES.add(404);
+ RETRY_CODES.add(403);
+ RETRY_CODES.add(503);
+ RETRY_CODES.add(500);
+ }
+
+ // keys to the maps are currently of the form "http://localhost:8983/solr"
+ // which should be equivalent to HttpSolrServer.getBaseURL()
+ private final Map<String, ServerWrapper> aliveServers = new LinkedHashMap<>();
+ // access to aliveServers should be synchronized on itself
+
+ protected final Map<String, ServerWrapper> zombieServers = new ConcurrentHashMap<>();
+
+ // changes to aliveServers are reflected in this array, no need to synchronize
+ private volatile ServerWrapper[] aliveServerList = new ServerWrapper[0];
+
+
+ private ScheduledExecutorService aliveCheckExecutor;
+
+ private final HttpClient httpClient;
+ private final boolean clientIsInternal;
+ private final AtomicInteger counter = new AtomicInteger(-1);
+
+ private static final SolrQuery solrQuery = new SolrQuery("*:*");
+ private volatile ResponseParser parser;
+ private volatile RequestWriter requestWriter;
+
+ private Set<String> queryParams = new HashSet<>();
+
+ static {
+ solrQuery.setRows(0);
+ /**
+ * Default sort (if we don't supply a sort) is by score and since
+ * we request 0 rows any sorting and scoring is not necessary.
+ * SolrQuery.DOCID schema-independently specifies a non-scoring sort.
+ * <code>_docid_ asc</code> sort is efficient,
+ * <code>_docid_ desc</code> sort is not, so choose ascending DOCID sort.
+ */
+ solrQuery.setSort(SolrQuery.DOCID, SolrQuery.ORDER.asc);
+ // not a top-level request, we are interested only in the server being sent to i.e. it need not distribute our request to further servers
+ solrQuery.setDistrib(false);
+ }
+
+ protected static class ServerWrapper {
+
+ final HttpSolrClient client;
+
+ long lastUsed; // last time used for a real request
+ long lastChecked; // last time checked for liveness
+
+ // "standard" servers are used by default. They normally live in the alive list
+ // and move to the zombie list when unavailable. When they become available again,
+ // they move back to the alive list.
+ boolean standard = true;
+
+ int failedPings = 0;
+
+ public ServerWrapper(HttpSolrClient client) {
+ this.client = client;
+ }
+
+ @Override
+ public String toString() {
+ return client.getBaseURL();
+ }
+
+ public String getKey() {
+ return client.getBaseURL();
+ }
+
+ @Override
+ public int hashCode() {
+ return this.getKey().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (!(obj instanceof ServerWrapper)) return false;
+ return this.getKey().equals(((ServerWrapper)obj).getKey());
+ }
+ }
+
+ public static class Req {
+ protected SolrRequest request;
+ protected List<String> servers;
+ protected int numDeadServersToTry;
+
+ public Req(SolrRequest request, List<String> servers) {
+ this.request = request;
+ this.servers = servers;
+ this.numDeadServersToTry = servers.size();
+ }
+
+ public SolrRequest getRequest() {
+ return request;
+ }
+ public List<String> getServers() {
+ return servers;
+ }
+
+ /** @return the number of dead servers to try if there are no live servers left */
+ public int getNumDeadServersToTry() {
+ return numDeadServersToTry;
+ }
+
+ /** @param numDeadServersToTry The number of dead servers to try if there are no live servers left.
+ * Defaults to the number of servers in this request. */
+ public void setNumDeadServersToTry(int numDeadServersToTry) {
+ this.numDeadServersToTry = numDeadServersToTry;
+ }
+ }
+
+ public static class Rsp {
+ protected String server;
+ protected NamedList<Object> rsp;
+
+ /** The response from the server */
+ public NamedList<Object> getResponse() {
+ return rsp;
+ }
+
+ /** The server that returned the response */
+ public String getServer() {
+ return server;
+ }
+ }
+
+ public LBHttpSolrClient(String... solrServerUrls) throws MalformedURLException {
+ this(null, solrServerUrls);
+ }
+
+ /** The provided httpClient should use a multi-threaded connection manager */
+ public LBHttpSolrClient(HttpClient httpClient, String... solrServerUrl) {
+ this(httpClient, new BinaryResponseParser(), solrServerUrl);
+ }
+
+ /** The provided httpClient should use a multi-threaded connection manager */
+ public LBHttpSolrClient(HttpClient httpClient, ResponseParser parser, String... solrServerUrl) {
+ clientIsInternal = (httpClient == null);
+ this.parser = parser;
+ if (httpClient == null) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ if (solrServerUrl.length > 1) {
+ // we prefer retrying another server
+ params.set(HttpClientUtil.PROP_USE_RETRY, false);
+ } else {
+ params.set(HttpClientUtil.PROP_USE_RETRY, true);
+ }
+ this.httpClient = HttpClientUtil.createClient(params);
+ } else {
+ this.httpClient = httpClient;
+ }
+ for (String s : solrServerUrl) {
+ ServerWrapper wrapper = new ServerWrapper(makeSolrClient(s));
+ aliveServers.put(wrapper.getKey(), wrapper);
+ }
+ updateAliveList();
+ }
+
+ public Set<String> getQueryParams() {
+ return queryParams;
+ }
+
+ /**
+ * Expert Method.
+ * @param queryParams set of param keys to only send via the query string
+ */
+ public void setQueryParams(Set<String> queryParams) {
+ this.queryParams = queryParams;
+ }
+ public void addQueryParams(String queryOnlyParam) {
+ this.queryParams.add(queryOnlyParam) ;
+ }
+
+ public static String normalize(String server) {
+ if (server.endsWith("/"))
+ server = server.substring(0, server.length() - 1);
+ return server;
+ }
+
+ protected HttpSolrClient makeSolrClient(String server) {
+ HttpSolrClient client = new HttpSolrClient(server, httpClient, parser);
+ if (requestWriter != null) {
+ client.setRequestWriter(requestWriter);
+ }
+ if (queryParams != null) {
+ client.setQueryParams(queryParams);
+ }
+ return client;
+ }
+
+ /**
+ * Tries to query a live server from the list provided in Req. Servers in the dead pool are skipped.
+ * If a request fails due to an IOException, the server is moved to the dead pool for a certain period of
+ * time, or until a test request on that server succeeds.
+ *
+ * Servers are queried in the exact order given (except servers currently in the dead pool are skipped).
+ * If no live servers from the provided list remain to be tried, a number of previously skipped dead servers will be tried.
+ * Req.getNumDeadServersToTry() controls how many dead servers will be tried.
+ *
+ * If no live servers are found a SolrServerException is thrown.
+ *
+ * @param req contains both the request as well as the list of servers to query
+ *
+ * @return the result of the request
+ *
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public Rsp request(Req req) throws SolrServerException, IOException {
+ Rsp rsp = new Rsp();
+ Exception ex = null;
+ boolean isUpdate = req.request instanceof IsUpdateRequest;
+ List<ServerWrapper> skipped = null;
+
+ long timeAllowedNano = getTimeAllowedInNanos(req.getRequest());
+ long timeOutTime = System.nanoTime() + timeAllowedNano;
+ for (String serverStr : req.getServers()) {
+ if(isTimeExceeded(timeAllowedNano, timeOutTime)) {
+ break;
+ }
+
+ serverStr = normalize(serverStr);
+ // if the server is currently a zombie, just skip to the next one
+ ServerWrapper wrapper = zombieServers.get(serverStr);
+ if (wrapper != null) {
+ // System.out.println("ZOMBIE SERVER QUERIED: " + serverStr);
+ final int numDeadServersToTry = req.getNumDeadServersToTry();
+ if (numDeadServersToTry > 0) {
+ if (skipped == null) {
+ skipped = new ArrayList<>(numDeadServersToTry);
+ skipped.add(wrapper);
+ }
+ else if (skipped.size() < numDeadServersToTry) {
+ skipped.add(wrapper);
+ }
+ }
+ continue;
+ }
+ rsp.server = serverStr;
+ HttpSolrClient client = makeSolrClient(serverStr);
+
+ ex = doRequest(client, req, rsp, isUpdate, false, null);
+ if (ex == null) {
+ return rsp; // SUCCESS
+ }
+ }
+
+ // try the servers we previously skipped
+ if (skipped != null) {
+ for (ServerWrapper wrapper : skipped) {
+ if(isTimeExceeded(timeAllowedNano, timeOutTime)) {
+ break;
+ }
+
+ ex = doRequest(wrapper.client, req, rsp, isUpdate, true, wrapper.getKey());
+ if (ex == null) {
+ return rsp; // SUCCESS
+ }
+ }
+ }
+
+
+ if (ex == null) {
+ throw new SolrServerException("No live SolrServers available to handle this request");
+ } else {
+ throw new SolrServerException("No live SolrServers available to handle this request:" + zombieServers.keySet(), ex);
+ }
+
+ }
+
+ protected Exception addZombie(HttpSolrClient server, Exception e) {
+
+ ServerWrapper wrapper;
+
+ wrapper = new ServerWrapper(server);
+ wrapper.lastUsed = System.currentTimeMillis();
+ wrapper.standard = false;
+ zombieServers.put(wrapper.getKey(), wrapper);
+ startAliveCheckExecutor();
+ return e;
+ }
+
+ protected Exception doRequest(HttpSolrClient client, Req req, Rsp rsp, boolean isUpdate,
+ boolean isZombie, String zombieKey) throws SolrServerException, IOException {
+ Exception ex = null;
+ try {
+ rsp.rsp = client.request(req.getRequest());
+ if (isZombie) {
+ zombieServers.remove(zombieKey);
+ }
+ } catch (SolrException e) {
+ // we retry on 404 or 403 or 503 or 500
+ // unless it's an update - then we only retry on connect exception
+ if (!isUpdate && RETRY_CODES.contains(e.code())) {
+ ex = (!isZombie) ? addZombie(client, e) : e;
+ } else {
+ // Server is alive but the request was likely malformed or invalid
+ if (isZombie) {
+ zombieServers.remove(zombieKey);
+ }
+ throw e;
+ }
+ } catch (SocketException e) {
+ if (!isUpdate || e instanceof ConnectException) {
+ ex = (!isZombie) ? addZombie(client, e) : e;
+ } else {
+ throw e;
+ }
+ } catch (SocketTimeoutException e) {
+ if (!isUpdate) {
+ ex = (!isZombie) ? addZombie(client, e) : e;
+ } else {
+ throw e;
+ }
+ } catch (SolrServerException e) {
+ Throwable rootCause = e.getRootCause();
+ if (!isUpdate && rootCause instanceof IOException) {
+ ex = (!isZombie) ? addZombie(client, e) : e;
+ } else if (isUpdate && rootCause instanceof ConnectException) {
+ ex = (!isZombie) ? addZombie(client, e) : e;
+ } else {
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new SolrServerException(e);
+ }
+
+ return ex;
+ }
+
+ private void updateAliveList() {
+ synchronized (aliveServers) {
+ aliveServerList = aliveServers.values().toArray(new ServerWrapper[aliveServers.size()]);
+ }
+ }
+
+ private ServerWrapper removeFromAlive(String key) {
+ synchronized (aliveServers) {
+ ServerWrapper wrapper = aliveServers.remove(key);
+ if (wrapper != null)
+ updateAliveList();
+ return wrapper;
+ }
+ }
+
+ private void addToAlive(ServerWrapper wrapper) {
+ synchronized (aliveServers) {
+ ServerWrapper prev = aliveServers.put(wrapper.getKey(), wrapper);
+ // TODO: warn if there was a previous entry?
+ updateAliveList();
+ }
+ }
+
+ public void addSolrServer(String server) throws MalformedURLException {
+ HttpSolrClient client = makeSolrClient(server);
+ addToAlive(new ServerWrapper(client));
+ }
+
+ public String removeSolrServer(String server) {
+ try {
+ server = new URL(server).toExternalForm();
+ } catch (MalformedURLException e) {
+ throw new RuntimeException(e);
+ }
+ if (server.endsWith("/")) {
+ server = server.substring(0, server.length() - 1);
+ }
+
+ // there is a small race condition here - if the server is in the process of being moved between
+ // lists, we could fail to remove it.
+ removeFromAlive(server);
+ zombieServers.remove(server);
+ return null;
+ }
+
+ public void setConnectionTimeout(int timeout) {
+ HttpClientUtil.setConnectionTimeout(httpClient, timeout);
+ }
+
+ /**
+ * set soTimeout (read timeout) on the underlying HttpConnectionManager. This is desirable for queries, but probably
+ * not for indexing.
+ */
+ public void setSoTimeout(int timeout) {
+ HttpClientUtil.setSoTimeout(httpClient, timeout);
+ }
+
+ @Override
+ public void close() {
+ shutdown();
+ }
+
+ @Override
+ @Deprecated
+ public void shutdown() {
+ if (aliveCheckExecutor != null) {
+ aliveCheckExecutor.shutdownNow();
+ }
+ if(clientIsInternal) {
+ HttpClientUtil.close(httpClient);
+ }
+ }
+
+ /**
+ * Tries to query a live server. A SolrServerException is thrown if all servers are dead.
+ * If the request failed due to IOException then the live server is moved to dead pool and the request is
+ * retried on another live server. After live servers are exhausted, any servers previously marked as dead
+ * will be tried before failing the request.
+ *
+ * @param request the SolrRequest.
+ *
+ * @return response
+ *
+ * @throws IOException If there is a low-level I/O error.
+ */
+ @Override
+ public NamedList<Object> request(final SolrRequest request)
+ throws SolrServerException, IOException {
+ Exception ex = null;
+ ServerWrapper[] serverList = aliveServerList;
+
+ int maxTries = serverList.length;
+ Map<String,ServerWrapper> justFailed = null;
+
+ long timeAllowedNano = getTimeAllowedInNanos(request);
+ long timeOutTime = System.nanoTime() + timeAllowedNano;
+ for (int attempts=0; attempts<maxTries; attempts++) {
+ if(isTimeExceeded(timeAllowedNano, timeOutTime)) {
+ break;
+ }
+
+ int count = counter.incrementAndGet() & Integer.MAX_VALUE;
+ ServerWrapper wrapper = serverList[count % serverList.length];
+ wrapper.lastUsed = System.currentTimeMillis();
+
+ try {
+ return wrapper.client.request(request);
+ } catch (SolrException e) {
+ // Server is alive but the request was malformed or invalid
+ throw e;
+ } catch (SolrServerException e) {
+ if (e.getRootCause() instanceof IOException) {
+ ex = e;
+ moveAliveToDead(wrapper);
+ if (justFailed == null) justFailed = new HashMap<>();
+ justFailed.put(wrapper.getKey(), wrapper);
+ } else {
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new SolrServerException(e);
+ }
+ }
+
+ // try other standard servers that we didn't try just now
+ for (ServerWrapper wrapper : zombieServers.values()) {
+ if(isTimeExceeded(timeAllowedNano, timeOutTime)) {
+ break;
+ }
+
+ if (wrapper.standard==false || justFailed!=null && justFailed.containsKey(wrapper.getKey())) continue;
+ try {
+ NamedList<Object> rsp = wrapper.client.request(request);
+ // remove from zombie list *before* adding to alive to avoid a race that could lose a server
+ zombieServers.remove(wrapper.getKey());
+ addToAlive(wrapper);
+ return rsp;
+ } catch (SolrException e) {
+ // Server is alive but the request was malformed or invalid
+ throw e;
+ } catch (SolrServerException e) {
+ if (e.getRootCause() instanceof IOException) {
+ ex = e;
+ // still dead
+ } else {
+ throw e;
+ }
+ } catch (Exception e) {
+ throw new SolrServerException(e);
+ }
+ }
+
+
+ if (ex == null) {
+ throw new SolrServerException("No live SolrServers available to handle this request");
+ } else {
+ throw new SolrServerException("No live SolrServers available to handle this request", ex);
+ }
+ }
+
+ /**
+ * @return time allowed in nanos, returns -1 if no time_allowed is specified.
+ */
+ private long getTimeAllowedInNanos(final SolrRequest req) {
+ SolrParams reqParams = req.getParams();
+ return reqParams == null ? -1 :
+ TimeUnit.NANOSECONDS.convert(reqParams.getInt(CommonParams.TIME_ALLOWED, -1), TimeUnit.MILLISECONDS);
+ }
+
+ private boolean isTimeExceeded(long timeAllowedNano, long timeOutTime) {
+ return timeAllowedNano > 0 && System.nanoTime() > timeOutTime;
+ }
+
+ /**
+ * Takes up one dead server and check for aliveness. The check is done in a roundrobin. Each server is checked for
+ * aliveness once in 'x' millis where x is decided by the setAliveCheckinterval() or it is defaulted to 1 minute
+ *
+ * @param zombieServer a server in the dead pool
+ */
+ private void checkAZombieServer(ServerWrapper zombieServer) {
+ long currTime = System.currentTimeMillis();
+ try {
+ zombieServer.lastChecked = currTime;
+ QueryResponse resp = zombieServer.client.query(solrQuery);
+ if (resp.getStatus() == 0) {
+ // server has come back up.
+ // make sure to remove from zombies before adding to alive to avoid a race condition
+ // where another thread could mark it down, move it back to zombie, and then we delete
+ // from zombie and lose it forever.
+ ServerWrapper wrapper = zombieServers.remove(zombieServer.getKey());
+ if (wrapper != null) {
+ wrapper.failedPings = 0;
+ if (wrapper.standard) {
+ addToAlive(wrapper);
+ }
+ } else {
+ // something else already moved the server from zombie to alive
+ }
+ }
+ } catch (Exception e) {
+ //Expected. The server is still down.
+ zombieServer.failedPings++;
+
+ // If the server doesn't belong in the standard set belonging to this load balancer
+ // then simply drop it after a certain number of failed pings.
+ if (!zombieServer.standard && zombieServer.failedPings >= NONSTANDARD_PING_LIMIT) {
+ zombieServers.remove(zombieServer.getKey());
+ }
+ }
+ }
+
+ private void moveAliveToDead(ServerWrapper wrapper) {
+ wrapper = removeFromAlive(wrapper.getKey());
+ if (wrapper == null)
+ return; // another thread already detected the failure and removed it
+ zombieServers.put(wrapper.getKey(), wrapper);
+ startAliveCheckExecutor();
+ }
+
+ private int interval = CHECK_INTERVAL;
+
+ /**
+ * LBHttpSolrServer keeps pinging the dead servers at fixed interval to find if it is alive. Use this to set that
+ * interval
+ *
+ * @param interval time in milliseconds
+ */
+ public void setAliveCheckInterval(int interval) {
+ if (interval <= 0) {
+ throw new IllegalArgumentException("Alive check interval must be " +
+ "positive, specified value = " + interval);
+ }
+ this.interval = interval;
+ }
+
+ private void startAliveCheckExecutor() {
+ // double-checked locking, but it's OK because we don't *do* anything with aliveCheckExecutor
+ // if it's not null.
+ if (aliveCheckExecutor == null) {
+ synchronized (this) {
+ if (aliveCheckExecutor == null) {
+ aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(
+ new SolrjNamedThreadFactory("aliveCheckExecutor"));
+ aliveCheckExecutor.scheduleAtFixedRate(
+ getAliveCheckRunner(new WeakReference<>(this)),
+ this.interval, this.interval, TimeUnit.MILLISECONDS);
+ }
+ }
+ }
+ }
+
+ private static Runnable getAliveCheckRunner(final WeakReference<LBHttpSolrClient> lbRef) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ LBHttpSolrClient lb = lbRef.get();
+ if (lb != null && lb.zombieServers != null) {
+ for (ServerWrapper zombieServer : lb.zombieServers.values()) {
+ lb.checkAZombieServer(zombieServer);
+ }
+ }
+ }
+ };
+ }
+
+ /**
+ * Return the HttpClient this instance uses.
+ */
+ public HttpClient getHttpClient() {
+ return httpClient;
+ }
+
+ public ResponseParser getParser() {
+ return parser;
+ }
+
+ /**
+ * Changes the {@link ResponseParser} that will be used for the internal
+ * SolrServer objects.
+ *
+ * @param parser Default Response Parser chosen to parse the response if the parser
+ * were not specified as part of the request.
+ * @see org.apache.solr.client.solrj.SolrRequest#getResponseParser()
+ */
+ public void setParser(ResponseParser parser) {
+ this.parser = parser;
+ }
+
+ /**
+ * Changes the {@link RequestWriter} that will be used for the internal
+ * SolrServer objects.
+ *
+ * @param requestWriter Default RequestWriter, used to encode requests sent to the server.
+ */
+ public void setRequestWriter(RequestWriter requestWriter) {
+ this.requestWriter = requestWriter;
+ }
+
+ public RequestWriter getRequestWriter() {
+ return requestWriter;
+ }
+
+ @Override
+ protected void finalize() throws Throwable {
+ try {
+ if(this.aliveCheckExecutor!=null)
+ this.aliveCheckExecutor.shutdownNow();
+ } finally {
+ super.finalize();
+ }
+ }
+
+ // defaults
+ private static final int CHECK_INTERVAL = 60 * 1000; //1 minute between checks
+ private static final int NONSTANDARD_PING_LIMIT = 5; // number of times we'll ping dead servers not in the server list
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
new file mode 100644
index 0000000..ee28241
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/LBHttpSolrServer.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.http.client.HttpClient;
+import org.apache.solr.client.solrj.ResponseParser;
+
+import java.net.MalformedURLException;
+
+/**
+ * @deprecated Use {@link org.apache.solr.client.solrj.impl.LBHttpSolrClient}
+ */
+@Deprecated
+public class LBHttpSolrServer extends LBHttpSolrClient {
+
+ public LBHttpSolrServer(String... solrServerUrls) throws MalformedURLException {
+ super(solrServerUrls);
+ }
+
+ public LBHttpSolrServer(HttpClient httpClient, String... solrServerUrl) {
+ super(httpClient, solrServerUrl);
+ }
+
+ public LBHttpSolrServer(HttpClient httpClient, ResponseParser parser, String... solrServerUrl) {
+ super(httpClient, parser, solrServerUrl);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/NoOpResponseParser.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/NoOpResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/NoOpResponseParser.java
new file mode 100644
index 0000000..267dd25
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/NoOpResponseParser.java
@@ -0,0 +1,83 @@
+package org.apache.solr.client.solrj.impl;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringWriter;
+
+/**
+ * Simply puts the entire response into an entry in a NamedList.
+ * This parser isn't parse response into a QueryResponse.
+ */
+public class NoOpResponseParser extends ResponseParser {
+
+ private String writerType = "xml";
+
+ public NoOpResponseParser() {
+ }
+
+ public NoOpResponseParser(String writerType) {
+ this.writerType = writerType;
+ }
+
+ @Override
+ public String getWriterType() {
+ return writerType;
+ }
+
+ public void setWriterType(String writerType) {
+ this.writerType = writerType;
+ }
+
+ @Override
+ public NamedList<Object> processResponse(Reader reader) {
+ try {
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(reader, writer);
+ String output = writer.toString();
+ NamedList<Object> list = new NamedList<>();
+ list.add("response", output);
+ return list;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
+ }
+ }
+
+ @Override
+ public NamedList<Object> processResponse(InputStream body, String encoding) {
+ try {
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(body, writer, encoding);
+ String output = writer.toString();
+ NamedList<Object> list = new NamedList<>();
+ list.add("response", output);
+ return list;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
new file mode 100644
index 0000000..14be1f3
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import org.apache.solr.client.solrj.StreamingResponseCallback;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.DataInputInputStream;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ * A BinaryResponseParser that sends callback events rather then build
+ * a large response
+ *
+ *
+ * @since solr 4.0
+ */
+public class StreamingBinaryResponseParser extends BinaryResponseParser {
+ final StreamingResponseCallback callback;
+
+ public StreamingBinaryResponseParser( StreamingResponseCallback cb )
+ {
+ this.callback = cb;
+ }
+
+ @Override
+ public NamedList<Object> processResponse(InputStream body, String encoding) {
+ try {
+ JavaBinCodec codec = new JavaBinCodec() {
+
+ @Override
+ public SolrDocument readSolrDocument(DataInputInputStream dis) throws IOException {
+ SolrDocument doc = super.readSolrDocument(dis);
+ callback.streamSolrDocument( doc );
+ return null;
+ }
+
+ @Override
+ public SolrDocumentList readSolrDocumentList(DataInputInputStream dis) throws IOException {
+ SolrDocumentList solrDocs = new SolrDocumentList();
+ List list = (List) readVal(dis);
+ solrDocs.setNumFound((Long) list.get(0));
+ solrDocs.setStart((Long) list.get(1));
+ solrDocs.setMaxScore((Float) list.get(2));
+
+ callback.streamDocListInfo(
+ solrDocs.getNumFound(),
+ solrDocs.getStart(),
+ solrDocs.getMaxScore() );
+
+ // Read the Array
+ tagByte = dis.readByte();
+ if( (tagByte >>> 5) != (ARR >>> 5) ) {
+ throw new RuntimeException( "doclist must have an array" );
+ }
+ int sz = readSize(dis);
+ for (int i = 0; i < sz; i++) {
+ // must be a SolrDocument
+ readVal( dis );
+ }
+ return solrDocs;
+ }
+ };
+
+ return (NamedList<Object>) codec.unmarshal(body);
+ }
+ catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
+ }
+ }
+}
[12/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java
new file mode 100644
index 0000000..f9177c8
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/XMLResponseParser.java
@@ -0,0 +1,465 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.DateUtil;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.XMLErrorLogger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public class XMLResponseParser extends ResponseParser
+{
+ public static final String XML_CONTENT_TYPE = "application/xml; charset=UTF-8";
+ public static Logger log = LoggerFactory.getLogger(XMLResponseParser.class);
+ private static final XMLErrorLogger xmllog = new XMLErrorLogger(log);
+
+ // reuse the factory among all parser instances so things like string caches
+ // won't be duplicated
+ static final XMLInputFactory factory;
+ static {
+ factory = XMLInputFactory.newInstance();
+ try {
+ // The java 1.6 bundled stax parser (sjsxp) does not currently have a thread-safe
+ // XMLInputFactory, as that implementation tries to cache and reuse the
+ // XMLStreamReader. Setting the parser-specific "reuse-instance" property to false
+ // prevents this.
+ // All other known open-source stax parsers (and the bea ref impl)
+ // have thread-safe factories.
+ factory.setProperty("reuse-instance", Boolean.FALSE);
+ }
+ catch( IllegalArgumentException ex ) {
+ // Other implementations will likely throw this exception since "reuse-instance"
+ // isimplementation specific.
+ log.debug( "Unable to set the 'reuse-instance' property for the input factory: "+factory );
+ }
+ factory.setXMLReporter(xmllog);
+ }
+
+ public XMLResponseParser() {}
+
+ @Override
+ public String getWriterType()
+ {
+ return "xml";
+ }
+
+ @Override
+ public String getContentType() {
+ return XML_CONTENT_TYPE;
+ }
+
+ @Override
+ public NamedList<Object> processResponse(Reader in) {
+ XMLStreamReader parser = null;
+ try {
+ parser = factory.createXMLStreamReader(in);
+ } catch (XMLStreamException e) {
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
+ }
+
+ return processResponse(parser);
+ }
+
+ @Override
+ public NamedList<Object> processResponse(InputStream in, String encoding)
+ {
+ XMLStreamReader parser = null;
+ try {
+ parser = factory.createXMLStreamReader(in, encoding);
+ } catch (XMLStreamException e) {
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
+ }
+
+ return processResponse(parser);
+ }
+
+ /**
+ * parse the text into a named list...
+ */
+ private NamedList<Object> processResponse(XMLStreamReader parser)
+ {
+ try {
+ NamedList<Object> response = null;
+ for (int event = parser.next();
+ event != XMLStreamConstants.END_DOCUMENT;
+ event = parser.next())
+ {
+ switch (event) {
+ case XMLStreamConstants.START_ELEMENT:
+
+ if( response != null ) {
+ throw new Exception( "already read the response!" );
+ }
+
+ // only top-level element is "response
+ String name = parser.getLocalName();
+ if( name.equals( "response" ) || name.equals( "result" ) ) {
+ response = readNamedList( parser );
+ }
+ else if( name.equals( "solr" ) ) {
+ return new SimpleOrderedMap<>();
+ }
+ else {
+ throw new Exception( "really needs to be response or result. " +
+ "not:"+parser.getLocalName() );
+ }
+ break;
+ }
+ }
+ return response;
+ }
+ catch( Exception ex ) {
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "parsing error", ex );
+ }
+ finally {
+ try {
+ parser.close();
+ }
+ catch( Exception ex ){}
+ }
+ }
+
+
+ protected enum KnownType {
+ STR (true) { @Override public String read( String txt ) { return txt; } },
+ INT (true) { @Override public Integer read( String txt ) { return Integer.valueOf(txt); } },
+ FLOAT (true) { @Override public Float read( String txt ) { return Float.valueOf(txt); } },
+ DOUBLE (true) { @Override public Double read( String txt ) { return Double.valueOf(txt); } },
+ LONG (true) { @Override public Long read( String txt ) { return Long.valueOf(txt); } },
+ BOOL (true) { @Override public Boolean read( String txt ) { return Boolean.valueOf(txt); } },
+ NULL (true) { @Override public Object read( String txt ) { return null; } },
+ DATE (true) {
+ @Override
+ public Date read( String txt ) {
+ try {
+ return DateUtil.parseDate(txt);
+ }
+ catch( Exception ex ) {
+ ex.printStackTrace();
+ }
+ return null;
+ }
+ },
+
+ ARR (false) { @Override public Object read( String txt ) { return null; } },
+ LST (false) { @Override public Object read( String txt ) { return null; } },
+ RESULT (false) { @Override public Object read( String txt ) { return null; } },
+ DOC (false) { @Override public Object read( String txt ) { return null; } };
+
+ final boolean isLeaf;
+
+ KnownType( boolean isLeaf )
+ {
+ this.isLeaf = isLeaf;
+ }
+
+ public abstract Object read( String txt );
+
+ public static KnownType get( String v )
+ {
+ if( v != null ) {
+ try {
+ return KnownType.valueOf( v.toUpperCase(Locale.ROOT) );
+ }
+ catch( Exception ex ) {}
+ }
+ return null;
+ }
+ };
+
+ protected NamedList<Object> readNamedList( XMLStreamReader parser ) throws XMLStreamException
+ {
+ if( XMLStreamConstants.START_ELEMENT != parser.getEventType() ) {
+ throw new RuntimeException( "must be start element, not: "+parser.getEventType() );
+ }
+
+ StringBuilder builder = new StringBuilder();
+ NamedList<Object> nl = new SimpleOrderedMap<>();
+ KnownType type = null;
+ String name = null;
+
+ // just eat up the events...
+ int depth = 0;
+ while( true )
+ {
+ switch (parser.next()) {
+ case XMLStreamConstants.START_ELEMENT:
+ depth++;
+ builder.setLength( 0 ); // reset the text
+ type = KnownType.get( parser.getLocalName() );
+ if( type == null ) {
+ throw new RuntimeException( "this must be known type! not: "+parser.getLocalName() );
+ }
+
+ name = null;
+ int cnt = parser.getAttributeCount();
+ for( int i=0; i<cnt; i++ ) {
+ if( "name".equals( parser.getAttributeLocalName( i ) ) ) {
+ name = parser.getAttributeValue( i );
+ break;
+ }
+ }
+
+ /** The name in a NamedList can actually be null
+ if( name == null ) {
+ throw new XMLStreamException( "requires 'name' attribute: "+parser.getLocalName(), parser.getLocation() );
+ }
+ **/
+
+ if( !type.isLeaf ) {
+ switch( type ) {
+ case LST: nl.add( name, readNamedList( parser ) ); depth--; continue;
+ case ARR: nl.add( name, readArray( parser ) ); depth--; continue;
+ case RESULT: nl.add( name, readDocuments( parser ) ); depth--; continue;
+ case DOC: nl.add( name, readDocument( parser ) ); depth--; continue;
+ }
+ throw new XMLStreamException( "branch element not handled!", parser.getLocation() );
+ }
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ if( --depth < 0 ) {
+ return nl;
+ }
+ //System.out.println( "NL:ELEM:"+type+"::"+name+"::"+builder );
+ nl.add( name, type.read( builder.toString().trim() ) );
+ break;
+
+ case XMLStreamConstants.SPACE: // TODO? should this be trimmed? make sure it only gets one/two space?
+ case XMLStreamConstants.CDATA:
+ case XMLStreamConstants.CHARACTERS:
+ builder.append( parser.getText() );
+ break;
+ }
+ }
+ }
+
+ protected List<Object> readArray( XMLStreamReader parser ) throws XMLStreamException
+ {
+ if( XMLStreamConstants.START_ELEMENT != parser.getEventType() ) {
+ throw new RuntimeException( "must be start element, not: "+parser.getEventType() );
+ }
+ if( !"arr".equals( parser.getLocalName().toLowerCase(Locale.ROOT) ) ) {
+ throw new RuntimeException( "must be 'arr', not: "+parser.getLocalName() );
+ }
+
+ StringBuilder builder = new StringBuilder();
+ KnownType type = null;
+
+ List<Object> vals = new ArrayList<>();
+
+ int depth = 0;
+ while( true )
+ {
+ switch (parser.next()) {
+ case XMLStreamConstants.START_ELEMENT:
+ depth++;
+ KnownType t = KnownType.get( parser.getLocalName() );
+ if( t == null ) {
+ throw new RuntimeException( "this must be known type! not: "+parser.getLocalName() );
+ }
+ if( type == null ) {
+ type = t;
+ }
+ /*** actually, there is no rule that arrays need the same type
+ else if( type != t && !(t == KnownType.NULL || type == KnownType.NULL)) {
+ throw new RuntimeException( "arrays must have the same type! ("+type+"!="+t+") "+parser.getLocalName() );
+ }
+ ***/
+ type = t;
+
+ builder.setLength( 0 ); // reset the text
+
+ if( !type.isLeaf ) {
+ switch( type ) {
+ case LST: vals.add( readNamedList( parser ) ); depth--; continue;
+ case ARR: vals.add( readArray( parser ) ); depth--; continue;
+ case RESULT: vals.add( readDocuments( parser ) ); depth--; continue;
+ case DOC: vals.add( readDocument( parser ) ); depth--; continue;
+ }
+ throw new XMLStreamException( "branch element not handled!", parser.getLocation() );
+ }
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ if( --depth < 0 ) {
+ return vals; // the last element is itself
+ }
+ //System.out.println( "ARR:"+type+"::"+builder );
+ Object val = type.read( builder.toString().trim() );
+ if( val == null && type != KnownType.NULL) {
+ throw new XMLStreamException( "error reading value:"+type, parser.getLocation() );
+ }
+ vals.add( val );
+ break;
+
+ case XMLStreamConstants.SPACE: // TODO? should this be trimmed? make sure it only gets one/two space?
+ case XMLStreamConstants.CDATA:
+ case XMLStreamConstants.CHARACTERS:
+ builder.append( parser.getText() );
+ break;
+ }
+ }
+ }
+
+ protected SolrDocumentList readDocuments( XMLStreamReader parser ) throws XMLStreamException
+ {
+ SolrDocumentList docs = new SolrDocumentList();
+
+ // Parse the attributes
+ for( int i=0; i<parser.getAttributeCount(); i++ ) {
+ String n = parser.getAttributeLocalName( i );
+ String v = parser.getAttributeValue( i );
+ if( "numFound".equals( n ) ) {
+ docs.setNumFound( Long.parseLong( v ) );
+ }
+ else if( "start".equals( n ) ) {
+ docs.setStart( Long.parseLong( v ) );
+ }
+ else if( "maxScore".equals( n ) ) {
+ docs.setMaxScore( Float.parseFloat( v ) );
+ }
+ }
+
+ // Read through each document
+ int event;
+ while( true ) {
+ event = parser.next();
+ if( XMLStreamConstants.START_ELEMENT == event ) {
+ if( !"doc".equals( parser.getLocalName() ) ) {
+ throw new RuntimeException( "should be doc! "+parser.getLocalName() + " :: " + parser.getLocation() );
+ }
+ docs.add( readDocument( parser ) );
+ }
+ else if ( XMLStreamConstants.END_ELEMENT == event ) {
+ return docs; // only happens once
+ }
+ }
+ }
+
+ protected SolrDocument readDocument( XMLStreamReader parser ) throws XMLStreamException
+ {
+ if( XMLStreamConstants.START_ELEMENT != parser.getEventType() ) {
+ throw new RuntimeException( "must be start element, not: "+parser.getEventType() );
+ }
+ if( !"doc".equals( parser.getLocalName().toLowerCase(Locale.ROOT) ) ) {
+ throw new RuntimeException( "must be 'lst', not: "+parser.getLocalName() );
+ }
+
+ SolrDocument doc = new SolrDocument();
+ StringBuilder builder = new StringBuilder();
+ KnownType type = null;
+ String name = null;
+
+ // just eat up the events...
+ int depth = 0;
+ while( true )
+ {
+ switch (parser.next()) {
+ case XMLStreamConstants.START_ELEMENT:
+ depth++;
+ builder.setLength( 0 ); // reset the text
+ type = KnownType.get( parser.getLocalName() );
+ if( type == null ) {
+ throw new RuntimeException( "this must be known type! not: "+parser.getLocalName() );
+ }
+
+ name = null;
+ int cnt = parser.getAttributeCount();
+ for( int i=0; i<cnt; i++ ) {
+ if( "name".equals( parser.getAttributeLocalName( i ) ) ) {
+ name = parser.getAttributeValue( i );
+ break;
+ }
+ }
+
+ //Nested documents
+ while( type == KnownType.DOC) {
+ doc.addChildDocument(readDocument(parser));
+ int event = parser.next();
+ if (event == XMLStreamConstants.END_ELEMENT) { //Doc ends
+ return doc;
+ }
+ }
+
+ if( name == null ) {
+ throw new XMLStreamException( "requires 'name' attribute: "+parser.getLocalName(), parser.getLocation() );
+ }
+
+ // Handle multi-valued fields
+ if( type == KnownType.ARR ) {
+ for( Object val : readArray( parser ) ) {
+ doc.addField( name, val );
+ }
+ depth--; // the array reading clears out the 'endElement'
+ } else if( type == KnownType.LST ) {
+ doc.addField( name, readNamedList( parser ) );
+ depth--;
+ } else if( !type.isLeaf ) {
+ System.out.println("nbot leaf!:" + type);
+
+ throw new XMLStreamException( "must be value or array", parser.getLocation() );
+ }
+ break;
+
+ case XMLStreamConstants.END_ELEMENT:
+ if( --depth < 0 ) {
+ return doc;
+ }
+ //System.out.println( "FIELD:"+type+"::"+name+"::"+builder );
+ Object val = type.read( builder.toString().trim() );
+ if( val == null ) {
+ throw new XMLStreamException( "error reading value:"+type, parser.getLocation() );
+ }
+ doc.addField( name, val );
+ break;
+
+ case XMLStreamConstants.SPACE: // TODO? should this be trimmed? make sure it only gets one/two space?
+ case XMLStreamConstants.CDATA:
+ case XMLStreamConstants.CHARACTERS:
+ builder.append( parser.getText() );
+ break;
+ }
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/package-info.java
new file mode 100644
index 0000000..491ef05
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Concrete implementations of client API classes.
+
+ */
+package org.apache.solr.client.solrj.impl;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/package-info.java
new file mode 100644
index 0000000..b88918e
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Primary APIs for communicating with a Solr Server from a Java client.
+ */
+package org.apache.solr.client.solrj;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
new file mode 100644
index 0000000..8fbc463
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
@@ -0,0 +1,144 @@
+package org.apache.solr.client.solrj.request;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.UpdateParams;
+
+
+/**
+ *
+ *
+ **/
+public abstract class AbstractUpdateRequest extends SolrRequest<UpdateResponse> implements IsUpdateRequest {
+ protected ModifiableSolrParams params;
+ protected int commitWithin = -1;
+
+ public enum ACTION {
+ COMMIT,
+ OPTIMIZE
+ }
+
+ public AbstractUpdateRequest(METHOD m, String path) {
+ super(m, path);
+ }
+
+ /** Sets appropriate parameters for the given ACTION */
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher ) {
+ return setAction(action, waitFlush, waitSearcher, 1);
+ }
+
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, boolean softCommit ) {
+ return setAction(action, waitFlush, waitSearcher, softCommit, 1);
+ }
+
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments ) {
+ return setAction(action, waitFlush, waitSearcher, false, maxSegments);
+ }
+
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, boolean softCommit, int maxSegments ) {
+ if (params == null)
+ params = new ModifiableSolrParams();
+
+ if( action == ACTION.OPTIMIZE ) {
+ params.set( UpdateParams.OPTIMIZE, "true" );
+ params.set(UpdateParams.MAX_OPTIMIZE_SEGMENTS, maxSegments);
+ }
+ else if( action == ACTION.COMMIT ) {
+ params.set( UpdateParams.COMMIT, "true" );
+ params.set( UpdateParams.SOFT_COMMIT, String.valueOf(softCommit));
+ }
+ params.set( UpdateParams.WAIT_SEARCHER, String.valueOf(waitSearcher));
+ return this;
+ }
+
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments , boolean softCommit, boolean expungeDeletes) {
+ setAction(action, waitFlush, waitSearcher,softCommit,maxSegments) ;
+ params.set(UpdateParams.EXPUNGE_DELETES, String.valueOf(expungeDeletes));
+ return this;
+ }
+
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
+ return setAction(action, waitFlush, waitSearcher,maxSegments,false,expungeDeletes);
+ }
+
+ public AbstractUpdateRequest setAction(ACTION action, boolean waitFlush, boolean waitSearcher, int maxSegments, boolean softCommit, boolean expungeDeletes, boolean openSearcher) {
+ setAction(action, waitFlush, waitSearcher, maxSegments, softCommit, expungeDeletes);
+ params.set(UpdateParams.OPEN_SEARCHER, String.valueOf(openSearcher));
+ return this;
+ }
+
+ /**
+ * @since Solr 1.4
+ */
+ public AbstractUpdateRequest rollback() {
+ if (params == null)
+ params = new ModifiableSolrParams();
+
+ params.set( UpdateParams.ROLLBACK, "true" );
+ return this;
+ }
+
+ public void setParam(String param, String value) {
+ if (params == null)
+ params = new ModifiableSolrParams();
+ params.set(param, value);
+ }
+
+ /** Sets the parameters for this update request, overwriting any previous */
+ public void setParams(ModifiableSolrParams params) {
+ this.params = params;
+ }
+
+ @Override
+ public ModifiableSolrParams getParams() {
+ return params;
+ }
+
+ @Override
+ protected UpdateResponse createResponse(SolrClient client) {
+ return new UpdateResponse();
+ }
+
+ public boolean isWaitSearcher() {
+ return params != null && params.getBool(UpdateParams.WAIT_SEARCHER, false);
+ }
+
+ public ACTION getAction() {
+ if (params==null) return null;
+ if (params.getBool(UpdateParams.COMMIT, false)) return ACTION.COMMIT;
+ if (params.getBool(UpdateParams.OPTIMIZE, false)) return ACTION.OPTIMIZE;
+ return null;
+ }
+
+ public void setWaitSearcher(boolean waitSearcher) {
+ setParam( UpdateParams.WAIT_SEARCHER, waitSearcher+"" );
+ }
+
+ public int getCommitWithin() {
+ return commitWithin;
+ }
+
+ public void setCommitWithin(int commitWithin) {
+ this.commitWithin = commitWithin;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
new file mode 100644
index 0000000..1ad0b26
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -0,0 +1,860 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * This class is experimental and subject to change.
+ *
+ * @since solr 4.5
+ */
+public class CollectionAdminRequest extends SolrRequest<CollectionAdminResponse> {
+
+ protected CollectionAction action = null;
+
+ private static String PROPERTY_PREFIX = "property.";
+
+ protected void setAction( CollectionAction action ) {
+ this.action = action;
+ }
+
+ public CollectionAdminRequest()
+ {
+ super( METHOD.GET, "/admin/collections" );
+ }
+
+ public CollectionAdminRequest( String path )
+ {
+ super( METHOD.GET, path );
+ }
+
+ @Override
+ public SolrParams getParams() {
+ if( action == null ) {
+ throw new RuntimeException( "no action specified!" );
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set( CoreAdminParams.ACTION, action.toString() );
+ return params;
+ }
+
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return null;
+ }
+
+ @Override
+ protected CollectionAdminResponse createResponse(SolrClient client) {
+ return new CollectionAdminResponse();
+ }
+
+ protected void addProperties(ModifiableSolrParams params, Properties props) {
+ Iterator<Map.Entry<Object, Object>> iter = props.entrySet().iterator();
+ while(iter.hasNext()) {
+ Map.Entry<Object, Object> prop = iter.next();
+ String key = (String) prop.getKey();
+ String value = (String) prop.getValue();
+ params.set(PROPERTY_PREFIX + key, value);
+ }
+ }
+
+ //---------------------------------------------------------------------------------------
+ //
+ //---------------------------------------------------------------------------------------
+
+ protected static class CollectionSpecificAdminRequest extends CollectionAdminRequest {
+ protected String collection = null;
+
+ public final void setCollectionName( String collectionName )
+ {
+ this.collection = collectionName;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set( CoreAdminParams.NAME, collection );
+ return params;
+ }
+
+
+ }
+
+ protected static class CollectionShardAdminRequest extends CollectionSpecificAdminRequest {
+ protected String shardName = null;
+
+ public void setShardName(String shard) { this.shardName = shard; }
+ public String getShardName() { return this.shardName; }
+
+ public ModifiableSolrParams getCommonParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set( "collection", collection );
+ params.set( "shard", shardName);
+ return params;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ return getCommonParams();
+ }
+ }
+
+ protected static class CollectionAdminRoleRequest extends CollectionAdminRequest {
+ private String node;
+ private String role;
+
+ public void setNode(String node) {
+ this.node = node;
+ }
+
+ public String getNode() {
+ return this.node;
+ }
+
+ public void setRole(String role) {
+ this.role = role;
+ }
+
+ public String getRole() {
+ return this.role;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set("role", this.role);
+ params.set("node", this.node);
+ return params;
+ }
+
+ }
+
+ /** Specific Collection API call implementations **/
+
+ // CREATE request
+ public static class Create extends CollectionSpecificAdminRequest {
+ protected String configName = null;
+ protected String createNodeSet = null;
+ protected String routerName;
+ protected String shards;
+ protected String routerField;
+ protected Integer numShards;
+ protected Integer maxShardsPerNode;
+ protected Integer replicationFactor;
+
+ private Properties properties;
+ protected Boolean autoAddReplicas;
+ protected Integer stateFormat;
+ protected String asyncId;
+
+
+ public Create() {
+ action = CollectionAction.CREATE;
+ }
+
+ public void setConfigName(String config) { this.configName = config; }
+ public void setCreateNodeSet(String nodeSet) { this.createNodeSet = nodeSet; }
+ public void setRouterName(String routerName) { this.routerName = routerName; }
+ public void setShards(String shards) { this.shards = shards; }
+ public void setRouterField(String routerField) { this.routerField = routerField; }
+ public void setNumShards(Integer numShards) {this.numShards = numShards;}
+ public void setMaxShardsPerNode(Integer numShards) { this.maxShardsPerNode = numShards; }
+ public void setAutoAddReplicas(boolean autoAddReplicas) { this.autoAddReplicas = autoAddReplicas; }
+ public void setReplicationFactor(Integer repl) { this.replicationFactor = repl; }
+ public void setStateFormat(Integer stateFormat) { this.stateFormat = stateFormat; }
+ public void setAsyncId(String asyncId) {
+ this.asyncId = asyncId;
+ }
+
+ public String getConfigName() { return configName; }
+ public String getCreateNodeSet() { return createNodeSet; }
+ public String getRouterName() { return routerName; }
+ public String getShards() { return shards; }
+ public Integer getNumShards() { return numShards; }
+ public Integer getMaxShardsPerNode() { return maxShardsPerNode; }
+ public Integer getReplicationFactor() { return replicationFactor; }
+ public Boolean getAutoAddReplicas() { return autoAddReplicas; }
+ public Integer getStateFormat() { return stateFormat; }
+ public String getAsyncId() {
+ return asyncId;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+
+ params.set( "collection.configName", configName);
+ params.set( "createNodeSet", createNodeSet);
+ if (numShards != null) {
+ params.set( ZkStateReader.NUM_SHARDS_PROP, numShards);
+ }
+ if (maxShardsPerNode != null) {
+ params.set( "maxShardsPerNode", maxShardsPerNode);
+ }
+ params.set( "router.name", routerName);
+ params.set("shards", shards);
+ if (routerField != null) {
+ params.set("router.field", routerField);
+ }
+ if (replicationFactor != null) {
+ params.set( "replicationFactor", replicationFactor);
+ }
+ params.set("async", asyncId);
+ if (autoAddReplicas != null) {
+ params.set(ZkStateReader.AUTO_ADD_REPLICAS, autoAddReplicas);
+ }
+ if(properties != null) {
+ addProperties(params, properties);
+ }
+ if (stateFormat != null) {
+ params.set(DocCollection.STATE_FORMAT, stateFormat);
+ }
+ return params;
+ }
+
+ }
+
+ // RELOAD request
+ public static class Reload extends CollectionSpecificAdminRequest {
+ public Reload() {
+ action = CollectionAction.RELOAD;
+ }
+ }
+
+ // DELETE request
+ public static class Delete extends CollectionSpecificAdminRequest {
+ public Delete() {
+ action = CollectionAction.DELETE;
+ }
+ }
+
+ // CREATESHARD request
+ public static class CreateShard extends CollectionShardAdminRequest {
+ protected String nodeSet;
+ private Properties properties;
+
+ public void setNodeSet(String nodeSet) {
+ this.nodeSet = nodeSet;
+ }
+
+ public String getNodeSet() {
+ return nodeSet;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ public CreateShard() {
+ action = CollectionAction.CREATESHARD;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = getCommonParams();
+ params.set( "createNodeSet", nodeSet);
+ if(properties != null) {
+ addProperties(params, properties);
+ }
+ return params;
+ }
+ }
+
+ // SPLITSHARD request
+ public static class SplitShard extends CollectionShardAdminRequest {
+ protected String ranges;
+ protected String splitKey;
+ protected String asyncId;
+
+ private Properties properties;
+
+ public SplitShard() {
+ action = CollectionAction.SPLITSHARD;
+ }
+
+ public void setRanges(String ranges) { this.ranges = ranges; }
+ public String getRanges() { return ranges; }
+
+ public void setSplitKey(String splitKey) {
+ this.splitKey = splitKey;
+ }
+
+ public String getSplitKey() {
+ return this.splitKey;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ public void setAsyncId(String asyncId) {
+ this.asyncId = asyncId;
+ }
+
+ public String getAsyncId() {
+ return asyncId;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = getCommonParams();
+ params.set( "ranges", ranges);
+
+ if(splitKey != null)
+ params.set("split.key", this.splitKey);
+
+ if(properties != null) {
+ addProperties(params, properties);
+ }
+
+ params.set("async", asyncId);
+ return params;
+ }
+ }
+
+ // DELETESHARD request
+ public static class DeleteShard extends CollectionShardAdminRequest {
+ public DeleteShard() {
+ action = CollectionAction.DELETESHARD;
+ }
+ }
+
+ // REQUESTSTATUS request
+ public static class RequestStatus extends CollectionAdminRequest {
+ protected String requestId = null;
+
+ public RequestStatus() {
+ action = CollectionAction.REQUESTSTATUS;
+ }
+
+ public void setRequestId(String requestId) {this.requestId = requestId; }
+ public String getRequestId() { return this.requestId; }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set("requestid", requestId);
+ return params;
+ }
+ }
+
+ // CREATEALIAS request
+ public static class CreateAlias extends CollectionAdminRequest {
+ protected String aliasName;
+ protected String aliasedCollections;
+
+ public CreateAlias() {
+ action = CollectionAction.CREATEALIAS;
+ }
+
+ public void setAliasName(String aliasName) {
+ this.aliasName = aliasName;
+ }
+
+ public String getAliasName() {
+ return aliasName;
+ }
+
+ public void setAliasedCollections(String alias) { this.aliasedCollections = alias; }
+ public String getAliasedCollections() { return this.aliasedCollections; }
+
+ @Deprecated
+ public void setCollectionName(String aliasName) {
+ this.aliasName = aliasName;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set("name", aliasName);
+ params.set( "collections", aliasedCollections );
+ return params;
+ }
+ }
+
+ // DELETEALIAS request
+ public static class DeleteAlias extends CollectionAdminRequest {
+ protected String aliasName;
+
+ public DeleteAlias() {
+ action = CollectionAction.DELETEALIAS;
+ }
+
+ public void setAliasName(String aliasName) {
+ this.aliasName = aliasName;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set("name", aliasName);
+ return params;
+ }
+ }
+
+ // ADDREPLICA request
+ public static class AddReplica extends CollectionShardAdminRequest {
+ private String node;
+ private String routeKey;
+ private String instanceDir;
+ private String dataDir;
+ private Properties properties;
+ private String asyncId;
+
+ public AddReplica() {
+ action = CollectionAction.ADDREPLICA;
+ }
+
+ public Properties getProperties() {
+ return properties;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ public String getNode() {
+ return node;
+ }
+
+ public void setNode(String node) {
+ this.node = node;
+ }
+
+ public String getRouteKey() {
+ return routeKey;
+ }
+
+ public void setRouteKey(String routeKey) {
+ this.routeKey = routeKey;
+ }
+
+ public String getInstanceDir() {
+ return instanceDir;
+ }
+
+ public void setInstanceDir(String instanceDir) {
+ this.instanceDir = instanceDir;
+ }
+
+ public String getDataDir() {
+ return dataDir;
+ }
+
+ public void setDataDir(String dataDir) {
+ this.dataDir = dataDir;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ if (shardName == null || shardName.isEmpty()) {
+ params.remove("shard");
+ if (routeKey == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Either shard or routeKey must be provided");
+ }
+ params.add(ShardParams._ROUTE_, routeKey);
+ }
+ params.set("async", asyncId);
+ if (node != null) {
+ params.add("node", node);
+ }
+ if (instanceDir != null) {
+ params.add("instanceDir", instanceDir);
+ }
+ if (dataDir != null) {
+ params.add("dataDir", dataDir);
+ }
+ if (properties != null) {
+ addProperties(params, properties);
+ }
+ return params;
+ }
+
+ public void setAsyncId(String asyncId) {
+ this.asyncId = asyncId;
+ }
+
+ public String getAsyncId() {
+ return asyncId;
+ }
+ }
+
+ // DELETEREPLICA request
+ public static class DeleteReplica extends CollectionShardAdminRequest {
+ private String replica;
+ private Boolean onlyIfDown;
+
+ public DeleteReplica() {
+ action = CollectionAction.DELETEREPLICA;
+ }
+
+ public void setReplica(String replica) {
+ this.replica = replica;
+ }
+
+ public String getReplica() {
+ return this.replica;
+ }
+
+ public void setOnlyIfDown(boolean onlyIfDown) {
+ this.onlyIfDown = onlyIfDown;
+ }
+
+ public Boolean getOnlyIfDown() {
+ return this.onlyIfDown;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set(ZkStateReader.REPLICA_PROP, this.replica);
+
+ if(onlyIfDown != null) {
+ params.set("onlyIfDown", this.onlyIfDown);
+ }
+ return params;
+ }
+ }
+
+ // CLUSTERPROP request
+ public static class ClusterProp extends CollectionAdminRequest {
+ private String propertyName;
+ private String propertyValue;
+
+ public ClusterProp() {
+ this.action = CollectionAction.CLUSTERPROP;
+ }
+
+ public void setPropertyName(String propertyName) {
+ this.propertyName = propertyName;
+ }
+
+ public String getPropertyName() {
+ return this.propertyName;
+ }
+
+ public void setPropertyValue(String propertyValue) {
+ this.propertyValue = propertyValue;
+ }
+
+ public String getPropertyValue() {
+ return this.propertyValue;
+ }
+
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.add("name", propertyName);
+ params.add("val", propertyValue);
+
+ return params;
+ }
+
+ }
+
+ // MIGRATE request
+ public static class Migrate extends CollectionSpecificAdminRequest {
+ private String targetCollection;
+ private String splitKey;
+ private Integer forwardTimeout;
+ private Properties properties;
+ private String asyncId;
+
+ public Migrate() {
+ action = CollectionAction.MIGRATE;
+ }
+
+ public void setTargetCollection(String targetCollection) {
+ this.targetCollection = targetCollection;
+ }
+
+ public String getTargetCollection() {
+ return this.targetCollection;
+ }
+
+ public void setSplitKey(String splitKey) {
+ this.splitKey = splitKey;
+ }
+
+ public String getSplitKey() {
+ return this.splitKey;
+ }
+
+ public void setForwardTimeout(int forwardTimeout) {
+ this.forwardTimeout = forwardTimeout;
+ }
+
+ public Integer getForwardTimeout() {
+ return this.forwardTimeout;
+ }
+
+ public void setProperties(Properties properties) {
+ this.properties = properties;
+ }
+
+ public Properties getProperties() {
+ return this.properties;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set( "collection", collection );
+ params.set("target.collection", targetCollection);
+ params.set("split.key", splitKey);
+ if(forwardTimeout != null) {
+ params.set("forward.timeout", forwardTimeout);
+ }
+ params.set("async", asyncId);
+
+ if(properties != null) {
+ addProperties(params, properties);
+ }
+
+ return params;
+ }
+
+ public void setAsyncId(String asyncId) {
+ this.asyncId = asyncId;
+ }
+
+ public String getAsyncId() {
+ return asyncId;
+ }
+ }
+
+ // ADDROLE request
+ public static class AddRole extends CollectionAdminRoleRequest {
+ public AddRole() {
+ action = CollectionAction.ADDROLE;
+ }
+ }
+
+ // REMOVEROLE request
+ public static class RemoveRole extends CollectionAdminRoleRequest {
+ public RemoveRole() {
+ action = CollectionAction.REMOVEROLE;
+ }
+ }
+
+ // OVERSEERSTATUS request
+ public static class OverseerStatus extends CollectionAdminRequest {
+ public OverseerStatus () {
+ action = CollectionAction.OVERSEERSTATUS;
+ }
+ }
+
+ // CLUSTERSTATUS request
+ public static class ClusterStatus extends CollectionShardAdminRequest {
+
+ public ClusterStatus () {
+ action = CollectionAction.CLUSTERSTATUS;
+ }
+
+ }
+
+ // LIST request
+ public static class List extends CollectionAdminRequest {
+ public List () {
+ action = CollectionAction.LIST;
+ }
+ }
+
+ // ADDREPLICAPROP request
+ public static class AddReplicaProp extends CollectionShardAdminRequest {
+ private String replica;
+ private String propertyName;
+ private String propertyValue;
+ private Boolean shardUnique;
+
+ public AddReplicaProp() {
+ action = CollectionAction.ADDREPLICAPROP;
+ }
+
+ public String getReplica() {
+ return replica;
+ }
+
+ public void setReplica(String replica) {
+ this.replica = replica;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+
+ public void setPropertyName(String propertyName) {
+ this.propertyName = propertyName;
+ }
+
+ public String getPropertyValue() {
+ return propertyValue;
+ }
+
+ public void setPropertyValue(String propertyValue) {
+ this.propertyValue = propertyValue;
+ }
+
+ public Boolean getShardUnique() {
+ return shardUnique;
+ }
+
+ public void setShardUnique(Boolean shardUnique) {
+ this.shardUnique = shardUnique;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set("replica", replica);
+ params.set("property", propertyName);
+ params.set("property.value", propertyValue);
+
+ if(shardUnique != null)
+ params.set("shardUnique", shardUnique);
+
+ return params;
+ }
+
+ }
+
+ // DELETEREPLICAPROP request
+ public static class DeleteReplicaProp extends CollectionShardAdminRequest {
+ private String replica;
+ private String propertyName;
+
+ public DeleteReplicaProp() {
+ this.action = CollectionAction.DELETEREPLICAPROP;
+ }
+
+ public String getReplica() {
+ return replica;
+ }
+
+ public void setReplica(String replica) {
+ this.replica = replica;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+
+ public void setPropertyName(String propertyName) {
+ this.propertyName = propertyName;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set("replica", replica);
+ params.set("property", propertyName);
+ return params;
+ }
+ }
+
+ // BALANCESHARDUNIQUE request
+ public static class BalanceShardUnique extends CollectionAdminRequest {
+ private String collection;
+ private String propertyName;
+ private Boolean onlyActiveNodes;
+ private Boolean shardUnique;
+
+ public BalanceShardUnique() {
+ this.action = CollectionAction.BALANCESHARDUNIQUE;
+ }
+
+ public String getPropertyName() {
+ return propertyName;
+ }
+
+ public void setPropertyName(String propertyName) {
+ this.propertyName = propertyName;
+ }
+
+ public Boolean getOnlyActiveNodes() {
+ return onlyActiveNodes;
+ }
+
+ public void setOnlyActiveNodes(Boolean onlyActiveNodes) {
+ this.onlyActiveNodes = onlyActiveNodes;
+ }
+
+ public Boolean getShardUnique() {
+ return shardUnique;
+ }
+
+ public void setShardUnique(Boolean shardUnique) {
+ this.shardUnique = shardUnique;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
+ params.set("collection", collection);
+ params.set("property", propertyName);
+ if(onlyActiveNodes != null)
+ params.set("onlyactivenodes", onlyActiveNodes);
+ if(shardUnique != null)
+ params.set("shardUnique", shardUnique);
+ return params;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/ContentStreamUpdateRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/ContentStreamUpdateRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/ContentStreamUpdateRequest.java
new file mode 100644
index 0000000..2697e3f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/ContentStreamUpdateRequest.java
@@ -0,0 +1,78 @@
+package org.apache.solr.client.solrj.request;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+
+import java.io.IOException;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+
+/**
+ * Basic functionality to upload a File or {@link org.apache.solr.common.util.ContentStream} to a Solr Cell or some
+ * other handler that takes ContentStreams (CSV)
+ * <p>
+ * See http://wiki.apache.org/solr/ExtractingRequestHandler<br>
+ * See http://wiki.apache.org/solr/UpdateCSV
+ *
+ *
+ **/
+public class ContentStreamUpdateRequest extends AbstractUpdateRequest {
+ List<ContentStream> contentStreams;
+
+ /**
+ *
+ * @param url The URL to send the {@link org.apache.solr.common.util.ContentStream} to in Solr.
+ */
+ public ContentStreamUpdateRequest(String url) {
+ super(METHOD.POST, url);
+ contentStreams = new ArrayList<>();
+ }
+
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return contentStreams;
+ }
+
+ /**
+ * Add a File to the {@link org.apache.solr.common.util.ContentStream}s.
+ * @param file The File to add.
+ * @throws IOException if there was an error with the file.
+ *
+ * @see #getContentStreams()
+ * @see org.apache.solr.common.util.ContentStreamBase.FileStream
+ */
+ public void addFile(File file, String contentType) throws IOException {
+ ContentStreamBase cs = new ContentStreamBase.FileStream(file);
+ cs.setContentType(contentType);
+ addContentStream(cs);
+ }
+
+ /**
+ * Add a {@link org.apache.solr.common.util.ContentStream} to {@link #getContentStreams()}
+ * @param contentStream The {@link org.apache.solr.common.util.ContentStream}
+ */
+ public void addContentStream(ContentStream contentStream){
+ contentStreams.add(contentStream);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
new file mode 100644
index 0000000..b0c0409
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/CoreAdminRequest.java
@@ -0,0 +1,593 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * This class is experimental and subject to change.
+ *
+ * @since solr 1.3
+ */
+public class CoreAdminRequest extends SolrRequest<CoreAdminResponse> {
+
+ protected String core = null;
+ protected String other = null;
+ protected boolean isIndexInfoNeeded = true;
+ protected CoreAdminParams.CoreAdminAction action = null;
+
+ //a create core request
+ public static class Create extends CoreAdminRequest {
+
+ protected String instanceDir;
+ protected String configName = null;
+ protected String schemaName = null;
+ protected String dataDir = null;
+ protected String ulogDir = null;
+ protected String configSet = null;
+ protected String collection;
+ private Integer numShards;
+ private String shardId;
+ private String roles;
+ private String coreNodeName;
+ private Boolean loadOnStartup;
+ private Boolean isTransient;
+ private String collectionConfigName;
+
+ public Create() {
+ action = CoreAdminAction.CREATE;
+ }
+
+ public void setInstanceDir(String instanceDir) { this.instanceDir = instanceDir; }
+ public void setSchemaName(String schema) { this.schemaName = schema; }
+ public void setConfigName(String config) { this.configName = config; }
+ public void setDataDir(String dataDir) { this.dataDir = dataDir; }
+ public void setUlogDir(String ulogDir) { this.ulogDir = ulogDir; }
+ public void setConfigSet(String configSet) {
+ this.configSet = configSet;
+ }
+ public void setCollection(String collection) { this.collection = collection; }
+ public void setNumShards(int numShards) {this.numShards = numShards;}
+ public void setShardId(String shardId) {this.shardId = shardId;}
+ public void setRoles(String roles) {this.roles = roles;}
+ public void setCoreNodeName(String coreNodeName) {this.coreNodeName = coreNodeName;}
+ public void setIsTransient(Boolean isTransient) { this.isTransient = isTransient; }
+ public void setIsLoadOnStartup(Boolean loadOnStartup) { this.loadOnStartup = loadOnStartup;}
+ public void setCollectionConfigName(String name) { this.collectionConfigName = name;}
+
+ public String getInstanceDir() { return instanceDir; }
+ public String getSchemaName() { return schemaName; }
+ public String getConfigName() { return configName; }
+ public String getDataDir() { return dataDir; }
+ public String getUlogDir() { return ulogDir; }
+ public String getConfigSet() {
+ return configSet;
+ }
+ public String getCollection() { return collection; }
+ public String getShardId() { return shardId; }
+ public String getRoles() { return roles; }
+ public String getCoreNodeName() { return coreNodeName; }
+ public Boolean getIsLoadOnStartup() { return loadOnStartup; }
+ public Boolean getIsTransient() { return isTransient; }
+ public String getCollectionConfigName() { return collectionConfigName;}
+
+ @Override
+ public SolrParams getParams() {
+ if( action == null ) {
+ throw new RuntimeException( "no action specified!" );
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set( CoreAdminParams.ACTION, action.toString() );
+ if( action.equals(CoreAdminAction.CREATE) ) {
+ params.set( CoreAdminParams.NAME, core );
+ } else {
+ params.set( CoreAdminParams.CORE, core );
+ }
+ params.set( CoreAdminParams.INSTANCE_DIR, instanceDir);
+ if (configName != null) {
+ params.set( CoreAdminParams.CONFIG, configName);
+ }
+ if (schemaName != null) {
+ params.set( CoreAdminParams.SCHEMA, schemaName);
+ }
+ if (dataDir != null) {
+ params.set( CoreAdminParams.DATA_DIR, dataDir);
+ }
+ if (ulogDir != null) {
+ params.set( CoreAdminParams.ULOG_DIR, ulogDir);
+ }
+ if (configSet != null) {
+ params.set( CoreAdminParams.CONFIGSET, configSet);
+ }
+ if (collection != null) {
+ params.set( CoreAdminParams.COLLECTION, collection);
+ }
+ if (numShards != null) {
+ params.set( ZkStateReader.NUM_SHARDS_PROP, numShards);
+ }
+ if (shardId != null) {
+ params.set( CoreAdminParams.SHARD, shardId);
+ }
+ if (roles != null) {
+ params.set( CoreAdminParams.ROLES, roles);
+ }
+ if (coreNodeName != null) {
+ params.set( CoreAdminParams.CORE_NODE_NAME, coreNodeName);
+ }
+
+ if (isTransient != null) {
+ params.set(CoreAdminParams.TRANSIENT, isTransient);
+ }
+
+ if (loadOnStartup != null) {
+ params.set(CoreAdminParams.LOAD_ON_STARTUP, loadOnStartup);
+ }
+
+ if (collectionConfigName != null) {
+ params.set("collection.configName", collectionConfigName);
+ }
+
+ return params;
+ }
+
+ }
+
+ public static class WaitForState extends CoreAdminRequest {
+ protected String nodeName;
+ protected String coreNodeName;
+ protected String state;
+ protected Boolean checkLive;
+ protected Boolean onlyIfLeader;
+ protected Boolean onlyIfLeaderActive;
+
+ public WaitForState() {
+ action = CoreAdminAction.PREPRECOVERY;
+ }
+
+ public void setNodeName(String nodeName) {
+ this.nodeName = nodeName;
+ }
+
+ public String getNodeName() {
+ return nodeName;
+ }
+
+ public String getCoreNodeName() {
+ return coreNodeName;
+ }
+
+ public void setCoreNodeName(String coreNodeName) {
+ this.coreNodeName = coreNodeName;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public void setState(String state) {
+ this.state = state;
+ }
+
+ public Boolean getCheckLive() {
+ return checkLive;
+ }
+
+ public void setCheckLive(Boolean checkLive) {
+ this.checkLive = checkLive;
+ }
+
+ public boolean isOnlyIfLeader() {
+ return onlyIfLeader;
+ }
+
+ public void setOnlyIfLeader(boolean onlyIfLeader) {
+ this.onlyIfLeader = onlyIfLeader;
+ }
+
+ public void setOnlyIfLeaderActive(boolean onlyIfLeaderActive) {
+ this.onlyIfLeaderActive = onlyIfLeaderActive;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ if( action == null ) {
+ throw new RuntimeException( "no action specified!" );
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set( CoreAdminParams.ACTION, action.toString() );
+
+ params.set( CoreAdminParams.CORE, core );
+
+ if (nodeName != null) {
+ params.set( "nodeName", nodeName);
+ }
+
+ if (coreNodeName != null) {
+ params.set( "coreNodeName", coreNodeName);
+ }
+
+ if (state != null) {
+ params.set( "state", state);
+ }
+
+ if (checkLive != null) {
+ params.set( "checkLive", checkLive);
+ }
+
+ if (onlyIfLeader != null) {
+ params.set( "onlyIfLeader", onlyIfLeader);
+ }
+
+ if (onlyIfLeaderActive != null) {
+ params.set( "onlyIfLeaderActive", onlyIfLeaderActive);
+ }
+
+ return params;
+ }
+
+ public String toString() {
+ if (action != null) {
+ return "WaitForState: "+getParams();
+ } else {
+ return super.toString();
+ }
+ }
+ }
+
+ public static class RequestRecovery extends CoreAdminRequest {
+
+ public RequestRecovery() {
+ action = CoreAdminAction.REQUESTRECOVERY;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ if( action == null ) {
+ throw new RuntimeException( "no action specified!" );
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set( CoreAdminParams.ACTION, action.toString() );
+
+ params.set( CoreAdminParams.CORE, core );
+
+ return params;
+ }
+ }
+
+ public static class RequestSyncShard extends CoreAdminRequest {
+ private String shard;
+ private String collection;
+
+ public RequestSyncShard() {
+ action = CoreAdminAction.REQUESTSYNCSHARD;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ if( action == null ) {
+ throw new RuntimeException( "no action specified!" );
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CoreAdminParams.ACTION, action.toString());
+ params.set("shard", shard);
+ params.set("collection", collection);
+ params.set(CoreAdminParams.CORE, core);
+ return params;
+ }
+
+ public String getShard() {
+ return shard;
+ }
+
+ public void setShard(String shard) {
+ this.shard = shard;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public void setCollection(String collection) {
+ this.collection = collection;
+ }
+ }
+
+ //a persist core request
+ public static class Persist extends CoreAdminRequest {
+ protected String fileName = null;
+
+ public Persist() {
+ action = CoreAdminAction.PERSIST;
+ }
+
+ public void setFileName(String name) {
+ fileName = name;
+ }
+ public String getFileName() {
+ return fileName;
+ }
+ @Override
+ public SolrParams getParams() {
+ if( action == null ) {
+ throw new RuntimeException( "no action specified!" );
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set( CoreAdminParams.ACTION, action.toString() );
+ if (fileName != null) {
+ params.set( CoreAdminParams.FILE, fileName);
+ }
+ return params;
+ }
+ }
+
+ public static class MergeIndexes extends CoreAdminRequest {
+ protected List<String> indexDirs;
+ protected List<String> srcCores;
+
+ public MergeIndexes() {
+ action = CoreAdminAction.MERGEINDEXES;
+ }
+
+ public void setIndexDirs(List<String> indexDirs) {
+ this.indexDirs = indexDirs;
+ }
+
+ public List<String> getIndexDirs() {
+ return indexDirs;
+ }
+
+ public List<String> getSrcCores() {
+ return srcCores;
+ }
+
+ public void setSrcCores(List<String> srcCores) {
+ this.srcCores = srcCores;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ if (action == null) {
+ throw new RuntimeException("no action specified!");
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CoreAdminParams.ACTION, action.toString());
+ params.set(CoreAdminParams.CORE, core);
+ if (indexDirs != null) {
+ for (String indexDir : indexDirs) {
+ params.add(CoreAdminParams.INDEX_DIR, indexDir);
+ }
+ }
+ if (srcCores != null) {
+ for (String srcCore : srcCores) {
+ params.add(CoreAdminParams.SRC_CORE, srcCore);
+ }
+ }
+ return params;
+ }
+ }
+
+ public static class Unload extends CoreAdminRequest {
+ protected boolean deleteIndex;
+ protected boolean deleteDataDir;
+ protected boolean deleteInstanceDir;
+
+ public Unload(boolean deleteIndex) {
+ action = CoreAdminAction.UNLOAD;
+ this.deleteIndex = deleteIndex;
+ }
+
+ public boolean isDeleteIndex() {
+ return deleteIndex;
+ }
+
+ public void setDeleteIndex(boolean deleteIndex) {
+ this.deleteIndex = deleteIndex;
+ }
+
+ public void setDeleteDataDir(boolean deleteDataDir) {
+ this.deleteDataDir = deleteDataDir;
+ }
+
+ public void setDeleteInstanceDir(boolean deleteInstanceDir){
+ this.deleteInstanceDir = deleteInstanceDir;
+ }
+
+ public boolean isDeleteDataDir() {
+ return deleteDataDir;
+ }
+
+ public boolean isDeleteInstanceDir() {
+ return deleteInstanceDir;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set(CoreAdminParams.DELETE_INDEX, deleteIndex);
+ params.set(CoreAdminParams.DELETE_DATA_DIR, deleteDataDir);
+ params.set(CoreAdminParams.DELETE_INSTANCE_DIR, deleteInstanceDir);
+ return params;
+ }
+
+ }
+
+ public CoreAdminRequest()
+ {
+ super( METHOD.GET, "/admin/cores" );
+ }
+
+ public CoreAdminRequest( String path )
+ {
+ super( METHOD.GET, path );
+ }
+
+ public final void setCoreName( String coreName )
+ {
+ this.core = coreName;
+ }
+
+ public final void setOtherCoreName( String otherCoreName )
+ {
+ this.other = otherCoreName;
+ }
+
+ public final void setIndexInfoNeeded(boolean isIndexInfoNeeded) {
+ this.isIndexInfoNeeded = isIndexInfoNeeded;
+ }
+
+ //---------------------------------------------------------------------------------------
+ //
+ //---------------------------------------------------------------------------------------
+
+ public void setAction( CoreAdminAction action )
+ {
+ this.action = action;
+ }
+
+ //---------------------------------------------------------------------------------------
+ //
+ //---------------------------------------------------------------------------------------
+
+ @Override
+ public SolrParams getParams()
+ {
+ if( action == null ) {
+ throw new RuntimeException( "no action specified!" );
+ }
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set( CoreAdminParams.ACTION, action.toString() );
+ params.set( CoreAdminParams.CORE, core );
+ params.set(CoreAdminParams.INDEX_INFO, (isIndexInfoNeeded ? "true" : "false"));
+ if (other != null) {
+ params.set(CoreAdminParams.OTHER, other);
+ }
+ return params;
+ }
+
+ //---------------------------------------------------------------------------------------
+ //
+ //---------------------------------------------------------------------------------------
+
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return null;
+ }
+
+ @Override
+ protected CoreAdminResponse createResponse(SolrClient client) {
+ return new CoreAdminResponse();
+ }
+
+ //---------------------------------------------------------------------------------------
+ //
+ //---------------------------------------------------------------------------------------
+
+ public static CoreAdminResponse reloadCore(String name, SolrClient client) throws SolrServerException, IOException
+ {
+ CoreAdminRequest req = new CoreAdminRequest();
+ req.setCoreName(name);
+ req.setAction(CoreAdminAction.RELOAD);
+ return req.process(client);
+ }
+
+ public static CoreAdminResponse unloadCore(String name, SolrClient client) throws SolrServerException, IOException
+ {
+ return unloadCore(name, false, client);
+ }
+
+ public static CoreAdminResponse unloadCore(String name, boolean deleteIndex, SolrClient client) throws SolrServerException, IOException {
+ return unloadCore(name, deleteIndex, false, client);
+ }
+
+ public static CoreAdminResponse unloadCore(String name, boolean deleteIndex, boolean deleteInstanceDir, SolrClient client) throws SolrServerException, IOException {
+ Unload req = new Unload(deleteIndex);
+ req.setCoreName(name);
+ req.setDeleteInstanceDir(deleteInstanceDir);
+ return req.process(client);
+ }
+
+ public static CoreAdminResponse renameCore(String coreName, String newName, SolrClient client ) throws SolrServerException, IOException
+ {
+ CoreAdminRequest req = new CoreAdminRequest();
+ req.setCoreName(coreName);
+ req.setOtherCoreName(newName);
+ req.setAction( CoreAdminAction.RENAME );
+ return req.process( client );
+ }
+
+ public static CoreAdminResponse getStatus( String name, SolrClient client ) throws SolrServerException, IOException
+ {
+ CoreAdminRequest req = new CoreAdminRequest();
+ req.setCoreName( name );
+ req.setAction( CoreAdminAction.STATUS );
+ return req.process( client );
+ }
+
+ public static CoreAdminResponse createCore( String name, String instanceDir, SolrClient client ) throws SolrServerException, IOException
+ {
+ return CoreAdminRequest.createCore(name, instanceDir, client, null, null);
+ }
+
+ public static CoreAdminResponse createCore( String name, String instanceDir, SolrClient client, String configFile, String schemaFile ) throws SolrServerException, IOException {
+ return createCore(name, instanceDir, client, configFile, schemaFile, null, null);
+ }
+
+ public static CoreAdminResponse createCore( String name, String instanceDir, SolrClient client, String configFile, String schemaFile, String dataDir, String tlogDir ) throws SolrServerException, IOException
+ {
+ CoreAdminRequest.Create req = new CoreAdminRequest.Create();
+ req.setCoreName( name );
+ req.setInstanceDir(instanceDir);
+ if (dataDir != null) {
+ req.setDataDir(dataDir);
+ }
+ if (tlogDir != null) {
+ req.setUlogDir(tlogDir);
+ }
+ if(configFile != null){
+ req.setConfigName(configFile);
+ }
+ if(schemaFile != null){
+ req.setSchemaName(schemaFile);
+ }
+ return req.process( client );
+ }
+
+ public static CoreAdminResponse mergeIndexes(String name,
+ String[] indexDirs, String[] srcCores, SolrClient client) throws SolrServerException,
+ IOException {
+ CoreAdminRequest.MergeIndexes req = new CoreAdminRequest.MergeIndexes();
+ req.setCoreName(name);
+ req.setIndexDirs(Arrays.asList(indexDirs));
+ req.setSrcCores(Arrays.asList(srcCores));
+ return req.process(client);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java
new file mode 100644
index 0000000..1efaaf5
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.util.Collection;
+
+/**
+ * Send arbitrary XML to a request handler
+ *
+ *
+ * @since solr 1.3
+ */
+public class DirectXmlRequest extends SolrRequest<UpdateResponse> implements IsUpdateRequest {
+
+ final String xml;
+ private SolrParams params;
+
+ public DirectXmlRequest( String path, String body )
+ {
+ super( METHOD.POST, path );
+ xml = body;
+ }
+
+ @Override
+ public Collection<ContentStream> getContentStreams() {
+ return ClientUtils.toContentStreams( xml, ClientUtils.TEXT_XML );
+ }
+
+ @Override
+ protected UpdateResponse createResponse(SolrClient client) {
+ return new UpdateResponse();
+ }
+
+ @Override
+ public SolrParams getParams() {
+ return params;
+ }
+
+
+ public void setParams(SolrParams params) {
+ this.params = params;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java
new file mode 100644
index 0000000..92e2064
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/DocumentAnalysisRequest.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.DocumentAnalysisResponse;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.AnalysisParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A request for the org.apache.solr.handler.DocumentAnalysisRequestHandler.
+ *
+ *
+ * @since solr 1.4
+ */
+public class DocumentAnalysisRequest extends SolrRequest<DocumentAnalysisResponse> {
+
+ private List<SolrInputDocument> documents = new ArrayList<>();
+ private String query;
+ private boolean showMatch = false;
+
+ /**
+ * Constructs a new request with a default uri of "/documentanalysis".
+ */
+ public DocumentAnalysisRequest() {
+ super(METHOD.POST, "/analysis/document");
+ }
+
+ /**
+ * Constructs a new request with the given request handler uri.
+ *
+ * @param uri The of the request handler.
+ */
+ public DocumentAnalysisRequest(String uri) {
+ super(METHOD.POST, uri);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return ClientUtils.toContentStreams(getXML(), ClientUtils.TEXT_XML);
+ }
+
+ @Override
+ protected DocumentAnalysisResponse createResponse(SolrClient client) {
+ return new DocumentAnalysisResponse();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public ModifiableSolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ if (query != null) {
+ params.add(AnalysisParams.QUERY, query);
+ params.add(AnalysisParams.SHOW_MATCH, String.valueOf(showMatch));
+ }
+ return params;
+ }
+
+ //================================================ Helper Methods ==================================================
+
+ /**
+ * Returns the xml be be set as the request body.
+ *
+ * @return The xml be be set as the request body.
+ *
+ * @throws IOException When constructing the xml fails
+ */
+ String getXML() throws IOException {
+ StringWriter writer = new StringWriter();
+ writer.write("<docs>");
+ for (SolrInputDocument document : documents) {
+ ClientUtils.writeXML(document, writer);
+ }
+ writer.write("</docs>");
+ writer.flush();
+
+ String xml = writer.toString();
+ return (xml.length() > 0) ? xml : null;
+ }
+
+
+ //============================================ Setter/Getter Methods ===============================================
+
+ /**
+ * Adds a document to be analyzed.
+ *
+ * @param doc The document to be analyzed.
+ *
+ * @return This DocumentAnalysisRequest (fluent interface support).
+ */
+ public DocumentAnalysisRequest addDocument(SolrInputDocument doc) {
+ documents.add(doc);
+ return this;
+ }
+
+ /**
+ * Adds a collection of documents to be analyzed.
+ *
+ * @param docs The documents to be analyzed.
+ *
+ * @return This DocumentAnalysisRequest (fluent interface support).
+ *
+ * @see #addDocument(org.apache.solr.common.SolrInputDocument)
+ */
+ public DocumentAnalysisRequest addDocuments(Collection<SolrInputDocument> docs) {
+ documents.addAll(docs);
+ return this;
+ }
+
+ /**
+ * Sets the query to be analyzed. By default the query is set to null, meaning no query analysis will be performed.
+ *
+ * @param query The query to be analyzed.
+ *
+ * @return This DocumentAnalysisRequest (fluent interface support).
+ */
+ public DocumentAnalysisRequest setQuery(String query) {
+ this.query = query;
+ return this;
+ }
+
+ /**
+ * Sets whether index time tokens that match query time tokens should be marked as a "match". By default this is set
+ * to {@code false}. Obviously, this flag is ignored if when the query is set to {@code null}.
+ *
+ * @param showMatch Sets whether index time tokens that match query time tokens should be marked as a "match".
+ *
+ * @return This DocumentAnalysisRequest (fluent interface support).
+ */
+ public DocumentAnalysisRequest setShowMatch(boolean showMatch) {
+ this.showMatch = showMatch;
+ return this;
+ }
+
+ /**
+ * Returns all documents that will be analyzed when processing the request.
+ *
+ * @return All documents that will be analyzed when processing the request.
+ *
+ * @see #addDocument(org.apache.solr.common.SolrInputDocument)
+ */
+ public List<SolrInputDocument> getDocuments() {
+ return documents;
+ }
+
+ /**
+ * Returns the query that will be analyzed when processing the request. May return {@code null} indicating that no
+ * query time analysis is taking place.
+ *
+ * @return The query that will be analyzed when processing the request.
+ *
+ * @see #setQuery(String)
+ */
+ public String getQuery() {
+ return query;
+ }
+
+ /**
+ * Returns whether index time tokens that match query time tokens will be marked as a "match".
+ *
+ * @return Whether index time tokens that match query time tokens will be marked as a "match".
+ *
+ * @see #setShowMatch(boolean)
+ */
+ public boolean isShowMatch() {
+ return showMatch;
+ }
+
+}
[15/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrQuery.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrQuery.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrQuery.java
new file mode 100644
index 0000000..bda6f34
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrQuery.java
@@ -0,0 +1,1109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.FacetParams;
+import org.apache.solr.common.params.HighlightParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.StatsParams;
+import org.apache.solr.common.params.TermsParams;
+import org.apache.solr.common.util.DateUtil;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+
+/**
+ * This is an augmented SolrParams with get/set/add fields for common fields used
+ * in the Standard and Dismax request handlers
+ *
+ *
+ * @since solr 1.3
+ */
+public class SolrQuery extends ModifiableSolrParams
+{
+ public static final String DOCID = "_docid_"; // duplicate of org.apache.solr.search.QueryParsing.DOCID which is not accessible from here
+
+ public enum ORDER { desc, asc;
+ public ORDER reverse() {
+ return (this == asc) ? desc : asc;
+ }
+ }
+
+ /** Maintains a map of current sorts */
+ private List<SortClause> sortClauses;
+
+ public SolrQuery() {
+ super();
+ }
+
+ /** Create a new SolrQuery
+ *
+ * @param q query string
+ */
+ public SolrQuery(String q) {
+ this();
+ this.set(CommonParams.Q, q);
+ }
+
+ /** enable/disable terms.
+ *
+ * @param b flag to indicate terms should be enabled. <br> if b==false, removes all other terms parameters
+ * @return Current reference (<i>this</i>)
+ */
+ public SolrQuery setTerms(boolean b) {
+ if (b) {
+ this.set(TermsParams.TERMS, true);
+ } else {
+ this.remove(TermsParams.TERMS);
+ this.remove(TermsParams.TERMS_FIELD);
+ this.remove(TermsParams.TERMS_LOWER);
+ this.remove(TermsParams.TERMS_UPPER);
+ this.remove(TermsParams.TERMS_UPPER_INCLUSIVE);
+ this.remove(TermsParams.TERMS_LOWER_INCLUSIVE);
+ this.remove(TermsParams.TERMS_LIMIT);
+ this.remove(TermsParams.TERMS_PREFIX_STR);
+ this.remove(TermsParams.TERMS_MINCOUNT);
+ this.remove(TermsParams.TERMS_MAXCOUNT);
+ this.remove(TermsParams.TERMS_RAW);
+ this.remove(TermsParams.TERMS_SORT);
+ this.remove(TermsParams.TERMS_REGEXP_STR);
+ this.remove(TermsParams.TERMS_REGEXP_FLAG);
+ }
+ return this;
+ }
+
+ public boolean getTerms() {
+ return this.getBool(TermsParams.TERMS, false);
+ }
+
+ public SolrQuery addTermsField(String field) {
+ this.add(TermsParams.TERMS_FIELD, field);
+ return this;
+ }
+
+ public String[] getTermsFields() {
+ return this.getParams(TermsParams.TERMS_FIELD);
+ }
+
+ public SolrQuery setTermsLower(String lower) {
+ this.set(TermsParams.TERMS_LOWER, lower);
+ return this;
+ }
+
+ public String getTermsLower() {
+ return this.get(TermsParams.TERMS_LOWER, "");
+ }
+
+ public SolrQuery setTermsUpper(String upper) {
+ this.set(TermsParams.TERMS_UPPER, upper);
+ return this;
+ }
+
+ public String getTermsUpper() {
+ return this.get(TermsParams.TERMS_UPPER, "");
+ }
+
+ public SolrQuery setTermsUpperInclusive(boolean b) {
+ this.set(TermsParams.TERMS_UPPER_INCLUSIVE, b);
+ return this;
+ }
+
+ public boolean getTermsUpperInclusive() {
+ return this.getBool(TermsParams.TERMS_UPPER_INCLUSIVE, false);
+ }
+
+ public SolrQuery setTermsLowerInclusive(boolean b) {
+ this.set(TermsParams.TERMS_LOWER_INCLUSIVE, b);
+ return this;
+ }
+
+ public boolean getTermsLowerInclusive() {
+ return this.getBool(TermsParams.TERMS_LOWER_INCLUSIVE, true);
+ }
+
+ public SolrQuery setTermsLimit(int limit) {
+ this.set(TermsParams.TERMS_LIMIT, limit);
+ return this;
+ }
+
+ public int getTermsLimit() {
+ return this.getInt(TermsParams.TERMS_LIMIT, 10);
+ }
+
+ public SolrQuery setTermsMinCount(int cnt) {
+ this.set(TermsParams.TERMS_MINCOUNT, cnt);
+ return this;
+ }
+
+ public int getTermsMinCount() {
+ return this.getInt(TermsParams.TERMS_MINCOUNT, 1);
+ }
+
+ public SolrQuery setTermsMaxCount(int cnt) {
+ this.set(TermsParams.TERMS_MAXCOUNT, cnt);
+ return this;
+ }
+
+ public int getTermsMaxCount() {
+ return this.getInt(TermsParams.TERMS_MAXCOUNT, -1);
+ }
+
+ public SolrQuery setTermsPrefix(String prefix) {
+ this.set(TermsParams.TERMS_PREFIX_STR, prefix);
+ return this;
+ }
+
+ public String getTermsPrefix() {
+ return this.get(TermsParams.TERMS_PREFIX_STR, "");
+ }
+
+ public SolrQuery setTermsRaw(boolean b) {
+ this.set(TermsParams.TERMS_RAW, b);
+ return this;
+ }
+
+ public boolean getTermsRaw() {
+ return this.getBool(TermsParams.TERMS_RAW, false);
+ }
+
+ public SolrQuery setTermsSortString(String type) {
+ this.set(TermsParams.TERMS_SORT, type);
+ return this;
+ }
+
+ public String getTermsSortString() {
+ return this.get(TermsParams.TERMS_SORT, TermsParams.TERMS_SORT_COUNT);
+ }
+
+ public SolrQuery setTermsRegex(String regex) {
+ this.set(TermsParams.TERMS_REGEXP_STR, regex);
+ return this;
+ }
+
+ public String getTermsRegex() {
+ return this.get(TermsParams.TERMS_REGEXP_STR);
+ }
+
+ public SolrQuery setTermsRegexFlag(String flag) {
+ this.add(TermsParams.TERMS_REGEXP_FLAG, flag);
+ return this;
+ }
+
+ public String[] getTermsRegexFlags() {
+ return this.getParams(TermsParams.TERMS_REGEXP_FLAG);
+ }
+
+ /** Add field(s) for facet computation.
+ *
+ * @param fields Array of field names from the IndexSchema
+ * @return this
+ */
+ public SolrQuery addFacetField(String ... fields) {
+ add(FacetParams.FACET_FIELD, fields);
+ this.set(FacetParams.FACET, true);
+ return this;
+ }
+
+ /** Add field(s) for pivot computation.
+ *
+ * pivot fields are comma separated
+ *
+ * @param fields Array of field names from the IndexSchema
+ * @return this
+ */
+ public SolrQuery addFacetPivotField(String ... fields) {
+ add(FacetParams.FACET_PIVOT, fields);
+ this.set(FacetParams.FACET, true);
+ return this;
+ }
+
+ /**
+ * Add a numeric range facet.
+ *
+ * @param field The field
+ * @param start The start of range
+ * @param end The end of the range
+ * @param gap The gap between each count
+ * @return this
+ */
+ public SolrQuery addNumericRangeFacet(String field, Number start, Number end, Number gap) {
+ add(FacetParams.FACET_RANGE, field);
+ add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_START), start.toString());
+ add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_END), end.toString());
+ add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_GAP), gap.toString());
+ this.set(FacetParams.FACET, true);
+ return this;
+ }
+
+ /**
+ * Add a numeric range facet.
+ *
+ * @param field The field
+ * @param start The start of range
+ * @param end The end of the range
+ * @param gap The gap between each count
+ * @return this
+ */
+ public SolrQuery addDateRangeFacet(String field, Date start, Date end, String gap) {
+ add(FacetParams.FACET_RANGE, field);
+ add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_START), DateUtil.getThreadLocalDateFormat().format(start));
+ add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_END), DateUtil.getThreadLocalDateFormat().format(end));
+ add(String.format(Locale.ROOT, "f.%s.%s", field, FacetParams.FACET_RANGE_GAP), gap);
+ this.set(FacetParams.FACET, true);
+ return this;
+ }
+
+ /**
+ * Add Interval Faceting on a field. All intervals for the same field should be included
+ * in the same call to this method.
+ * For syntax documentation see <a href="https://wiki.apache.org/solr/SimpleFacetParameters#Interval_Faceting">Solr wiki</a>.
+ * <br>
+ * Key substitution, filter exclusions or other local params on the field are not supported when using this method,
+ * if this is needed, use the lower level {@link #add} method.<br>
+ * Key substitution IS supported on intervals when using this method.
+ *
+ *
+ * @param field the field to add facet intervals. Must be an existing field and can't be null
+ * @param intervals Intervals to be used for faceting. It can be an empty array, but it can't
+ * be <code>null</code>
+ * @return this
+ */
+ public SolrQuery addIntervalFacets(String field, String[] intervals) {
+ if (intervals == null) {
+ throw new IllegalArgumentException("Can't add null intervals");
+ }
+ if (field == null) {
+ throw new IllegalArgumentException("Field can't be null");
+ }
+ set(FacetParams.FACET, true);
+ add(FacetParams.FACET_INTERVAL, field);
+ for (String interval:intervals) {
+ add(String.format(Locale.ROOT, "f.%s.facet.interval.set", field), interval);
+ }
+ return this;
+ }
+
+ /**
+ * Remove all Interval Facets on a field
+ *
+ * @param field the field to remove from facet intervals
+ * @return Array of current intervals for <code>field</code>
+ */
+ public String[] removeIntervalFacets(String field) {
+ while(remove(FacetParams.FACET_INTERVAL, field)){};
+ return remove(String.format(Locale.ROOT, "f.%s.facet.interval.set", field));
+ }
+
+ /** get the facet fields
+ *
+ * @return string array of facet fields or null if not set/empty
+ */
+ public String[] getFacetFields() {
+ return this.getParams(FacetParams.FACET_FIELD);
+ }
+
+ /** remove a facet field
+ *
+ * @param name Name of the facet field to be removed.
+ *
+ * @return true, if the item was removed. <br>
+ * false, if the facet field was null or did not exist.
+ */
+ public boolean removeFacetField(String name) {
+ boolean b = this.remove(FacetParams.FACET_FIELD, name);
+ if (this.get(FacetParams.FACET_FIELD) == null && this.get(FacetParams.FACET_QUERY) == null) {
+ this.setFacet(false);
+ }
+ return b;
+ }
+
+ /** enable/disable faceting.
+ *
+ * @param b flag to indicate faceting should be enabled. <br> if b==false, removes all other faceting parameters
+ * @return Current reference (<i>this</i>)
+ */
+ public SolrQuery setFacet(boolean b) {
+ if (b) {
+ this.set(FacetParams.FACET, true);
+ } else {
+ this.remove(FacetParams.FACET);
+ this.remove(FacetParams.FACET_MINCOUNT);
+ this.remove(FacetParams.FACET_FIELD);
+ this.remove(FacetParams.FACET_LIMIT);
+ this.remove(FacetParams.FACET_MISSING);
+ this.remove(FacetParams.FACET_OFFSET);
+ this.remove(FacetParams.FACET_PREFIX);
+ this.remove(FacetParams.FACET_QUERY);
+ this.remove(FacetParams.FACET_SORT);
+ this.remove(FacetParams.FACET_ZEROS);
+ this.remove(FacetParams.FACET_PREFIX); // does not include the individual fields...
+ this.remove(FacetParams.FACET_INTERVAL); // does not remove interval parameters
+ }
+ return this;
+ }
+
+ public SolrQuery setFacetPrefix( String prefix )
+ {
+ this.set( FacetParams.FACET_PREFIX, prefix );
+ return this;
+ }
+
+ public SolrQuery setFacetPrefix( String field, String prefix )
+ {
+ this.set( "f."+field+"."+FacetParams.FACET_PREFIX, prefix );
+ return this;
+ }
+
+ /** add a faceting query
+ *
+ * @param f facet query
+ */
+ public SolrQuery addFacetQuery(String f) {
+ this.add(FacetParams.FACET_QUERY, f);
+ this.set(FacetParams.FACET, true);
+ return this;
+ }
+
+ /** get facet queries
+ *
+ * @return all facet queries or null if not set/empty
+ */
+ public String[] getFacetQuery() {
+ return this.getParams(FacetParams.FACET_QUERY);
+ }
+
+ /** remove a facet query
+ *
+ * @param q the facet query to remove
+ * @return true if the facet query was removed false otherwise
+ */
+ public boolean removeFacetQuery(String q) {
+ boolean b = this.remove(FacetParams.FACET_QUERY, q);
+ if (this.get(FacetParams.FACET_FIELD) == null && this.get(FacetParams.FACET_QUERY) == null) {
+ this.setFacet(false);
+ }
+ return b;
+ }
+
+ /** set the facet limit
+ *
+ * @param lim number facet items to return
+ */
+ public SolrQuery setFacetLimit(int lim) {
+ this.set(FacetParams.FACET_LIMIT, lim);
+ return this;
+ }
+
+ /** get current facet limit
+ *
+ * @return facet limit or default of 25
+ */
+ public int getFacetLimit() {
+ return this.getInt(FacetParams.FACET_LIMIT, 25);
+ }
+
+ /** set facet minimum count
+ *
+ * @param cnt facets having less that cnt hits will be excluded from teh facet list
+ */
+ public SolrQuery setFacetMinCount(int cnt) {
+ this.set(FacetParams.FACET_MINCOUNT, cnt);
+ return this;
+ }
+
+ /** get facet minimum count
+ *
+ * @return facet minimum count or default of 1
+ */
+ public int getFacetMinCount() {
+ return this.getInt(FacetParams.FACET_MINCOUNT, 1);
+ }
+
+ /**
+ * Sets facet missing boolean flag
+ *
+ * @param v flag to indicate the field of {@link FacetParams#FACET_MISSING} .
+ * @return this
+ */
+ public SolrQuery setFacetMissing(Boolean v) {
+ this.set(FacetParams.FACET_MISSING, v);
+ return this;
+ }
+
+ /** get facet sort
+ *
+ * @return facet sort or default of {@link FacetParams#FACET_SORT_COUNT}
+ */
+ public String getFacetSortString() {
+ return this.get(FacetParams.FACET_SORT, FacetParams.FACET_SORT_COUNT);
+ }
+
+
+ /** set facet sort
+ *
+ * @param sort sort facets
+ * @return this
+ */
+ public SolrQuery setFacetSort(String sort) {
+ this.set(FacetParams.FACET_SORT, sort);
+ return this;
+ }
+
+ /** add highlight field
+ *
+ * @param f field to enable for highlighting
+ */
+ public SolrQuery addHighlightField(String f) {
+ this.add(HighlightParams.FIELDS, f);
+ this.set(HighlightParams.HIGHLIGHT, true);
+ return this;
+ }
+
+ /** remove a field for highlighting
+ *
+ * @param f field name to not highlight
+ * @return <i>true</i>, if removed, <br> <i>false</i>, otherwise
+ */
+ public boolean removeHighlightField(String f) {
+ boolean b = this.remove(HighlightParams.FIELDS, f);
+ if (this.get(HighlightParams.FIELDS) == null) {
+ this.setHighlight(false);
+ }
+ return b;
+ }
+
+ /** get list of highlighted fields
+ *
+ * @return Array of highlight fields or null if not set/empty
+ */
+ public String[] getHighlightFields() {
+ return this.getParams(HighlightParams.FIELDS);
+ }
+
+ public SolrQuery setHighlightSnippets(int num) {
+ this.set(HighlightParams.SNIPPETS, num);
+ return this;
+ }
+
+ public int getHighlightSnippets() {
+ return this.getInt(HighlightParams.SNIPPETS, 1);
+ }
+
+ public SolrQuery setHighlightFragsize(int num) {
+ this.set(HighlightParams.FRAGSIZE, num);
+ return this;
+ }
+
+ public int getHighlightFragsize() {
+ return this.getInt(HighlightParams.FRAGSIZE, 100);
+ }
+
+ public SolrQuery setHighlightRequireFieldMatch(boolean flag) {
+ this.set(HighlightParams.FIELD_MATCH, flag);
+ return this;
+ }
+
+ public boolean getHighlightRequireFieldMatch() {
+ return this.getBool(HighlightParams.FIELD_MATCH, false);
+ }
+
+ public SolrQuery setHighlightSimplePre(String f) {
+ this.set(HighlightParams.SIMPLE_PRE, f);
+ return this;
+ }
+
+ public String getHighlightSimplePre() {
+ return this.get(HighlightParams.SIMPLE_PRE, "");
+ }
+
+ public SolrQuery setHighlightSimplePost(String f) {
+ this.set(HighlightParams.SIMPLE_POST, f);
+ return this;
+ }
+
+ public String getHighlightSimplePost() {
+ return this.get(HighlightParams.SIMPLE_POST, "");
+ }
+
+ /**
+ * Gets the raw sort field, as it will be sent to Solr.
+ * <p>
+ * The returned sort field will always contain a serialized version
+ * of the sort string built using {@link #setSort(SortClause)},
+ * {@link #addSort(SortClause)}, {@link #addOrUpdateSort(SortClause)},
+ * {@link #removeSort(SortClause)}, {@link #clearSorts()} and
+ * {@link #setSorts(List)}.
+ */
+ public String getSortField() {
+ return this.get(CommonParams.SORT);
+ }
+
+ /**
+ * Clears current sort information.
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery clearSorts() {
+ sortClauses = null;
+ serializeSorts();
+ return this;
+ }
+
+ /**
+ * Replaces the current sort information.
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery setSorts(List<SortClause> value) {
+ sortClauses = new ArrayList<>(value);
+ serializeSorts();
+ return this;
+ }
+
+ /**
+ * Gets an a list of current sort clauses.
+ *
+ * @return an immutable list of current sort clauses
+ * @since 4.2
+ */
+ public List<SortClause> getSorts() {
+ if (sortClauses == null) return Collections.emptyList();
+ else return Collections.unmodifiableList(sortClauses);
+ }
+
+ /**
+ * Replaces the current sort information with a single sort clause
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery setSort(String field, ORDER order) {
+ return setSort(new SortClause(field, order));
+ }
+
+ /**
+ * Replaces the current sort information with a single sort clause
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery setSort(SortClause sortClause) {
+ clearSorts();
+ return addSort(sortClause);
+ }
+
+ /**
+ * Adds a single sort clause to the end of the current sort information.
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery addSort(String field, ORDER order) {
+ return addSort(new SortClause(field, order));
+ }
+
+ /**
+ * Adds a single sort clause to the end of the query.
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery addSort(SortClause sortClause) {
+ if (sortClauses == null) sortClauses = new ArrayList<>();
+ sortClauses.add(sortClause);
+ serializeSorts();
+ return this;
+ }
+
+ /**
+ * Updates or adds a single sort clause to the query.
+ * If the field is already used for sorting, the order
+ * of the existing field is modified; otherwise, it is
+ * added to the end.
+ * <p>
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery addOrUpdateSort(String field, ORDER order) {
+ return addOrUpdateSort(new SortClause(field, order));
+ }
+
+ /**
+ * Updates or adds a single sort field specification to the current sort
+ * information. If the sort field already exist in the sort information map,
+ * its position is unchanged and the sort order is set; if it does not exist,
+ * it is appended at the end with the specified order..
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery addOrUpdateSort(SortClause sortClause) {
+ if (sortClauses != null) {
+ for (int index=0 ; index<sortClauses.size() ; index++) {
+ SortClause existing = sortClauses.get(index);
+ if (existing.getItem().equals(sortClause.getItem())) {
+ sortClauses.set(index, sortClause);
+ serializeSorts();
+ return this;
+ }
+ }
+ }
+ return addSort(sortClause);
+ }
+
+ /**
+ * Removes a single sort field from the current sort information.
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery removeSort(SortClause sortClause) {
+ return removeSort(sortClause.getItem());
+ }
+
+ /**
+ * Removes a single sort field from the current sort information.
+ *
+ * @return the modified SolrQuery object, for easy chaining
+ * @since 4.2
+ */
+ public SolrQuery removeSort(String itemName) {
+ if (sortClauses != null) {
+ for (SortClause existing : sortClauses) {
+ if (existing.getItem().equals(itemName)) {
+ sortClauses.remove(existing);
+ if (sortClauses.isEmpty()) sortClauses = null;
+ serializeSorts();
+ break;
+ }
+ }
+ }
+ return this;
+ }
+
+ private void serializeSorts() {
+ if (sortClauses == null || sortClauses.isEmpty()) {
+ remove(CommonParams.SORT);
+ } else {
+ StringBuilder sb = new StringBuilder();
+ for (SortClause sortClause : sortClauses) {
+ if (sb.length() > 0) sb.append(",");
+ sb.append(sortClause.getItem());
+ sb.append(" ");
+ sb.append(sortClause.getOrder());
+ }
+ set(CommonParams.SORT, sb.toString());
+ }
+ }
+
+ public void setGetFieldStatistics( boolean v )
+ {
+ this.set( StatsParams.STATS, v );
+ }
+
+ public void setGetFieldStatistics( String field )
+ {
+ this.set( StatsParams.STATS, true );
+ this.add( StatsParams.STATS_FIELD, field );
+ }
+
+
+ public void addGetFieldStatistics( String ... field )
+ {
+ this.set( StatsParams.STATS, true );
+ this.add( StatsParams.STATS_FIELD, field );
+ }
+
+ public void addStatsFieldFacets( String field, String ... facets )
+ {
+ if( field == null ) {
+ this.add( StatsParams.STATS_FACET, facets );
+ }
+ else {
+ for( String f : facets ) {
+ this.add( "f."+field+"."+StatsParams.STATS_FACET, f );
+ }
+ }
+ }
+
+ public void addStatsFieldCalcDistinct(String field, boolean calcDistinct) {
+ if (field == null) {
+ this.add(StatsParams.STATS_CALC_DISTINCT, Boolean.toString(calcDistinct));
+ } else {
+ this.add("f." + field + "." + StatsParams.STATS_CALC_DISTINCT, Boolean.toString(calcDistinct));
+ }
+ }
+
+ public SolrQuery setFilterQueries(String ... fq) {
+ this.set(CommonParams.FQ, fq);
+ return this;
+ }
+
+ public SolrQuery addFilterQuery(String ... fq) {
+ this.add(CommonParams.FQ, fq);
+ return this;
+ }
+
+ public boolean removeFilterQuery(String fq) {
+ return this.remove(CommonParams.FQ, fq);
+ }
+
+ public String[] getFilterQueries() {
+ return this.getParams(CommonParams.FQ);
+ }
+
+ public boolean getHighlight() {
+ return this.getBool(HighlightParams.HIGHLIGHT, false);
+ }
+
+ public SolrQuery setHighlight(boolean b) {
+ if (b) {
+ this.set(HighlightParams.HIGHLIGHT, true);
+ } else {
+ this.remove(HighlightParams.HIGHLIGHT);
+ this.remove(HighlightParams.FIELD_MATCH);
+ this.remove(HighlightParams.FIELDS);
+ this.remove(HighlightParams.FORMATTER);
+ this.remove(HighlightParams.FRAGSIZE);
+ this.remove(HighlightParams.SIMPLE_POST);
+ this.remove(HighlightParams.SIMPLE_PRE);
+ this.remove(HighlightParams.SNIPPETS);
+ }
+ return this;
+ }
+
+ public SolrQuery setFields(String ... fields) {
+ if( fields == null || fields.length == 0 ) {
+ this.remove( CommonParams.FL );
+ return this;
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append( fields[0] );
+ for( int i=1; i<fields.length; i++ ) {
+ sb.append( ',' );
+ sb.append( fields[i] );
+ }
+ this.set(CommonParams.FL, sb.toString() );
+ return this;
+ }
+
+ public SolrQuery addField(String field) {
+ return addValueToParam(CommonParams.FL, field);
+ }
+
+ public String getFields() {
+ String fields = this.get(CommonParams.FL);
+ if (fields!=null && fields.equals("score")) {
+ fields = "*, score";
+ }
+ return fields;
+ }
+
+ private static Pattern scorePattern = Pattern.compile("(^|[, ])score");
+
+ public SolrQuery setIncludeScore(boolean includeScore) {
+ String fields = get(CommonParams.FL,"*");
+ if (includeScore) {
+ if (!scorePattern.matcher(fields).find()) {
+ this.set(CommonParams.FL, fields+",score");
+ }
+ } else {
+ this.set(CommonParams.FL, scorePattern.matcher(fields).replaceAll(""));
+ }
+ return this;
+ }
+
+ public SolrQuery setQuery(String query) {
+ this.set(CommonParams.Q, query);
+ return this;
+ }
+
+ public String getQuery() {
+ return this.get(CommonParams.Q);
+ }
+
+ public SolrQuery setRows(Integer rows) {
+ if( rows == null ) {
+ this.remove( CommonParams.ROWS );
+ }
+ else {
+ this.set(CommonParams.ROWS, rows);
+ }
+ return this;
+ }
+
+ public Integer getRows()
+ {
+ return this.getInt(CommonParams.ROWS);
+ }
+
+ public void setShowDebugInfo(boolean showDebugInfo) {
+ this.set(CommonParams.DEBUG_QUERY, String.valueOf(showDebugInfo));
+ }
+
+ public void setDistrib(boolean val) {
+ this.set(CommonParams.DISTRIB, String.valueOf(val));
+ }
+
+
+ public SolrQuery setStart(Integer start) {
+ if( start == null ) {
+ this.remove( CommonParams.START );
+ }
+ else {
+ this.set(CommonParams.START, start);
+ }
+ return this;
+ }
+
+ public Integer getStart()
+ {
+ return this.getInt(CommonParams.START);
+ }
+
+ /**
+ * The Request Handler to use (see the solrconfig.xml), which is stored in the "qt" parameter.
+ * Normally it starts with a '/' and if so it will be used by
+ * {@link org.apache.solr.client.solrj.request.QueryRequest#getPath()} in the URL instead of the "qt" parameter.
+ * If this is left blank, then the default of "/select" is assumed.
+ *
+ * @param qt The Request Handler name corresponding to one in solrconfig.xml on the server.
+ * @return this
+ */
+ public SolrQuery setRequestHandler(String qt) {
+ this.set(CommonParams.QT, qt);
+ return this;
+ }
+
+ public String getRequestHandler() {
+ return this.get(CommonParams.QT);
+ }
+
+ /**
+ * @return this
+ * @see ModifiableSolrParams#set(String,String[])
+ */
+ public SolrQuery setParam(String name, String ... values) {
+ this.set(name, values);
+ return this;
+ }
+
+ /**
+ * @return this
+ * @see org.apache.solr.common.params.ModifiableSolrParams#set(String, boolean)
+ */
+ public SolrQuery setParam(String name, boolean value) {
+ this.set(name, value);
+ return this;
+ }
+
+ /** get a deep copy of this object **/
+ public SolrQuery getCopy() {
+ SolrQuery q = new SolrQuery();
+ for (String name : this.getParameterNames()) {
+ q.setParam(name, this.getParams(name));
+ }
+ return q;
+ }
+
+ /**
+ * Set the maximum time allowed for this query. If the query takes more time
+ * than the specified milliseconds, a timeout occurs and partial (or no)
+ * results may be returned.
+ *
+ * If given Integer is null, then this parameter is removed from the request
+ *
+ *@param milliseconds the time in milliseconds allowed for this query
+ */
+ public SolrQuery setTimeAllowed(Integer milliseconds) {
+ if (milliseconds == null) {
+ this.remove(CommonParams.TIME_ALLOWED);
+ } else {
+ this.set(CommonParams.TIME_ALLOWED, milliseconds);
+ }
+ return this;
+ }
+
+ /**
+ * Get the maximum time allowed for this query.
+ */
+ public Integer getTimeAllowed() {
+ return this.getInt(CommonParams.TIME_ALLOWED);
+ }
+
+ ///////////////////////
+ // Utility functions
+ ///////////////////////
+
+ private String toSortString(String field, ORDER order) {
+ return field.trim() + ' ' + String.valueOf(order).trim();
+ }
+
+ private String join(String a, String b, String sep) {
+ StringBuilder sb = new StringBuilder();
+ if (a!=null && a.length()>0) {
+ sb.append(a);
+ sb.append(sep);
+ }
+ if (b!=null && b.length()>0) {
+ sb.append(b);
+ }
+ return sb.toString().trim();
+ }
+
+ private SolrQuery addValueToParam(String name, String value) {
+ String tmp = this.get(name);
+ tmp = join(tmp, value, ",");
+ this.set(name, tmp);
+ return this;
+ }
+
+ private String join(String[] vals, String sep, String removeVal) {
+ StringBuilder sb = new StringBuilder();
+ for (int i=0; i<vals.length; i++) {
+ if (!vals[i].equals(removeVal)) {
+ if (sb.length() > 0) {
+ sb.append(sep);
+ }
+ sb.append(vals[i]);
+ }
+ }
+ return sb.toString().trim();
+ }
+
+ /**
+ * A single sort clause, encapsulating what to sort and the sort order.
+ * <p>
+ * The item specified can be "anything sortable" by solr; some examples
+ * include a simple field name, the constant string {@code score}, and functions
+ * such as {@code sum(x_f, y_f)}.
+ * <p>
+ * A SortClause can be created through different mechanisms:
+ * <PRE><code>
+ * new SortClause("product", SolrQuery.ORDER.asc);
+ * new SortClause("product", "asc");
+ * SortClause.asc("product");
+ * SortClause.desc("product");
+ * </code></PRE>
+ */
+ public static class SortClause implements java.io.Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final String item;
+ private final ORDER order;
+
+ /**
+ * Creates a SortClause based on item and order
+ * @param item item to sort on
+ * @param order direction to sort
+ */
+ public SortClause(String item, ORDER order) {
+ this.item = item;
+ this.order = order;
+ }
+
+ /**
+ * Creates a SortClause based on item and order
+ * @param item item to sort on
+ * @param order string value for direction to sort
+ */
+ public SortClause(String item, String order) {
+ this(item, ORDER.valueOf(order));
+ }
+
+ /**
+ * Creates an ascending SortClause for an item
+ * @param item item to sort on
+ */
+ public static SortClause create (String item, ORDER order) {
+ return new SortClause(item, order);
+ }
+
+ /**
+ * Creates a SortClause based on item and order
+ * @param item item to sort on
+ * @param order string value for direction to sort
+ */
+ public static SortClause create(String item, String order) {
+ return new SortClause(item, ORDER.valueOf(order));
+ }
+
+ /**
+ * Creates an ascending SortClause for an item
+ * @param item item to sort on
+ */
+ public static SortClause asc (String item) {
+ return new SortClause(item, ORDER.asc);
+ }
+
+ /**
+ * Creates a decending SortClause for an item
+ * @param item item to sort on
+ */
+ public static SortClause desc (String item) {
+ return new SortClause(item, ORDER.desc);
+ }
+
+ /**
+ * Gets the item to sort, typically a function or a fieldname
+ * @return item to sort
+ */
+ public String getItem() {
+ return item;
+ }
+
+ /**
+ * Gets the order to sort
+ * @return order to sort
+ */
+ public ORDER getOrder() {
+ return order;
+ }
+
+ public boolean equals(Object other){
+ if (this == other) return true;
+ if (!(other instanceof SortClause)) return false;
+ final SortClause that = (SortClause) other;
+ return this.getItem().equals(that.getItem()) && this.getOrder().equals(that.getOrder());
+ }
+
+ public int hashCode(){
+ return this.getItem().hashCode();
+ }
+
+ /**
+ * Gets a human readable description of the sort clause.
+ * <p>
+ * The returned string is not suitable for passing to Solr,
+ * but may be useful in debug output and the like.
+ * @return a description of the current sort clause
+ */
+ public String toString() {
+ return "[" + getClass().getSimpleName() + ": item=" + getItem() + "; order=" + getOrder() + "]";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrRequest.java
new file mode 100644
index 0000000..6d4efb6
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrRequest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public abstract class SolrRequest<T extends SolrResponse> implements Serializable {
+
+ public enum METHOD {
+ GET,
+ POST,
+ PUT
+ };
+
+ private METHOD method = METHOD.GET;
+ private String path = null;
+
+ private ResponseParser responseParser;
+ private StreamingResponseCallback callback;
+ private Set<String> queryParams;
+
+ //---------------------------------------------------------
+ //---------------------------------------------------------
+
+ public SolrRequest( METHOD m, String path )
+ {
+ this.method = m;
+ this.path = path;
+ }
+
+ //---------------------------------------------------------
+ //---------------------------------------------------------
+
+ public METHOD getMethod() {
+ return method;
+ }
+ public void setMethod(METHOD method) {
+ this.method = method;
+ }
+
+ public String getPath() {
+ return path;
+ }
+ public void setPath(String path) {
+ this.path = path;
+ }
+
+ /**
+ *
+ * @return The {@link org.apache.solr.client.solrj.ResponseParser}
+ */
+ public ResponseParser getResponseParser() {
+ return responseParser;
+ }
+
+ /**
+ * Optionally specify how the Response should be parsed. Not all server implementations require a ResponseParser
+ * to be specified.
+ * @param responseParser The {@link org.apache.solr.client.solrj.ResponseParser}
+ */
+ public void setResponseParser(ResponseParser responseParser) {
+ this.responseParser = responseParser;
+ }
+
+ public StreamingResponseCallback getStreamingResponseCallback() {
+ return callback;
+ }
+
+ public void setStreamingResponseCallback(StreamingResponseCallback callback) {
+ this.callback = callback;
+ }
+
+ /**
+ * Parameter keys that are sent via the query string
+ */
+ public Set<String> getQueryParams() {
+ return this.queryParams;
+ }
+
+ public void setQueryParams(Set<String> queryParams) {
+ this.queryParams = queryParams;
+ }
+
+ public abstract SolrParams getParams();
+
+ public abstract Collection<ContentStream> getContentStreams() throws IOException;
+
+ /**
+ * Create a new SolrResponse to hold the response from the server
+ * @param client the {@link SolrClient} the request will be sent to
+ */
+ protected abstract T createResponse(SolrClient client);
+
+ /**
+ * Send this request to a {@link SolrClient} and return the response
+ * @param client the SolrClient to communicate with
+ * @return the response
+ * @throws SolrServerException if there is an error on the Solr server
+ * @throws IOException if there is a communication error
+ */
+ public final T process(SolrClient client) throws SolrServerException, IOException {
+ long startTime = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
+ T res = createResponse(client);
+ res.setResponse(client.request(this));
+ long endTime = TimeUnit.MILLISECONDS.convert(System.nanoTime(), TimeUnit.NANOSECONDS);
+ res.setElapsedTime(endTime - startTime);
+ return res;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrResponse.java
new file mode 100644
index 0000000..244a757
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrResponse.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public abstract class SolrResponse implements Serializable {
+ public abstract long getElapsedTime();
+
+ public abstract void setResponse(NamedList<Object> rsp);
+
+ public abstract void setElapsedTime(long elapsedTime);
+
+ public abstract NamedList<Object> getResponse();
+
+ public static byte[] serializable(SolrResponse response) {
+ try {
+ ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+ ObjectOutputStream outputStream = new ObjectOutputStream(byteStream);
+ outputStream.writeObject(response);
+ return byteStream.toByteArray();
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ public static SolrResponse deserialize(byte[] bytes) {
+ try {
+ ByteArrayInputStream byteStream = new ByteArrayInputStream(bytes);
+ ObjectInputStream inputStream = new ObjectInputStream(byteStream);
+ return (SolrResponse) inputStream.readObject();
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServer.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServer.java
new file mode 100644
index 0000000..0238214
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServer.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+/**
+ * @deprecated Use {@link org.apache.solr.client.solrj.SolrClient}
+ */
+@Deprecated
+public abstract class SolrServer extends SolrClient {
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServerException.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServerException.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServerException.java
new file mode 100644
index 0000000..ed16a43
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrServerException.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+/** Exception to catch all types of communication / parsing issues associated with talking to SOLR
+ *
+ *
+ * @since solr 1.3
+ */
+public class SolrServerException extends Exception {
+
+ private static final long serialVersionUID = -3371703521752000294L;
+
+ public SolrServerException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public SolrServerException(String message) {
+ super(message);
+ }
+
+ public SolrServerException(Throwable cause) {
+ super(cause);
+ }
+
+ public Throwable getRootCause() {
+ Throwable t = this;
+ while (true) {
+ Throwable cause = t.getCause();
+ if (cause!=null) {
+ t = cause;
+ } else {
+ break;
+ }
+ }
+ return t;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/StreamingResponseCallback.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/StreamingResponseCallback.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/StreamingResponseCallback.java
new file mode 100644
index 0000000..2ba79c8
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/StreamingResponseCallback.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import org.apache.solr.common.SolrDocument;
+
+/**
+ * A callback interface for streaming response
+ *
+ * @since solr 4.0
+ */
+public abstract class StreamingResponseCallback {
+ /*
+ * Called for each SolrDocument in the response
+ */
+ public abstract void streamSolrDocument( SolrDocument doc );
+
+ /*
+ * Called at the beginning of each DocList (and SolrDocumentList)
+ */
+ public abstract void streamDocListInfo( long numFound, long start, Float maxScore );
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/BindingException.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/BindingException.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/BindingException.java
new file mode 100644
index 0000000..cd10138
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/BindingException.java
@@ -0,0 +1,29 @@
+package org.apache.solr.client.solrj.beans;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class BindingException extends RuntimeException {
+
+ public BindingException(String message) {
+ super(message);
+ }
+
+ public BindingException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java
new file mode 100644
index 0000000..bff6c6a
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/DocumentObjectBinder.java
@@ -0,0 +1,470 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.beans;
+
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+
+import java.lang.reflect.*;
+import java.util.*;
+import java.util.regex.Pattern;
+import java.util.concurrent.ConcurrentHashMap;
+import java.nio.ByteBuffer;
+
+/**
+ * A class to map objects to and from solr documents.
+ *
+ *
+ * @since solr 1.3
+ */
+public class DocumentObjectBinder {
+
+ private final Map<Class, List<DocField>> infocache = new ConcurrentHashMap<>();
+
+ public DocumentObjectBinder() {
+ }
+
+ public <T> List<T> getBeans(Class<T> clazz, SolrDocumentList solrDocList) {
+ List<DocField> fields = getDocFields(clazz);
+ List<T> result = new ArrayList<>(solrDocList.size());
+
+ for (SolrDocument sdoc : solrDocList) {
+ result.add(getBean(clazz, fields, sdoc));
+ }
+ return result;
+ }
+
+ public <T> T getBean(Class<T> clazz, SolrDocument solrDoc) {
+ return getBean(clazz, null, solrDoc);
+ }
+
+ private <T> T getBean(Class<T> clazz, List<DocField> fields, SolrDocument solrDoc) {
+ if (fields == null) {
+ fields = getDocFields(clazz);
+ }
+
+ try {
+ T obj = clazz.newInstance();
+ for (DocField docField : fields) {
+ docField.inject(obj, solrDoc);
+ }
+ return obj;
+ } catch (Exception e) {
+ throw new BindingException("Could not instantiate object of " + clazz, e);
+ }
+ }
+
+ public SolrInputDocument toSolrInputDocument(Object obj) {
+ List<DocField> fields = getDocFields(obj.getClass());
+ if (fields.isEmpty()) {
+ throw new BindingException("class: " + obj.getClass() + " does not define any fields.");
+ }
+
+ SolrInputDocument doc = new SolrInputDocument();
+ for (DocField field : fields) {
+ if (field.dynamicFieldNamePatternMatcher != null &&
+ field.get(obj) != null &&
+ field.isContainedInMap) {
+ Map<String, Object> mapValue = (Map<String, Object>) field.get(obj);
+
+ for (Map.Entry<String, Object> e : mapValue.entrySet()) {
+ doc.setField(e.getKey(), e.getValue(), 1.0f);
+ }
+ } else {
+ if (field.child != null) {
+ addChild(obj, field, doc);
+ } else {
+ doc.setField(field.name, field.get(obj), 1.0f);
+ }
+ }
+ }
+ return doc;
+ }
+
+ private void addChild(Object obj, DocField field, SolrInputDocument doc) {
+ Object val = field.get(obj);
+ if (val == null) return;
+ if (val instanceof Collection) {
+ Collection collection = (Collection) val;
+ for (Object o : collection) {
+ SolrInputDocument child = toSolrInputDocument(o);
+ doc.addChildDocument(child);
+ }
+ } else if (val.getClass().isArray()) {
+ Object[] objs = (Object[]) val;
+ for (Object o : objs) doc.addChildDocument(toSolrInputDocument(o));
+ } else {
+ doc.addChildDocument(toSolrInputDocument(val));
+ }
+ }
+
+ private List<DocField> getDocFields(Class clazz) {
+ List<DocField> fields = infocache.get(clazz);
+ if (fields == null) {
+ synchronized(infocache) {
+ infocache.put(clazz, fields = collectInfo(clazz));
+ }
+ }
+ return fields;
+ }
+
+ private List<DocField> collectInfo(Class clazz) {
+ List<DocField> fields = new ArrayList<>();
+ Class superClazz = clazz;
+ List<AccessibleObject> members = new ArrayList<>();
+
+ while (superClazz != null && superClazz != Object.class) {
+ members.addAll(Arrays.asList(superClazz.getDeclaredFields()));
+ members.addAll(Arrays.asList(superClazz.getDeclaredMethods()));
+ superClazz = superClazz.getSuperclass();
+ }
+ boolean childFieldFound = false;
+ for (AccessibleObject member : members) {
+ if (member.isAnnotationPresent(Field.class)) {
+ member.setAccessible(true);
+ DocField df = new DocField(member);
+ if (df.child != null) {
+ if (childFieldFound)
+ throw new BindingException(clazz.getName() + " cannot have more than one Field with child=true");
+ childFieldFound = true;
+ }
+ fields.add(df);
+ }
+ }
+ return fields;
+ }
+
+ private class DocField {
+ private Field annotation;
+ private String name;
+ private java.lang.reflect.Field field;
+ private Method setter;
+ private Method getter;
+ private Class type;
+ private boolean isArray;
+ private boolean isList;
+ private List<DocField> child;
+
+ /*
+ * dynamic fields may use a Map based data structure to bind a given field.
+ * if a mapping is done using, "Map<String, List<String>> foo", <code>isContainedInMap</code>
+ * is set to <code>TRUE</code> as well as <code>isList</code> is set to <code>TRUE</code>
+ */
+ private boolean isContainedInMap;
+ private Pattern dynamicFieldNamePatternMatcher;
+
+ public DocField(AccessibleObject member) {
+ if (member instanceof java.lang.reflect.Field) {
+ field = (java.lang.reflect.Field) member;
+ } else {
+ setter = (Method) member;
+ }
+ annotation = member.getAnnotation(Field.class);
+ storeName(annotation);
+ storeType();
+
+ // Look for a matching getter
+ if (setter != null) {
+ String gname = setter.getName();
+ if (gname.startsWith("set")) {
+ gname = "get" + gname.substring(3);
+ try {
+ getter = setter.getDeclaringClass().getMethod(gname, (Class[]) null);
+ } catch (Exception ex) {
+ // no getter -- don't worry about it...
+ if (type == Boolean.class) {
+ gname = "is" + setter.getName().substring(3);
+ try {
+ getter = setter.getDeclaringClass().getMethod(gname, (Class[]) null);
+ } catch(Exception ex2) {
+ // no getter -- don't worry about it...
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private void storeName(Field annotation) {
+ if (annotation.value().equals(DEFAULT)) {
+ if (field != null) {
+ name = field.getName();
+ } else {
+ String setterName = setter.getName();
+ if (setterName.startsWith("set") && setterName.length() > 3) {
+ name = setterName.substring(3, 4).toLowerCase(Locale.ROOT) + setterName.substring(4);
+ } else {
+ name = setter.getName();
+ }
+ }
+ } else if (annotation.value().indexOf('*') >= 0) { //dynamic fields are annotated as @Field("categories_*")
+ //if the field was annotated as a dynamic field, convert the name into a pattern
+ //the wildcard (*) is supposed to be either a prefix or a suffix, hence the use of replaceFirst
+ name = annotation.value().replaceFirst("\\*", "\\.*");
+ dynamicFieldNamePatternMatcher = Pattern.compile("^"+name+"$");
+ } else {
+ name = annotation.value();
+ }
+ }
+
+ private void storeType() {
+ if (field != null) {
+ type = field.getType();
+ } else {
+ Class[] params = setter.getParameterTypes();
+ if (params.length != 1) {
+ throw new BindingException("Invalid setter method. Must have one and only one parameter");
+ }
+ type = params[0];
+ }
+
+ if (type == Collection.class || type == List.class || type == ArrayList.class) {
+ isList = true;
+ if (annotation.child()) {
+ populateChild(field.getGenericType());
+ } else {
+ type = Object.class;
+ }
+ } else if (type == byte[].class) {
+ //no op
+ } else if (type.isArray()) {
+ isArray = true;
+ if (annotation.child()) {
+ populateChild(type.getComponentType());
+ } else {
+ type = type.getComponentType();
+ }
+ } else if (type == Map.class || type == HashMap.class) { //corresponding to the support for dynamicFields
+ if (annotation.child()) throw new BindingException("Map should is not a valid type for a child document");
+ isContainedInMap = true;
+ //assigned a default type
+ type = Object.class;
+ if (field != null) {
+ if (field.getGenericType() instanceof ParameterizedType) {
+ //check what are the generic values
+ ParameterizedType parameterizedType = (ParameterizedType) field.getGenericType();
+ Type[] types = parameterizedType.getActualTypeArguments();
+ if (types != null && types.length == 2 && types[0] == String.class) {
+ //the key should always be String
+ //Raw and primitive types
+ if (types[1] instanceof Class) {
+ //the value could be multivalued then it is a List, Collection, ArrayList
+ if (types[1] == Collection.class || types[1] == List.class || types[1] == ArrayList.class) {
+ type = Object.class;
+ isList = true;
+ } else {
+ //else assume it is a primitive and put in the source type itself
+ type = (Class) types[1];
+ }
+ } else if (types[1] instanceof ParameterizedType) { //Of all the Parameterized types, only List is supported
+ Type rawType = ((ParameterizedType) types[1]).getRawType();
+ if (rawType == Collection.class || rawType == List.class || rawType == ArrayList.class) {
+ type = Object.class;
+ isList = true;
+ }
+ } else if (types[1] instanceof GenericArrayType) { //Array types
+ type = (Class) ((GenericArrayType) types[1]).getGenericComponentType();
+ isArray = true;
+ } else { //Throw an Exception if types are not known
+ throw new BindingException("Allowed type for values of mapping a dynamicField are : " +
+ "Object, Object[] and List");
+ }
+ }
+ }
+ }
+ } else {
+ if (annotation.child()) {
+ populateChild(type);
+ }
+ }
+ }
+
+ private void populateChild(Type typ) {
+ if (typ == null) {
+ throw new RuntimeException("no type information available for" + (field == null ? setter : field));
+ }
+ if (typ.getClass() == Class.class) {//of type class
+ type = (Class) typ;
+ } else if (typ instanceof ParameterizedType) {
+ type = (Class) ((ParameterizedType) typ).getActualTypeArguments()[0];
+ } else {
+ throw new BindingException("Invalid type information available for" + (field == null ? setter : field));
+
+ }
+ child = getDocFields(type);
+ }
+
+ /**
+ * Called by the {@link #inject} method to read the value(s) for a field
+ * This method supports reading of all "matching" fieldName's in the <code>SolrDocument</code>
+ *
+ * Returns <code>SolrDocument.getFieldValue</code> for regular fields,
+ * and <code>Map<String, List<Object>></code> for a dynamic field. The key is all matching fieldName's.
+ */
+ @SuppressWarnings("unchecked")
+ private Object getFieldValue(SolrDocument solrDocument) {
+ if (child != null) {
+ List<SolrDocument> children = solrDocument.getChildDocuments();
+ if (children == null || children.isEmpty()) return null;
+ if (isList) {
+ ArrayList list = new ArrayList(children.size());
+ for (SolrDocument c : children) {
+ list.add(getBean(type, child, c));
+ }
+ return list;
+ } else if (isArray) {
+ Object[] arr = (Object[]) Array.newInstance(type, children.size());
+ for (int i = 0; i < children.size(); i++) {
+ arr[i] = getBean(type, child, children.get(i));
+ }
+ return arr;
+
+ } else {
+ return getBean(type, child, children.get(0));
+ }
+ }
+ Object fieldValue = solrDocument.getFieldValue(name);
+ if (fieldValue != null) {
+ //this is not a dynamic field. so return the value
+ return fieldValue;
+ }
+
+ if (dynamicFieldNamePatternMatcher == null) {
+ return null;
+ }
+
+ //reading dynamic field values
+ Map<String, Object> allValuesMap = null;
+ List allValuesList = null;
+ if (isContainedInMap) {
+ allValuesMap = new HashMap<>();
+ } else {
+ allValuesList = new ArrayList();
+ }
+
+ for (String field : solrDocument.getFieldNames()) {
+ if (dynamicFieldNamePatternMatcher.matcher(field).find()) {
+ Object val = solrDocument.getFieldValue(field);
+ if (val == null) {
+ continue;
+ }
+
+ if (isContainedInMap) {
+ if (isList) {
+ if (!(val instanceof List)) {
+ List al = new ArrayList();
+ al.add(val);
+ val = al;
+ }
+ } else if (isArray) {
+ if (!(val instanceof List)) {
+ Object[] arr = (Object[]) Array.newInstance(type, 1);
+ arr[0] = val;
+ val = arr;
+ } else {
+ val = Array.newInstance(type, ((List) val).size());
+ }
+ }
+ allValuesMap.put(field, val);
+ } else {
+ if (val instanceof Collection) {
+ allValuesList.addAll((Collection) val);
+ } else {
+ allValuesList.add(val);
+ }
+ }
+ }
+ }
+ if (isContainedInMap) {
+ return allValuesMap.isEmpty() ? null : allValuesMap;
+ } else {
+ return allValuesList.isEmpty() ? null : allValuesList;
+ }
+ }
+
+ <T> void inject(T obj, SolrDocument sdoc) {
+ Object val = getFieldValue(sdoc);
+ if(val == null) {
+ return;
+ }
+
+ if (isArray && !isContainedInMap) {
+ List list;
+ if (val.getClass().isArray()) {
+ set(obj, val);
+ return;
+ } else if (val instanceof List) {
+ list = (List) val;
+ } else {
+ list = new ArrayList();
+ list.add(val);
+ }
+ set(obj, list.toArray((Object[]) Array.newInstance(type, list.size())));
+ } else if (isList && !isContainedInMap) {
+ if (!(val instanceof List)) {
+ List list = new ArrayList();
+ list.add(val);
+ val = list;
+ }
+ set(obj, val);
+ } else if (isContainedInMap) {
+ if (val instanceof Map) {
+ set(obj, val);
+ }
+ } else {
+ set(obj, val);
+ }
+
+ }
+
+ private void set(Object obj, Object v) {
+ if (v != null && type == ByteBuffer.class && v.getClass() == byte[].class) {
+ v = ByteBuffer.wrap((byte[]) v);
+ }
+ try {
+ if (field != null) {
+ field.set(obj, v);
+ } else if (setter != null) {
+ setter.invoke(obj, v);
+ }
+ }
+ catch (Exception e) {
+ throw new BindingException("Exception while setting value : " + v + " on " + (field != null ? field : setter), e);
+ }
+ }
+
+ public Object get(final Object obj) {
+ if (field != null) {
+ try {
+ return field.get(obj);
+ } catch (Exception e) {
+ throw new BindingException("Exception while getting value: " + field, e);
+ }
+ } else if (getter == null) {
+ throw new BindingException("Missing getter for field: " + name + " -- You can only call the 'get' for fields that have a field of 'get' method");
+ }
+
+ try {
+ return getter.invoke(obj, (Object[]) null);
+ } catch (Exception e) {
+ throw new BindingException("Exception while getting value: " + getter, e);
+ }
+ }
+ }
+ public static final String DEFAULT = "#default";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java
new file mode 100644
index 0000000..39f6752
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/Field.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.beans;
+
+import static java.lang.annotation.ElementType.*;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+import static org.apache.solr.client.solrj.beans.DocumentObjectBinder.DEFAULT;
+
+import java.lang.annotation.Target;
+import java.lang.annotation.Retention;
+
+
+/**
+ * This class can be used to annotate a field or a setter an any class
+ * and SlrJ would help you convert to SolrInputDocument and from SolrDocument
+ *
+ * @since solr 1.3
+ */
+@Target({FIELD, METHOD})
+@Retention(RUNTIME)
+public @interface Field {
+ boolean child() default false;
+ String value() default DEFAULT;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java
new file mode 100644
index 0000000..890005f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/beans/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Annotation based mapping of client objects to Solr documents.
+ */
+package org.apache.solr.client.solrj.beans;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java
new file mode 100644
index 0000000..67274c2
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryRequestWriter.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.common.util.ContentStream;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * A RequestWriter which writes requests in the javabin format
+ *
+ *
+ * @see org.apache.solr.client.solrj.request.RequestWriter
+ * @since solr 1.4
+ */
+public class BinaryRequestWriter extends RequestWriter {
+
+ @Override
+ public Collection<ContentStream> getContentStreams(SolrRequest req) throws IOException {
+ if (req instanceof UpdateRequest) {
+ UpdateRequest updateRequest = (UpdateRequest) req;
+ if (isNull(updateRequest.getDocuments()) &&
+ isNull(updateRequest.getDeleteByIdMap()) &&
+ isNull(updateRequest.getDeleteQuery())
+ && (updateRequest.getDocIterator() == null) ) {
+ return null;
+ }
+ List<ContentStream> l = new ArrayList<>();
+ l.add(new LazyContentStream(updateRequest));
+ return l;
+ } else {
+ return super.getContentStreams(req);
+ }
+
+ }
+
+
+ @Override
+ public String getUpdateContentType() {
+ return "application/javabin";
+ }
+
+ @Override
+ public ContentStream getContentStream(final UpdateRequest request) throws IOException {
+ final BAOS baos = new BAOS();
+ new JavaBinUpdateRequestCodec().marshal(request, baos);
+
+ return new ContentStream() {
+ @Override
+ public String getName() {
+ return null;
+ }
+
+ @Override
+ public String getSourceInfo() {
+ return "javabin";
+ }
+
+ @Override
+ public String getContentType() {
+ return "application/javabin";
+ }
+
+ @Override
+ public Long getSize() // size if we know it, otherwise null
+ {
+ return new Long(baos.size());
+ }
+
+ @Override
+ public InputStream getStream() {
+ return new ByteArrayInputStream(baos.getbuf(), 0, baos.size());
+ }
+
+ @Override
+ public Reader getReader() {
+ throw new RuntimeException("No reader available . this is a binarystream");
+ }
+ };
+ }
+
+
+ @Override
+ public void write(SolrRequest request, OutputStream os) throws IOException {
+ if (request instanceof UpdateRequest) {
+ UpdateRequest updateRequest = (UpdateRequest) request;
+ new JavaBinUpdateRequestCodec().marshal(updateRequest, os);
+ }
+ }
+
+ /*
+ * A hack to get access to the protected internal buffer and avoid an additional copy
+ */
+ class BAOS extends ByteArrayOutputStream {
+ byte[] getbuf() {
+ return super.buf;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java
new file mode 100644
index 0000000..4f3caf1
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/BinaryResponseParser.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.JavaBinCodec;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+
+/**
+ *
+ * @since solr 1.3
+ */
+public class BinaryResponseParser extends ResponseParser {
+ public static final String BINARY_CONTENT_TYPE = "application/octet-stream";
+
+ @Override
+ public String getWriterType() {
+ return "javabin";
+ }
+
+ @Override
+ public NamedList<Object> processResponse(InputStream body, String encoding) {
+ try {
+ return (NamedList<Object>) new JavaBinCodec().unmarshal(body);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);
+
+ }
+ }
+
+ @Override
+ public String getContentType() {
+ return BINARY_CONTENT_TYPE;
+ }
+
+ @Override
+ public String getVersion() {
+ return "2";
+ }
+
+ @Override
+ public NamedList<Object> processResponse(Reader reader) {
+ throw new RuntimeException("Cannot handle character stream");
+ }
+}
[09/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/ClientUtils.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/ClientUtils.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/ClientUtils.java
new file mode 100644
index 0000000..f2e4036
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/ClientUtils.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.util;
+
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.DateUtil;
+import org.apache.solr.common.util.XML;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.net.URLEncoder;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+
+/**
+ *
+ * @since solr 1.3
+ */
+public class ClientUtils
+{
+ // Standard Content types
+ public static final String TEXT_XML = "application/xml; charset=UTF-8";
+
+ /**
+ * Take a string and make it an iterable ContentStream
+ */
+ public static Collection<ContentStream> toContentStreams( final String str, final String contentType )
+ {
+ if( str == null )
+ return null;
+
+ ArrayList<ContentStream> streams = new ArrayList<>( 1 );
+ ContentStreamBase ccc = new ContentStreamBase.StringStream( str );
+ ccc.setContentType( contentType );
+ streams.add( ccc );
+ return streams;
+ }
+
+ /**
+ * @param d SolrDocument to convert
+ * @return a SolrInputDocument with the same fields and values as the
+ * SolrDocument. All boosts are 1.0f
+ */
+ public static SolrInputDocument toSolrInputDocument( SolrDocument d )
+ {
+ SolrInputDocument doc = new SolrInputDocument();
+ for( String name : d.getFieldNames() ) {
+ doc.addField( name, d.getFieldValue(name), 1.0f );
+ }
+ return doc;
+ }
+
+ /**
+ * @param d SolrInputDocument to convert
+ * @return a SolrDocument with the same fields and values as the SolrInputDocument
+ */
+ public static SolrDocument toSolrDocument(SolrInputDocument d) {
+ SolrDocument doc = new SolrDocument();
+ for (SolrInputField field : d) {
+ doc.setField(field.getName(), field.getValue());
+ }
+ if (d.getChildDocuments() != null) {
+ for (SolrInputDocument in : d.getChildDocuments()) {
+ doc.addChildDocument(toSolrDocument(in));
+ }
+
+ }
+ return doc;
+ }
+
+ //------------------------------------------------------------------------
+ //------------------------------------------------------------------------
+
+ public static void writeXML( SolrInputDocument doc, Writer writer ) throws IOException
+ {
+ writer.write("<doc boost=\""+doc.getDocumentBoost()+"\">");
+
+ for( SolrInputField field : doc ) {
+ float boost = field.getBoost();
+ String name = field.getName();
+
+ for( Object v : field ) {
+ String update = null;
+
+ if (v instanceof Map) {
+ // currently only supports a single value
+ for (Entry<Object,Object> entry : ((Map<Object,Object>)v).entrySet()) {
+ update = entry.getKey().toString();
+ v = entry.getValue();
+ if (v instanceof Collection) {
+ Collection values = (Collection) v;
+ for (Object value : values) {
+ writeVal(writer, boost, name, value, update);
+ boost = 1.0f;
+ }
+ } else {
+ writeVal(writer, boost, name, v, update);
+ boost = 1.0f;
+ }
+ }
+ } else {
+ writeVal(writer, boost, name, v, update);
+ // only write the boost for the first multi-valued field
+ // otherwise, the used boost is the product of all the boost values
+ boost = 1.0f;
+ }
+ }
+ }
+
+ if (doc.hasChildDocuments()) {
+ for (SolrInputDocument childDocument : doc.getChildDocuments()) {
+ writeXML(childDocument, writer);
+ }
+ }
+
+ writer.write("</doc>");
+ }
+
+ private static void writeVal(Writer writer, float boost, String name, Object v, String update) throws IOException {
+ if (v instanceof Date) {
+ v = DateUtil.getThreadLocalDateFormat().format( (Date)v );
+ } else if (v instanceof byte[]) {
+ byte[] bytes = (byte[]) v;
+ v = Base64.byteArrayToBase64(bytes, 0, bytes.length);
+ } else if (v instanceof ByteBuffer) {
+ ByteBuffer bytes = (ByteBuffer) v;
+ v = Base64.byteArrayToBase64(bytes.array(), bytes.position(),bytes.limit() - bytes.position());
+ }
+
+ if (update == null) {
+ if( boost != 1.0f ) {
+ XML.writeXML(writer, "field", v.toString(), "name", name, "boost", boost);
+ } else if (v != null) {
+ XML.writeXML(writer, "field", v.toString(), "name", name );
+ }
+ } else {
+ if( boost != 1.0f ) {
+ XML.writeXML(writer, "field", v.toString(), "name", name, "boost", boost, "update", update);
+ } else {
+ if (v == null) {
+ XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true);
+ } else {
+ XML.writeXML(writer, "field", v.toString(), "name", name, "update", update);
+ }
+ }
+ }
+ }
+
+
+ public static String toXML( SolrInputDocument doc )
+ {
+ StringWriter str = new StringWriter();
+ try {
+ writeXML( doc, str );
+ }
+ catch( Exception ex ){}
+ return str.toString();
+ }
+
+ //---------------------------------------------------------------------------------------
+
+ /**
+ * See: <a href="https://www.google.com/?gws_rd=ssl#q=lucene+query+parser+syntax">Lucene query parser syntax</a>
+ * for more information on Escaping Special Characters
+ */
+ // NOTE: its broken to link to any lucene-queryparser.jar docs, not in classpath!!!!!
+ public static String escapeQueryChars(String s) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < s.length(); i++) {
+ char c = s.charAt(i);
+ // These characters are part of the query syntax and must be escaped
+ if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':'
+ || c == '^' || c == '[' || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~'
+ || c == '*' || c == '?' || c == '|' || c == '&' || c == ';' || c == '/'
+ || Character.isWhitespace(c)) {
+ sb.append('\\');
+ }
+ sb.append(c);
+ }
+ return sb.toString();
+ }
+
+ public static String toQueryString( SolrParams params, boolean xml ) {
+ StringBuilder sb = new StringBuilder(128);
+ try {
+ String amp = xml ? "&" : "&";
+ boolean first=true;
+ Iterator<String> names = params.getParameterNamesIterator();
+ while( names.hasNext() ) {
+ String key = names.next();
+ String[] valarr = params.getParams( key );
+ if( valarr == null ) {
+ sb.append( first?"?":amp );
+ sb.append(key);
+ first=false;
+ }
+ else {
+ for (String val : valarr) {
+ sb.append( first? "?":amp );
+ sb.append(key);
+ if( val != null ) {
+ sb.append('=');
+ sb.append( URLEncoder.encode( val, "UTF-8" ) );
+ }
+ first=false;
+ }
+ }
+ }
+ }
+ catch (IOException e) {throw new RuntimeException(e);} // can't happen
+ return sb.toString();
+ }
+
+ /** Constructs a slices map from a collection of slices and handles disambiguation if multiple collections are being queried simultaneously */
+ public static void addSlices(Map<String,Slice> target, String collectionName, Collection<Slice> slices, boolean multiCollection) {
+ for (Slice slice : slices) {
+ String key = slice.getName();
+ if (multiCollection) key = collectionName + "_" + key;
+ target.put(key, slice);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/package-info.java
new file mode 100644
index 0000000..9464970
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/util/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Utilities for Solr client applications.
+ */
+package org.apache.solr.client.solrj.util;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/EnumFieldValue.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/EnumFieldValue.java b/ranger_solrj/src/main/java/org/apache/solr/common/EnumFieldValue.java
new file mode 100644
index 0000000..50d1fb0
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/EnumFieldValue.java
@@ -0,0 +1,116 @@
+package org.apache.solr.common;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Serializable;
+
+/**
+ * Represents a Enum field value, which includes integer value (indicating the sort order) and string (displayed) value.
+ * Note: this class has a natural ordering that is inconsistent with equals
+ */
+
+public final class EnumFieldValue implements Serializable, Comparable<EnumFieldValue> {
+ private final Integer intValue;
+ private final String stringValue;
+
+ @Override
+ public int hashCode() {
+ int result = intValue != null ? intValue.hashCode() : 0;
+ result = 31 * result + (stringValue != null ? stringValue.hashCode() : 0);
+ return result;
+ }
+
+ public EnumFieldValue(Integer intValue, String stringValue) {
+ this.intValue = intValue;
+ this.stringValue = stringValue;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null)
+ return false;
+ if (!(obj instanceof EnumFieldValue))
+ return false;
+
+ EnumFieldValue otherEnumFieldValue = (EnumFieldValue) obj;
+ return equalsIntegers(intValue, otherEnumFieldValue.intValue) && equalStrings(stringValue, otherEnumFieldValue.stringValue);
+ }
+
+ /**
+ * @return string (displayed) value
+ */
+ @Override
+ public String toString() {
+ return stringValue;
+ }
+
+ /**
+ * @return integer value (indicating the sort order)
+ */
+ public Integer toInt() {
+ return intValue;
+ }
+
+ @Override
+ public int compareTo(EnumFieldValue o) {
+ if (o == null)
+ return 1;
+ return compareIntegers(intValue, o.intValue);
+ }
+
+ private boolean equalStrings(String str1, String str2) {
+ if ((str1 == null) && (str2 == null))
+ return true;
+
+ if (str1 == null)
+ return false;
+
+ if (str2 == null)
+ return false;
+
+ return str1.equals(str2);
+ }
+
+ private boolean equalsIntegers(Integer int1, Integer int2) {
+ if ((int1 == null) && (int2 == null))
+ return true;
+
+ if (int1 == null)
+ return false;
+
+ if (int2 == null)
+ return false;
+
+ return int1.equals(int2);
+ }
+
+ private int compareIntegers(Integer int1, Integer int2) {
+ if ((int1 == null) && (int2 == null))
+ return 0;
+
+ if (int1 == null)
+ return -1;
+
+ if (int2 == null)
+ return 1;
+
+ return int1.compareTo(int2);
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocument.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocument.java b/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocument.java
new file mode 100644
index 0000000..b7e4465
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocument.java
@@ -0,0 +1,396 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.common.util.NamedList;
+
+
+/**
+ * A concrete representation of a document within a Solr index. Unlike a lucene
+ * Document, a SolrDocument may have an Object value matching the type defined in
+ * schema.xml
+ *
+ * For indexing documents, use the SolrInputDocument that contains extra information
+ * for document and field boosting.
+ *
+ *
+ * @since solr 1.3
+ */
+public class SolrDocument implements Map<String,Object>, Iterable<Map.Entry<String, Object>>, Serializable
+{
+ private final Map<String,Object> _fields;
+
+ private List<SolrDocument> _childDocuments;
+
+ public SolrDocument()
+ {
+ _fields = new LinkedHashMap<>();
+ }
+
+ /**
+ * @return a list of field names defined in this document - this Collection is directly backed by this SolrDocument.
+ * @see #keySet
+ */
+ public Collection<String> getFieldNames() {
+ return this.keySet();
+ }
+
+ ///////////////////////////////////////////////////////////////////
+ // Add / Set / Remove Fields
+ ///////////////////////////////////////////////////////////////////
+
+ /**
+ * Remove all fields from the document
+ */
+ @Override
+ public void clear()
+ {
+ _fields.clear();
+
+ if(_childDocuments != null) {
+ _childDocuments.clear();
+ }
+ }
+
+ /**
+ * Remove all fields with the name
+ */
+ public boolean removeFields(String name)
+ {
+ return this.remove( name ) != null;
+ }
+
+ /**
+ * Set a field with the given object. If the object is an Array, it will
+ * set multiple fields with the included contents. This will replace any existing
+ * field with the given name
+ */
+ @SuppressWarnings("unchecked")
+ public void setField(String name, Object value)
+ {
+ if( value instanceof Object[] ) {
+ value = new ArrayList(Arrays.asList( (Object[])value ));
+ }
+ else if( value instanceof Collection ) {
+ // nothing
+ }
+ else if( value instanceof NamedList ) {
+ // nothing
+ }
+ else if( value instanceof Iterable ) {
+ ArrayList<Object> lst = new ArrayList<>();
+ for( Object o : (Iterable)value ) {
+ lst.add( o );
+ }
+ value = lst;
+ }
+ _fields.put(name, value);
+ }
+
+ /**
+ * This will add a field to the document. If fields already exist with this
+ * name it will append value to the collection. If the value is Collection,
+ * each value will be added independently.
+ *
+ * The class type of value and the name parameter should match schema.xml.
+ * schema.xml can be found in conf directory under the solr home by default.
+ *
+ * @param name Name of the field, should match one of the field names defined under "fields" tag in schema.xml.
+ * @param value Value of the field, should be of same class type as defined by "type" attribute of the corresponding field in schema.xml.
+ */
+ @SuppressWarnings("unchecked")
+ public void addField(String name, Object value)
+ {
+ Object existing = _fields.get(name);
+ if (existing == null) {
+ if( value instanceof Collection ) {
+ Collection<Object> c = new ArrayList<>( 3 );
+ for ( Object o : (Collection<Object>)value ) {
+ c.add(o);
+ }
+ this.setField( name, c );
+ } else {
+ this.setField( name, value );
+ }
+ return;
+ }
+
+ Collection<Object> vals = null;
+ if( existing instanceof Collection ) {
+ vals = (Collection<Object>)existing;
+ }
+ else {
+ vals = new ArrayList<>( 3 );
+ vals.add( existing );
+ }
+
+ // Add the values to the collection
+ if( value instanceof Iterable ) {
+ for( Object o : (Iterable<Object>)value ) {
+ vals.add( o );
+ }
+ }
+ else if( value instanceof Object[] ) {
+ for( Object o : (Object[])value ) {
+ vals.add( o );
+ }
+ }
+ else {
+ vals.add( value );
+ }
+ _fields.put( name, vals );
+ }
+
+ ///////////////////////////////////////////////////////////////////
+ // Get the field values
+ ///////////////////////////////////////////////////////////////////
+
+ /**
+ * returns the first value for a field
+ */
+ public Object getFirstValue(String name) {
+ Object v = _fields.get( name );
+ if (v == null || !(v instanceof Collection)) return v;
+ Collection c = (Collection)v;
+ if (c.size() > 0 ) {
+ return c.iterator().next();
+ }
+ return null;
+ }
+
+ /**
+ * Get the value or collection of values for a given field.
+ */
+ public Object getFieldValue(String name) {
+ return _fields.get( name );
+ }
+
+ /**
+ * Get a collection of values for a given field name
+ */
+ @SuppressWarnings("unchecked")
+ public Collection<Object> getFieldValues(String name) {
+ Object v = _fields.get( name );
+ if( v instanceof Collection ) {
+ return (Collection<Object>)v;
+ }
+ if( v != null ) {
+ ArrayList<Object> arr = new ArrayList<>(1);
+ arr.add( v );
+ return arr;
+ }
+ return null;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SolrDocument"+_fields;
+ }
+
+ /**
+ * Iterate of String->Object keys
+ */
+ @Override
+ public Iterator<Entry<String, Object>> iterator() {
+ return _fields.entrySet().iterator();
+ }
+
+ //-----------------------------------------------------------------------------------------
+ // JSTL Helpers
+ //-----------------------------------------------------------------------------------------
+
+ /**
+ * Expose a Map interface to the solr field value collection.
+ */
+ public Map<String,Collection<Object>> getFieldValuesMap()
+ {
+ return new Map<String,Collection<Object>>() {
+ /** Get the field Value */
+ @Override
+ public Collection<Object> get(Object key) {
+ return getFieldValues( (String)key );
+ }
+
+ // Easily Supported methods
+ @Override
+ public boolean containsKey(Object key) { return _fields.containsKey( key ); }
+ @Override
+ public Set<String> keySet() { return _fields.keySet(); }
+ @Override
+ public int size() { return _fields.size(); }
+ @Override
+ public boolean isEmpty() { return _fields.isEmpty(); }
+
+ // Unsupported operations. These are not necessary for JSTL
+ @Override
+ public void clear() { throw new UnsupportedOperationException(); }
+ @Override
+ public boolean containsValue(Object value) {throw new UnsupportedOperationException();}
+ @Override
+ public Set<java.util.Map.Entry<String, Collection<Object>>> entrySet() {throw new UnsupportedOperationException();}
+ @Override
+ public void putAll(Map<? extends String, ? extends Collection<Object>> t) {throw new UnsupportedOperationException();}
+ @Override
+ public Collection<Collection<Object>> values() {throw new UnsupportedOperationException();}
+ @Override
+ public Collection<Object> put(String key, Collection<Object> value) {throw new UnsupportedOperationException();}
+ @Override
+ public Collection<Object> remove(Object key) {throw new UnsupportedOperationException();}
+ @Override
+ public String toString() {return _fields.toString();}
+ };
+ }
+
+ /**
+ * Expose a Map interface to the solr fields. This function is useful for JSTL
+ */
+ public Map<String,Object> getFieldValueMap() {
+ return new Map<String,Object>() {
+ /** Get the field Value */
+ @Override
+ public Object get(Object key) {
+ return getFirstValue( (String)key );
+ }
+
+ // Easily Supported methods
+ @Override
+ public boolean containsKey(Object key) { return _fields.containsKey( key ); }
+ @Override
+ public Set<String> keySet() { return _fields.keySet(); }
+ @Override
+ public int size() { return _fields.size(); }
+ @Override
+ public boolean isEmpty() { return _fields.isEmpty(); }
+
+ // Unsupported operations. These are not necessary for JSTL
+ @Override
+ public void clear() { throw new UnsupportedOperationException(); }
+ @Override
+ public boolean containsValue(Object value) {throw new UnsupportedOperationException();}
+ @Override
+ public Set<java.util.Map.Entry<String, Object>> entrySet() {throw new UnsupportedOperationException();}
+ @Override
+ public void putAll(Map<? extends String, ? extends Object> t) {throw new UnsupportedOperationException();}
+ @Override
+ public Collection<Object> values() {throw new UnsupportedOperationException();}
+ @Override
+ public Collection<Object> put(String key, Object value) {throw new UnsupportedOperationException();}
+ @Override
+ public Collection<Object> remove(Object key) {throw new UnsupportedOperationException();}
+ @Override
+ public String toString() {return _fields.toString();}
+ };
+ }
+
+ //---------------------------------------------------
+ // MAP interface
+ //---------------------------------------------------
+
+ @Override
+ public boolean containsKey(Object key) {
+ return _fields.containsKey(key);
+ }
+
+ @Override
+ public boolean containsValue(Object value) {
+ return _fields.containsValue(value);
+ }
+
+ @Override
+ public Set<Entry<String, Object>> entrySet() {
+ return _fields.entrySet();
+ }
+ //TODO: Shouldn't the input parameter here be a String? The _fields map requires a String.
+ @Override
+ public Object get(Object key) {
+ return _fields.get(key);
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return _fields.isEmpty();
+ }
+
+ @Override
+ public Set<String> keySet() {
+ return _fields.keySet();
+ }
+
+ @Override
+ public Object put(String key, Object value) {
+ return _fields.put(key, value);
+ }
+
+ @Override
+ public void putAll(Map<? extends String, ? extends Object> t) {
+ _fields.putAll( t );
+ }
+
+ @Override
+ public Object remove(Object key) {
+ return _fields.remove(key);
+ }
+
+ @Override
+ public int size() {
+ return _fields.size();
+ }
+
+ @Override
+ public Collection<Object> values() {
+ return _fields.values();
+ }
+
+ public void addChildDocument(SolrDocument child) {
+ if (_childDocuments == null) {
+ _childDocuments = new ArrayList<>();
+ }
+ _childDocuments.add(child);
+ }
+
+ public void addChildDocuments(Collection<SolrDocument> childs) {
+ for (SolrDocument child : childs) {
+ addChildDocument(child);
+ }
+ }
+
+ /** Returns the list of child documents, or null if none. */
+ public List<SolrDocument> getChildDocuments() {
+ return _childDocuments;
+ }
+
+ public boolean hasChildDocuments() {
+ boolean isEmpty = (_childDocuments == null || _childDocuments.isEmpty());
+ return !isEmpty;
+ }
+
+ public int getChildDocumentCount() {
+ return _childDocuments.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocumentList.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocumentList.java b/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocumentList.java
new file mode 100644
index 0000000..d803e7d
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/SolrDocumentList.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.util.ArrayList;
+
+
+/**
+ * Represent a list of SolrDocuments returned from a search. This includes
+ * position and offset information.
+ *
+ *
+ * @since solr 1.3
+ */
+public class SolrDocumentList extends ArrayList<SolrDocument>
+{
+ private long numFound = 0;
+ private long start = 0;
+ private Float maxScore = null;
+
+ public Float getMaxScore() {
+ return maxScore;
+ }
+
+ public void setMaxScore(Float maxScore) {
+ this.maxScore = maxScore;
+ }
+
+ public long getNumFound() {
+ return numFound;
+ }
+
+ public void setNumFound(long numFound) {
+ this.numFound = numFound;
+ }
+
+ public long getStart() {
+ return start;
+ }
+
+ public void setStart(long start) {
+ this.start = start;
+ }
+
+ @Override
+ public String toString() {
+ return "{numFound="+numFound
+ +",start="+start
+ + (maxScore!=null ? ",maxScore="+maxScore : "")
+ +",docs="+super.toString()
+ +"}";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/SolrException.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/SolrException.java b/ranger_solrj/src/main/java/org/apache/solr/common/SolrException.java
new file mode 100644
index 0000000..02bbc04
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/SolrException.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.CharArrayWriter;
+import java.io.PrintWriter;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.solr.common.util.NamedList;
+import org.slf4j.Logger;
+
+/**
+ *
+ */
+public class SolrException extends RuntimeException {
+
+ /**
+ * This list of valid HTTP Status error codes that Solr may return in
+ * the case of a "Server Side" error.
+ *
+ * @since solr 1.2
+ */
+ public enum ErrorCode {
+ BAD_REQUEST( 400 ),
+ UNAUTHORIZED( 401 ),
+ FORBIDDEN( 403 ),
+ NOT_FOUND( 404 ),
+ CONFLICT( 409 ),
+ UNSUPPORTED_MEDIA_TYPE( 415 ),
+ SERVER_ERROR( 500 ),
+ SERVICE_UNAVAILABLE( 503 ),
+ INVALID_STATE( 510 ),
+ UNKNOWN(0);
+ public final int code;
+
+ private ErrorCode( int c )
+ {
+ code = c;
+ }
+ public static ErrorCode getErrorCode(int c){
+ for (ErrorCode err : values()) {
+ if(err.code == c) return err;
+ }
+ return UNKNOWN;
+ }
+ };
+
+ public SolrException(ErrorCode code, String msg) {
+ super(msg);
+ this.code = code.code;
+ }
+ public SolrException(ErrorCode code, String msg, Throwable th) {
+ super(msg, th);
+ this.code = code.code;
+ }
+
+ public SolrException(ErrorCode code, Throwable th) {
+ super(th);
+ this.code = code.code;
+ }
+
+ /**
+ * Constructor that can set arbitrary http status code. Not for
+ * use in Solr, but may be used by clients in subclasses to capture
+ * errors returned by the servlet container or other HTTP proxies.
+ */
+ protected SolrException(int code, String msg, Throwable th) {
+ super(msg, th);
+ this.code = code;
+ }
+
+ int code=0;
+ protected NamedList<String> metadata;
+
+ /**
+ * The HTTP Status code associated with this Exception. For SolrExceptions
+ * thrown by Solr "Server Side", this should valid {@link ErrorCode},
+ * however client side exceptions may contain an arbitrary error code based
+ * on the behavior of the Servlet Container hosting Solr, or any HTTP
+ * Proxies that may exist between the client and the server.
+ *
+ * @return The HTTP Status code associated with this Exception
+ */
+ public int code() { return code; }
+
+ public void setMetadata(NamedList<String> metadata) {
+ this.metadata = metadata;
+ }
+
+ public NamedList<String> getMetadata() {
+ return metadata;
+ }
+
+ public String getMetadata(String key) {
+ return (metadata != null && key != null) ? metadata.get(key) : null;
+ }
+
+ public void setMetadata(String key, String value) {
+ if (key == null || value == null)
+ throw new IllegalArgumentException("Exception metadata cannot be null!");
+
+ if (metadata == null)
+ metadata = new NamedList<String>();
+ metadata.add(key, value);
+ }
+
+ public void log(Logger log) { log(log,this); }
+ public static void log(Logger log, Throwable e) {
+ String stackTrace = toStr(e);
+ String ignore = doIgnore(e, stackTrace);
+ if (ignore != null) {
+ log.info(ignore);
+ return;
+ }
+ log.error(stackTrace);
+
+ }
+
+ public static void log(Logger log, String msg, Throwable e) {
+ String stackTrace = msg + ':' + toStr(e);
+ String ignore = doIgnore(e, stackTrace);
+ if (ignore != null) {
+ log.info(ignore);
+ return;
+ }
+ log.error(stackTrace);
+ }
+
+ public static void log(Logger log, String msg) {
+ String stackTrace = msg;
+ String ignore = doIgnore(null, stackTrace);
+ if (ignore != null) {
+ log.info(ignore);
+ return;
+ }
+ log.error(stackTrace);
+ }
+
+ // public String toString() { return toStr(this); } // oops, inf loop
+ @Override
+ public String toString() { return super.toString(); }
+
+ public static String toStr(Throwable e) {
+ CharArrayWriter cw = new CharArrayWriter();
+ PrintWriter pw = new PrintWriter(cw);
+ e.printStackTrace(pw);
+ pw.flush();
+ return cw.toString();
+
+/** This doesn't work for some reason!!!!!
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ e.printStackTrace(pw);
+ pw.flush();
+ System.out.println("The STRING:" + sw.toString());
+ return sw.toString();
+**/
+ }
+
+
+ /** For test code - do not log exceptions that match any of the regular expressions in ignorePatterns */
+ public static Set<String> ignorePatterns;
+
+ /** Returns null if this exception does not match any ignore patterns, or a message string to use if it does. */
+ public static String doIgnore(Throwable t, String m) {
+ if (ignorePatterns == null || m == null) return null;
+ if (t != null && t instanceof AssertionError) return null;
+
+ for (String regex : ignorePatterns) {
+ Pattern pattern = Pattern.compile(regex);
+ Matcher matcher = pattern.matcher(m);
+
+ if (matcher.find()) return "Ignoring exception matching " + regex;
+ }
+
+ return null;
+ }
+
+ public static Throwable getRootCause(Throwable t) {
+ while (true) {
+ Throwable cause = t.getCause();
+ if (cause!=null) {
+ t = cause;
+ } else {
+ break;
+ }
+ }
+ return t;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputDocument.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputDocument.java b/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputDocument.java
new file mode 100644
index 0000000..a51efbf
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputDocument.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Represent the field and boost information needed to construct and index
+ * a Lucene Document. Like the SolrDocument, the field values should
+ * match those specified in schema.xml
+ *
+ *
+ * @since solr 1.3
+ */
+public class SolrInputDocument implements Map<String,SolrInputField>, Iterable<SolrInputField>, Serializable
+{
+ private final Map<String,SolrInputField> _fields;
+ private float _documentBoost = 1.0f;
+ private List<SolrInputDocument> _childDocuments;
+
+ public SolrInputDocument() {
+ _fields = new LinkedHashMap<>();
+ }
+
+ public SolrInputDocument(Map<String,SolrInputField> fields) {
+ _fields = fields;
+ }
+
+ /**
+ * Remove all fields and boosts from the document
+ */
+ @Override
+ public void clear()
+ {
+ if( _fields != null ) {
+ _fields.clear();
+ }
+ _childDocuments = null;
+ }
+
+ ///////////////////////////////////////////////////////////////////
+ // Add / Set fields
+ ///////////////////////////////////////////////////////////////////
+
+ /**
+ * Add a field with implied null value for boost.
+ *
+ * The class type of value and the name parameter should match schema.xml.
+ * schema.xml can be found in conf directory under the solr home by default.
+ *
+ * @param name Name of the field, should match one of the field names defined under "fields" tag in schema.xml.
+ * @param value Value of the field, should be of same class type as defined by "type" attribute of the corresponding field in schema.xml.
+ * @see #addField(String, Object, float)
+ */
+ public void addField(String name, Object value)
+ {
+ addField(name, value, 1.0f );
+ }
+
+ /** Get the first value for a field.
+ *
+ * @param name name of the field to fetch
+ * @return first value of the field or null if not present
+ */
+ public Object getFieldValue(String name)
+ {
+ SolrInputField field = getField(name);
+ Object o = null;
+ if (field!=null) o = field.getFirstValue();
+ return o;
+ }
+
+ /** Get all the values for a field.
+ *
+ * @param name name of the field to fetch
+ * @return value of the field or null if not set
+ */
+ public Collection<Object> getFieldValues(String name)
+ {
+ SolrInputField field = getField(name);
+ if (field!=null) {
+ return field.getValues();
+ }
+ return null;
+ }
+
+ /** Get all field names.
+ *
+ * @return Set of all field names.
+ */
+ public Collection<String> getFieldNames()
+ {
+ return _fields.keySet();
+ }
+
+ /** Set a field with implied null value for boost.
+ *
+ * @see #setField(String, Object, float)
+ * @param name name of the field to set
+ * @param value value of the field
+ */
+ public void setField(String name, Object value)
+ {
+ setField(name, value, 1.0f );
+ }
+
+ public void setField(String name, Object value, float boost )
+ {
+ SolrInputField field = new SolrInputField( name );
+ _fields.put( name, field );
+ field.setValue( value, boost );
+ }
+
+ /**
+ * Adds a field with the given name, value and boost. If a field with the
+ * name already exists, then the given value is appended to the value of that
+ * field, with the new boost. If the value is a collection, then each of its
+ * values will be added to the field.
+ *
+ * The class type of value and the name parameter should match schema.xml.
+ * schema.xml can be found in conf directory under the solr home by default.
+ *
+ * @param name Name of the field, should match one of the field names defined under "fields" tag in schema.xml.
+ * @param value Value of the field, should be of same class type as defined by "type" attribute of the corresponding field in schema.xml.
+ * @param boost Boost value for the field
+ */
+ public void addField(String name, Object value, float boost )
+ {
+ SolrInputField field = _fields.get( name );
+ if( field == null || field.value == null ) {
+ setField(name, value, boost);
+ }
+ else {
+ field.addValue( value, boost );
+ }
+ }
+
+ /**
+ * Remove a field from the document
+ *
+ * @param name The field name whose field is to be removed from the document
+ * @return the previous field with <tt>name</tt>, or
+ * <tt>null</tt> if there was no field for <tt>key</tt>.
+ */
+ public SolrInputField removeField(String name) {
+ return _fields.remove( name );
+ }
+
+ ///////////////////////////////////////////////////////////////////
+ // Get the field values
+ ///////////////////////////////////////////////////////////////////
+
+ public SolrInputField getField( String field )
+ {
+ return _fields.get( field );
+ }
+
+ @Override
+ public Iterator<SolrInputField> iterator() {
+ return _fields.values().iterator();
+ }
+
+ public float getDocumentBoost() {
+ return _documentBoost;
+ }
+
+ public void setDocumentBoost(float documentBoost) {
+ _documentBoost = documentBoost;
+ }
+
+ @Override
+ public String toString()
+ {
+ return "SolrInputDocument(fields: " + _fields.values()
+ + ( _childDocuments == null ? "" : (", children: " + _childDocuments) )
+ + ")";
+ }
+
+ public SolrInputDocument deepCopy() {
+ SolrInputDocument clone = new SolrInputDocument();
+ Set<Entry<String,SolrInputField>> entries = _fields.entrySet();
+ for (Map.Entry<String,SolrInputField> fieldEntry : entries) {
+ clone._fields.put(fieldEntry.getKey(), fieldEntry.getValue().deepCopy());
+ }
+ clone._documentBoost = _documentBoost;
+
+ if (_childDocuments != null) {
+ clone._childDocuments = new ArrayList<>(_childDocuments.size());
+ for (SolrInputDocument child : _childDocuments) {
+ clone._childDocuments.add(child.deepCopy());
+ }
+ }
+
+ return clone;
+ }
+
+ //---------------------------------------------------
+ // MAP interface
+ //---------------------------------------------------
+
+ @Override
+ public boolean containsKey(Object key) {
+ return _fields.containsKey(key);
+ }
+
+ @Override
+ public boolean containsValue(Object value) {
+ return _fields.containsValue(value);
+ }
+
+ @Override
+ public Set<Entry<String, SolrInputField>> entrySet() {
+ return _fields.entrySet();
+ }
+
+ @Override
+ public SolrInputField get(Object key) {
+ return _fields.get(key);
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return _fields.isEmpty();
+ }
+
+ @Override
+ public Set<String> keySet() {
+ return _fields.keySet();
+ }
+
+ @Override
+ public SolrInputField put(String key, SolrInputField value) {
+ return _fields.put(key, value);
+ }
+
+ @Override
+ public void putAll(Map<? extends String, ? extends SolrInputField> t) {
+ _fields.putAll( t );
+ }
+
+ @Override
+ public SolrInputField remove(Object key) {
+ return _fields.remove(key);
+ }
+
+ @Override
+ public int size() {
+ return _fields.size();
+ }
+
+ @Override
+ public Collection<SolrInputField> values() {
+ return _fields.values();
+ }
+
+ public void addChildDocument(SolrInputDocument child) {
+ if (_childDocuments == null) {
+ _childDocuments = new ArrayList<>();
+ }
+ _childDocuments.add(child);
+ }
+
+ public void addChildDocuments(Collection<SolrInputDocument> childs) {
+ for (SolrInputDocument child : childs) {
+ addChildDocument(child);
+ }
+ }
+
+ /** Returns the list of child documents, or null if none. */
+ public List<SolrInputDocument> getChildDocuments() {
+ return _childDocuments;
+ }
+
+ public boolean hasChildDocuments() {
+ boolean isEmpty = (_childDocuments == null || _childDocuments.isEmpty());
+ return !isEmpty;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputField.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputField.java b/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputField.java
new file mode 100644
index 0000000..02b6856
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/SolrInputField.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+
+/**
+ *
+ * @since solr 1.3
+ */
+public class SolrInputField implements Iterable<Object>, Serializable
+{
+ String name;
+ Object value = null;
+ float boost = 1.0f;
+
+ public SolrInputField( String n )
+ {
+ this.name = n;
+ }
+
+ //---------------------------------------------------------------
+ //---------------------------------------------------------------
+
+ /**
+ * Set the value for a field. Arrays will be converted to a collection. If
+ * a collection is given, then that collection will be used as the backing
+ * collection for the values.
+ */
+ public void setValue(Object v, float b) {
+ boost = b;
+
+ if( v instanceof Object[] ) {
+ Object[] arr = (Object[])v;
+ Collection<Object> c = new ArrayList<>( arr.length );
+ for( Object o : arr ) {
+ c.add( o );
+ }
+ value = c;
+ }
+ else {
+ value = v;
+ }
+ }
+
+ /**
+ * Add values to a field. If the added value is a collection, each value
+ * will be added individually.
+ */
+ @SuppressWarnings("unchecked")
+ public void addValue(Object v, float b) {
+ if( value == null ) {
+ if ( v instanceof Collection ) {
+ Collection<Object> c = new ArrayList<>( 3 );
+ for ( Object o : (Collection<Object>)v ) {
+ c.add( o );
+ }
+ setValue(c, b);
+ } else {
+ setValue(v, b);
+ }
+
+ return;
+ }
+
+ // The lucene API and solr XML field specification make it possible to set boosts
+ // on multi-value fields even though lucene indexing does not support this.
+ // To keep behavior consistent with what happens in the lucene index, we accumulate
+ // the product of all boosts specified for this field.
+ boost *= b;
+
+ Collection<Object> vals = null;
+ if( value instanceof Collection ) {
+ vals = (Collection<Object>)value;
+ }
+ else {
+ vals = new ArrayList<>( 3 );
+ vals.add( value );
+ value = vals;
+ }
+
+ // Add the new values to a collection
+ if( v instanceof Iterable ) {
+ for( Object o : (Iterable<Object>)v ) {
+ vals.add( o );
+ }
+ }
+ else if( v instanceof Object[] ) {
+ for( Object o : (Object[])v ) {
+ vals.add( o );
+ }
+ }
+ else {
+ vals.add( v );
+ }
+ }
+
+ //---------------------------------------------------------------
+ //---------------------------------------------------------------
+
+ @SuppressWarnings("unchecked")
+ public Object getFirstValue() {
+ if( value instanceof Collection ) {
+ Collection c = (Collection<Object>)value;
+ if( c.size() > 0 ) {
+ return c.iterator().next();
+ }
+ return null;
+ }
+ return value;
+ }
+
+ /**
+ * @return the value for this field. If the field has multiple values, this
+ * will be a collection.
+ */
+ public Object getValue() {
+ return value;
+ }
+
+ /**
+ * @return the values for this field. This will return a collection even
+ * if the field is not multi-valued
+ */
+ @SuppressWarnings("unchecked")
+ public Collection<Object> getValues() {
+ if( value instanceof Collection ) {
+ return (Collection<Object>)value;
+ }
+ if( value != null ) {
+ Collection<Object> vals = new ArrayList<>(1);
+ vals.add( value );
+ return vals;
+ }
+ return null;
+ }
+
+ /**
+ * @return the number of values for this field
+ */
+ public int getValueCount() {
+ if( value instanceof Collection ) {
+ return ((Collection)value).size();
+ }
+ return (value == null) ? 0 : 1;
+ }
+
+ //---------------------------------------------------------------
+ //---------------------------------------------------------------
+
+ public float getBoost() {
+ return boost;
+ }
+
+ public void setBoost(float boost) {
+ this.boost = boost;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Iterator<Object> iterator() {
+ if( value instanceof Collection ) {
+ return ((Collection)value).iterator();
+ }
+ return new Iterator<Object>() {
+ boolean nxt = (value!=null);
+
+ @Override
+ public boolean hasNext() {
+ return nxt;
+ }
+
+ @Override
+ public Object next() {
+ nxt = false;
+ return value;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ @Override
+ public String toString()
+ {
+ return name + ((boost == 1.0) ? "=" : ("("+boost+")=")) + value;
+ }
+
+ public SolrInputField deepCopy() {
+ SolrInputField clone = new SolrInputField(name);
+ clone.boost = boost;
+ // We can't clone here, so we rely on simple primitives
+ if (value instanceof Collection) {
+ Collection<Object> values = (Collection<Object>) value;
+ Collection<Object> cloneValues = new ArrayList<>(values.size());
+ cloneValues.addAll(values);
+ clone.value = cloneValues;
+ } else {
+ clone.value = value;
+ }
+ return clone;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/StringUtils.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/StringUtils.java b/ranger_solrj/src/main/java/org/apache/solr/common/StringUtils.java
new file mode 100644
index 0000000..00fef63
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/StringUtils.java
@@ -0,0 +1,26 @@
+package org.apache.solr.common;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class StringUtils {
+
+ public static boolean isEmpty(String s) {
+ return (s == null) || s.isEmpty();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Aliases.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Aliases.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Aliases.java
new file mode 100644
index 0000000..1d18323
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Aliases.java
@@ -0,0 +1,63 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class Aliases {
+
+ private Map<String,Map<String,String>> aliasMap;
+
+ public Aliases(Map<String,Map<String,String>> aliasMap) {
+ this.aliasMap = aliasMap;
+ }
+
+ public Aliases() {
+ this.aliasMap = new HashMap<>();
+ }
+
+ public Map<String,String> getCollectionAliasMap() {
+ Map<String,String> cam = aliasMap.get("collection");
+ if (cam == null) return null;
+ return Collections.unmodifiableMap(cam);
+ }
+
+ public Map<String,Map<String,String>> getAliasMap() {
+ return Collections.unmodifiableMap(aliasMap);
+ }
+
+ public int collectionAliasSize() {
+ Map<String,String> cam = aliasMap.get("collection");
+ if (cam == null) return 0;
+ return cam.size();
+ }
+
+ @Override
+ public String toString() {
+ return "Aliases [aliasMap=" + aliasMap + "]";
+ }
+
+ public String getCollectionAlias(String collectionName) {
+ Map<String,String> cam = aliasMap.get("collection");
+ if (cam == null) return null;
+ return cam.get(collectionName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/BeforeReconnect.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/BeforeReconnect.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/BeforeReconnect.java
new file mode 100644
index 0000000..44379a4
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/BeforeReconnect.java
@@ -0,0 +1,22 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+public interface BeforeReconnect {
+ public void command();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClosableThread.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClosableThread.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClosableThread.java
new file mode 100644
index 0000000..1a19cbd
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClosableThread.java
@@ -0,0 +1,27 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @deprecated because this class is no longer used internally and will be removed
+ */
+@Deprecated
+public interface ClosableThread {
+ public void close();
+ public boolean isClosed();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterState.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterState.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterState.java
new file mode 100644
index 0000000..1b14360
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterState.java
@@ -0,0 +1,397 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.noggit.JSONWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+/**
+ * Immutable state of the cloud. Normally you can get the state by using
+ * {@link ZkStateReader#getClusterState()}.
+ * @lucene.experimental
+ */
+public class ClusterState implements JSONWriter.Writable {
+ private static Logger log = LoggerFactory.getLogger(ClusterState.class);
+
+ private final Integer znodeVersion;
+
+ private final Map<String, CollectionRef> collectionStates;
+ private Set<String> liveNodes;
+
+ /**
+ * Use this constr when ClusterState is meant for consumption.
+ */
+ public ClusterState(Integer znodeVersion, Set<String> liveNodes,
+ Map<String, DocCollection> collectionStates) {
+ this(liveNodes, getRefMap(collectionStates),znodeVersion);
+ }
+
+ private static Map<String, CollectionRef> getRefMap(Map<String, DocCollection> collectionStates) {
+ Map<String, CollectionRef> collRefs = new LinkedHashMap<>(collectionStates.size());
+ for (Entry<String, DocCollection> entry : collectionStates.entrySet()) {
+ final DocCollection c = entry.getValue();
+ collRefs.put(entry.getKey(), new CollectionRef(c));
+ }
+ return collRefs;
+ }
+
+ /**Use this if all the collection states are not readily available and some needs to be lazily loaded
+ */
+ public ClusterState(Set<String> liveNodes, Map<String, CollectionRef> collectionStates, Integer znodeVersion){
+ this.znodeVersion = znodeVersion;
+ this.liveNodes = new HashSet<>(liveNodes.size());
+ this.liveNodes.addAll(liveNodes);
+ this.collectionStates = new LinkedHashMap<>(collectionStates);
+ }
+
+
+ /**
+ * Returns a new cluster state object modified with the given collection.
+ *
+ * @param collectionName the name of the modified (or deleted) collection
+ * @param collection the collection object. A null value deletes the collection from the state
+ * @return the updated cluster state which preserves the current live nodes and zk node version
+ */
+ public ClusterState copyWith(String collectionName, DocCollection collection) {
+ ClusterState result = new ClusterState(liveNodes, new LinkedHashMap<>(collectionStates), znodeVersion);
+ if (collection == null) {
+ result.collectionStates.remove(collectionName);
+ } else {
+ result.collectionStates.put(collectionName, new CollectionRef(collection));
+ }
+ return result;
+ }
+
+
+ /**
+ * Get the lead replica for specific collection, or null if one currently doesn't exist.
+ */
+ public Replica getLeader(String collection, String sliceName) {
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) return null;
+ Slice slice = coll.getSlice(sliceName);
+ if (slice == null) return null;
+ return slice.getLeader();
+ }
+ private Replica getReplica(DocCollection coll, String replicaName) {
+ if (coll == null) return null;
+ for (Slice slice : coll.getSlices()) {
+ Replica replica = slice.getReplica(replicaName);
+ if (replica != null) return replica;
+ }
+ return null;
+ }
+
+ public boolean hasCollection(String coll) {
+ return collectionStates.containsKey(coll) ;
+ }
+
+ /**
+ * Gets the replica by the core name (assuming the slice is unknown) or null if replica is not found.
+ * If the slice is known, do not use this method.
+ * coreNodeName is the same as replicaName
+ */
+ public Replica getReplica(final String collection, final String coreNodeName) {
+ return getReplica(getCollectionOrNull(collection), coreNodeName);
+ }
+
+ /**
+ * Get the named Slice for collection, or null if not found.
+ */
+ public Slice getSlice(String collection, String sliceName) {
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) return null;
+ return coll.getSlice(sliceName);
+ }
+
+ public Map<String, Slice> getSlicesMap(String collection) {
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) return null;
+ return coll.getSlicesMap();
+ }
+
+ public Map<String, Slice> getActiveSlicesMap(String collection) {
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) return null;
+ return coll.getActiveSlicesMap();
+ }
+
+ public Collection<Slice> getSlices(String collection) {
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) return null;
+ return coll.getSlices();
+ }
+
+ public Collection<Slice> getActiveSlices(String collection) {
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) return null;
+ return coll.getActiveSlices();
+ }
+
+
+ /**
+ * Get the named DocCollection object, or throw an exception if it doesn't exist.
+ */
+ public DocCollection getCollection(String collection) {
+ DocCollection coll = getCollectionOrNull(collection);
+ if (coll == null) throw new SolrException(ErrorCode.BAD_REQUEST, "Could not find collection : " + collection);
+ return coll;
+ }
+
+ public CollectionRef getCollectionRef(String coll) {
+ return collectionStates.get(coll);
+ }
+
+ public DocCollection getCollectionOrNull(String coll) {
+ CollectionRef ref = collectionStates.get(coll);
+ return ref == null? null:ref.get();
+ }
+
+ /**
+ * Get collection names.
+ */
+ public Set<String> getCollections() {
+ return collectionStates.keySet();
+ }
+
+
+ /**
+ * Get names of the currently live nodes.
+ */
+ public Set<String> getLiveNodes() {
+ return Collections.unmodifiableSet(liveNodes);
+ }
+
+ public String getShardId(String nodeName, String coreName) {
+ return getShardId(null, nodeName, coreName);
+ }
+
+ public String getShardId(String collectionName, String nodeName, String coreName) {
+ Collection<CollectionRef> states = collectionStates.values();
+ if (collectionName != null) {
+ CollectionRef c = collectionStates.get(collectionName);
+ if (c != null) states = Collections.singletonList( c );
+ }
+
+ for (CollectionRef ref : states) {
+ DocCollection coll = ref.get();
+ if(coll == null) continue;// this collection go tremoved in between, skip
+ for (Slice slice : coll.getSlices()) {
+ for (Replica replica : slice.getReplicas()) {
+ // TODO: for really large clusters, we could 'index' on this
+ String rnodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
+ String rcore = replica.getStr(ZkStateReader.CORE_NAME_PROP);
+ if (nodeName.equals(rnodeName) && coreName.equals(rcore)) {
+ return slice.getName();
+ }
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Check if node is alive.
+ */
+ public boolean liveNodesContain(String name) {
+ return liveNodes.contains(name);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("live nodes:" + liveNodes);
+ sb.append(" collections:" + collectionStates);
+ return sb.toString();
+ }
+
+ public static ClusterState load(Integer version, byte[] bytes, Set<String> liveNodes) {
+ return load(version, bytes, liveNodes, ZkStateReader.CLUSTER_STATE);
+ }
+ /**
+ * Create ClusterState from json string that is typically stored in zookeeper.
+ *
+ * @param version zk version of the clusterstate.json file (bytes)
+ * @param bytes clusterstate.json as a byte array
+ * @param liveNodes list of live nodes
+ * @return the ClusterState
+ */
+ public static ClusterState load(Integer version, byte[] bytes, Set<String> liveNodes, String znode) {
+ // System.out.println("######## ClusterState.load:" + (bytes==null ? null : new String(bytes)));
+ if (bytes == null || bytes.length == 0) {
+ return new ClusterState(version, liveNodes, Collections.<String, DocCollection>emptyMap());
+ }
+ Map<String, Object> stateMap = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
+ Map<String,CollectionRef> collections = new LinkedHashMap<>(stateMap.size());
+ for (Entry<String, Object> entry : stateMap.entrySet()) {
+ String collectionName = entry.getKey();
+ DocCollection coll = collectionFromObjects(collectionName, (Map<String,Object>)entry.getValue(), version, znode);
+ collections.put(collectionName, new CollectionRef(coll));
+ }
+
+ return new ClusterState( liveNodes, collections,version);
+ }
+
+
+ public static Aliases load(byte[] bytes) {
+ if (bytes == null || bytes.length == 0) {
+ return new Aliases();
+ }
+ Map<String,Map<String,String>> aliasMap = (Map<String,Map<String,String>>) ZkStateReader.fromJSON(bytes);
+
+ return new Aliases(aliasMap);
+ }
+
+ private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version, String znode) {
+ Map<String,Object> props;
+ Map<String,Slice> slices;
+
+ Map<String,Object> sliceObjs = (Map<String,Object>)objs.get(DocCollection.SHARDS);
+ if (sliceObjs == null) {
+ // legacy format from 4.0... there was no separate "shards" level to contain the collection shards.
+ slices = makeSlices(objs);
+ props = Collections.emptyMap();
+ } else {
+ slices = makeSlices(sliceObjs);
+ props = new HashMap<>(objs);
+ objs.remove(DocCollection.SHARDS);
+ }
+
+ Object routerObj = props.get(DocCollection.DOC_ROUTER);
+ DocRouter router;
+ if (routerObj == null) {
+ router = DocRouter.DEFAULT;
+ } else if (routerObj instanceof String) {
+ // back compat with Solr4.4
+ router = DocRouter.getDocRouter((String)routerObj);
+ } else {
+ Map routerProps = (Map)routerObj;
+ router = DocRouter.getDocRouter((String) routerProps.get("name"));
+ }
+
+ return new DocCollection(name, slices, props, router, version, znode);
+ }
+
+ private static Map<String,Slice> makeSlices(Map<String,Object> genericSlices) {
+ if (genericSlices == null) return Collections.emptyMap();
+ Map<String,Slice> result = new LinkedHashMap<>(genericSlices.size());
+ for (Map.Entry<String,Object> entry : genericSlices.entrySet()) {
+ String name = entry.getKey();
+ Object val = entry.getValue();
+ if (val instanceof Slice) {
+ result.put(name, (Slice)val);
+ } else if (val instanceof Map) {
+ result.put(name, new Slice(name, null, (Map<String,Object>)val));
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public void write(JSONWriter jsonWriter) {
+ LinkedHashMap<String , DocCollection> map = new LinkedHashMap<>();
+ for (Entry<String, CollectionRef> e : collectionStates.entrySet()) {
+ // using this class check to avoid fetching from ZK in case of lazily loaded collection
+ if (e.getValue().getClass() == CollectionRef.class) {
+ // check if it is a lazily loaded collection outside of clusterstate.json
+ DocCollection coll = e.getValue().get();
+ if (coll.getStateFormat() == 1) {
+ map.put(coll.getName(),coll);
+ }
+ }
+ }
+ jsonWriter.write(map);
+ }
+
+ /**
+ * The version of clusterstate.json in ZooKeeper.
+ *
+ * @return null if ClusterState was created for publication, not consumption
+ */
+ public Integer getZkClusterStateVersion() {
+ return znodeVersion;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + ((znodeVersion == null) ? 0 : znodeVersion.hashCode());
+ result = prime * result + ((liveNodes == null) ? 0 : liveNodes.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) return true;
+ if (obj == null) return false;
+ if (getClass() != obj.getClass()) return false;
+ ClusterState other = (ClusterState) obj;
+ if (znodeVersion == null) {
+ if (other.znodeVersion != null) return false;
+ } else if (!znodeVersion.equals(other.znodeVersion)) return false;
+ if (liveNodes == null) {
+ if (other.liveNodes != null) return false;
+ } else if (!liveNodes.equals(other.liveNodes)) return false;
+ return true;
+ }
+
+
+
+ /**
+ * Internal API used only by ZkStateReader
+ */
+ void setLiveNodes(Set<String> liveNodes){
+ this.liveNodes = liveNodes;
+ }
+
+ /**For internal use only
+ */
+ Map<String, CollectionRef> getCollectionStates() {
+ return collectionStates;
+ }
+
+ public static class CollectionRef {
+ private final DocCollection coll;
+
+ public CollectionRef(DocCollection coll) {
+ this.coll = coll;
+ }
+
+ public DocCollection get(){
+ return coll;
+ }
+
+ public boolean isLazilyLoaded() { return false; }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterStateUtil.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterStateUtil.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterStateUtil.java
new file mode 100644
index 0000000..fe74884
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ClusterStateUtil.java
@@ -0,0 +1,230 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ClusterStateUtil {
+ private static Logger log = LoggerFactory.getLogger(ClusterStateUtil.class);
+
+ private static final int TIMEOUT_POLL_MS = 1000;
+
+ /**
+ * Wait to see *all* cores live and active.
+ *
+ * @param zkStateReader
+ * to use for ClusterState
+ * @param timeoutInMs
+ * how long to wait before giving up
+ * @return false if timed out
+ */
+ public static boolean waitForAllActiveAndLive(ZkStateReader zkStateReader, int timeoutInMs) {
+ return waitForAllActiveAndLive(zkStateReader, null, timeoutInMs);
+ }
+
+ /**
+ * Wait to see *all* cores live and active.
+ *
+ * @param zkStateReader
+ * to use for ClusterState
+ * @param collection to look at
+ * @param timeoutInMs
+ * how long to wait before giving up
+ * @return false if timed out
+ */
+ public static boolean waitForAllActiveAndLive(ZkStateReader zkStateReader, String collection,
+ int timeoutInMs) {
+ long timeout = System.nanoTime()
+ + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
+ boolean success = false;
+ while (System.nanoTime() < timeout) {
+ success = true;
+ ClusterState clusterState = zkStateReader.getClusterState();
+ if (clusterState != null) {
+ Set<String> collections;
+ if (collection != null) {
+ collections = Collections.singleton(collection);
+ } else {
+ collections = clusterState.getCollections();
+ }
+ for (String coll : collections) {
+ DocCollection docCollection = clusterState.getCollection(coll);
+ Collection<Slice> slices = docCollection.getSlices();
+ for (Slice slice : slices) {
+ // only look at active shards
+ if (slice.getState().equals(Slice.ACTIVE)) {
+ Collection<Replica> replicas = slice.getReplicas();
+ for (Replica replica : replicas) {
+ // on a live node?
+ boolean live = clusterState.liveNodesContain(replica
+ .getNodeName());
+ String state = replica.getStr(ZkStateReader.STATE_PROP);
+ if (!live || !state.equals(ZkStateReader.ACTIVE)) {
+ // fail
+ success = false;
+ }
+ }
+ }
+ }
+ }
+ if (!success) {
+ try {
+ Thread.sleep(TIMEOUT_POLL_MS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Interrupted");
+ }
+ }
+ }
+ }
+
+ return success;
+ }
+
+ /**
+ * Wait to see an entry in the ClusterState with a specific coreNodeName and
+ * baseUrl.
+ *
+ * @param zkStateReader
+ * to use for ClusterState
+ * @param collection
+ * to look in
+ * @param coreNodeName
+ * to wait for
+ * @param baseUrl
+ * to wait for
+ * @param timeoutInMs
+ * how long to wait before giving up
+ * @return false if timed out
+ */
+ public static boolean waitToSeeLive(ZkStateReader zkStateReader,
+ String collection, String coreNodeName, String baseUrl,
+ int timeoutInMs) {
+ long timeout = System.nanoTime()
+ + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
+
+ while (System.nanoTime() < timeout) {
+ log.debug("waiting to see replica just created live collection={} replica={} baseUrl={}",
+ collection, coreNodeName, baseUrl);
+ ClusterState clusterState = zkStateReader.getClusterState();
+ if (clusterState != null) {
+ DocCollection docCollection = clusterState.getCollection(collection);
+ Collection<Slice> slices = docCollection.getSlices();
+ for (Slice slice : slices) {
+ // only look at active shards
+ if (slice.getState().equals(Slice.ACTIVE)) {
+ Collection<Replica> replicas = slice.getReplicas();
+ for (Replica replica : replicas) {
+ // on a live node?
+ boolean live = clusterState.liveNodesContain(replica.getNodeName());
+ String rcoreNodeName = replica.getName();
+ String rbaseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
+ if (live && coreNodeName.equals(rcoreNodeName)
+ && baseUrl.equals(rbaseUrl)) {
+ // found it
+ return true;
+ }
+ }
+ }
+ }
+ try {
+ Thread.sleep(TIMEOUT_POLL_MS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Interrupted");
+ }
+ }
+ }
+
+ log.error("Timed out waiting to see replica just created in cluster state. Continuing...");
+ return false;
+ }
+
+ public static boolean waitForAllNotLive(ZkStateReader zkStateReader, int timeoutInMs) {
+ return waitForAllNotLive(zkStateReader, null, timeoutInMs);
+ }
+
+
+ public static boolean waitForAllNotLive(ZkStateReader zkStateReader,
+ String collection, int timeoutInMs) {
+ long timeout = System.nanoTime()
+ + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
+ boolean success = false;
+ while (System.nanoTime() < timeout) {
+ success = true;
+ ClusterState clusterState = zkStateReader.getClusterState();
+ if (clusterState != null) {
+ Set<String> collections;
+ if (collection == null) {
+ collections = clusterState.getCollections();
+ } else {
+ collections = Collections.singleton(collection);
+ }
+ for (String coll : collections) {
+ DocCollection docCollection = clusterState.getCollection(coll);
+ Collection<Slice> slices = docCollection.getSlices();
+ for (Slice slice : slices) {
+ // only look at active shards
+ if (slice.getState().equals(Slice.ACTIVE)) {
+ Collection<Replica> replicas = slice.getReplicas();
+ for (Replica replica : replicas) {
+ // on a live node?
+ boolean live = clusterState.liveNodesContain(replica
+ .getNodeName());
+ if (live) {
+ // fail
+ success = false;
+ }
+ }
+ }
+ }
+ }
+ if (!success) {
+ try {
+ Thread.sleep(TIMEOUT_POLL_MS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Interrupted");
+ }
+ }
+ }
+ }
+
+ return success;
+ }
+
+ public static boolean isAutoAddReplicas(ZkStateReader reader, String collection) {
+ ClusterState clusterState = reader.getClusterState();
+ if (clusterState != null) {
+ DocCollection docCollection = clusterState.getCollectionOrNull(collection);
+ if (docCollection != null) {
+ return docCollection.getAutoAddReplicas();
+ }
+ }
+ return false;
+ }
+
+}
[10/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
new file mode 100644
index 0000000..9c542d7
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldAnalysisResponse.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.common.util.NamedList;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A response that is returned by processing the {@link org.apache.solr.client.solrj.request.FieldAnalysisRequest}.
+ * Holds a map of {@link Analysis} objects per field name as well as a map of {@link Analysis} objects per field type.
+ *
+ *
+ * @since solr 1.4
+ */
+public class FieldAnalysisResponse extends AnalysisResponseBase {
+
+ private Map<String, Analysis> analysisByFieldTypeName = new HashMap<>();
+ private Map<String, Analysis> analysisByFieldName = new HashMap<>();
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setResponse(NamedList<Object> response) {
+ super.setResponse(response);
+
+ @SuppressWarnings("unchecked")
+ NamedList<NamedList<NamedList<NamedList<List<NamedList<Object>>>>>> analysisNL
+ = (NamedList<NamedList<NamedList<NamedList<List<NamedList<Object>>>>>>) response.get("analysis");
+
+ for (Map.Entry<String, NamedList<NamedList<List<NamedList<Object>>>>> entry
+ : analysisNL.get("field_types")) {
+
+ analysisByFieldTypeName.put(entry.getKey(), buildAnalysis(entry.getValue()));
+ }
+
+ for (Map.Entry<String, NamedList<NamedList<List<NamedList<Object>>>>> entry
+ : analysisNL.get("field_names")) {
+
+ analysisByFieldName.put(entry.getKey(), buildAnalysis(entry.getValue()));
+ }
+ }
+
+ private Analysis buildAnalysis(NamedList<NamedList<List<NamedList<Object>>>> value) {
+ Analysis analysis = new Analysis();
+
+ NamedList<List<NamedList<Object>>> queryNL = value.get("query");
+ List<AnalysisPhase> phases = (queryNL == null) ? null : buildPhases(queryNL);
+ analysis.setQueryPhases(phases);
+
+ NamedList<List<NamedList<Object>>> indexNL = value.get("index");
+ phases = buildPhases(indexNL);
+ analysis.setIndexPhases(phases);
+
+ return analysis;
+ }
+
+ /**
+ * Returns the number of field type analyses.
+ *
+ * @return The number of field type analyses.
+ */
+ public int getFieldTypeAnalysisCount() {
+ return analysisByFieldTypeName.size();
+ }
+
+ /**
+ * Returns the analysis for the given field type or {@code null} if no such analysis exists.
+ *
+ * @param fieldTypeName The name of the field type.
+ *
+ * @return The analysis for the given field type.
+ */
+ public Analysis getFieldTypeAnalysis(String fieldTypeName) {
+ return analysisByFieldTypeName.get(fieldTypeName);
+ }
+
+ /**
+ * Returns all field type analyses with their associated field types.
+ *
+ * @return All field type analyses with their associated field types.
+ */
+ public Iterable<Map.Entry<String, Analysis>> getAllFieldTypeAnalysis() {
+ return analysisByFieldTypeName.entrySet();
+ }
+
+ /**
+ * Returns the number of field name analyses.
+ *
+ * @return The number of field name analyses.
+ */
+ public int getFieldNameAnalysisCount() {
+ return analysisByFieldName.size();
+ }
+
+ /**
+ * Returns the analysis for the given field name or {@code null} if no such analysis exists.
+ *
+ * @param fieldName The field name.
+ *
+ * @return The analysis for the given field name.
+ */
+ public Analysis getFieldNameAnalysis(String fieldName) {
+ return analysisByFieldName.get(fieldName);
+ }
+
+ /**
+ * Returns all field name analysese with their associated field names.
+ *
+ * @return all field name analysese with their associated field names.
+ */
+ public Iterable<Map.Entry<String, Analysis>> getAllFieldNameAnalysis() {
+ return analysisByFieldName.entrySet();
+ }
+
+
+ //================================================= Inner Classes ==================================================
+
+ /**
+ * The analysis of a field. Holds a list of all the query time analysis phases (if a query analysis was requested)
+ * as well as index time phases.
+ */
+ public static class Analysis {
+
+ private List<AnalysisPhase> queryPhases;
+ private List<AnalysisPhase> indexPhases;
+
+ /**
+ * This class should only be instantiated internally.
+ */
+ private Analysis() {
+ }
+
+ /**
+ * Returns the number of query time analysis phases in this analysis or
+ * {@code -1} if query time analysis doesn't exist.
+ *
+ * @return Returns the number of query time analysis phases in this
+ * analysis or {@code -1} if query time analysis doesn't exist.
+ */
+ public int getQueryPhasesCount() {
+ return queryPhases == null ? -1 : queryPhases.size();
+ }
+
+ /**
+ * Returns the query time analysis phases for this analysis or {@code null}
+ * if query time analysis doesn't exist.
+ *
+ *
+ * @return The query time analysis phases for this analysis or {@code null}
+ * if query time analysis doesn't exist.
+ *
+ */
+ public Iterable<AnalysisPhase> getQueryPhases() {
+ return queryPhases;
+ }
+
+ /**
+ * Returns the index time analysis phases for this analysis.
+ *
+ * @return The index time analysis phases for this analysis.
+ */
+ public int getIndexPhasesCount() {
+ return indexPhases.size();
+ }
+
+ /**
+ * Returns the index time analysis phases for this analysis.
+ *
+ * @return The index time analysis phases for this analysis.
+ */
+ public Iterable<AnalysisPhase> getIndexPhases() {
+ return indexPhases;
+ }
+
+ private void setQueryPhases(List<AnalysisPhase> queryPhases) {
+ this.queryPhases = queryPhases;
+ }
+
+ private void setIndexPhases(List<AnalysisPhase> indexPhases) {
+ this.indexPhases = indexPhases;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java
new file mode 100644
index 0000000..9685832
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FieldStatsInfo.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.common.util.NamedList;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Holds stats info
+ *
+ *
+ * @since solr 1.4
+ */
+public class FieldStatsInfo implements Serializable {
+ final String name;
+
+ Object min;
+ Object max;
+ Object sum;
+ Long count;
+ Long countDistinct;
+ Collection<Object> distinctValues;
+ Long missing;
+ Object mean = null;
+ Double sumOfSquares = null;
+ Double stddev = null;
+
+ Map<String,List<FieldStatsInfo>> facets;
+
+ public FieldStatsInfo( NamedList<Object> nl, String fname )
+ {
+ name = fname;
+
+ for( Map.Entry<String, Object> entry : nl ) {
+ if( "min".equals( entry.getKey() ) ) {
+ min = entry.getValue();
+ }
+ else if( "max".equals( entry.getKey() ) ) {
+ max = entry.getValue();
+ }
+ else if( "sum".equals( entry.getKey() ) ) {
+ sum = entry.getValue();
+ }
+ else if( "count".equals( entry.getKey() ) ) {
+ count = (Long)entry.getValue();
+ }
+ else if ("countDistinct".equals(entry.getKey())) {
+ countDistinct = (Long) entry.getValue();
+ }
+ else if ("distinctValues".equals(entry.getKey())) {
+ distinctValues = (Collection<Object>) entry.getValue();
+ }
+ else if( "missing".equals( entry.getKey() ) ) {
+ missing = (Long)entry.getValue();
+ }
+ else if( "mean".equals( entry.getKey() ) ) {
+ mean = entry.getValue();
+ }
+ else if( "sumOfSquares".equals( entry.getKey() ) ) {
+ sumOfSquares = (Double)entry.getValue();
+ }
+ else if( "stddev".equals( entry.getKey() ) ) {
+ stddev = (Double)entry.getValue();
+ }
+ else if( "facets".equals( entry.getKey() ) ) {
+ @SuppressWarnings("unchecked")
+ NamedList<Object> fields = (NamedList<Object>)entry.getValue();
+ facets = new HashMap<>();
+ for( Map.Entry<String, Object> ev : fields ) {
+ List<FieldStatsInfo> vals = new ArrayList<>();
+ facets.put( ev.getKey(), vals );
+ @SuppressWarnings("unchecked")
+ NamedList<NamedList<Object>> vnl = (NamedList<NamedList<Object>>) ev.getValue();
+ for( int i=0; i<vnl.size(); i++ ) {
+ String n = vnl.getName(i);
+ vals.add( new FieldStatsInfo( vnl.getVal(i), n ) );
+ }
+ }
+ }
+ else {
+ throw new RuntimeException( "unknown key: "+entry.getKey() + " ["+entry.getValue()+"]" );
+ }
+ }
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append( name );
+ sb.append( ": {" );
+ if( min != null ) {
+ sb.append( " min:").append( min );
+ }
+ if( max != null ) {
+ sb.append( " max:").append( max );
+ }
+ if( sum != null ) {
+ sb.append( " sum:").append( sum );
+ }
+ if( count != null ) {
+ sb.append( " count:").append( count );
+ }
+ if (countDistinct != null) {
+ sb.append(" countDistinct:").append(countDistinct);
+ }
+ if (distinctValues != null) {
+ sb.append(" distinctValues:").append(distinctValues);
+ }
+ if( missing != null ) {
+ sb.append( " missing:").append( missing );
+ }
+ if( mean != null ) {
+ sb.append( " mean:").append( mean );
+ }
+ if( stddev != null ) {
+ sb.append( " stddev:").append(stddev);
+ }
+ sb.append( " }" );
+ return sb.toString();
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Object getMin() {
+ return min;
+ }
+
+ public Object getMax() {
+ return max;
+ }
+
+ public Object getSum() {
+ return sum;
+ }
+
+ public Long getCount() {
+ return count;
+ }
+
+ public Long getCountDistinct() {
+ return countDistinct;
+ }
+
+ public Collection<Object> getDistinctValues() {
+ return distinctValues;
+ }
+
+ public Long getMissing() {
+ return missing;
+ }
+
+ public Object getMean() {
+ return mean;
+ }
+
+ public Double getStddev() {
+ return stddev;
+ }
+
+ public Double getSumOfSquares() {
+ return sumOfSquares;
+ }
+
+ public Map<String, List<FieldStatsInfo>> getFacets() {
+ return facets;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/Group.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/Group.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/Group.java
new file mode 100644
index 0000000..2ee0389
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/Group.java
@@ -0,0 +1,69 @@
+package org.apache.solr.client.solrj.response;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.SolrDocumentList;
+
+import java.io.Serializable;
+
+/**
+ * Represents a group. A group contains a common group value that all documents inside the group share and
+ * documents that belong to this group.
+ *
+ * A group value can be a field value, function result or a query string depending on the {@link GroupCommand}.
+ * In case of a field value or a function result the value is always a indexed value.
+ *
+ * @since solr 3.4
+ */
+public class Group implements Serializable {
+
+ private final String _groupValue;
+ private final SolrDocumentList _result;
+
+ /**
+ * Creates a Group instance.
+ *
+ * @param groupValue The common group value (indexed value) that all documents share.
+ * @param result The documents to be displayed that belong to this group
+ */
+ public Group(String groupValue, SolrDocumentList result) {
+ _groupValue = groupValue;
+ _result = result;
+ }
+
+ /**
+ * Returns the common group value that all documents share inside this group.
+ * This is an indexed value, not a stored value.
+ *
+ * @return the common group value
+ */
+ public String getGroupValue() {
+ return _groupValue;
+ }
+
+ /**
+ * Returns the documents to be displayed that belong to this group.
+ * How many documents are returned depend on the <code>group.offset</code> and <code>group.limit</code> parameters.
+ *
+ * @return the documents to be displayed that belong to this group
+ */
+ public SolrDocumentList getResult() {
+ return _result;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupCommand.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupCommand.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupCommand.java
new file mode 100644
index 0000000..c2c8127
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupCommand.java
@@ -0,0 +1,125 @@
+package org.apache.solr.client.solrj.response;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * This class represents the result of a group command.
+ * This can be the result of the following parameter:
+ * <ul>
+ * <li> group.field
+ * <li> group.func
+ * <li> group.query
+ * </ul>
+ *
+ * An instance of this class contains:
+ * <ul>
+ * <li> The name of this command. This can be the field, function or query grouped by.
+ * <li> The total number of documents that have matched.
+ * <li> The total number of groups that have matched.
+ * <li> The groups to be displayed. Depending on the start and rows parameter.
+ * </ul>
+ *
+ * In case of <code>group.query</code> only one group is present and ngroups is always <code>null</code>.
+ *
+ * @since solr 3.4
+ */
+public class GroupCommand implements Serializable {
+
+ private final String _name;
+ private final List<Group> _values = new ArrayList<>();
+ private final int _matches;
+ private final Integer _ngroups;
+
+ /**
+ * Creates a GroupCommand instance
+ *
+ * @param name The name of this command
+ * @param matches The total number of documents found for this command
+ */
+ public GroupCommand(String name, int matches) {
+ _name = name;
+ _matches = matches;
+ _ngroups = null;
+ }
+
+ /**
+ * Creates a GroupCommand instance.
+ *
+ * @param name The name of this command
+ * @param matches The total number of documents found for this command
+ * @param nGroups The total number of groups found for this command.
+ */
+ public GroupCommand(String name, int matches, int nGroups) {
+ _name = name;
+ _matches = matches;
+ _ngroups = nGroups;
+ }
+
+ /**
+ * Returns the name of this command. This can be the field, function or query grouped by.
+ *
+ * @return the name of this command
+ */
+ public String getName() {
+ return _name;
+ }
+
+ /**
+ * Adds a group to this command.
+ *
+ * @param group A group to be added
+ */
+ public void add(Group group) {
+ _values.add(group);
+ }
+
+ /**
+ * Returns the groups to be displayed.
+ * The number of groups returned depend on the <code>start</code> and <code>rows</code> parameters.
+ *
+ * @return the groups to be displayed.
+ */
+ public List<Group> getValues() {
+ return _values;
+ }
+
+ /**
+ * Returns the total number of documents found for this command.
+ *
+ * @return the total number of documents found for this command.
+ */
+ public int getMatches() {
+ return _matches;
+ }
+
+ /**
+ * Returns the total number of groups found for this command.
+ * Returns <code>null</code> if the <code>group.ngroups</code> parameter is unset or <code>false</code> or
+ * if this is a group command query (parameter = <code>group.query</code>).
+ *
+ * @return the total number of groups found for this command.
+ */
+ public Integer getNGroups() {
+ return _ngroups;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupResponse.java
new file mode 100644
index 0000000..210bb5e
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/GroupResponse.java
@@ -0,0 +1,56 @@
+package org.apache.solr.client.solrj.response;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Overall grouping result. Contains a list of {@link GroupCommand} instances that is the result of
+ * one the following parameters:
+ * <ul>
+ * <li>group.field
+ * <li>group.func
+ * <li>group.query
+ * </ul>
+ *
+ * @since solr 3.4
+ */
+public class GroupResponse implements Serializable {
+
+ private final List<GroupCommand> _values = new ArrayList<>();
+
+ /**
+ * Adds a grouping command to the response.
+ *
+ * @param command The grouping command to add
+ */
+ public void add(GroupCommand command) {
+ _values.add(command);
+ }
+
+ /**
+ * Returns all grouping commands.
+ *
+ * @return all grouping commands
+ */
+ public List<GroupCommand> getValues() {
+ return _values;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/IntervalFacet.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/IntervalFacet.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/IntervalFacet.java
new file mode 100644
index 0000000..1f91fe8
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/IntervalFacet.java
@@ -0,0 +1,85 @@
+package org.apache.solr.client.solrj.response;
+
+import java.util.List;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Objects of this class will contain the result of all the intervals defined
+ * for a specific field.
+ */
+public class IntervalFacet {
+
+ /**
+ * The field for which interval facets where calculated
+ */
+ private final String field;
+
+ /**
+ * The list of interval facets calculated for {@link #field}
+ */
+ private final List<Count> intervals;
+
+ IntervalFacet(String field, List<Count> values) {
+ this.field = field;
+ this.intervals = values;
+ }
+
+ /**
+ * @return The field for which interval facets where calculated
+ */
+ public String getField() {
+ return field;
+ }
+
+ /**
+ * @return The list of interval facets calculated for {@link #field}
+ */
+ public List<Count> getIntervals() {
+ return intervals;
+ }
+
+ /**
+ * Holds counts for facet intervals defined in a field
+ */
+ public static class Count {
+ /**
+ * The key of this interval. This is the original
+ * interval string or the value of the "key" local
+ * param
+ */
+ private final String key;
+ /**
+ * The count of this interval
+ */
+ private final int count;
+
+ Count(String key, int count) {
+ super();
+ this.key = key;
+ this.count = count;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public int getCount() {
+ return count;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/LukeResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/LukeResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/LukeResponse.java
new file mode 100644
index 0000000..5d2f328
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/LukeResponse.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.common.luke.FieldFlag;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.Serializable;
+import java.util.*;
+
+
+/**
+ * This is an incomplete representation of the data returned from Luke
+ *
+ *
+ * @since solr 1.3
+ */
+public class LukeResponse extends SolrResponseBase {
+
+ public static class FieldTypeInfo implements Serializable {
+ String name;
+ String className;
+ boolean tokenized;
+ String analyzer;
+ List<String> fields;
+
+
+ public FieldTypeInfo(String name) {
+ this.name = name;
+ fields = Collections.emptyList();
+ }
+
+
+ public String getAnalyzer() {
+ return analyzer;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+
+ public List<String> getFields() {
+ return fields;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean isTokenized() {
+ return tokenized;
+ }/*
+ Sample:
+ types={ignored={fields=null,tokenized=false,analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer@f94934},
+ integer={fields=null,tokenized=false,analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer@3525a2},
+ sfloat={fields=[price, weight],tokenized=false,analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer@39cf9c},
+ text_ws={fields=[cat],tokenized=true,analyzer=TokenizerChain(org.apache.solr.analysis.WhitespaceTokenizerFactory@6d3ca2)},
+ alphaOnlySort={fields=[alphaNameSort],tokenized=true,analyzer=TokenizerChain(org.apache.solr.analysis.KeywordTokenizerFactory@a7bd3b,
+ org.apache.solr.analysis.LowerCaseFilterFactory@78aae2, org.apache.solr.analysis.TrimFilterFactory@1b16a7,
+ org.apache.solr.analysis.PatternReplaceFilterFactory@6c6b08)},date={fields=[timestamp],tokenized=false,
+ analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer@e6e42e},sint={fields=[popularity],
+ tokenized=false,analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer@8ea21d},
+ boolean={fields=[inStock],tokenized=false,analyzer=org.apache.solr.schema.BoolField$1@354949},
+ textTight={fields=[sku],tokenized=true,analyzer=TokenizerChain(org.apache.solr.analysis.WhitespaceTokenizerFactory@5e88f7,
+ org.apache.solr.analysis.SynonymFilterFactory@723646, org.apache.solr.analysis.StopFilterFactory@492ff1,
+ org.apache.solr.analysis.WordDelimiterFilterFactory@eaabad, org.apache.solr.analysis.LowerCaseFilterFactory@ad1355,
+ org.apache.solr.analysis.EnglishPorterFilterFactory@d03a00, org.apache.solr.analysis.RemoveDuplicatesTokenFilterFactory@900079)},
+ long={fields=null,tokenized=false,analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer@f3b83},
+ double={fields=null,tokenized=false,analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer@c2b07},
+
+ */
+
+ @SuppressWarnings("unchecked")
+ public void read(NamedList<Object> nl) {
+ for (Map.Entry<String, Object> entry : nl) {
+ String key = entry.getKey();
+ if ("fields".equals(key) && entry.getValue() != null) {
+ List<String> theFields = (List<String>) entry.getValue();
+ fields = new ArrayList<>(theFields);
+ } else if ("tokenized".equals(key) == true) {
+ tokenized = Boolean.parseBoolean(entry.getValue().toString());
+ } else if ("analyzer".equals(key) == true) {
+ analyzer = entry.getValue().toString();
+ } else if ("className".equals(key) == true) {
+ className = entry.getValue().toString();
+ }
+ }
+ }
+ }
+
+ public static class FieldInfo implements Serializable {
+ String name;
+ String type;
+ String schema;
+ int docs;
+ int distinct;
+ EnumSet<FieldFlag> flags;
+ boolean cacheableFaceting;
+ NamedList<Integer> topTerms;
+
+ public FieldInfo(String n) {
+ name = n;
+ }
+
+ @SuppressWarnings("unchecked")
+ public void read(NamedList<Object> nl) {
+ for (Map.Entry<String, Object> entry : nl) {
+ if ("type".equals(entry.getKey())) {
+ type = (String) entry.getValue();
+ }
+ if ("flags".equals(entry.getKey())) {
+ flags = parseFlags((String) entry.getValue());
+ } else if ("schema".equals(entry.getKey())) {
+ schema = (String) entry.getValue();
+ } else if ("docs".equals(entry.getKey())) {
+ docs = (Integer) entry.getValue();
+ } else if ("distinct".equals(entry.getKey())) {
+ distinct = (Integer) entry.getValue();
+ } else if ("cacheableFaceting".equals(entry.getKey())) {
+ cacheableFaceting = (Boolean) entry.getValue();
+ } else if ("topTerms".equals(entry.getKey())) {
+ topTerms = (NamedList<Integer>) entry.getValue();
+ }
+ }
+ }
+
+ public static EnumSet<FieldFlag> parseFlags(String flagStr) {
+ EnumSet<FieldFlag> result = EnumSet.noneOf(FieldFlag.class);
+ char[] chars = flagStr.toCharArray();
+ for (int i = 0; i < chars.length; i++) {
+ if (chars[i] != '-') {
+ FieldFlag flag = FieldFlag.getFlag(chars[i]);
+ result.add(flag);
+ }
+ }
+ return result;
+ }
+
+ public EnumSet<FieldFlag> getFlags() {
+ return flags;
+ }
+
+ public boolean isCacheableFaceting() {
+ return cacheableFaceting;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public int getDistinct() {
+ return distinct;
+ }
+
+ public int getDocs() {
+ return docs;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public NamedList<Integer> getTopTerms() {
+ return topTerms;
+ }
+ }
+
+ private NamedList<Object> indexInfo;
+ private Map<String, FieldInfo> fieldInfo;
+ private Map<String, FieldTypeInfo> fieldTypeInfo;
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public void setResponse(NamedList<Object> res) {
+ super.setResponse(res);
+
+ // Parse indexinfo
+ indexInfo = (NamedList<Object>) res.get("index");
+
+ NamedList<Object> schema = (NamedList<Object>) res.get("schema");
+ NamedList<Object> flds = (NamedList<Object>) res.get("fields");
+ if (flds == null && schema != null ) {
+ flds = (NamedList<Object>) schema.get("fields");
+ }
+ if (flds != null) {
+ fieldInfo = new HashMap<>();
+ for (Map.Entry<String, Object> field : flds) {
+ FieldInfo f = new FieldInfo(field.getKey());
+ f.read((NamedList<Object>) field.getValue());
+ fieldInfo.put(field.getKey(), f);
+ }
+ }
+
+ if( schema != null ) {
+ NamedList<Object> fldTypes = (NamedList<Object>) schema.get("types");
+ if (fldTypes != null) {
+ fieldTypeInfo = new HashMap<>();
+ for (Map.Entry<String, Object> fieldType : fldTypes) {
+ FieldTypeInfo ft = new FieldTypeInfo(fieldType.getKey());
+ ft.read((NamedList<Object>) fieldType.getValue());
+ fieldTypeInfo.put(fieldType.getKey(), ft);
+ }
+ }
+ }
+ }
+
+ //----------------------------------------------------------------
+ //----------------------------------------------------------------
+
+ public String getIndexDirectory() {
+ if (indexInfo == null) return null;
+ return (String) indexInfo.get("directory");
+ }
+
+ public Integer getNumDocs() {
+ if (indexInfo == null) return null;
+ return (Integer) indexInfo.get("numDocs");
+ }
+
+ public Integer getMaxDoc() {
+ if (indexInfo == null) return null;
+ return (Integer) indexInfo.get("maxDoc");
+ }
+
+ public Integer getNumTerms() {
+ if (indexInfo == null) return null;
+ return (Integer) indexInfo.get("numTerms");
+ }
+
+ public Map<String, FieldTypeInfo> getFieldTypeInfo() {
+ return fieldTypeInfo;
+ }
+
+ public FieldTypeInfo getFieldTypeInfo(String name) {
+ return fieldTypeInfo.get(name);
+ }
+
+ public NamedList<Object> getIndexInfo() {
+ return indexInfo;
+ }
+
+ public Map<String, FieldInfo> getFieldInfo() {
+ return fieldInfo;
+ }
+
+ public FieldInfo getFieldInfo(String f) {
+ return fieldInfo.get(f);
+ }
+
+ //----------------------------------------------------------------
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/PivotField.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/PivotField.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/PivotField.java
new file mode 100644
index 0000000..3b084f6
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/PivotField.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import java.io.PrintStream;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+public class PivotField implements Serializable
+{
+ final String _field;
+ final Object _value;
+ final int _count;
+ final List<PivotField> _pivot;
+ final Map<String,FieldStatsInfo> _statsInfo;
+
+ /**
+ * @deprecated Use {@link #PivotField(String,Object,int,List,Map)} with a null <code>statsInfo</code>
+ */
+ @Deprecated
+ public PivotField( String f, Object v, int count, List<PivotField> pivot) {
+ this(f, v, count, pivot, null);
+ }
+
+ public PivotField( String f, Object v, int count, List<PivotField> pivot, Map<String,FieldStatsInfo> statsInfo)
+ {
+ _field = f;
+ _value = v;
+ _count = count;
+ _pivot = pivot;
+ _statsInfo = statsInfo;
+ }
+
+ public String getField() {
+ return _field;
+ }
+
+ public Object getValue() {
+ return _value;
+ }
+
+ public int getCount() {
+ return _count;
+ }
+
+ public List<PivotField> getPivot() {
+ return _pivot;
+ }
+
+ public Map<String,FieldStatsInfo> getFieldStatsInfo() {
+ return _statsInfo;
+ }
+
+ @Override
+ public String toString()
+ {
+ return _field + ":" + _value + " ["+_count+"] "+_pivot;
+ }
+
+ public void write( PrintStream out, int indent )
+ {
+ for( int i=0; i<indent; i++ ) {
+ out.print( " " );
+ }
+ out.print( _field + "=" + _value + " ("+_count+")" );
+ if (null != _statsInfo) {
+ out.print( "->stats:[" );
+ for( FieldStatsInfo fieldStatsInfo : _statsInfo.values() ) {
+ out.print(fieldStatsInfo.toString());
+ out.print(",");
+ }
+ out.print("]");
+ }
+ out.println();
+ if( _pivot != null ) {
+ for( PivotField p : _pivot ) {
+ p.write( out, indent+1 );
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/QueryResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/QueryResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/QueryResponse.java
new file mode 100644
index 0000000..89cd971
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/QueryResponse.java
@@ -0,0 +1,586 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.beans.DocumentObjectBinder;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CursorMarkParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+@SuppressWarnings("unchecked")
+public class QueryResponse extends SolrResponseBase
+{
+ // Direct pointers to known types
+ private NamedList<Object> _header = null;
+ private SolrDocumentList _results = null;
+ private NamedList<ArrayList> _sortvalues = null;
+ private NamedList<Object> _facetInfo = null;
+ private NamedList<Object> _debugInfo = null;
+ private NamedList<Object> _highlightingInfo = null;
+ private NamedList<Object> _spellInfo = null;
+ private NamedList<Object> _statsInfo = null;
+ private NamedList<NamedList<Number>> _termsInfo = null;
+ private String _cursorMarkNext = null;
+
+ // Grouping response
+ private NamedList<Object> _groupedInfo = null;
+ private GroupResponse _groupResponse = null;
+
+ private NamedList<Object> _expandedInfo = null;
+ private Map<String, SolrDocumentList> _expandedResults = null;
+
+ // Facet stuff
+ private Map<String,Integer> _facetQuery = null;
+ private List<FacetField> _facetFields = null;
+ private List<FacetField> _limitingFacets = null;
+ private List<FacetField> _facetDates = null;
+ private List<RangeFacet> _facetRanges = null;
+ private NamedList<List<PivotField>> _facetPivot = null;
+ private List<IntervalFacet> _intervalFacets = null;
+
+ // Highlight Info
+ private Map<String,Map<String,List<String>>> _highlighting = null;
+
+ // SpellCheck Response
+ private SpellCheckResponse _spellResponse = null;
+
+ // Terms Response
+ private TermsResponse _termsResponse = null;
+
+ // Field stats Response
+ private Map<String,FieldStatsInfo> _fieldStatsInfo = null;
+
+ // Debug Info
+ private Map<String,Object> _debugMap = null;
+ private Map<String,String> _explainMap = null;
+
+ // utility variable used for automatic binding -- it should not be serialized
+ private transient final SolrClient solrClient;
+
+ public QueryResponse(){
+ solrClient = null;
+ }
+
+ /**
+ * Utility constructor to set the solrServer and namedList
+ */
+ public QueryResponse( NamedList<Object> res , SolrClient solrClient){
+ this.setResponse( res );
+ this.solrClient = solrClient;
+ }
+
+ public QueryResponse(SolrClient solrClient) {
+ this.solrClient = solrClient;
+ }
+
+ @Override
+ public void setResponse( NamedList<Object> res )
+ {
+ super.setResponse( res );
+
+ // Look for known things
+ for( int i=0; i<res.size(); i++ ) {
+ String n = res.getName( i );
+ if( "responseHeader".equals( n ) ) {
+ _header = (NamedList<Object>) res.getVal( i );
+ }
+ else if( "response".equals( n ) ) {
+ _results = (SolrDocumentList) res.getVal( i );
+ }
+ else if( "sort_values".equals( n ) ) {
+ _sortvalues = (NamedList<ArrayList>) res.getVal( i );
+ }
+ else if( "facet_counts".equals( n ) ) {
+ _facetInfo = (NamedList<Object>) res.getVal( i );
+ // extractFacetInfo inspects _results, so defer calling it
+ // in case it hasn't been populated yet.
+ }
+ else if( "debug".equals( n ) ) {
+ _debugInfo = (NamedList<Object>) res.getVal( i );
+ extractDebugInfo( _debugInfo );
+ }
+ else if( "grouped".equals( n ) ) {
+ _groupedInfo = (NamedList<Object>) res.getVal( i );
+ extractGroupedInfo( _groupedInfo );
+ }
+ else if("expanded".equals(n)) {
+ _expandedResults = (Map<String, SolrDocumentList>) res.getVal( i );
+ }
+ else if( "highlighting".equals( n ) ) {
+ _highlightingInfo = (NamedList<Object>) res.getVal( i );
+ extractHighlightingInfo( _highlightingInfo );
+ }
+ else if ( "spellcheck".equals( n ) ) {
+ _spellInfo = (NamedList<Object>) res.getVal( i );
+ extractSpellCheckInfo( _spellInfo );
+ }
+ else if ( "stats".equals( n ) ) {
+ _statsInfo = (NamedList<Object>) res.getVal( i );
+ extractStatsInfo( _statsInfo );
+ }
+ else if ( "terms".equals( n ) ) {
+ _termsInfo = (NamedList<NamedList<Number>>) res.getVal( i );
+ extractTermsInfo( _termsInfo );
+ }
+ else if ( CursorMarkParams.CURSOR_MARK_NEXT.equals( n ) ) {
+ _cursorMarkNext = (String) res.getVal( i );
+ }
+ }
+ if(_facetInfo != null) extractFacetInfo( _facetInfo );
+ }
+
+ private void extractSpellCheckInfo(NamedList<Object> spellInfo) {
+ _spellResponse = new SpellCheckResponse(spellInfo);
+ }
+
+ private void extractTermsInfo(NamedList<NamedList<Number>> termsInfo) {
+ _termsResponse = new TermsResponse(termsInfo);
+ }
+
+ private void extractStatsInfo(NamedList<Object> info) {
+ _fieldStatsInfo = extractFieldStatsInfo(info);
+ }
+
+ private Map<String, FieldStatsInfo> extractFieldStatsInfo(NamedList<Object> info) {
+ if( info != null ) {
+ Map<String, FieldStatsInfo> fieldStatsInfoMap = new TreeMap<>();
+ NamedList<NamedList<Object>> ff = (NamedList<NamedList<Object>>) info.get( "stats_fields" );
+ if( ff != null ) {
+ for( Map.Entry<String,NamedList<Object>> entry : ff ) {
+ NamedList<Object> v = entry.getValue();
+ if( v != null ) {
+ fieldStatsInfoMap.put( entry.getKey(),
+ new FieldStatsInfo( v, entry.getKey() ) );
+ }
+ }
+ }
+ return fieldStatsInfoMap;
+ }
+ return null;
+ }
+
+ private void extractDebugInfo( NamedList<Object> debug )
+ {
+ _debugMap = new LinkedHashMap<>(); // keep the order
+ for( Map.Entry<String, Object> info : debug ) {
+ _debugMap.put( info.getKey(), info.getValue() );
+ }
+
+ // Parse out interesting bits from the debug info
+ _explainMap = new HashMap<>();
+ NamedList<String> explain = (NamedList<String>)_debugMap.get( "explain" );
+ if( explain != null ) {
+ for( Map.Entry<String, String> info : explain ) {
+ String key = info.getKey();
+ _explainMap.put( key, info.getValue() );
+ }
+ }
+ }
+
+ private void extractGroupedInfo( NamedList<Object> info ) {
+ if ( info != null ) {
+ _groupResponse = new GroupResponse();
+ int size = info.size();
+ for (int i=0; i < size; i++) {
+ String fieldName = info.getName(i);
+ Object fieldGroups = info.getVal(i);
+ SimpleOrderedMap<Object> simpleOrderedMap = (SimpleOrderedMap<Object>) fieldGroups;
+
+ Object oMatches = simpleOrderedMap.get("matches");
+ Object oNGroups = simpleOrderedMap.get("ngroups");
+ Object oGroups = simpleOrderedMap.get("groups");
+ Object queryCommand = simpleOrderedMap.get("doclist");
+ if (oMatches == null) {
+ continue;
+ }
+
+ if (oGroups != null) {
+ Integer iMatches = (Integer) oMatches;
+ ArrayList<Object> groupsArr = (ArrayList<Object>) oGroups;
+ GroupCommand groupedCommand;
+ if (oNGroups != null) {
+ Integer iNGroups = (Integer) oNGroups;
+ groupedCommand = new GroupCommand(fieldName, iMatches, iNGroups);
+ } else {
+ groupedCommand = new GroupCommand(fieldName, iMatches);
+ }
+
+ for (Object oGrp : groupsArr) {
+ SimpleOrderedMap grpMap = (SimpleOrderedMap) oGrp;
+ Object sGroupValue = grpMap.get( "groupValue");
+ SolrDocumentList doclist = (SolrDocumentList) grpMap.get( "doclist");
+ Group group = new Group(sGroupValue != null ? sGroupValue.toString() : null, doclist) ;
+ groupedCommand.add(group);
+ }
+
+ _groupResponse.add(groupedCommand);
+ } else if (queryCommand != null) {
+ Integer iMatches = (Integer) oMatches;
+ GroupCommand groupCommand;
+ if (oNGroups != null) {
+ Integer iNGroups = (Integer) oNGroups;
+ groupCommand = new GroupCommand(fieldName, iMatches, iNGroups);
+ } else {
+ groupCommand = new GroupCommand(fieldName, iMatches);
+ }
+ SolrDocumentList docList = (SolrDocumentList) queryCommand;
+ groupCommand.add(new Group(fieldName, docList));
+ _groupResponse.add(groupCommand);
+ }
+ }
+ }
+ }
+
+ private void extractHighlightingInfo( NamedList<Object> info )
+ {
+ _highlighting = new HashMap<>();
+ for( Map.Entry<String, Object> doc : info ) {
+ Map<String,List<String>> fieldMap = new HashMap<>();
+ _highlighting.put( doc.getKey(), fieldMap );
+
+ NamedList<List<String>> fnl = (NamedList<List<String>>)doc.getValue();
+ for( Map.Entry<String, List<String>> field : fnl ) {
+ fieldMap.put( field.getKey(), field.getValue() );
+ }
+ }
+ }
+
+ private void extractFacetInfo( NamedList<Object> info )
+ {
+ // Parse the queries
+ _facetQuery = new LinkedHashMap<>();
+ NamedList<Integer> fq = (NamedList<Integer>) info.get( "facet_queries" );
+ if (fq != null) {
+ for( Map.Entry<String, Integer> entry : fq ) {
+ _facetQuery.put( entry.getKey(), entry.getValue() );
+ }
+ }
+
+ // Parse the facet info into fields
+ // TODO?? The list could be <int> or <long>? If always <long> then we can switch to <Long>
+ NamedList<NamedList<Number>> ff = (NamedList<NamedList<Number>>) info.get( "facet_fields" );
+ if( ff != null ) {
+ _facetFields = new ArrayList<>( ff.size() );
+ _limitingFacets = new ArrayList<>( ff.size() );
+
+ long minsize = _results == null ? Long.MAX_VALUE :_results.getNumFound();
+ for( Map.Entry<String,NamedList<Number>> facet : ff ) {
+ FacetField f = new FacetField( facet.getKey() );
+ for( Map.Entry<String, Number> entry : facet.getValue() ) {
+ f.add( entry.getKey(), entry.getValue().longValue() );
+ }
+
+ _facetFields.add( f );
+ FacetField nl = f.getLimitingFields( minsize );
+ if( nl.getValueCount() > 0 ) {
+ _limitingFacets.add( nl );
+ }
+ }
+ }
+
+ //Parse date facets
+ NamedList<NamedList<Object>> df = (NamedList<NamedList<Object>>) info.get("facet_dates");
+ if (df != null) {
+ // System.out.println(df);
+ _facetDates = new ArrayList<>( df.size() );
+ for (Map.Entry<String, NamedList<Object>> facet : df) {
+ // System.out.println("Key: " + facet.getKey() + " Value: " + facet.getValue());
+ NamedList<Object> values = facet.getValue();
+ String gap = (String) values.get("gap");
+ Date end = (Date) values.get("end");
+ FacetField f = new FacetField(facet.getKey(), gap, end);
+
+ for (Map.Entry<String, Object> entry : values) {
+ try {
+ f.add(entry.getKey(), Long.parseLong(entry.getValue().toString()));
+ } catch (NumberFormatException e) {
+ //Ignore for non-number responses which are already handled above
+ }
+ }
+
+ _facetDates.add(f);
+ }
+ }
+
+ //Parse range facets
+ NamedList<NamedList<Object>> rf = (NamedList<NamedList<Object>>) info.get("facet_ranges");
+ if (rf != null) {
+ _facetRanges = new ArrayList<>( rf.size() );
+ for (Map.Entry<String, NamedList<Object>> facet : rf) {
+ NamedList<Object> values = facet.getValue();
+ Object rawGap = values.get("gap");
+
+ RangeFacet rangeFacet;
+ if (rawGap instanceof Number) {
+ Number gap = (Number) rawGap;
+ Number start = (Number) values.get("start");
+ Number end = (Number) values.get("end");
+
+ Number before = (Number) values.get("before");
+ Number after = (Number) values.get("after");
+ Number between = (Number) values.get("between");
+
+ rangeFacet = new RangeFacet.Numeric(facet.getKey(), start, end, gap, before, after, between);
+ } else {
+ String gap = (String) rawGap;
+ Date start = (Date) values.get("start");
+ Date end = (Date) values.get("end");
+
+ Number before = (Number) values.get("before");
+ Number after = (Number) values.get("after");
+ Number between = (Number) values.get("between");
+
+ rangeFacet = new RangeFacet.Date(facet.getKey(), start, end, gap, before, after, between);
+ }
+
+ NamedList<Integer> counts = (NamedList<Integer>) values.get("counts");
+ for (Map.Entry<String, Integer> entry : counts) {
+ rangeFacet.addCount(entry.getKey(), entry.getValue());
+ }
+
+ _facetRanges.add(rangeFacet);
+ }
+ }
+
+ //Parse pivot facets
+ NamedList pf = (NamedList) info.get("facet_pivot");
+ if (pf != null) {
+ _facetPivot = new NamedList<>();
+ for( int i=0; i<pf.size(); i++ ) {
+ _facetPivot.add( pf.getName(i), readPivots( (List<NamedList>)pf.getVal(i) ) );
+ }
+ }
+
+ //Parse interval facets
+ NamedList<NamedList<Object>> intervalsNL = (NamedList<NamedList<Object>>) info.get("facet_intervals");
+ if (intervalsNL != null) {
+ _intervalFacets = new ArrayList<>(intervalsNL.size());
+ for (Map.Entry<String, NamedList<Object>> intervalField : intervalsNL) {
+ String field = intervalField.getKey();
+ List<IntervalFacet.Count> counts = new ArrayList<IntervalFacet.Count>(intervalField.getValue().size());
+ for (Map.Entry<String, Object> interval : intervalField.getValue()) {
+ counts.add(new IntervalFacet.Count(interval.getKey(), (Integer)interval.getValue()));
+ }
+ _intervalFacets.add(new IntervalFacet(field, counts));
+ }
+ }
+ }
+
+ protected List<PivotField> readPivots( List<NamedList> list )
+ {
+ ArrayList<PivotField> values = new ArrayList<>( list.size() );
+ for( NamedList nl : list ) {
+ // NOTE, this is cheating, but we know the order they are written in, so no need to check
+ assert "field".equals(nl.getName(0));
+ String f = (String)nl.getVal( 0 );
+ assert "value".equals(nl.getName(1));
+ Object v = nl.getVal( 1 );
+ assert "count".equals(nl.getName(2));
+ int cnt = ((Integer)nl.getVal( 2 )).intValue();
+
+ List<PivotField> subPivots = null;
+ Map<String,FieldStatsInfo> fieldStatsInfos = null;
+
+ if (4 <= nl.size()) {
+ for(int index = 3; index < nl.size(); index++) {
+ final String key = nl.getName(index);
+ final Object val = nl.getVal(index);
+ switch (key) {
+
+ case "pivot": {
+ assert null != val : "Server sent back 'null' for sub pivots?";
+ assert val instanceof List : "Server sent non-List for sub pivots?";
+
+ subPivots = readPivots( (List<NamedList>) val );
+ break;
+ }
+ case "stats": {
+ assert null != val : "Server sent back 'null' for stats?";
+ assert val instanceof NamedList : "Server sent non-NamedList for stats?";
+
+ fieldStatsInfos = extractFieldStatsInfo((NamedList<Object>) val);
+ break;
+ }
+ default:
+ throw new RuntimeException( "unknown key in pivot: "+ key+ " ["+val+"]");
+
+ }
+ }
+ }
+
+ values.add( new PivotField( f, v, cnt, subPivots, fieldStatsInfos ) );
+ }
+ return values;
+ }
+
+ //------------------------------------------------------
+ //------------------------------------------------------
+
+ /**
+ * Remove the field facet info
+ */
+ public void removeFacets() {
+ _facetFields = new ArrayList<>();
+ }
+
+ //------------------------------------------------------
+ //------------------------------------------------------
+
+ public NamedList<Object> getHeader() {
+ return _header;
+ }
+
+ public SolrDocumentList getResults() {
+ return _results;
+ }
+
+ public NamedList<ArrayList> getSortValues(){
+ return _sortvalues;
+ }
+
+ public Map<String, Object> getDebugMap() {
+ return _debugMap;
+ }
+
+ public Map<String, String> getExplainMap() {
+ return _explainMap;
+ }
+
+ public Map<String,Integer> getFacetQuery() {
+ return _facetQuery;
+ }
+
+ public Map<String, SolrDocumentList> getExpandedResults(){
+ return this._expandedResults;
+ }
+
+ /**
+ * Returns the {@link GroupResponse} containing the group commands.
+ * A group command can be the result of one of the following parameters:
+ * <ul>
+ * <li>group.field
+ * <li>group.func
+ * <li>group.query
+ * </ul>
+ *
+ * @return the {@link GroupResponse} containing the group commands
+ */
+ public GroupResponse getGroupResponse() {
+ return _groupResponse;
+ }
+
+ public Map<String, Map<String, List<String>>> getHighlighting() {
+ return _highlighting;
+ }
+
+ public SpellCheckResponse getSpellCheckResponse() {
+ return _spellResponse;
+ }
+
+ public TermsResponse getTermsResponse() {
+ return _termsResponse;
+ }
+
+ /**
+ * See also: {@link #getLimitingFacets()}
+ */
+ public List<FacetField> getFacetFields() {
+ return _facetFields;
+ }
+
+ public List<FacetField> getFacetDates() {
+ return _facetDates;
+ }
+
+ public List<RangeFacet> getFacetRanges() {
+ return _facetRanges;
+ }
+
+ public NamedList<List<PivotField>> getFacetPivot() {
+ return _facetPivot;
+ }
+
+ public List<IntervalFacet> getIntervalFacets() {
+ return _intervalFacets;
+ }
+
+ /** get
+ *
+ * @param name the name of the
+ * @return the FacetField by name or null if it does not exist
+ */
+ public FacetField getFacetField(String name) {
+ if (_facetFields==null) return null;
+ for (FacetField f : _facetFields) {
+ if (f.getName().equals(name)) return f;
+ }
+ return null;
+ }
+
+ public FacetField getFacetDate(String name) {
+ if (_facetDates == null)
+ return null;
+ for (FacetField f : _facetDates)
+ if (f.getName().equals(name))
+ return f;
+ return null;
+ }
+
+ /**
+ * @return a list of FacetFields where the count is less then
+ * then #getResults() {@link SolrDocumentList#getNumFound()}
+ *
+ * If you want all results exactly as returned by solr, use:
+ * {@link #getFacetFields()}
+ */
+ public List<FacetField> getLimitingFacets() {
+ return _limitingFacets;
+ }
+
+ public <T> List<T> getBeans(Class<T> type){
+ return solrClient == null ?
+ new DocumentObjectBinder().getBeans(type,_results):
+ solrClient.getBinder().getBeans(type, _results);
+ }
+
+ public Map<String, FieldStatsInfo> getFieldStatsInfo() {
+ return _fieldStatsInfo;
+ }
+
+ public String getNextCursorMark() {
+ return _cursorMarkNext;
+ }
+}
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/RangeFacet.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/RangeFacet.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/RangeFacet.java
new file mode 100644
index 0000000..52b4e6b
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/RangeFacet.java
@@ -0,0 +1,126 @@
+package org.apache.solr.client.solrj.response;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a range facet result
+ */
+public abstract class RangeFacet<B, G> {
+
+ private final String name;
+ private final List<Count> counts = new ArrayList<>();
+
+ private final B start;
+ private final B end;
+ private final G gap;
+
+ private final Number before;
+ private final Number after;
+ private final Number between;
+
+ protected RangeFacet(String name, B start, B end, G gap, Number before, Number after, Number between) {
+ this.name = name;
+ this.start = start;
+ this.end = end;
+ this.gap = gap;
+ this.before = before;
+ this.after = after;
+ this.between = between;
+ }
+
+ public void addCount(String value, int count) {
+ counts.add(new Count(value, count, this));
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<Count> getCounts() {
+ return counts;
+ }
+
+ public B getStart() {
+ return start;
+ }
+
+ public B getEnd() {
+ return end;
+ }
+
+ public G getGap() {
+ return gap;
+ }
+
+ public Number getBefore() {
+ return before;
+ }
+
+ public Number getAfter() {
+ return after;
+ }
+
+ public Number getBetween() {
+ return between;
+ }
+
+ public static class Numeric extends RangeFacet<Number, Number> {
+
+ public Numeric(String name, Number start, Number end, Number gap, Number before, Number after, Number between) {
+ super(name, start, end, gap, before, after, between);
+ }
+
+ }
+
+ public static class Date extends RangeFacet<java.util.Date, String> {
+
+ public Date(String name, java.util.Date start, java.util.Date end, String gap, Number before, Number after, Number between) {
+ super(name, start, end, gap, before, after, between);
+ }
+
+ }
+
+ public static class Count {
+
+ private final String value;
+ private final int count;
+ private final RangeFacet rangeFacet;
+
+ public Count(String value, int count, RangeFacet rangeFacet) {
+ this.value = value;
+ this.count = count;
+ this.rangeFacet = rangeFacet;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public RangeFacet getRangeFacet() {
+ return rangeFacet;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrPingResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrPingResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrPingResponse.java
new file mode 100644
index 0000000..cbd892f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrPingResponse.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public class SolrPingResponse extends SolrResponseBase
+{
+ // nothing special now...
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrResponseBase.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrResponseBase.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrResponseBase.java
new file mode 100644
index 0000000..ca59e54
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SolrResponseBase.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public class SolrResponseBase extends SolrResponse
+{
+ private long elapsedTime = -1;
+ private NamedList<Object> response = null;
+ private String requestUrl = null;
+
+ @Override
+ public long getElapsedTime() {
+ return elapsedTime;
+ }
+
+ public void setElapsedTime(long elapsedTime) {
+ this.elapsedTime = elapsedTime;
+ }
+
+ @Override
+ public NamedList<Object> getResponse() {
+ return response;
+ }
+
+ @Override
+ public void setResponse(NamedList<Object> response) {
+ this.response = response;
+ }
+
+ @Override
+ public String toString() {
+ return response.toString();
+ }
+
+ public NamedList getResponseHeader() {
+ return (NamedList) response.get("responseHeader");
+ }
+
+ // these two methods are based on the logic in SolrCore.setResponseHeaderValues(...)
+ public int getStatus() {
+ NamedList header = getResponseHeader();
+ if (header != null) {
+ return (Integer) header.get("status");
+ }
+ else {
+ return 0;
+ }
+ }
+
+ public int getQTime() {
+ NamedList header = getResponseHeader();
+ if (header != null) {
+ return (Integer) header.get("QTime");
+ }
+ else {
+ return 0;
+ }
+ }
+
+ public String getRequestUrl() {
+ return requestUrl;
+ }
+
+ public void setRequestUrl(String requestUrl) {
+ this.requestUrl = requestUrl;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SpellCheckResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SpellCheckResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SpellCheckResponse.java
new file mode 100644
index 0000000..a3ae45d
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/SpellCheckResponse.java
@@ -0,0 +1,273 @@
+package org.apache.solr.client.solrj.response;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.util.NamedList;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Encapsulates responses from SpellCheckComponent
+ *
+ *
+ * @since solr 1.3
+ */
+public class SpellCheckResponse {
+ private boolean correctlySpelled;
+ private List<Collation> collations;
+ private List<Suggestion> suggestions = new ArrayList<>();
+ Map<String, Suggestion> suggestionMap = new LinkedHashMap<>();
+
+ public SpellCheckResponse(NamedList<Object> spellInfo) {
+ @SuppressWarnings("unchecked")
+ NamedList<Object> sugg = (NamedList<Object>) spellInfo.get("suggestions");
+ if (sugg == null) {
+ correctlySpelled = true;
+ return;
+ }
+ for (int i = 0; i < sugg.size(); i++) {
+ String n = sugg.getName(i);
+ @SuppressWarnings("unchecked")
+ Suggestion s = new Suggestion(n, (NamedList<Object>) sugg.getVal(i));
+ suggestionMap.put(n, s);
+ suggestions.add(s);
+ }
+
+ Boolean correctlySpelled = (Boolean) spellInfo.get("correctlySpelled");
+ if (correctlySpelled != null) {
+ this.correctlySpelled = correctlySpelled;
+ }
+
+ @SuppressWarnings("unchecked")
+ NamedList<Object> coll = (NamedList<Object>) spellInfo.get("collations");
+ if (coll != null) {
+ // The 'collationInternalRank' values are ignored so we only care 'collation's.
+ List<Object> collationInfo = coll.getAll("collation");
+ collations = new ArrayList<>(collationInfo.size());
+ for (Object o : collationInfo) {
+ if (o instanceof String) {
+ collations.add(new Collation()
+ .setCollationQueryString((String) o));
+ } else if (o instanceof NamedList) {
+ @SuppressWarnings("unchecked")
+ NamedList<Object> expandedCollation = (NamedList<Object>) o;
+ String collationQuery
+ = (String) expandedCollation.get("collationQuery");
+ int hits = (Integer) expandedCollation.get("hits");
+ @SuppressWarnings("unchecked")
+ NamedList<String> misspellingsAndCorrections
+ = (NamedList<String>) expandedCollation.get("misspellingsAndCorrections");
+
+ Collation collation = new Collation();
+ collation.setCollationQueryString(collationQuery);
+ collation.setNumberOfHits(hits);
+
+ for (int ii = 0; ii < misspellingsAndCorrections.size(); ii++) {
+ String misspelling = misspellingsAndCorrections.getName(ii);
+ String correction = misspellingsAndCorrections.getVal(ii);
+ collation.addMisspellingsAndCorrection(new Correction(
+ misspelling, correction));
+ }
+ collations.add(collation);
+ } else {
+ throw new AssertionError(
+ "Should get Lists of Strings or List of NamedLists here.");
+ }
+ }
+ }
+ }
+
+ public boolean isCorrectlySpelled() {
+ return correctlySpelled;
+ }
+
+ public List<Suggestion> getSuggestions() {
+ return suggestions;
+ }
+
+ public Map<String, Suggestion> getSuggestionMap() {
+ return suggestionMap;
+ }
+
+ public Suggestion getSuggestion(String token) {
+ return suggestionMap.get(token);
+ }
+
+ public String getFirstSuggestion(String token) {
+ Suggestion s = suggestionMap.get(token);
+ if (s==null || s.getAlternatives().isEmpty()) return null;
+ return s.getAlternatives().get(0);
+ }
+
+ /**
+ * <p>
+ * Return the first collated query string. For convenience and backwards-compatibility. Use getCollatedResults() for full data.
+ * </p>
+ * @return first collated query string
+ */
+ public String getCollatedResult() {
+ return collations==null || collations.size()==0 ? null : collations.get(0).collationQueryString;
+ }
+
+ /**
+ * <p>
+ * Return all collations.
+ * Will include # of hits and misspelling-to-correction details if "spellcheck.collateExtendedResults was true.
+ * </p>
+ * @return all collations
+ */
+ public List<Collation> getCollatedResults() {
+ return collations;
+ }
+
+ public static class Suggestion {
+ private String token;
+ private int numFound;
+ private int startOffset;
+ private int endOffset;
+ private int originalFrequency;
+ private List<String> alternatives = new ArrayList<>();
+ private List<Integer> alternativeFrequencies;
+
+ public Suggestion(String token, NamedList<Object> suggestion) {
+ this.token = token;
+ for (int i = 0; i < suggestion.size(); i++) {
+ String n = suggestion.getName(i);
+
+ if ("numFound".equals(n)) {
+ numFound = (Integer) suggestion.getVal(i);
+ } else if ("startOffset".equals(n)) {
+ startOffset = (Integer) suggestion.getVal(i);
+ } else if ("endOffset".equals(n)) {
+ endOffset = (Integer) suggestion.getVal(i);
+ } else if ("origFreq".equals(n)) {
+ originalFrequency = (Integer) suggestion.getVal(i);
+ } else if ("suggestion".equals(n)) {
+ @SuppressWarnings("unchecked")
+ List list = (List)suggestion.getVal(i);
+ if (list.size() > 0 && list.get(0) instanceof NamedList) {
+ // extended results detected
+ @SuppressWarnings("unchecked")
+ List<NamedList> extended = (List<NamedList>)list;
+ alternativeFrequencies = new ArrayList<>();
+ for (NamedList nl : extended) {
+ alternatives.add((String)nl.get("word"));
+ alternativeFrequencies.add((Integer)nl.get("freq"));
+ }
+ } else {
+ @SuppressWarnings("unchecked")
+ List<String> alts = (List<String>) list;
+ alternatives.addAll(alts);
+ }
+ }
+ }
+ }
+
+ public String getToken() {
+ return token;
+ }
+
+ public int getNumFound() {
+ return numFound;
+ }
+
+ public int getStartOffset() {
+ return startOffset;
+ }
+
+ public int getEndOffset() {
+ return endOffset;
+ }
+
+ public int getOriginalFrequency() {
+ return originalFrequency;
+ }
+
+ /** The list of alternatives */
+ public List<String> getAlternatives() {
+ return alternatives;
+ }
+
+ /** The frequencies of the alternatives in the corpus, or null if this information was not returned */
+ public List<Integer> getAlternativeFrequencies() {
+ return alternativeFrequencies;
+ }
+
+ }
+
+ public class Collation {
+ private String collationQueryString;
+ private List<Correction> misspellingsAndCorrections = new ArrayList<>();
+ private long numberOfHits;
+
+ public long getNumberOfHits() {
+ return numberOfHits;
+ }
+
+ public void setNumberOfHits(long numberOfHits) {
+ this.numberOfHits = numberOfHits;
+ }
+
+ public String getCollationQueryString() {
+ return collationQueryString;
+ }
+
+ public Collation setCollationQueryString(String collationQueryString) {
+ this.collationQueryString = collationQueryString;
+ return this;
+ }
+
+ public List<Correction> getMisspellingsAndCorrections() {
+ return misspellingsAndCorrections;
+ }
+
+ public Collation addMisspellingsAndCorrection(Correction correction) {
+ this.misspellingsAndCorrections.add(correction);
+ return this;
+ }
+
+ }
+
+ public class Correction {
+ private String original;
+ private String correction;
+
+ public Correction(String original, String correction) {
+ this.original = original;
+ this.correction = correction;
+ }
+
+ public String getOriginal() {
+ return original;
+ }
+
+ public void setOriginal(String original) {
+ this.original = original;
+ }
+
+ public String getCorrection() {
+ return correction;
+ }
+
+ public void setCorrection(String correction) {
+ this.correction = correction;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/TermsResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/TermsResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/TermsResponse.java
new file mode 100644
index 0000000..5d8c0b7
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/TermsResponse.java
@@ -0,0 +1,89 @@
+package org.apache.solr.client.solrj.response;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.util.NamedList;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Encapsulates responses from TermsComponent
+ */
+public class TermsResponse {
+ private Map<String, List<Term>> termMap = new HashMap<>();
+
+ public TermsResponse(NamedList<NamedList<Number>> termsInfo) {
+ for (int i = 0; i < termsInfo.size(); i++) {
+ String fieldName = termsInfo.getName(i);
+ List<Term> itemList = new ArrayList<>();
+ NamedList<Number> items = termsInfo.getVal(i);
+
+ for (int j = 0; j < items.size(); j++) {
+ Term t = new Term(items.getName(j), items.getVal(j).longValue());
+ itemList.add(t);
+ }
+
+ termMap.put(fieldName, itemList);
+ }
+ }
+
+ /**
+ * Get's the term list for a given field
+ *
+ * @return the term list or null if no terms for the given field exist
+ */
+ public List<Term> getTerms(String field) {
+ return termMap.get(field);
+ }
+
+ public Map<String, List<Term>> getTermMap() {
+ return termMap;
+ }
+
+ public static class Term {
+ private String term;
+ private long frequency;
+
+ public Term(String term, long frequency) {
+ this.term = term;
+ this.frequency = frequency;
+ }
+
+ public String getTerm() {
+ return term;
+ }
+
+ public void setTerm(String term) {
+ this.term = term;
+ }
+
+ public long getFrequency() {
+ return frequency;
+ }
+
+ public void setFrequency(long frequency) {
+ this.frequency = frequency;
+ }
+
+ public void addFrequency(long frequency) {
+ this.frequency += frequency;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/UpdateResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/UpdateResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/UpdateResponse.java
new file mode 100644
index 0000000..7a62bc0
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/UpdateResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+
+/**
+ * TODO -- mostly a stub until we have a defined output format
+ *
+ *
+ * @since solr 1.3
+ */
+public class UpdateResponse extends SolrResponseBase
+{
+ // nothing special now...
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/package-info.java
new file mode 100644
index 0000000..e2bbe3a
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Convenience classes for dealing with various types of Solr responses.
+
+ */
+package org.apache.solr.client.solrj.response;
+
+
[08/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/CompositeIdRouter.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/CompositeIdRouter.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/CompositeIdRouter.java
new file mode 100644
index 0000000..5b1652d
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/CompositeIdRouter.java
@@ -0,0 +1,327 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Hash;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+//
+// user!uniqueid
+// app!user!uniqueid
+// user/4!uniqueid
+// app/2!user/4!uniqueid
+//
+public class CompositeIdRouter extends HashBasedRouter {
+ public static final String NAME = "compositeId";
+
+ public static final String SEPARATOR = "!";
+
+ // separator used to optionally specify number of bits to allocate toward first part.
+ public static final int bitsSeparator = '/';
+ private int bits = 16;
+
+ @Override
+ public int sliceHash(String id, SolrInputDocument doc, SolrParams params, DocCollection collection) {
+ String shardFieldName = getRouteField(collection);
+ if (shardFieldName != null && doc != null) {
+ Object o = doc.getFieldValue(shardFieldName);
+ if (o == null)
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No value for :" + shardFieldName + ". Unable to identify shard");
+ id = o.toString();
+ }
+ if (id.indexOf(SEPARATOR) < 0) {
+ return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
+ }
+
+ return new KeyParser(id).getHash();
+ }
+
+
+ /**
+ * Get Range for a given CompositeId based route key
+ *
+ * @param routeKey to return Range for
+ * @return Range for given routeKey
+ */
+ public Range keyHashRange(String routeKey) {
+ if (routeKey.indexOf(SEPARATOR) < 0) {
+ int hash = sliceHash(routeKey, null, null, null);
+ return new Range(hash, hash);
+ }
+
+ return new KeyParser(routeKey).getRange();
+ }
+
+ @Override
+ public Collection<Slice> getSearchSlicesSingle(String shardKey, SolrParams params, DocCollection collection) {
+ if (shardKey == null) {
+ // search across whole collection
+ // TODO: this may need modification in the future when shard splitting could cause an overlap
+ return collection.getActiveSlices();
+ }
+ String id = shardKey;
+
+ if (shardKey.indexOf(SEPARATOR) < 0) {
+ // shardKey is a simple id, so don't do a range
+ return Collections.singletonList(hashToSlice(Hash.murmurhash3_x86_32(id, 0, id.length(), 0), collection));
+ }
+
+ Range completeRange = new KeyParser(id).getRange();
+
+ List<Slice> targetSlices = new ArrayList<>(1);
+ for (Slice slice : collection.getActiveSlices()) {
+ Range range = slice.getRange();
+ if (range != null && range.overlaps(completeRange)) {
+ targetSlices.add(slice);
+ }
+ }
+
+ return targetSlices;
+ }
+
+ public List<Range> partitionRangeByKey(String key, Range range) {
+ List<Range> result = new ArrayList<>(3);
+ Range keyRange = keyHashRange(key);
+ if (!keyRange.overlaps(range)) {
+ throw new IllegalArgumentException("Key range does not overlap given range");
+ }
+ if (keyRange.equals(range)) {
+ return Collections.singletonList(keyRange);
+ } else if (keyRange.isSubsetOf(range)) {
+ result.add(new Range(range.min, keyRange.min - 1));
+ result.add(keyRange);
+ result.add((new Range(keyRange.max + 1, range.max)));
+ } else if (range.includes(keyRange.max)) {
+ result.add(new Range(range.min, keyRange.max));
+ result.add(new Range(keyRange.max + 1, range.max));
+ } else {
+ result.add(new Range(range.min, keyRange.min - 1));
+ result.add(new Range(keyRange.min, range.max));
+ }
+ return result;
+ }
+
+ @Override
+ public List<Range> partitionRange(int partitions, Range range) {
+ int min = range.min;
+ int max = range.max;
+
+ assert max >= min;
+ if (partitions == 0) return Collections.EMPTY_LIST;
+ long rangeSize = (long) max - (long) min;
+ long rangeStep = Math.max(1, rangeSize / partitions);
+
+ List<Range> ranges = new ArrayList<>(partitions);
+
+ long start = min;
+ long end = start;
+
+ // keep track of the idealized target to avoid accumulating rounding errors
+ long targetStart = min;
+ long targetEnd = targetStart;
+
+ // Round to avoid splitting hash domains across ranges if such rounding is not significant.
+ // With default bits==16, one would need to create more than 4000 shards before this
+ // becomes false by default.
+ int mask = 0x0000ffff;
+ boolean round = rangeStep >= (1 << bits) * 16;
+
+ while (end < max) {
+ targetEnd = targetStart + rangeStep;
+ end = targetEnd;
+
+ if (round && ((end & mask) != mask)) {
+ // round up or down?
+ int increment = 1 << bits; // 0x00010000
+ long roundDown = (end | mask) - increment;
+ long roundUp = (end | mask) + increment;
+ if (end - roundDown < roundUp - end && roundDown > start) {
+ end = roundDown;
+ } else {
+ end = roundUp;
+ }
+ }
+
+ // make last range always end exactly on MAX_VALUE
+ if (ranges.size() == partitions - 1) {
+ end = max;
+ }
+ ranges.add(new Range((int) start, (int) end));
+ start = end + 1L;
+ targetStart = targetEnd + 1L;
+ }
+
+ return ranges;
+ }
+
+ /**
+ * Helper class to calculate parts, masks etc for an id.
+ */
+ static class KeyParser {
+ String key;
+ int[] numBits;
+ int[] hashes;
+ int[] masks;
+ boolean triLevel;
+ int pieces;
+
+ public KeyParser(final String key) {
+ this.key = key;
+ List<String> partsList = new ArrayList<>(3);
+ int firstSeparatorPos = key.indexOf(SEPARATOR);
+ if (-1 == firstSeparatorPos) {
+ partsList.add(key);
+ } else {
+ partsList.add(key.substring(0, firstSeparatorPos));
+ int lastPos = key.length() - 1;
+ // Don't make any more parts if the first separator is the last char
+ if (firstSeparatorPos < lastPos) {
+ int secondSeparatorPos = key.indexOf(SEPARATOR, firstSeparatorPos + 1);
+ if (-1 == secondSeparatorPos) {
+ partsList.add(key.substring(firstSeparatorPos + 1));
+ } else if (secondSeparatorPos == lastPos) {
+ // Don't make any more parts if the key has exactly two separators and
+ // they're the last two chars - back-compatibility with the behavior of
+ // String.split() - see SOLR-6257.
+ if (firstSeparatorPos < secondSeparatorPos - 1) {
+ partsList.add(key.substring(firstSeparatorPos + 1, secondSeparatorPos));
+ }
+ } else { // The second separator is not the last char
+ partsList.add(key.substring(firstSeparatorPos + 1, secondSeparatorPos));
+ partsList.add(key.substring(secondSeparatorPos + 1));
+ }
+ // Ignore any further separators beyond the first two
+ }
+ }
+ pieces = partsList.size();
+ String[] parts = partsList.toArray(new String[pieces]);
+ numBits = new int[2];
+ if (key.endsWith("!") && pieces < 3)
+ pieces++;
+ hashes = new int[pieces];
+
+ if (pieces == 3) {
+ numBits[0] = 8;
+ numBits[1] = 8;
+ triLevel = true;
+ } else {
+ numBits[0] = 16;
+ triLevel = false;
+ }
+
+ for (int i = 0; i < pieces; i++) {
+ if (i < pieces - 1) {
+ int commaIdx = parts[i].indexOf(bitsSeparator);
+
+ if (commaIdx > 0) {
+ numBits[i] = getNumBits(parts[i], commaIdx);
+ parts[i] = parts[i].substring(0, commaIdx);
+ }
+ }
+ //Last component of an ID that ends with a '!'
+ if(i >= parts.length)
+ hashes[i] = Hash.murmurhash3_x86_32("", 0, "".length(), 0);
+ else
+ hashes[i] = Hash.murmurhash3_x86_32(parts[i], 0, parts[i].length(), 0);
+ }
+ masks = getMasks();
+ }
+
+ Range getRange() {
+ int lowerBound;
+ int upperBound;
+
+ if (triLevel) {
+ lowerBound = hashes[0] & masks[0] | hashes[1] & masks[1];
+ upperBound = lowerBound | masks[2];
+ } else {
+ lowerBound = hashes[0] & masks[0];
+ upperBound = lowerBound | masks[1];
+ }
+ // If the upper bits are 0xF0000000, the range we want to cover is
+ // 0xF0000000 0xFfffffff
+
+ if ((masks[0] == 0 && !triLevel) || (masks[0] == 0 && masks[1] == 0 && triLevel)) {
+ // no bits used from first part of key.. the code above will produce 0x000000000->0xffffffff
+ // which only works on unsigned space, but we're using signed space.
+ lowerBound = Integer.MIN_VALUE;
+ upperBound = Integer.MAX_VALUE;
+ }
+ Range r = new Range(lowerBound, upperBound);
+ return r;
+ }
+
+ /**
+ * Get bit masks for routing based on routing level
+ */
+ private int[] getMasks() {
+ int[] masks;
+ if (triLevel)
+ masks = getBitMasks(numBits[0], numBits[1]);
+ else
+ masks = getBitMasks(numBits[0]);
+
+ return masks;
+ }
+
+ private int[] getBitMasks(int firstBits, int secondBits) {
+ // java can't shift 32 bits
+ int[] masks = new int[3];
+ masks[0] = firstBits == 0 ? 0 : (-1 << (32 - firstBits));
+ masks[1] = (firstBits + secondBits) == 0 ? 0 : (-1 << (32 - firstBits - secondBits));
+ masks[1] = masks[0] ^ masks[1];
+ masks[2] = (firstBits + secondBits) == 32 ? 0 : ~(masks[0] | masks[1]);
+ return masks;
+ }
+
+ private int getNumBits(String firstPart, int commaIdx) {
+ int v = 0;
+ for (int idx = commaIdx + 1; idx < firstPart.length(); idx++) {
+ char ch = firstPart.charAt(idx);
+ if (ch < '0' || ch > '9') return -1;
+ v = v * 10 + (ch - '0');
+ }
+ return v > 32 ? -1 : v;
+ }
+
+ private int[] getBitMasks(int firstBits) {
+ // java can't shift 32 bits
+ int[] masks;
+ masks = new int[2];
+ masks[0] = firstBits == 0 ? 0 : (-1 << (32 - firstBits));
+ masks[1] = firstBits == 32 ? 0 : (-1 >>> firstBits);
+ return masks;
+ }
+
+ int getHash() {
+ int result = hashes[0] & masks[0];
+
+ for (int i = 1; i < pieces; i++)
+ result = result | (hashes[i] & masks[i]);
+ return result;
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ConnectionManager.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ConnectionManager.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ConnectionManager.java
new file mode 100644
index 0000000..fd43f00
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ConnectionManager.java
@@ -0,0 +1,237 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.solr.common.SolrException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ConnectionManager implements Watcher {
+ protected static final Logger log = LoggerFactory
+ .getLogger(ConnectionManager.class);
+
+ private final String name;
+
+ private volatile boolean connected = false;
+
+ private final ZkClientConnectionStrategy connectionStrategy;
+
+ private final String zkServerAddress;
+
+ private final SolrZkClient client;
+
+ private final OnReconnect onReconnect;
+ private final BeforeReconnect beforeReconnect;
+
+ private volatile boolean isClosed = false;
+
+ // Track the likely expired state
+ private static class LikelyExpiredState {
+ private static LikelyExpiredState NOT_EXPIRED = new LikelyExpiredState(StateType.NOT_EXPIRED, 0);
+ private static LikelyExpiredState EXPIRED = new LikelyExpiredState(StateType.EXPIRED, 0);
+
+ public enum StateType {
+ NOT_EXPIRED, // definitely not expired
+ EXPIRED, // definitely expired
+ TRACKING_TIME // not sure, tracking time of last disconnect
+ }
+
+ private StateType stateType;
+ private long lastDisconnectTime;
+ public LikelyExpiredState(StateType stateType, long lastDisconnectTime) {
+ this.stateType = stateType;
+ this.lastDisconnectTime = lastDisconnectTime;
+ }
+
+ public boolean isLikelyExpired(long timeToExpire) {
+ return stateType == StateType.EXPIRED
+ || ( stateType == StateType.TRACKING_TIME && (System.nanoTime() - lastDisconnectTime > TimeUnit.NANOSECONDS.convert(timeToExpire, TimeUnit.MILLISECONDS)));
+ }
+ }
+
+ private volatile LikelyExpiredState likelyExpiredState = LikelyExpiredState.EXPIRED;
+
+ public ConnectionManager(String name, SolrZkClient client, String zkServerAddress, ZkClientConnectionStrategy strat, OnReconnect onConnect, BeforeReconnect beforeReconnect) {
+ this.name = name;
+ this.client = client;
+ this.connectionStrategy = strat;
+ this.zkServerAddress = zkServerAddress;
+ this.onReconnect = onConnect;
+ this.beforeReconnect = beforeReconnect;
+ }
+
+ private synchronized void connected() {
+ connected = true;
+ likelyExpiredState = LikelyExpiredState.NOT_EXPIRED;
+ notifyAll();
+ }
+
+ private synchronized void disconnected() {
+ connected = false;
+ // record the time we expired unless we are already likely expired
+ if (!likelyExpiredState.isLikelyExpired(0)) {
+ likelyExpiredState = new LikelyExpiredState(LikelyExpiredState.StateType.TRACKING_TIME, System.nanoTime());
+ }
+ notifyAll();
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ if (log.isInfoEnabled()) {
+ log.info("Watcher " + this + " name:" + name + " got event " + event
+ + " path:" + event.getPath() + " type:" + event.getType());
+ }
+
+ if (isClosed) {
+ log.info("Client->ZooKeeper status change trigger but we are already closed");
+ return;
+ }
+
+ KeeperState state = event.getState();
+
+ if (state == KeeperState.SyncConnected) {
+ connected();
+ connectionStrategy.connected();
+ } else if (state == KeeperState.Expired) {
+ // we don't call disconnected here, because we know we are expired
+ connected = false;
+ likelyExpiredState = LikelyExpiredState.EXPIRED;
+
+ log.info("Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...");
+
+ if (beforeReconnect != null) {
+ try {
+ beforeReconnect.command();
+ } catch (Exception e) {
+ log.warn("Exception running beforeReconnect command", e);
+ }
+ }
+
+ try {
+ connectionStrategy.reconnect(zkServerAddress,
+ client.getZkClientTimeout(), this,
+ new ZkClientConnectionStrategy.ZkUpdate() {
+ @Override
+ public void update(SolrZooKeeper keeper) {
+ try {
+ waitForConnected(Long.MAX_VALUE);
+ } catch (Exception e1) {
+ closeKeeper(keeper);
+ throw new RuntimeException(e1);
+ }
+
+ log.info("Connection with ZooKeeper reestablished.");
+ try {
+ client.updateKeeper(keeper);
+ } catch (InterruptedException e) {
+ closeKeeper(keeper);
+ Thread.currentThread().interrupt();
+ // we must have been asked to stop
+ throw new RuntimeException(e);
+ } catch (Exception t) {
+ closeKeeper(keeper);
+ throw new RuntimeException(t);
+ }
+
+ if (onReconnect != null) {
+ onReconnect.command();
+ }
+ }
+ });
+ } catch (Exception e) {
+ SolrException.log(log, "", e);
+ }
+ log.info("Connected:" + connected);
+ } else if (state == KeeperState.Disconnected) {
+ log.info("zkClient has disconnected");
+ disconnected();
+ connectionStrategy.disconnected();
+ } else if (state == KeeperState.AuthFailed) {
+ log.warn("zkClient received AuthFailed");
+ }
+ }
+
+ public synchronized boolean isConnected() {
+ return !isClosed && connected;
+ }
+
+ // we use a volatile rather than sync
+ // to avoid possible deadlock on shutdown
+ public void close() {
+ this.isClosed = true;
+ this.likelyExpiredState = LikelyExpiredState.EXPIRED;
+ }
+
+ public boolean isLikelyExpired() {
+ return isClosed || likelyExpiredState.isLikelyExpired((long) (client.getZkClientTimeout() * 0.90));
+ }
+
+ public synchronized void waitForConnected(long waitForConnection)
+ throws TimeoutException {
+ log.info("Waiting for client to connect to ZooKeeper");
+ long expire = System.nanoTime() + TimeUnit.NANOSECONDS.convert(waitForConnection, TimeUnit.MILLISECONDS);
+ long left = 1;
+ while (!connected && left > 0) {
+ if (isClosed) {
+ break;
+ }
+ try {
+ wait(500);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ left = expire - System.nanoTime();
+ }
+ if (!connected) {
+ throw new TimeoutException("Could not connect to ZooKeeper " + zkServerAddress + " within " + waitForConnection + " ms");
+ }
+ log.info("Client is connected to ZooKeeper");
+ }
+
+ public synchronized void waitForDisconnected(long timeout)
+ throws InterruptedException, TimeoutException {
+ long expire = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS);
+ long left = timeout;
+ while (connected && left > 0) {
+ wait(left);
+ left = expire - System.nanoTime();
+ }
+ if (connected) {
+ throw new TimeoutException("Did not disconnect");
+ }
+ }
+
+ private void closeKeeper(SolrZooKeeper keeper) {
+ try {
+ keeper.close();
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java
new file mode 100644
index 0000000..c2f4a31
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultConnectionStrategy.java
@@ -0,0 +1,75 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import java.io.IOException;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.solr.common.SolrException;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TODO: improve backoff retry impl
+ */
+public class DefaultConnectionStrategy extends ZkClientConnectionStrategy {
+
+ private static Logger log = LoggerFactory.getLogger(DefaultConnectionStrategy.class);
+
+ @Override
+ public void connect(String serverAddress, int timeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException {
+ SolrZooKeeper zk = createSolrZooKeeper(serverAddress, timeout, watcher);
+ boolean success = false;
+ try {
+ updater.update(zk);
+ success = true;
+ } finally {
+ if (!success) {
+ zk.close();
+ }
+ }
+ }
+
+ @Override
+ public void reconnect(final String serverAddress, final int zkClientTimeout,
+ final Watcher watcher, final ZkUpdate updater) throws IOException {
+ log.info("Connection expired - starting a new one...");
+ SolrZooKeeper zk = createSolrZooKeeper(serverAddress, zkClientTimeout, watcher);
+ boolean success = false;
+ try {
+ updater
+ .update(zk);
+ success = true;
+ log.info("Reconnected to ZooKeeper");
+ } catch (Exception e) {
+ SolrException.log(log, "Reconnect to ZooKeeper failed", e);
+ log.info("Reconnect to ZooKeeper failed");
+ } finally {
+ if (!success) {
+ try {
+ zk.close();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkACLProvider.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkACLProvider.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkACLProvider.java
new file mode 100644
index 0000000..9b0301a
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkACLProvider.java
@@ -0,0 +1,45 @@
+package org.apache.solr.common.cloud;
+
+import java.util.List;
+
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class DefaultZkACLProvider implements ZkACLProvider {
+
+ private List<ACL> globalACLsToAdd;
+
+ @Override
+ public List<ACL> getACLsToAdd(String zNodePath) {
+ // In default (simple) implementation use the same set of ACLs for all znodes
+ if (globalACLsToAdd == null) {
+ synchronized (this) {
+ if (globalACLsToAdd == null) globalACLsToAdd = createGlobalACLsToAdd();
+ }
+ }
+ return globalACLsToAdd;
+
+ }
+
+ protected List<ACL> createGlobalACLsToAdd() {
+ return ZooDefs.Ids.OPEN_ACL_UNSAFE;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java
new file mode 100644
index 0000000..ca09068
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DefaultZkCredentialsProvider.java
@@ -0,0 +1,41 @@
+package org.apache.solr.common.cloud;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class DefaultZkCredentialsProvider implements ZkCredentialsProvider {
+
+ private Collection<ZkCredentials> zkCredentials;
+
+ @Override
+ public Collection<ZkCredentials> getCredentials() {
+ if (zkCredentials == null) {
+ synchronized (this) {
+ if (zkCredentials == null) zkCredentials = createCredentials();
+ }
+ }
+ return zkCredentials;
+ }
+
+ protected Collection<ZkCredentials> createCredentials() {
+ return new ArrayList<ZkCredentials>();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocCollection.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocCollection.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocCollection.java
new file mode 100644
index 0000000..a5e4252
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocCollection.java
@@ -0,0 +1,201 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.noggit.JSONUtil;
+import org.noggit.JSONWriter;
+
+/**
+ * Models a Collection in zookeeper (but that Java name is obviously taken, hence "DocCollection")
+ */
+public class DocCollection extends ZkNodeProps {
+ public static final String DOC_ROUTER = "router";
+ public static final String SHARDS = "shards";
+ public static final String STATE_FORMAT = "stateFormat";
+ private int znodeVersion = -1; // sentinel
+
+ private final String name;
+ private final Map<String, Slice> slices;
+ private final Map<String, Slice> activeSlices;
+ private final DocRouter router;
+ private final String znode;
+
+ private final Integer replicationFactor;
+ private final Integer maxShardsPerNode;
+ private final boolean autoAddReplicas;
+
+
+ public DocCollection(String name, Map<String, Slice> slices, Map<String, Object> props, DocRouter router) {
+ this(name, slices, props, router, -1, ZkStateReader.CLUSTER_STATE);
+ }
+
+ /**
+ * @param name The name of the collection
+ * @param slices The logical shards of the collection. This is used directly and a copy is not made.
+ * @param props The properties of the slice. This is used directly and a copy is not made.
+ */
+ public DocCollection(String name, Map<String, Slice> slices, Map<String, Object> props, DocRouter router, int zkVersion, String znode) {
+ super(props==null ? props = new HashMap<String,Object>() : props);
+ this.znodeVersion = zkVersion;
+ this.name = name;
+
+ this.slices = slices;
+ this.activeSlices = new HashMap<>();
+ Object replicationFactorObject = (Object) props.get(ZkStateReader.REPLICATION_FACTOR);
+ if (replicationFactorObject != null) {
+ this.replicationFactor = Integer.parseInt(replicationFactorObject.toString());
+ } else {
+ this.replicationFactor = null;
+ }
+ Object maxShardsPerNodeObject = (Object) props.get(ZkStateReader.MAX_SHARDS_PER_NODE);
+ if (maxShardsPerNodeObject != null) {
+ this.maxShardsPerNode = Integer.parseInt(maxShardsPerNodeObject.toString());
+ } else {
+ this.maxShardsPerNode = null;
+ }
+ Object autoAddReplicasObject = (Object) props.get(ZkStateReader.AUTO_ADD_REPLICAS);
+ if (autoAddReplicasObject != null) {
+ this.autoAddReplicas = Boolean.parseBoolean(autoAddReplicasObject.toString());
+ } else {
+ this.autoAddReplicas = false;
+ }
+
+ Iterator<Map.Entry<String, Slice>> iter = slices.entrySet().iterator();
+
+ while (iter.hasNext()) {
+ Map.Entry<String, Slice> slice = iter.next();
+ if (slice.getValue().getState().equals(Slice.ACTIVE))
+ this.activeSlices.put(slice.getKey(), slice.getValue());
+ }
+ this.router = router;
+ this.znode = znode == null? ZkStateReader.CLUSTER_STATE : znode;
+ assert name != null && slices != null;
+ }
+
+ /**Use this to make an exact copy of DocCollection with a new set of Slices and every other property as is
+ * @param slices the new set of Slices
+ * @return the resulting DocCollection
+ */
+ public DocCollection copyWithSlices(Map<String, Slice> slices){
+ return new DocCollection(getName(), slices, propMap, router, znodeVersion,znode);
+ }
+
+ /**
+ * Return collection name.
+ */
+ public String getName() {
+ return name;
+ }
+
+ public Slice getSlice(String sliceName) {
+ return slices.get(sliceName);
+ }
+
+ /**
+ * Gets the list of all slices for this collection.
+ */
+ public Collection<Slice> getSlices() {
+ return slices.values();
+ }
+
+
+ /**
+ * Return the list of active slices for this collection.
+ */
+ public Collection<Slice> getActiveSlices() {
+ return activeSlices.values();
+ }
+
+ /**
+ * Get the map of all slices (sliceName->Slice) for this collection.
+ */
+ public Map<String, Slice> getSlicesMap() {
+ return slices;
+ }
+
+ /**
+ * Get the map of active slices (sliceName->Slice) for this collection.
+ */
+ public Map<String, Slice> getActiveSlicesMap() {
+ return activeSlices;
+ }
+
+ public int getZNodeVersion(){
+ return znodeVersion;
+ }
+
+ public int getStateFormat() {
+ return ZkStateReader.CLUSTER_STATE.equals(znode) ? 1 : 2;
+ }
+ /**
+ * @return replication factor for this collection or null if no
+ * replication factor exists.
+ */
+ public Integer getReplicationFactor() {
+ return replicationFactor;
+ }
+
+ public boolean getAutoAddReplicas() {
+ return autoAddReplicas;
+ }
+
+ public int getMaxShardsPerNode() {
+ if (maxShardsPerNode == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, ZkStateReader.MAX_SHARDS_PER_NODE + " is not in the cluster state.");
+ }
+ return maxShardsPerNode;
+ }
+
+ public String getZNode(){
+ return znode;
+ }
+
+
+ public DocRouter getRouter() {
+ return router;
+ }
+
+ @Override
+ public String toString() {
+ return "DocCollection("+name+")=" + JSONUtil.toJSON(this);
+ }
+
+ @Override
+ public void write(JSONWriter jsonWriter) {
+ LinkedHashMap<String, Object> all = new LinkedHashMap<>(slices.size() + 1);
+ all.putAll(propMap);
+ all.put(SHARDS, slices);
+ jsonWriter.write(all);
+ }
+
+ public Replica getReplica(String coreNodeName) {
+ for (Slice slice : slices.values()) {
+ Replica replica = slice.getReplica(coreNodeName);
+ if (replica != null) return replica;
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocRouter.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocRouter.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocRouter.java
new file mode 100644
index 0000000..24d00dd
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/DocRouter.java
@@ -0,0 +1,227 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.StrUtils;
+import org.noggit.JSONWriter;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
+
+/**
+ * Class to partition int range into n ranges.
+ * @lucene.experimental
+ */
+public abstract class DocRouter {
+ public static final String DEFAULT_NAME = CompositeIdRouter.NAME;
+ public static final DocRouter DEFAULT = new CompositeIdRouter();
+
+
+ public static DocRouter getDocRouter(String routerName) {
+ DocRouter router = routerMap.get(routerName);
+ if (router != null) return router;
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown document router '"+ routerName + "'");
+ }
+
+ protected String getRouteField(DocCollection coll){
+ if(coll == null) return null;
+ Object o = coll.get(DOC_ROUTER);
+ if (o instanceof String) {
+ return null;
+ //old format. cannot have a routefield. Ignore it
+ }
+ Map m = (Map) o;
+ if(m == null) return null;
+ return (String) m.get("field");
+
+ }
+
+ public static Map<String,Object> getRouterSpec(ZkNodeProps props){
+ Map<String,Object> map = new LinkedHashMap<>();
+ for (String s : props.keySet()) {
+ if(s.startsWith("router.")){
+ map.put(s.substring(7), props.get(s));
+ }
+ }
+ Object o = props.get("router");
+ if (o instanceof String) {
+ map.put("name", o);
+ } else if (map.get("name") == null) {
+ map.put("name", DEFAULT_NAME);
+ }
+ return map;
+ }
+
+ // currently just an implementation detail...
+ private final static Map<String, DocRouter> routerMap;
+ static {
+ routerMap = new HashMap<>();
+ PlainIdRouter plain = new PlainIdRouter();
+ // instead of doing back compat this way, we could always convert the clusterstate on first read to "plain" if it doesn't have any properties.
+ routerMap.put(null, plain); // back compat with 4.0
+ routerMap.put(PlainIdRouter.NAME, plain);
+ routerMap.put(CompositeIdRouter.NAME, DEFAULT_NAME.equals(CompositeIdRouter.NAME) ? DEFAULT : new CompositeIdRouter());
+ routerMap.put(ImplicitDocRouter.NAME, new ImplicitDocRouter());
+ // NOTE: careful that the map keys (the static .NAME members) are filled in by making them final
+ }
+
+
+ // Hash ranges can't currently "wrap" - i.e. max must be greater or equal to min.
+ // TODO: ranges may not be all contiguous in the future (either that or we will
+ // need an extra class to model a collection of ranges)
+ public static class Range implements JSONWriter.Writable, Comparable<Range> {
+ public int min; // inclusive
+ public int max; // inclusive
+
+ public Range(int min, int max) {
+ assert min <= max;
+ this.min = min;
+ this.max = max;
+ }
+
+ public boolean includes(int hash) {
+ return hash >= min && hash <= max;
+ }
+
+ public boolean isSubsetOf(Range superset) {
+ return superset.min <= min && superset.max >= max;
+ }
+
+ public boolean overlaps(Range other) {
+ return includes(other.min) || includes(other.max) || isSubsetOf(other);
+ }
+
+ @Override
+ public String toString() {
+ return Integer.toHexString(min) + '-' + Integer.toHexString(max);
+ }
+
+
+ @Override
+ public int hashCode() {
+ // difficult numbers to hash... only the highest bits will tend to differ.
+ // ranges will only overlap during a split, so we can just hash the lower range.
+ return (min>>28) + (min>>25) + (min>>21) + min;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj.getClass() != getClass()) return false;
+ Range other = (Range)obj;
+ return this.min == other.min && this.max == other.max;
+ }
+
+ @Override
+ public void write(JSONWriter writer) {
+ writer.write(toString());
+ }
+
+ @Override
+ public int compareTo(Range that) {
+ int mincomp = Integer.valueOf(this.min).compareTo(that.min);
+ return mincomp == 0 ? Integer.valueOf(this.max).compareTo(that.max) : mincomp;
+ }
+ }
+
+ public Range fromString(String range) {
+ int middle = range.indexOf('-');
+ String minS = range.substring(0, middle);
+ String maxS = range.substring(middle+1);
+ long min = Long.parseLong(minS, 16); // use long to prevent the parsing routines from potentially worrying about overflow
+ long max = Long.parseLong(maxS, 16);
+ return new Range((int)min, (int)max);
+ }
+
+ public Range fullRange() {
+ return new Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Returns the range for each partition
+ */
+ public List<Range> partitionRange(int partitions, Range range) {
+ int min = range.min;
+ int max = range.max;
+
+ assert max >= min;
+ if (partitions == 0) return Collections.EMPTY_LIST;
+ long rangeSize = (long)max - (long)min;
+ long rangeStep = Math.max(1, rangeSize / partitions);
+
+ List<Range> ranges = new ArrayList<>(partitions);
+
+ long start = min;
+ long end = start;
+
+ while (end < max) {
+ end = start + rangeStep;
+ // make last range always end exactly on MAX_VALUE
+ if (ranges.size() == partitions - 1) {
+ end = max;
+ }
+ ranges.add(new Range((int)start, (int)end));
+ start = end + 1L;
+ }
+
+ return ranges;
+ }
+
+ /** Returns the Slice that the document should reside on, or null if there is not enough information */
+ public abstract Slice getTargetSlice(String id, SolrInputDocument sdoc, String route, SolrParams params, DocCollection collection);
+
+ /** This method is consulted to determine what slices should be queried for a request when
+ * an explicit shards parameter was not used.
+ * This method only accepts a single shard key (or null). If you have a comma separated list of shard keys,
+ * call getSearchSlices
+ **/
+ public abstract Collection<Slice> getSearchSlicesSingle(String shardKey, SolrParams params, DocCollection collection);
+
+ public abstract boolean isTargetSlice(String id, SolrInputDocument sdoc, SolrParams params, String shardId, DocCollection collection);
+
+
+ /** This method is consulted to determine what slices should be queried for a request when
+ * an explicit shards parameter was not used.
+ * This method accepts a multi-valued shardKeys parameter (normally comma separated from the shard.keys request parameter)
+ * and aggregates the slices returned by getSearchSlicesSingle for each shardKey.
+ **/
+ public Collection<Slice> getSearchSlices(String shardKeys, SolrParams params, DocCollection collection) {
+ if (shardKeys == null || shardKeys.indexOf(',') < 0) {
+ return getSearchSlicesSingle(shardKeys, params, collection);
+ }
+
+ List<String> shardKeyList = StrUtils.splitSmart(shardKeys, ",", true);
+ HashSet<Slice> allSlices = new HashSet<>();
+ for (String shardKey : shardKeyList) {
+ allSlices.addAll( getSearchSlicesSingle(shardKey, params, collection) );
+ }
+ return allSlices;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/HashBasedRouter.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/HashBasedRouter.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/HashBasedRouter.java
new file mode 100644
index 0000000..4132f01
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/HashBasedRouter.java
@@ -0,0 +1,81 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Hash;
+
+import java.util.Collection;
+import java.util.Collections;
+
+public abstract class HashBasedRouter extends DocRouter {
+
+ @Override
+ public Slice getTargetSlice(String id, SolrInputDocument sdoc, String route, SolrParams params, DocCollection collection) {
+ int hash;
+ if (route != null) {
+ hash = sliceHash(route, sdoc, params, collection);
+ } else {
+ if (id == null) id = getId(sdoc, params);
+ hash = sliceHash(id, sdoc, params, collection);
+ }
+ return hashToSlice(hash, collection);
+ }
+
+ @Override
+ public boolean isTargetSlice(String id, SolrInputDocument sdoc, SolrParams params, String shardId, DocCollection collection) {
+ if (id == null) id = getId(sdoc, params);
+ int hash = sliceHash(id, sdoc, params, collection);
+ Range range = collection.getSlice(shardId).getRange();
+ return range != null && range.includes(hash);
+ }
+
+ public int sliceHash(String id, SolrInputDocument sdoc, SolrParams params, DocCollection collection) {
+ return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
+ }
+
+ protected String getId(SolrInputDocument sdoc, SolrParams params) {
+ Object idObj = sdoc.getFieldValue("id"); // blech
+ String id = idObj != null ? idObj.toString() : "null"; // should only happen on client side
+ return id;
+ }
+
+ protected Slice hashToSlice(int hash, DocCollection collection) {
+ for (Slice slice : collection.getActiveSlices()) {
+ Range range = slice.getRange();
+ if (range != null && range.includes(hash)) return slice;
+ }
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No active slice servicing hash code " + Integer.toHexString(hash) + " in " + collection);
+ }
+
+
+ @Override
+ public Collection<Slice> getSearchSlicesSingle(String shardKey, SolrParams params, DocCollection collection) {
+ if (shardKey == null) {
+ // search across whole collection
+ // TODO: this may need modification in the future when shard splitting could cause an overlap
+ return collection.getActiveSlices();
+ }
+
+ // use the shardKey as an id for plain hashing
+ Slice slice = getTargetSlice(shardKey, null, null, params, collection);
+ return slice == null ? Collections.<Slice>emptyList() : Collections.singletonList(slice);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ImplicitDocRouter.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ImplicitDocRouter.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ImplicitDocRouter.java
new file mode 100644
index 0000000..3b2e83d
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ImplicitDocRouter.java
@@ -0,0 +1,104 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import static org.apache.solr.common.params.ShardParams._ROUTE_;
+
+/** This document router is for custom sharding
+ */
+public class ImplicitDocRouter extends DocRouter {
+
+ public static final String NAME = "implicit";
+
+ private static Logger log = LoggerFactory
+ .getLogger(ImplicitDocRouter.class);
+
+ @Override
+ public Slice getTargetSlice(String id, SolrInputDocument sdoc, String route, SolrParams params, DocCollection collection) {
+ String shard = null;
+
+ if (route != null) // if a route is already passed in, try to use it
+ shard = route;
+ else if (sdoc != null) {
+ String f = getRouteField(collection);
+ if(f !=null) {
+ Object o = sdoc.getFieldValue(f);
+ if (o != null) shard = o.toString();
+ else throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No value for field "+f +" in " + sdoc);
+ }
+ if(shard == null) {
+ Object o = sdoc.getFieldValue(_ROUTE_);
+ if (o != null) {
+ shard = o.toString();
+ }
+ }
+ }
+
+ if (shard == null) {
+ shard = params.get(_ROUTE_);
+ }
+
+ if (shard != null) {
+
+ Slice slice = collection.getSlice(shard);
+ if (slice == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shard called =" + shard + " in " + collection);
+ }
+ return slice;
+ }
+
+ return null; // no shard specified... use default.
+ }
+
+ @Override
+ public boolean isTargetSlice(String id, SolrInputDocument sdoc, SolrParams params, String shardId, DocCollection collection) {
+
+ // todo : how to handle this?
+ return false;
+ }
+
+ @Override
+ public Collection<Slice> getSearchSlicesSingle(String shardKey, SolrParams params, DocCollection collection) {
+
+ if (shardKey == null) {
+ return collection.getActiveSlices();
+ }
+
+ // assume the shardKey is just a slice name
+ Slice slice = collection.getSlice(shardKey);
+ if (slice == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "implicit router can't find shard " + shardKey + " in collection " + collection.getName());
+ }
+
+ return Collections.singleton(slice);
+ }
+
+ @Override
+ public List<Range> partitionRange(int partitions, Range range) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/OnReconnect.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/OnReconnect.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/OnReconnect.java
new file mode 100644
index 0000000..e447982
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/OnReconnect.java
@@ -0,0 +1,22 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+public interface OnReconnect {
+ public void command();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/PlainIdRouter.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/PlainIdRouter.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/PlainIdRouter.java
new file mode 100644
index 0000000..bd14089
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/PlainIdRouter.java
@@ -0,0 +1,23 @@
+package org.apache.solr.common.cloud;
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class PlainIdRouter extends HashBasedRouter {
+ public static final String NAME = "plain";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Replica.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Replica.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Replica.java
new file mode 100644
index 0000000..ecd4837
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Replica.java
@@ -0,0 +1,48 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.noggit.JSONUtil;
+
+import java.util.Map;
+
+
+public class Replica extends ZkNodeProps {
+ private final String name;
+ private final String nodeName;
+
+ public Replica(String name, Map<String,Object> propMap) {
+ super(propMap);
+ this.name = name;
+ nodeName = (String)propMap.get(ZkStateReader.NODE_NAME_PROP);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ /** The name of the node this replica resides on */
+ public String getNodeName() {
+ return nodeName;
+ }
+
+ @Override
+ public String toString() {
+ return name + ':' + JSONUtil.toJSON(propMap, -1); // small enough, keep it on one line (i.e. no indent)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/RoutingRule.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/RoutingRule.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/RoutingRule.java
new file mode 100644
index 0000000..69f810f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/RoutingRule.java
@@ -0,0 +1,71 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.noggit.JSONUtil;
+
+/**
+ * Used for routing docs with particular keys into another collection
+ */
+public class RoutingRule extends ZkNodeProps {
+ private final List<DocRouter.Range> routeRanges;
+ private final String routeRangesStr;
+ private final String targetCollectionName;
+ private final Long expireAt;
+
+ public RoutingRule(String routeKey, Map<String, Object> propMap) {
+ super(propMap);
+ this.routeRangesStr = (String) propMap.get("routeRanges");
+ String[] rangesArr = this.routeRangesStr.split(",");
+ if (rangesArr != null && rangesArr.length > 0) {
+ this.routeRanges = new ArrayList<>();
+ for (String r : rangesArr) {
+ routeRanges.add(DocRouter.DEFAULT.fromString(r));
+ }
+ } else {
+ this.routeRanges = null;
+ }
+ this.targetCollectionName = (String) propMap.get("targetCollection");
+ this.expireAt = Long.parseLong((String) propMap.get("expireAt"));
+ }
+
+ public List<DocRouter.Range> getRouteRanges() {
+ return routeRanges;
+ }
+
+ public String getTargetCollectionName() {
+ return targetCollectionName;
+ }
+
+ public Long getExpireAt() {
+ return expireAt;
+ }
+
+ public String getRouteRangesStr() {
+ return routeRangesStr;
+ }
+
+ @Override
+ public String toString() {
+ return JSONUtil.toJSON(propMap, -1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SaslZkACLProvider.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SaslZkACLProvider.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SaslZkACLProvider.java
new file mode 100644
index 0000000..86ab25f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SaslZkACLProvider.java
@@ -0,0 +1,49 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+
+/**
+ * ZkACLProvider that gives all permissions for the user specified in System
+ * property "solr.authorization.superuser" (default: "solr") when using sasl,
+ * and gives read permissions for anyone else. Designed for a setup where
+ * configurations have already been set up and will not be modified, or
+ * where configuration changes are controlled via Solr APIs.
+ */
+public class SaslZkACLProvider extends DefaultZkACLProvider {
+
+ private static String superUser = System.getProperty("solr.authorization.superuser", "solr");
+
+ @Override
+ protected List<ACL> createGlobalACLsToAdd() {
+ List<ACL> result = new ArrayList<ACL>();
+ result.add(new ACL(ZooDefs.Perms.ALL, new Id("sasl", superUser)));
+ result.add(new ACL(ZooDefs.Perms.READ, ZooDefs.Ids.ANYONE_ID_UNSAFE));
+
+ if (result.isEmpty()) {
+ result = super.createGlobalACLsToAdd();
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Slice.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Slice.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Slice.java
new file mode 100644
index 0000000..70abf6e
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/Slice.java
@@ -0,0 +1,196 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.noggit.JSONUtil;
+import org.noggit.JSONWriter;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * A Slice contains immutable information about a logical shard (all replicas that share the same shard id).
+ */
+public class Slice extends ZkNodeProps {
+ public static String REPLICAS = "replicas";
+ public static String RANGE = "range";
+ public static String STATE = "state";
+ public static String LEADER = "leader"; // FUTURE: do we want to record the leader as a slice property in the JSON (as opposed to isLeader as a replica property?)
+ public static String ACTIVE = "active";
+ public static String INACTIVE = "inactive";
+ public static String CONSTRUCTION = "construction";
+ public static String RECOVERY = "recovery";
+ public static String PARENT = "parent";
+
+ private final String name;
+ private final DocRouter.Range range;
+ private final Integer replicationFactor; // FUTURE: optional per-slice override of the collection replicationFactor
+ private final Map<String,Replica> replicas;
+ private final Replica leader;
+ private final String state;
+ private final String parent;
+ private final Map<String, RoutingRule> routingRules;
+
+ /**
+ * @param name The name of the slice
+ * @param replicas The replicas of the slice. This is used directly and a copy is not made. If null, replicas will be constructed from props.
+ * @param props The properties of the slice - a shallow copy will always be made.
+ */
+ public Slice(String name, Map<String,Replica> replicas, Map<String,Object> props) {
+ super( props==null ? new LinkedHashMap<String,Object>(2) : new LinkedHashMap<>(props));
+ this.name = name;
+
+ Object rangeObj = propMap.get(RANGE);
+ if (propMap.containsKey(STATE) && propMap.get(STATE) != null)
+ this.state = (String) propMap.get(STATE);
+ else {
+ this.state = ACTIVE; //Default to ACTIVE
+ propMap.put(STATE, this.state);
+ }
+ DocRouter.Range tmpRange = null;
+ if (rangeObj instanceof DocRouter.Range) {
+ tmpRange = (DocRouter.Range)rangeObj;
+ } else if (rangeObj != null) {
+ // Doesn't support custom implementations of Range, but currently not needed.
+ tmpRange = DocRouter.DEFAULT.fromString(rangeObj.toString());
+ }
+ range = tmpRange;
+
+ /** debugging. this isn't an error condition for custom sharding.
+ if (range == null) {
+ System.out.println("###### NO RANGE for " + name + " props=" + props);
+ }
+ **/
+
+ if (propMap.containsKey(PARENT) && propMap.get(PARENT) != null)
+ this.parent = (String) propMap.get(PARENT);
+ else
+ this.parent = null;
+
+ replicationFactor = null; // future
+
+ // add the replicas *after* the other properties (for aesthetics, so it's easy to find slice properties in the JSON output)
+ this.replicas = replicas != null ? replicas : makeReplicas((Map<String,Object>)propMap.get(REPLICAS));
+ propMap.put(REPLICAS, this.replicas);
+
+ Map<String, Object> rules = (Map<String, Object>) propMap.get("routingRules");
+ if (rules != null) {
+ this.routingRules = new HashMap<>();
+ for (Map.Entry<String, Object> entry : rules.entrySet()) {
+ Object o = entry.getValue();
+ if (o instanceof Map) {
+ Map map = (Map) o;
+ RoutingRule rule = new RoutingRule(entry.getKey(), map);
+ routingRules.put(entry.getKey(), rule);
+ } else {
+ routingRules.put(entry.getKey(), (RoutingRule) o);
+ }
+ }
+ } else {
+ this.routingRules = null;
+ }
+
+ leader = findLeader();
+ }
+
+
+ private Map<String,Replica> makeReplicas(Map<String,Object> genericReplicas) {
+ if (genericReplicas == null) return new HashMap<>(1);
+ Map<String,Replica> result = new LinkedHashMap<>(genericReplicas.size());
+ for (Map.Entry<String,Object> entry : genericReplicas.entrySet()) {
+ String name = entry.getKey();
+ Object val = entry.getValue();
+ Replica r;
+ if (val instanceof Replica) {
+ r = (Replica)val;
+ } else {
+ r = new Replica(name, (Map<String,Object>)val);
+ }
+ result.put(name, r);
+ }
+ return result;
+ }
+
+ private Replica findLeader() {
+ for (Replica replica : replicas.values()) {
+ if (replica.getStr(LEADER) != null) return replica;
+ }
+ return null;
+ }
+
+ /**
+ * Return slice name (shard id).
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * Gets the list of replicas for this slice.
+ */
+ public Collection<Replica> getReplicas() {
+ return replicas.values();
+ }
+
+ /**
+ * Get the map of coreNodeName to replicas for this slice.
+ */
+ public Map<String, Replica> getReplicasMap() {
+ return replicas;
+ }
+
+ public Map<String,Replica> getReplicasCopy() {
+ return new LinkedHashMap<>(replicas);
+ }
+
+ public Replica getLeader() {
+ return leader;
+ }
+
+ public Replica getReplica(String replicaName) {
+ return replicas.get(replicaName);
+ }
+
+ public DocRouter.Range getRange() {
+ return range;
+ }
+
+ public String getState() {
+ return state;
+ }
+
+ public String getParent() {
+ return parent;
+ }
+
+ public Map<String, RoutingRule> getRoutingRules() {
+ return routingRules;
+ }
+
+ @Override
+ public String toString() {
+ return name + ':' + JSONUtil.toJSON(propMap);
+ }
+
+ @Override
+ public void write(JSONWriter jsonWriter) {
+ jsonWriter.write(propMap);
+ }
+}
[16/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
Support for Solr as Audit Destination.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/40aa090d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/40aa090d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/40aa090d
Branch: refs/heads/master
Commit: 40aa090d7b23a524b9900f5b722d02b48c06b947
Parents: 8e6acd5
Author: Don Bosco Durai <bo...@apache.org>
Authored: Tue Mar 17 14:17:30 2015 -0700
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Tue Mar 17 14:17:30 2015 -0700
----------------------------------------------------------------------
agents-audit/pom.xml | 28 +-
.../audit/provider/AuditProviderFactory.java | 23 +-
.../audit/provider/BaseAuditProvider.java | 5 +-
.../provider/kafka/KafkaAuditProvider.java | 2 +-
.../audit/provider/solr/SolrAuditProvider.java | 275 ++++
hbase-agent/conf/ranger-hbase-audit-changes.cfg | 12 +
hbase-agent/conf/ranger-hbase-audit.xml | 48 +
hbase-agent/scripts/install.properties | 6 +
hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 11 +
hdfs-agent/conf/ranger-hdfs-audit.xml | 48 +
hdfs-agent/scripts/install.properties | 7 +
hive-agent/conf/ranger-hive-audit-changes.cfg | 12 +
hive-agent/conf/ranger-hive-audit.xml | 48 +
hive-agent/scripts/install.properties | 6 +
knox-agent/conf/ranger-knox-audit-changes.cfg | 12 +
knox-agent/conf/ranger-knox-audit.xml | 48 +
knox-agent/scripts/install.properties | 6 +
plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 12 +
plugin-yarn/conf/ranger-yarn-audit.xml | 48 +
plugin-yarn/scripts/install.properties | 6 +
pom.xml | 9 +
ranger_solrj/.gitignore | 1 +
ranger_solrj/pom.xml | 55 +
.../solr/client/solrj/ResponseParser.java | 53 +
.../apache/solr/client/solrj/SolrClient.java | 416 ++++++
.../org/apache/solr/client/solrj/SolrQuery.java | 1109 ++++++++++++++++
.../apache/solr/client/solrj/SolrRequest.java | 137 ++
.../apache/solr/client/solrj/SolrResponse.java | 65 +
.../apache/solr/client/solrj/SolrServer.java | 25 +
.../solr/client/solrj/SolrServerException.java | 54 +
.../client/solrj/StreamingResponseCallback.java | 37 +
.../client/solrj/beans/BindingException.java | 29 +
.../solrj/beans/DocumentObjectBinder.java | 470 +++++++
.../apache/solr/client/solrj/beans/Field.java | 38 +
.../solr/client/solrj/beans/package-info.java | 23 +
.../client/solrj/impl/BinaryRequestWriter.java | 120 ++
.../client/solrj/impl/BinaryResponseParser.java | 64 +
.../solr/client/solrj/impl/CloudSolrClient.java | 1232 ++++++++++++++++++
.../solr/client/solrj/impl/CloudSolrServer.java | 61 +
.../solrj/impl/ConcurrentUpdateSolrClient.java | 492 +++++++
.../solrj/impl/ConcurrentUpdateSolrServer.java | 46 +
.../client/solrj/impl/HttpClientConfigurer.java | 97 ++
.../solr/client/solrj/impl/HttpClientUtil.java | 365 ++++++
.../solr/client/solrj/impl/HttpSolrClient.java | 821 ++++++++++++
.../solr/client/solrj/impl/HttpSolrServer.java | 41 +
.../client/solrj/impl/LBHttpSolrClient.java | 730 +++++++++++
.../client/solrj/impl/LBHttpSolrServer.java | 43 +
.../client/solrj/impl/NoOpResponseParser.java | 83 ++
.../impl/StreamingBinaryResponseParser.java | 91 ++
.../client/solrj/impl/XMLResponseParser.java | 465 +++++++
.../solr/client/solrj/impl/package-info.java | 24 +
.../apache/solr/client/solrj/package-info.java | 23 +
.../solrj/request/AbstractUpdateRequest.java | 144 ++
.../solrj/request/CollectionAdminRequest.java | 860 ++++++++++++
.../request/ContentStreamUpdateRequest.java | 78 ++
.../client/solrj/request/CoreAdminRequest.java | 593 +++++++++
.../client/solrj/request/DirectXmlRequest.java | 66 +
.../solrj/request/DocumentAnalysisRequest.java | 199 +++
.../solrj/request/FieldAnalysisRequest.java | 270 ++++
.../client/solrj/request/IsUpdateRequest.java | 26 +
.../request/JavaBinUpdateRequestCodec.java | 251 ++++
.../solr/client/solrj/request/LukeRequest.java | 120 ++
.../solr/client/solrj/request/QueryRequest.java | 89 ++
.../client/solrj/request/RequestWriter.java | 146 +++
.../solr/client/solrj/request/SolrPing.java | 111 ++
.../client/solrj/request/UpdateRequest.java | 463 +++++++
.../solr/client/solrj/request/package-info.java | 23 +
.../solrj/response/AnalysisResponseBase.java | 252 ++++
.../solrj/response/CollectionAdminResponse.java | 79 ++
.../solrj/response/CoreAdminResponse.java | 58 +
.../response/DocumentAnalysisResponse.java | 258 ++++
.../solr/client/solrj/response/FacetField.java | 176 +++
.../solrj/response/FieldAnalysisResponse.java | 204 +++
.../client/solrj/response/FieldStatsInfo.java | 191 +++
.../solr/client/solrj/response/Group.java | 69 +
.../client/solrj/response/GroupCommand.java | 125 ++
.../client/solrj/response/GroupResponse.java | 56 +
.../client/solrj/response/IntervalFacet.java | 85 ++
.../client/solrj/response/LukeResponse.java | 270 ++++
.../solr/client/solrj/response/PivotField.java | 97 ++
.../client/solrj/response/QueryResponse.java | 586 +++++++++
.../solr/client/solrj/response/RangeFacet.java | 126 ++
.../client/solrj/response/SolrPingResponse.java | 28 +
.../client/solrj/response/SolrResponseBase.java | 91 ++
.../solrj/response/SpellCheckResponse.java | 273 ++++
.../client/solrj/response/TermsResponse.java | 89 ++
.../client/solrj/response/UpdateResponse.java | 30 +
.../client/solrj/response/package-info.java | 24 +
.../solr/client/solrj/util/ClientUtils.java | 251 ++++
.../solr/client/solrj/util/package-info.java | 23 +
.../org/apache/solr/common/EnumFieldValue.java | 116 ++
.../org/apache/solr/common/SolrDocument.java | 396 ++++++
.../apache/solr/common/SolrDocumentList.java | 68 +
.../org/apache/solr/common/SolrException.java | 208 +++
.../apache/solr/common/SolrInputDocument.java | 301 +++++
.../org/apache/solr/common/SolrInputField.java | 232 ++++
.../org/apache/solr/common/StringUtils.java | 26 +
.../org/apache/solr/common/cloud/Aliases.java | 63 +
.../solr/common/cloud/BeforeReconnect.java | 22 +
.../solr/common/cloud/ClosableThread.java | 27 +
.../apache/solr/common/cloud/ClusterState.java | 397 ++++++
.../solr/common/cloud/ClusterStateUtil.java | 230 ++++
.../solr/common/cloud/CompositeIdRouter.java | 327 +++++
.../solr/common/cloud/ConnectionManager.java | 237 ++++
.../common/cloud/DefaultConnectionStrategy.java | 75 ++
.../solr/common/cloud/DefaultZkACLProvider.java | 45 +
.../cloud/DefaultZkCredentialsProvider.java | 41 +
.../apache/solr/common/cloud/DocCollection.java | 201 +++
.../org/apache/solr/common/cloud/DocRouter.java | 227 ++++
.../solr/common/cloud/HashBasedRouter.java | 81 ++
.../solr/common/cloud/ImplicitDocRouter.java | 104 ++
.../apache/solr/common/cloud/OnReconnect.java | 22 +
.../apache/solr/common/cloud/PlainIdRouter.java | 23 +
.../org/apache/solr/common/cloud/Replica.java | 48 +
.../apache/solr/common/cloud/RoutingRule.java | 71 +
.../solr/common/cloud/SaslZkACLProvider.java | 49 +
.../org/apache/solr/common/cloud/Slice.java | 196 +++
.../apache/solr/common/cloud/SolrZkClient.java | 736 +++++++++++
.../apache/solr/common/cloud/SolrZooKeeper.java | 103 ++
...ParamsAllAndReadonlyDigestZkACLProvider.java | 89 ++
...tCredentialsDigestZkCredentialsProvider.java | 60 +
.../apache/solr/common/cloud/ZkACLProvider.java | 28 +
.../cloud/ZkClientConnectionStrategy.java | 113 ++
.../apache/solr/common/cloud/ZkCmdExecutor.java | 111 ++
.../solr/common/cloud/ZkConfigManager.java | 145 +++
.../solr/common/cloud/ZkCoreNodeProps.java | 74 ++
.../common/cloud/ZkCredentialsProvider.java | 45 +
.../apache/solr/common/cloud/ZkNodeProps.java | 154 +++
.../apache/solr/common/cloud/ZkOperation.java | 37 +
.../apache/solr/common/cloud/ZkStateReader.java | 925 +++++++++++++
.../solr/common/cloud/ZooKeeperException.java | 33 +
.../apache/solr/common/cloud/package-info.java | 23 +
.../org/apache/solr/common/luke/FieldFlag.java | 70 +
.../apache/solr/common/luke/package-info.java | 23 +
.../org/apache/solr/common/package-info.java | 23 +
.../solr/common/params/AnalysisParams.java | 60 +
.../solr/common/params/AppendedSolrParams.java | 55 +
.../solr/common/params/CollectionParams.java | 74 ++
.../apache/solr/common/params/CommonParams.java | 228 ++++
.../solr/common/params/CoreAdminParams.java | 151 +++
.../solr/common/params/CursorMarkParams.java | 48 +
.../solr/common/params/DefaultSolrParams.java | 68 +
.../apache/solr/common/params/DisMaxParams.java | 78 ++
.../apache/solr/common/params/EventParams.java | 29 +
.../apache/solr/common/params/ExpandParams.java | 32 +
.../apache/solr/common/params/FacetParams.java | 405 ++++++
.../apache/solr/common/params/GroupParams.java | 71 +
.../solr/common/params/HighlightParams.java | 82 ++
.../solr/common/params/MapSolrParams.java | 88 ++
.../common/params/ModifiableSolrParams.java | 210 +++
.../solr/common/params/MoreLikeThisParams.java | 74 ++
.../solr/common/params/MultiMapSolrParams.java | 92 ++
.../common/params/QueryElevationParams.java | 53 +
.../solr/common/params/RequiredSolrParams.java | 155 +++
.../apache/solr/common/params/ShardParams.java | 56 +
.../apache/solr/common/params/SimpleParams.java | 50 +
.../apache/solr/common/params/SolrParams.java | 363 ++++++
.../solr/common/params/SpatialParams.java | 41 +
.../solr/common/params/SpellingParams.java | 174 +++
.../apache/solr/common/params/StatsParams.java | 28 +
.../solr/common/params/TermVectorParams.java | 66 +
.../apache/solr/common/params/TermsParams.java | 120 ++
.../apache/solr/common/params/UpdateParams.java | 72 +
.../apache/solr/common/params/package-info.java | 22 +
.../org/apache/solr/common/util/Base64.java | 153 +++
.../org/apache/solr/common/util/ByteUtils.java | 126 ++
.../apache/solr/common/util/ContentStream.java | 81 ++
.../solr/common/util/ContentStreamBase.java | 260 ++++
.../solr/common/util/DataInputInputStream.java | 27 +
.../org/apache/solr/common/util/DateUtil.java | 260 ++++
.../apache/solr/common/util/ExecutorUtil.java | 64 +
.../solr/common/util/FastInputStream.java | 253 ++++
.../solr/common/util/FastOutputStream.java | 233 ++++
.../java/org/apache/solr/common/util/Hash.java | 422 ++++++
.../org/apache/solr/common/util/IOUtils.java | 37 +
.../apache/solr/common/util/IteratorChain.java | 87 ++
.../apache/solr/common/util/JavaBinCodec.java | 820 ++++++++++++
.../solr/common/util/JsonRecordReader.java | 586 +++++++++
.../org/apache/solr/common/util/NamedList.java | 708 ++++++++++
.../solr/common/util/ObjectReleaseTracker.java | 62 +
.../org/apache/solr/common/util/RetryUtil.java | 43 +
.../solr/common/util/SimpleOrderedMap.java | 67 +
.../common/util/SolrjNamedThreadFactory.java | 50 +
.../org/apache/solr/common/util/StrUtils.java | 309 +++++
.../org/apache/solr/common/util/URLUtil.java | 50 +
.../java/org/apache/solr/common/util/XML.java | 207 +++
.../apache/solr/common/util/XMLErrorLogger.java | 84 ++
.../apache/solr/common/util/package-info.java | 23 +
ranger_solrj/src/main/java/overview.html | 21 +
security-admin/pom.xml | 5 +
security-admin/scripts/install.properties | 10 +
security-admin/scripts/setup.sh | 48 +-
.../java/org/apache/ranger/biz/AssetMgr.java | 11 +-
.../org/apache/ranger/biz/RangerBizUtil.java | 434 +++---
.../java/org/apache/ranger/biz/XAuditMgr.java | 41 +-
.../apache/ranger/db/RangerDaoManagerBase.java | 3 +
.../ranger/solr/SolrAccessAuditsService.java | 253 ++++
.../java/org/apache/ranger/solr/SolrMgr.java | 99 ++
.../java/org/apache/ranger/solr/SolrUtil.java | 327 +++++
src/main/assembly/hbase-agent.xml | 6 +
src/main/assembly/hdfs-agent.xml | 6 +
src/main/assembly/hive-agent.xml | 6 +
src/main/assembly/knox-agent.xml | 6 +
src/main/assembly/storm-agent.xml | 6 +
storm-agent/conf/ranger-storm-audit-changes.cfg | 13 +
storm-agent/conf/ranger-storm-audit.xml | 48 +
storm-agent/scripts/install.properties | 6 +
207 files changed, 30674 insertions(+), 219 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/pom.xml
----------------------------------------------------------------------
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index e54ec36..6715575 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -63,9 +63,29 @@
<version>${log4j.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.kafka</groupId>
- <artifactId>kafka_2.10</artifactId>
- <version>0.8.2.0</version>
-</dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${kafka.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger_solrj</artifactId>
+ <version>${ranger.solrj.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpcomponent.httpmime.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ <version>${noggit.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index 9fbe29f..bb8fa6d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider;
import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider;
+import org.apache.ranger.audit.provider.solr.SolrAuditProvider;
/*
@@ -43,6 +44,7 @@ public class AuditProviderFactory {
private static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled";
private static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled" ;
private static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
+ private static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
private static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024;
private static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000;
@@ -99,8 +101,9 @@ public class AuditProviderFactory {
boolean isAuditToHdfsEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false);
boolean isAuditToLog4jEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
boolean isAuditToKafkaEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false);
+ boolean isAuditToSolrEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false);
- if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled)) {
+ if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled)) {
LOG.info("AuditProviderFactory: Audit not enabled..");
mProvider = getDefaultProvider();
@@ -111,6 +114,7 @@ public class AuditProviderFactory {
List<AuditProvider> providers = new ArrayList<AuditProvider>();
if(isAuditToDbEnabled) {
+ LOG.info("DbAuditProvider is enabled");
DbAuditProvider dbProvider = new DbAuditProvider();
boolean isAuditToDbAsync = BaseAuditProvider.getBooleanProperty(props, DbAuditProvider.AUDIT_DB_IS_ASYNC_PROP, false);
@@ -128,6 +132,8 @@ public class AuditProviderFactory {
}
if(isAuditToHdfsEnabled) {
+ LOG.info("HdfsAuditProvider is enabled");
+
HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
boolean isAuditToHdfsAsync = BaseAuditProvider.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false);
@@ -156,7 +162,20 @@ public class AuditProviderFactory {
providers.add(kafkaProvider);
}
}
-
+
+ if(isAuditToSolrEnabled) {
+ LOG.info("SolrAuditProvider is enabled");
+ SolrAuditProvider solrProvider = new SolrAuditProvider();
+ solrProvider.init(props);
+
+ if( solrProvider.isAsync()) {
+ AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", solrProvider.getMaxQueueSize(), solrProvider.getMaxFlushInterval(), solrProvider);
+ providers.add(asyncProvider);
+ } else {
+ providers.add(solrProvider);
+ }
+ }
+
if(isAuditToLog4jEnabled) {
Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
index 14e6220..a068b8f 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
@@ -40,7 +40,7 @@ public abstract class BaseAuditProvider implements AuditProvider {
private int maxQueueSize = AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT;
private int maxFlushInterval = AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT;
-
+ protected Properties props = null;
public BaseAuditProvider() {
}
@@ -48,7 +48,8 @@ public abstract class BaseAuditProvider implements AuditProvider {
@Override
public void init(Properties props) {
LOG.info("BaseAuditProvider.init()");
-
+ this.props = props;
+
mLogFailureReportMinIntervalInMs = getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
index 54e73ea..0ec8790 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
@@ -117,7 +117,7 @@ public class KafkaAuditProvider extends BaseAuditProvider {
}
} catch (Throwable t) {
LOG.error("Error sending message to Kafka topic. topic=" + topic
- + ", message=" + message);
+ + ", message=" + message, t);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
new file mode 100644
index 0000000..1b463e6
--- /dev/null
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.provider.solr;
+
+import java.util.Date;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.BaseAuditProvider;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrInputDocument;
+
+public class SolrAuditProvider extends BaseAuditProvider {
+ private static final Log LOG = LogFactory.getLog(SolrAuditProvider.class);
+
+ public static final String AUDIT_MAX_QUEUE_SIZE_PROP = "xasecure.audit.solr.async.max.queue.size";
+ public static final String AUDIT_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.solr.async.max.flush.interval.ms";
+ public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms";
+
+ static final Object lock = new Object();
+ SolrClient solrClient = null;
+ Date lastConnectTime = null;
+ long lastFailTime = 0;
+
+ int retryWaitTime = 30000;
+
+ public SolrAuditProvider() {
+ }
+
+ @Override
+ public void init(Properties props) {
+ LOG.info("init() called");
+ super.init(props);
+
+ setMaxQueueSize(BaseAuditProvider.getIntProperty(props,
+ AUDIT_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT));
+ setMaxFlushInterval(BaseAuditProvider.getIntProperty(props,
+ AUDIT_MAX_QUEUE_SIZE_PROP,
+ AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT));
+ retryWaitTime = BaseAuditProvider.getIntProperty(props,
+ AUDIT_RETRY_WAIT_PROP, retryWaitTime);
+ }
+
+ void connect() {
+ if (solrClient == null) {
+ synchronized (lock) {
+ if (solrClient == null) {
+ String solrURL = BaseAuditProvider.getStringProperty(props,
+ "xasecure.audit.solr.solr_url");
+
+ if (lastConnectTime != null) {
+ // Let's wait for enough time before retrying
+ long diff = System.currentTimeMillis()
+ - lastConnectTime.getTime();
+ if (diff < retryWaitTime) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignore connecting to solr url="
+ + solrURL + ", lastConnect=" + diff
+ + "ms");
+ }
+ return;
+ }
+ }
+ lastConnectTime = new Date();
+
+ if (solrURL == null || solrURL.isEmpty()) {
+ LOG.fatal("Solr URL for Audit is empty");
+ return;
+ }
+
+ try {
+ // TODO: Need to support SolrCloud also
+ solrClient = new HttpSolrClient(solrURL);
+ if (solrClient instanceof HttpSolrClient) {
+ HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient;
+ httpSolrClient.setAllowCompression(true);
+ httpSolrClient.setConnectionTimeout(1000);
+ // solrClient.setSoTimeout(10000);
+ httpSolrClient.setMaxRetries(1);
+ }
+ } catch (Throwable t) {
+ LOG.fatal("Can't connect to Solr server. URL="
+ + solrURL, t);
+ }
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger.
+ * audit.model.AuditEventBase)
+ */
+ @Override
+ public void log(AuditEventBase event) {
+ if (!(event instanceof AuthzAuditEvent)) {
+ LOG.error(event.getClass().getName()
+ + " audit event class type is not supported");
+ return;
+ }
+ AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+ // TODO: This should be done at a higher level
+
+ if (authzEvent.getAgentHostname() == null) {
+ authzEvent.setAgentHostname(MiscUtil.getHostname());
+ }
+
+ if (authzEvent.getLogType() == null) {
+ authzEvent.setLogType("RangerAudit");
+ }
+
+ if (authzEvent.getEventId() == null) {
+ authzEvent.setEventId(MiscUtil.generateUniqueId());
+ }
+
+ try {
+ if (solrClient == null) {
+ connect();
+ if (solrClient == null) {
+ // Solr is still not initialized. So need to throw error
+ return;
+ }
+ }
+
+ if (lastFailTime > 0) {
+ long diff = System.currentTimeMillis() - lastFailTime;
+ if (diff < retryWaitTime) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignore sending audit. lastConnect=" + diff
+ + " ms");
+ }
+ return;
+ }
+ }
+ // Convert AuditEventBase to Solr document
+ SolrInputDocument document = toSolrDoc(authzEvent);
+ UpdateResponse response = solrClient.add(document);
+ if (response.getStatus() != 0) {
+ lastFailTime = System.currentTimeMillis();
+
+ // System.out.println("Response=" + response.toString()
+ // + ", status= " + response.getStatus() + ", event="
+ // + event);
+ // throw new Exception("Aborting. event=" + event +
+ // ", response="
+ // + response.toString());
+ } else {
+ lastFailTime = 0;
+ }
+
+ } catch (Throwable t) {
+ LOG.error("Error sending message to Solr", t);
+ }
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#start()
+ */
+ @Override
+ public void start() {
+ connect();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#stop()
+ */
+ @Override
+ public void stop() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete()
+ */
+ @Override
+ public void waitToComplete() {
+ // TODO Auto-generated method stub
+
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#isFlushPending()
+ */
+ @Override
+ public boolean isFlushPending() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#getLastFlushTime()
+ */
+ @Override
+ public long getLastFlushTime() {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+ */
+ @Override
+ public void flush() {
+ // TODO Auto-generated method stub
+
+ }
+
+ SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", auditEvent.getEventId());
+ doc.addField("access", auditEvent.getAccessType());
+ doc.addField("enforcer", auditEvent.getAclEnforcer());
+ doc.addField("agent", auditEvent.getAgentId());
+ doc.addField("repo", auditEvent.getRepositoryName());
+ doc.addField("sess", auditEvent.getSessionId());
+ doc.addField("reqUser", auditEvent.getUser());
+ doc.addField("reqData", auditEvent.getRequestData());
+ doc.addField("resource", auditEvent.getResourcePath());
+ doc.addField("cliIP", auditEvent.getClientIP());
+ doc.addField("logType", auditEvent.getLogType());
+ doc.addField("result", auditEvent.getAccessResult());
+ doc.addField("policy", auditEvent.getPolicyId());
+ doc.addField("repoType", auditEvent.getRepositoryType());
+ doc.addField("resType", auditEvent.getResourceType());
+ doc.addField("reason", auditEvent.getResultReason());
+ doc.addField("action", auditEvent.getAction());
+ doc.addField("evtTime", auditEvent.getEventTime());
+ return doc;
+ }
+
+ public boolean isAsync() {
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
index e5c381a..221d20a 100644
--- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg
+++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/ranger-hbase-audit.xml b/hbase-agent/conf/ranger-hbase-audit.xml
index b39696b..e5bfb89 100644
--- a/hbase-agent/conf/ranger-hbase-audit.xml
+++ b/hbase-agent/conf/ranger-hbase-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index 5a81ad4..7ff29c9 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
index e5c381a..8d31016 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
+++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
@@ -31,3 +31,14 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml
index d26345d..09114ad 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit.xml
+++ b/hdfs-agent/conf/ranger-hdfs-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 93790e3..2e1b61a 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -89,6 +89,13 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg
index 9fa7608..2d6d414 100644
--- a/hive-agent/conf/ranger-hive-audit-changes.cfg
+++ b/hive-agent/conf/ranger-hive-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml
index d011b24..e753336 100644
--- a/hive-agent/conf/ranger-hive-audit.xml
+++ b/hive-agent/conf/ranger-hive-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index bbd1849..75b1b5d 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg
index 7ae334e..f97d10f 100644
--- a/knox-agent/conf/ranger-knox-audit-changes.cfg
+++ b/knox-agent/conf/ranger-knox-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/ranger-knox-audit.xml b/knox-agent/conf/ranger-knox-audit.xml
index 0fdcefc..6f0adb9 100644
--- a/knox-agent/conf/ranger-knox-audit.xml
+++ b/knox-agent/conf/ranger-knox-audit.xml
@@ -183,4 +183,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index d821c5d..ecd9813 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -92,6 +92,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
index 4f2c5a2..e0dbea2 100644
--- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
+++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS
xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists
+#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists
+#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit.xml
----------------------------------------------------------------------
diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml
index c0096a4..f1e9687 100644
--- a/plugin-yarn/conf/ranger-yarn-audit.xml
+++ b/plugin-yarn/conf/ranger-yarn-audit.xml
@@ -184,4 +184,52 @@
<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
<value>30000</value>
</property>
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.ranger.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/scripts/install.properties
----------------------------------------------------------------------
diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties
index d2d1ffe..bbe9f7f 100644
--- a/plugin-yarn/scripts/install.properties
+++ b/plugin-yarn/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
#
# SSL Client Certificate Information
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ef39d37..00c8339 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
<module>knox-agent</module>
<module>storm-agent</module>
<module>plugin-yarn</module>
+ <module>ranger_solrj</module>
<module>security-admin</module>
<module>ugsync</module>
<module>unixauthclient</module>
@@ -129,6 +130,9 @@
<hamcrest.all.version>1.3</hamcrest.all.version>
<hbase.version>0.99.2</hbase.version>
<hive.version>1.2.0-SNAPSHOT</hive.version>
+ <httpcomponent.httpmime.version>4.2.5</httpcomponent.httpmime.version>
+ <httpcomponent.httpclient.version>4.2.5</httpcomponent.httpclient.version>
+ <httpcomponent.httpcore.version>4.2.5</httpcomponent.httpcore.version>
<calcite.version>0.9.2-incubating</calcite.version>
<tez.version>0.5.2</tez.version>
<javassist.version>3.12.1.GA</javassist.version>
@@ -138,16 +142,20 @@
<jersey-bundle.version>1.17.1</jersey-bundle.version>
<jersey-client.version>2.6</jersey-client.version>
<junit.version>4.11</junit.version>
+ <kafka.version>0.8.2.0</kafka.version>
<mockito.version>1.8.4</mockito.version>
<hamcrest-version>1.3</hamcrest-version>
<knox.gateway.version>0.5.0</knox.gateway.version>
<local.lib.dir>${project.basedir}/../lib/local</local.lib.dir>
<log4j.version>1.2.17</log4j.version>
<mysql-connector-java.version>5.1.31</mysql-connector-java.version>
+ <noggit.version>0.6</noggit.version>
<owasp-java-html-sanitizer.version>r239</owasp-java-html-sanitizer.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<security-agent-install-dir>hadoop-security/plugins</security-agent-install-dir>
<slf4j-api.version>1.7.5</slf4j-api.version>
+ <!--<solr.version>5.0.0</solr.version>-->
+ <ranger.solrj.version>0.4.0</ranger.solrj.version>
<springframework.spring.version>2.5.6</springframework.spring.version>
<!--
<springframework.spring.version>3.1.3.RELEASE</springframework.spring.version>
@@ -162,6 +170,7 @@
<tomcat.commons.el.version>5.5.23</tomcat.commons.el.version>
<tomcat.embed.version>7.0.55</tomcat.embed.version>
<velocity.version>1.7</velocity.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
<powermock.version>1.5.6</powermock.version>
<aspectj.version>1.8.2</aspectj.version>
<findbugs.plugin.version>3.0.0</findbugs.plugin.version>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/.gitignore
----------------------------------------------------------------------
diff --git a/ranger_solrj/.gitignore b/ranger_solrj/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/ranger_solrj/.gitignore
@@ -0,0 +1 @@
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/pom.xml
----------------------------------------------------------------------
diff --git a/ranger_solrj/pom.xml b/ranger_solrj/pom.xml
new file mode 100644
index 0000000..2b86140
--- /dev/null
+++ b/ranger_solrj/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>0.4.0</version>
+ </parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger_solrj</artifactId>
+ <version>0.4.0</version>
+ <name>ranger_solrj</name>
+ <url>http://maven.apache.org</url>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>${commons.io.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponent.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponent.httpcore.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpmime</artifactId>
+ <version>${httpcomponent.httpmime.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.noggit</groupId>
+ <artifactId>noggit</artifactId>
+ <version>${noggit.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j-api.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
new file mode 100644
index 0000000..d5c3b38
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import java.io.Reader;
+import java.io.InputStream;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public abstract class ResponseParser
+{
+ public abstract String getWriterType(); // for example: wt=XML, JSON, etc
+
+ public abstract NamedList<Object> processResponse(InputStream body, String encoding);
+
+ public abstract NamedList<Object> processResponse(Reader reader);
+
+ /**
+ * A well behaved ResponseParser will return its content-type.
+ *
+ * @return the content-type this parser expects to parse
+ */
+ public String getContentType() {
+ return null;
+ }
+
+ /**
+ * @return the version param passed to solr
+ */
+ public String getVersion()
+ {
+ return "2.2";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
new file mode 100644
index 0000000..28b7f4f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import org.apache.solr.client.solrj.SolrRequest.METHOD;
+import org.apache.solr.client.solrj.beans.DocumentObjectBinder;
+import org.apache.solr.client.solrj.impl.StreamingBinaryResponseParser;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.SolrPing;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Abstraction through which all communication with a Solr server may be routed
+ *
+ * @since 5.0, replaced {@code SolrServer}
+ */
+public abstract class SolrClient implements Serializable, Closeable {
+
+ private static final long serialVersionUID = 1L;
+ private DocumentObjectBinder binder;
+
+ /**
+ * Adds a collection of documents
+ * @param docs the collection of documents
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException {
+ return add(docs, -1);
+ }
+
+ /**
+ * Adds a collection of documents, specifying max time before they become committed
+ * @param docs the collection of documents
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.add(docs);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Adds a collection of beans
+ * @param beans the collection of beans
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse addBeans(Collection<?> beans) throws SolrServerException, IOException {
+ return addBeans(beans, -1);
+ }
+
+ /**
+ * Adds a collection of beans specifying max time before they become committed
+ * @param beans the collection of beans
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException {
+ DocumentObjectBinder binder = this.getBinder();
+ ArrayList<SolrInputDocument> docs = new ArrayList<>(beans.size());
+ for (Object bean : beans) {
+ docs.add(binder.toSolrInputDocument(bean));
+ }
+ return add(docs, commitWithinMs);
+ }
+
+ /**
+ * Adds a single document
+ * @param doc the input document
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException {
+ return add(doc, -1);
+ }
+
+ /**
+ * Adds a single document specifying max time before it becomes committed
+ * @param doc the input document
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.add(doc);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Adds a single bean
+ * @param obj the input bean
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse addBean(Object obj) throws IOException, SolrServerException {
+ return addBean(obj, -1);
+ }
+
+ /**
+ * Adds a single bean specifying max time before it becomes committed
+ * @param obj the input bean
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since solr 3.5
+ */
+ public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException {
+ return add(getBinder().toSolrInputDocument(obj),commitWithinMs);
+ }
+
+ /**
+ * Performs an explicit commit, causing pending documents to be committed for indexing
+ * <p>
+ * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse commit() throws SolrServerException, IOException {
+ return commit(true, true);
+ }
+
+ /**
+ * Performs an explicit optimize, causing a merge of all segments to one.
+ * <p>
+ * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
+ * <p>
+ * Note: In most cases it is not required to do explicit optimize
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse optimize() throws SolrServerException, IOException {
+ return optimize(true, true, 1);
+ }
+
+ /**
+ * Performs an explicit commit, causing pending documents to be committed for indexing
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
+ return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher).process( this );
+ }
+
+ /**
+ * Performs an explicit commit, causing pending documents to be committed for indexing
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @param softCommit makes index changes visible while neither fsync-ing index files nor writing a new index descriptor
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException {
+ return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher, softCommit).process( this );
+ }
+
+ /**
+ * Performs an explicit optimize, causing a merge of all segments to one.
+ * <p>
+ * Note: In most cases it is not required to do explicit optimize
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
+ return optimize(waitFlush, waitSearcher, 1);
+ }
+
+ /**
+ * Performs an explicit optimize, causing a merge of all segments to one.
+ * <p>
+ * Note: In most cases it is not required to do explicit optimize
+ * @param waitFlush block until index changes are flushed to disk
+ * @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
+ * @param maxSegments optimizes down to at most this number of segments
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException {
+ return new UpdateRequest().setAction(UpdateRequest.ACTION.OPTIMIZE, waitFlush, waitSearcher, maxSegments).process( this );
+ }
+
+ /**
+ * Performs a rollback of all non-committed documents pending.
+ * <p>
+ * Note that this is not a true rollback as in databases. Content you have previously
+ * added may have been committed due to autoCommit, buffer full, other client performing
+ * a commit etc.
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse rollback() throws SolrServerException, IOException {
+ return new UpdateRequest().rollback().process( this );
+ }
+
+ /**
+ * Deletes a single document by unique ID
+ * @param id the ID of the document to delete
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse deleteById(String id) throws SolrServerException, IOException {
+ return deleteById(id, -1);
+ }
+
+ /**
+ * Deletes a single document by unique ID, specifying max time before commit
+ * @param id the ID of the document to delete
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since 3.6
+ */
+ public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.deleteById(id);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Deletes a list of documents by unique ID
+ * @param ids the list of document IDs to delete
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException {
+ return deleteById(ids, -1);
+ }
+
+ /**
+ * Deletes a list of documents by unique ID, specifying max time before commit
+ * @param ids the list of document IDs to delete
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since 3.6
+ */
+ public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.deleteById(ids);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Deletes documents from the index based on a query
+ * @param query the query expressing what documents to delete
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException {
+ return deleteByQuery(query, -1);
+ }
+
+ /**
+ * Deletes documents from the index based on a query, specifying max time before commit
+ * @param query the query expressing what documents to delete
+ * @param commitWithinMs max time (in ms) before a commit will happen
+ * @throws IOException If there is a low-level I/O error.
+ * @since 3.6
+ */
+ public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ req.deleteByQuery(query);
+ req.setCommitWithin(commitWithinMs);
+ return req.process(this);
+ }
+
+ /**
+ * Issues a ping request to check if the server is alive
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public SolrPingResponse ping() throws SolrServerException, IOException {
+ return new SolrPing().process(this);
+ }
+
+ /**
+ * Performs a query to the Solr server
+ * @param params an object holding all key/value parameters to send along the request
+ */
+ public QueryResponse query(SolrParams params) throws SolrServerException, IOException {
+ return new QueryRequest(params).process(this);
+ }
+
+ /**
+ * Performs a query to the Solr server
+ * @param params an object holding all key/value parameters to send along the request
+ * @param method specifies the HTTP method to use for the request, such as GET or POST
+ */
+ public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException, IOException {
+ return new QueryRequest(params, method).process(this);
+ }
+
+ /**
+ * Query solr, and stream the results. Unlike the standard query, this will
+ * send events for each Document rather then add them to the QueryResponse.
+ *
+ * Although this function returns a 'QueryResponse' it should be used with care
+ * since it excludes anything that was passed to callback. Also note that
+ * future version may pass even more info to the callback and may not return
+ * the results in the QueryResponse.
+ *
+ * @since solr 4.0
+ */
+ public QueryResponse queryAndStreamResponse(SolrParams params, StreamingResponseCallback callback) throws SolrServerException, IOException
+ {
+ ResponseParser parser = new StreamingBinaryResponseParser(callback);
+ QueryRequest req = new QueryRequest(params);
+ req.setStreamingResponseCallback(callback);
+ req.setResponseParser(parser);
+ return req.process(this);
+ }
+
+ /**
+ * Retrieves the SolrDocument associated with the given identifier.
+ *
+ * @return retrieved SolrDocument, null if no document is found.
+ */
+ public SolrDocument getById(String id) throws SolrServerException, IOException {
+ return getById(id, null);
+ }
+
+ /**
+ * Retrieves the SolrDocument associated with the given identifier and uses
+ * the SolrParams to execute the request.
+ *
+ * @return retrieved SolrDocument, null if no document is found.
+ */
+ public SolrDocument getById(String id, SolrParams params) throws SolrServerException, IOException {
+ SolrDocumentList docs = getById(Arrays.asList(id), params);
+ if (!docs.isEmpty()) {
+ return docs.get(0);
+ }
+ return null;
+ }
+
+ /**
+ * Retrieves the SolrDocuments associated with the given identifiers.
+ * If a document was not found, it will not be added to the SolrDocumentList.
+ */
+ public SolrDocumentList getById(Collection<String> ids) throws SolrServerException, IOException {
+ return getById(ids, null);
+ }
+
+ /**
+ * Retrieves the SolrDocuments associated with the given identifiers and uses
+ * the SolrParams to execute the request.
+ * If a document was not found, it will not be added to the SolrDocumentList.
+ */
+ public SolrDocumentList getById(Collection<String> ids, SolrParams params) throws SolrServerException, IOException {
+ if (ids == null || ids.isEmpty()) {
+ throw new IllegalArgumentException("Must provide an identifier of a document to retrieve.");
+ }
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams(params);
+ if (StringUtils.isEmpty(reqParams.get(CommonParams.QT))) {
+ reqParams.set(CommonParams.QT, "/get");
+ }
+ reqParams.set("ids", (String[]) ids.toArray());
+
+ return query(reqParams).getResults();
+ }
+
+ /**
+ * SolrServer implementations need to implement how a request is actually processed
+ */
+ public abstract NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException;
+
+ public DocumentObjectBinder getBinder() {
+ if(binder == null){
+ binder = new DocumentObjectBinder();
+ }
+ return binder;
+ }
+
+ /**
+ * Release allocated resources.
+ *
+ * @since solr 4.0
+ * @deprecated Use close() instead.
+ */
+ @Deprecated
+ public abstract void shutdown();
+
+ //@SuppressWarnings("deprecation")
+ public void close() throws IOException {
+ shutdown();
+ }
+}
[11/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java
new file mode 100644
index 0000000..66ef535
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/FieldAnalysisRequest.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.FieldAnalysisResponse;
+import org.apache.solr.common.params.AnalysisParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * A request for the org.apache.solr.handler.FieldAnalysisRequestHandler.
+ *
+ *
+ * @since solr.14
+ */
+public class FieldAnalysisRequest extends SolrRequest<FieldAnalysisResponse> {
+
+ private String fieldValue;
+ private String query;
+ private boolean showMatch;
+ private List<String> fieldNames;
+ private List<String> fieldTypes;
+
+ /**
+ * Constructs a new FieldAnalysisRequest with a default uri of "/fieldanalysis".
+ */
+ public FieldAnalysisRequest() {
+ super(METHOD.GET, "/analysis/field");
+ }
+
+ /**
+ * Constructs a new FieldAnalysisRequest with a given uri.
+ *
+ * @param uri the uri of the request handler.
+ */
+ public FieldAnalysisRequest(String uri) {
+ super(METHOD.GET, uri);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return null;
+ }
+
+ @Override
+ protected FieldAnalysisResponse createResponse(SolrClient client) {
+ if (fieldTypes == null && fieldNames == null) {
+ throw new IllegalStateException("At least one field type or field name need to be specified");
+ }
+ if (fieldValue == null) {
+ throw new IllegalStateException("The field value must be set");
+ }
+ return new FieldAnalysisResponse();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(AnalysisParams.FIELD_VALUE, fieldValue);
+ if (query != null) {
+ params.add(AnalysisParams.QUERY, query);
+ params.add(AnalysisParams.SHOW_MATCH, String.valueOf(showMatch));
+ }
+ if (fieldNames != null) {
+ String fieldNameValue = listToCommaDelimitedString(fieldNames);
+ params.add(AnalysisParams.FIELD_NAME, fieldNameValue);
+ }
+ if (fieldTypes != null) {
+ String fieldTypeValue = listToCommaDelimitedString(fieldTypes);
+ params.add(AnalysisParams.FIELD_TYPE, fieldTypeValue);
+ }
+ return params;
+ }
+
+ //================================================ Helper Methods ==================================================
+
+ /**
+ * Convers the given list of string to a comma-separated string.
+ *
+ * @param list The list of string.
+ *
+ * @return The comma-separated string.
+ */
+ static String listToCommaDelimitedString(List<String> list) {
+ StringBuilder result = new StringBuilder();
+ for (String str : list) {
+ if (result.length() > 0) {
+ result.append(",");
+ }
+ result.append(str);
+ }
+ return result.toString();
+ }
+
+
+ //============================================ Setter/Getter Methods ===============================================
+
+ /**
+ * Sets the field value to be analyzed.
+ *
+ * @param fieldValue The field value to be analyzed.
+ *
+ * @return This FieldAnalysisRequest (fluent interface support).
+ */
+ public FieldAnalysisRequest setFieldValue(String fieldValue) {
+ this.fieldValue = fieldValue;
+ return this;
+ }
+
+ /**
+ * Returns the field value that will be analyzed when this request is processed.
+ *
+ * @return The field value that will be analyzed when this request is processed.
+ */
+ public String getFieldValue() {
+ return fieldValue;
+ }
+
+ /**
+ * Sets the query to be analyzed. May be {@code null} indicated that no query analysis should take place.
+ *
+ * @param query The query to be analyzed.
+ *
+ * @return This FieldAnalysisRequest (fluent interface support).
+ */
+ public FieldAnalysisRequest setQuery(String query) {
+ this.query = query;
+ return this;
+ }
+
+ /**
+ * Returns the query that will be analyzed. May return {@code null} indicating that no query analysis will be
+ * performed.
+ *
+ * @return The query that will be analyzed. May return {@code null} indicating that no query analysis will be
+ * performed.
+ */
+ public String getQuery() {
+ return query;
+ }
+
+ /**
+ * Sets whether index time tokens that match query time tokens should be marked as a "match". By default this is set
+ * to {@code false}. Obviously, this flag is ignored if when the query is set to {@code null}.
+ *
+ * @param showMatch Sets whether index time tokens that match query time tokens should be marked as a "match".
+ *
+ * @return This FieldAnalysisRequest (fluent interface support).
+ */
+ public FieldAnalysisRequest setShowMatch(boolean showMatch) {
+ this.showMatch = showMatch;
+ return this;
+ }
+
+ /**
+ * Returns whether index time tokens that match query time tokens should be marked as a "match".
+ *
+ * @return Whether index time tokens that match query time tokens should be marked as a "match".
+ *
+ * @see #setShowMatch(boolean)
+ */
+ public boolean isShowMatch() {
+ return showMatch;
+ }
+
+ /**
+ * Adds the given field name for analysis.
+ *
+ * @param fieldName A field name on which the analysis should be performed.
+ *
+ * @return this FieldAnalysisRequest (fluent interface support).
+ */
+ public FieldAnalysisRequest addFieldName(String fieldName) {
+ if (fieldNames == null) {
+ fieldNames = new LinkedList<>();
+ }
+ fieldNames.add(fieldName);
+ return this;
+ }
+
+ /**
+ * Sets the field names on which the analysis should be performed.
+ *
+ * @param fieldNames The field names on which the analysis should be performed.
+ *
+ * @return this FieldAnalysisRequest (fluent interface support).
+ */
+ public FieldAnalysisRequest setFieldNames(List<String> fieldNames) {
+ this.fieldNames = fieldNames;
+ return this;
+ }
+
+ /**
+ * Returns a list of field names the analysis should be performed on. May return {@code null} indicating that no
+ * analysis will be performed on field names.
+ *
+ * @return The field names the analysis should be performed on.
+ */
+ public List<String> getFieldNames() {
+ return fieldNames;
+ }
+
+ /**
+ * Adds the given field type for analysis.
+ *
+ * @param fieldTypeName A field type name on which analysis should be performed.
+ *
+ * @return This FieldAnalysisRequest (fluent interface support).
+ */
+ public FieldAnalysisRequest addFieldType(String fieldTypeName) {
+ if (fieldTypes == null) {
+ fieldTypes = new LinkedList<>();
+ }
+ fieldTypes.add(fieldTypeName);
+ return this;
+ }
+
+/**
+ * Sets the field types on which analysis should be performed.
+ *
+ * @param fieldTypes The field type names on which analysis should be performed.
+ *
+ * @return This FieldAnalysisRequest (fluent interface support).
+ */
+ public FieldAnalysisRequest setFieldTypes(List<String> fieldTypes) {
+ this.fieldTypes = fieldTypes;
+ return this;
+ }
+
+
+ /**
+ * Returns a list of field types the analysis should be performed on. May return {@code null} indicating that no
+ * analysis will be peformed on field types.
+ *
+ * @return The field types the analysis should be performed on.
+ */
+ public List<String> getFieldTypes() {
+ return fieldTypes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/IsUpdateRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/IsUpdateRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/IsUpdateRequest.java
new file mode 100644
index 0000000..ec49e30
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/IsUpdateRequest.java
@@ -0,0 +1,26 @@
+package org.apache.solr.client.solrj.request;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ * Marker class so that we can determine which requests are updates.
+ */
+public interface IsUpdateRequest {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
new file mode 100644
index 0000000..7bb64b4
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/JavaBinUpdateRequestCodec.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.request;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.DataInputInputStream;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ * Provides methods for marshalling an UpdateRequest to a NamedList which can be serialized in the javabin format and
+ * vice versa.
+ *
+ *
+ * @see org.apache.solr.common.util.JavaBinCodec
+ * @since solr 1.4
+ */
+public class JavaBinUpdateRequestCodec {
+
+ /**
+ * Converts an UpdateRequest to a NamedList which can be serialized to the given OutputStream in the javabin format
+ *
+ * @param updateRequest the UpdateRequest to be written out
+ * @param os the OutputStream to which the request is to be written
+ *
+ * @throws IOException in case of an exception during marshalling or writing to the stream
+ */
+ public void marshal(UpdateRequest updateRequest, OutputStream os) throws IOException {
+ NamedList nl = new NamedList();
+ NamedList params = solrParamsToNamedList(updateRequest.getParams());
+ if (updateRequest.getCommitWithin() != -1) {
+ params.add("commitWithin", updateRequest.getCommitWithin());
+ }
+ Iterator<SolrInputDocument> docIter = null;
+
+ if(updateRequest.getDocIterator() != null){
+ docIter = updateRequest.getDocIterator();
+ }
+
+ Map<SolrInputDocument,Map<String,Object>> docMap = updateRequest.getDocumentsMap();
+
+ nl.add("params", params);// 0: params
+ if (updateRequest.getDeleteByIdMap() != null) {
+ nl.add("delByIdMap", updateRequest.getDeleteByIdMap());
+ }
+ nl.add("delByQ", updateRequest.getDeleteQuery());
+
+ if (docMap != null) {
+ nl.add("docsMap", docMap.entrySet().iterator());
+ } else {
+ if (updateRequest.getDocuments() != null) {
+ docIter = updateRequest.getDocuments().iterator();
+ }
+ nl.add("docs", docIter);
+ }
+ JavaBinCodec codec = new JavaBinCodec();
+ codec.marshal(nl, os);
+ }
+
+ /**
+ * Reads a NamedList from the given InputStream, converts it into a SolrInputDocument and passes it to the given
+ * StreamingUpdateHandler
+ *
+ * @param is the InputStream from which to read
+ * @param handler an instance of StreamingUpdateHandler to which SolrInputDocuments are streamed one by one
+ *
+ * @return the UpdateRequest
+ *
+ * @throws IOException in case of an exception while reading from the input stream or unmarshalling
+ */
+ public UpdateRequest unmarshal(InputStream is, final StreamingUpdateHandler handler) throws IOException {
+ final UpdateRequest updateRequest = new UpdateRequest();
+ List<List<NamedList>> doclist;
+ List<Entry<SolrInputDocument,Map<Object,Object>>> docMap;
+ List<String> delById;
+ Map<String,Map<String,Object>> delByIdMap;
+ List<String> delByQ;
+ final NamedList[] namedList = new NamedList[1];
+ JavaBinCodec codec = new JavaBinCodec() {
+
+ // NOTE: this only works because this is an anonymous inner class
+ // which will only ever be used on a single stream -- if this class
+ // is ever refactored, this will not work.
+ private boolean seenOuterMostDocIterator = false;
+
+ @Override
+ public NamedList readNamedList(DataInputInputStream dis) throws IOException {
+ int sz = readSize(dis);
+ NamedList nl = new NamedList();
+ if (namedList[0] == null) {
+ namedList[0] = nl;
+ }
+ for (int i = 0; i < sz; i++) {
+ String name = (String) readVal(dis);
+ Object val = readVal(dis);
+ nl.add(name, val);
+ }
+ return nl;
+ }
+
+ @Override
+ public List readIterator(DataInputInputStream fis) throws IOException {
+
+ // default behavior for reading any regular Iterator in the stream
+ if (seenOuterMostDocIterator) return super.readIterator(fis);
+
+ // special treatment for first outermost Iterator
+ // (the list of documents)
+ seenOuterMostDocIterator = true;
+ return readOuterMostDocIterator(fis);
+ }
+
+ private List readOuterMostDocIterator(DataInputInputStream fis) throws IOException {
+ NamedList params = (NamedList) namedList[0].get("params");
+ updateRequest.setParams(new ModifiableSolrParams(SolrParams.toSolrParams(params)));
+ if (handler == null) return super.readIterator(fis);
+ Integer commitWithin = null;
+ Boolean overwrite = null;
+ while (true) {
+ Object o = readVal(fis);
+ if (o == END_OBJ) break;
+ SolrInputDocument sdoc = null;
+ if (o instanceof List) {
+ sdoc = listToSolrInputDocument((List<NamedList>) o);
+ } else if (o instanceof NamedList) {
+ UpdateRequest req = new UpdateRequest();
+ req.setParams(new ModifiableSolrParams(SolrParams.toSolrParams((NamedList) o)));
+ handler.update(null, req, null, null);
+ } else if (o instanceof Map.Entry){
+ sdoc = (SolrInputDocument) ((Map.Entry) o).getKey();
+ Map p = (Map) ((Map.Entry) o).getValue();
+ if (p != null) {
+ commitWithin = (Integer) p.get(UpdateRequest.COMMIT_WITHIN);
+ overwrite = (Boolean) p.get(UpdateRequest.OVERWRITE);
+ }
+ } else {
+
+ sdoc = (SolrInputDocument) o;
+ }
+ handler.update(sdoc, updateRequest, commitWithin, overwrite);
+ }
+ return Collections.EMPTY_LIST;
+ }
+
+ };
+
+ codec.unmarshal(is);
+
+ // NOTE: if the update request contains only delete commands the params
+ // must be loaded now
+ if(updateRequest.getParams()==null) {
+ NamedList params = (NamedList) namedList[0].get("params");
+ if(params!=null) {
+ updateRequest.setParams(new ModifiableSolrParams(SolrParams.toSolrParams(params)));
+ }
+ }
+ delById = (List<String>) namedList[0].get("delById");
+ delByIdMap = (Map<String,Map<String,Object>>) namedList[0].get("delByIdMap");
+ delByQ = (List<String>) namedList[0].get("delByQ");
+ doclist = (List) namedList[0].get("docs");
+ Object docsMapObj = namedList[0].get("docsMap");
+
+ if (docsMapObj instanceof Map) {//SOLR-5762
+ docMap = new ArrayList(((Map)docsMapObj).entrySet());
+ } else {
+ docMap = (List<Entry<SolrInputDocument, Map<Object, Object>>>) docsMapObj;
+ }
+
+
+ // we don't add any docs, because they were already processed
+ // deletes are handled later, and must be passed back on the UpdateRequest
+
+ if (delById != null) {
+ for (String s : delById) {
+ updateRequest.deleteById(s);
+ }
+ }
+ if (delByIdMap != null) {
+ for (Map.Entry<String,Map<String,Object>> entry : delByIdMap.entrySet()) {
+ Map<String,Object> params = entry.getValue();
+ if (params != null) {
+ Long version = (Long) params.get(UpdateRequest.VER);
+ if (params.containsKey(UpdateRequest.ROUTE))
+ updateRequest.deleteById(entry.getKey(), (String) params.get(UpdateRequest.ROUTE));
+ else
+ updateRequest.deleteById(entry.getKey(), version);
+ } else {
+ updateRequest.deleteById(entry.getKey());
+ }
+
+ }
+ }
+ if (delByQ != null) {
+ for (String s : delByQ) {
+ updateRequest.deleteByQuery(s);
+ }
+ }
+
+ return updateRequest;
+ }
+
+ private SolrInputDocument listToSolrInputDocument(List<NamedList> namedList) {
+ SolrInputDocument doc = new SolrInputDocument();
+ for (int i = 0; i < namedList.size(); i++) {
+ NamedList nl = namedList.get(i);
+ if (i == 0) {
+ doc.setDocumentBoost(nl.getVal(0) == null ? 1.0f : (Float) nl.getVal(0));
+ } else {
+ doc.addField((String) nl.getVal(0),
+ nl.getVal(1),
+ nl.getVal(2) == null ? 1.0f : (Float) nl.getVal(2));
+ }
+ }
+ return doc;
+ }
+
+ private NamedList solrParamsToNamedList(SolrParams params) {
+ if (params == null) return new NamedList();
+ return params.toNamedList();
+ }
+
+ public static interface StreamingUpdateHandler {
+ public void update(SolrInputDocument document, UpdateRequest req, Integer commitWithin, Boolean override);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/LukeRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/LukeRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/LukeRequest.java
new file mode 100644
index 0000000..82b5330
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/LukeRequest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.LukeResponse;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public class LukeRequest extends SolrRequest<LukeResponse> {
+
+ private List<String> fields;
+ private int numTerms = -1;
+ private boolean showSchema = false;
+
+ public LukeRequest()
+ {
+ super( METHOD.GET, "/admin/luke" );
+ }
+
+ public LukeRequest( String path )
+ {
+ super( METHOD.GET, path );
+ }
+
+ //---------------------------------------------------------------------------------
+ //---------------------------------------------------------------------------------
+
+ public void addField( String f )
+ {
+ if( fields == null ) {
+ fields = new ArrayList<>();
+ }
+ fields.add( f );
+ }
+
+ public void setFields( List<String> f )
+ {
+ fields = f;
+ }
+
+ //---------------------------------------------------------------------------------
+ //---------------------------------------------------------------------------------
+
+ public boolean isShowSchema() {
+ return showSchema;
+ }
+
+ public void setShowSchema(boolean showSchema) {
+ this.showSchema = showSchema;
+ }
+
+ public int getNumTerms() {
+ return numTerms;
+ }
+
+ /**
+ * the number of terms to return for a given field. If the number is 0, it will not traverse the terms.
+ */
+ public void setNumTerms(int count) {
+ this.numTerms = count;
+ }
+
+ //---------------------------------------------------------------------------------
+ //---------------------------------------------------------------------------------
+
+ @Override
+ public Collection<ContentStream> getContentStreams() {
+ return null;
+ }
+
+ @Override
+ protected LukeResponse createResponse(SolrClient client) {
+ return new LukeResponse();
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ if( fields != null && fields.size() > 0 ) {
+ params.add( CommonParams.FL, fields.toArray( new String[fields.size()] ) );
+ }
+ if( numTerms >= 0 ) {
+ params.add( "numTerms", numTerms+"" );
+ }
+ if (showSchema) {
+ params.add("show", "schema");
+ }
+ return params;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/QueryRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/QueryRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/QueryRequest.java
new file mode 100644
index 0000000..9d47f99
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/QueryRequest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.util.Collection;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public class QueryRequest extends SolrRequest<QueryResponse> {
+
+ private SolrParams query;
+
+ public QueryRequest()
+ {
+ super( METHOD.GET, null );
+ }
+
+ public QueryRequest( SolrParams q )
+ {
+ super( METHOD.GET, null );
+ query = q;
+ }
+
+ public QueryRequest( SolrParams q, METHOD method )
+ {
+ super( method, null );
+ query = q;
+ }
+
+ /**
+ * Use the params 'QT' parameter if it exists
+ */
+ @Override
+ public String getPath() {
+ String qt = query == null ? null : query.get( CommonParams.QT );
+ if( qt == null ) {
+ qt = super.getPath();
+ }
+ if( qt != null && qt.startsWith( "/" ) ) {
+ return qt;
+ }
+ return "/select";
+ }
+
+ //---------------------------------------------------------------------------------
+ //---------------------------------------------------------------------------------
+
+ @Override
+ public Collection<ContentStream> getContentStreams() {
+ return null;
+ }
+
+ @Override
+ protected QueryResponse createResponse(SolrClient client) {
+ return new QueryResponse(client);
+ }
+
+ @Override
+ public SolrParams getParams() {
+ return query;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/RequestWriter.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/RequestWriter.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/RequestWriter.java
new file mode 100644
index 0000000..15872b2
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/RequestWriter.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+
+import java.io.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * A RequestWriter is used to write requests to Solr.
+ * <p>
+ * A subclass can override the methods in this class to supply a custom format in which a request can be sent.
+ *
+ *
+ * @since solr 1.4
+ */
+public class RequestWriter {
+ public static final Charset UTF_8 = StandardCharsets.UTF_8;
+
+ public Collection<ContentStream> getContentStreams(SolrRequest req) throws IOException {
+ if (req instanceof UpdateRequest) {
+ UpdateRequest updateRequest = (UpdateRequest) req;
+ if (isEmpty(updateRequest)) return null;
+ List<ContentStream> l = new ArrayList<>();
+ l.add(new LazyContentStream(updateRequest));
+ return l;
+ }
+ return req.getContentStreams();
+ }
+
+ private boolean isEmpty(UpdateRequest updateRequest) {
+ return isNull(updateRequest.getDocuments()) &&
+ isNull(updateRequest.getDeleteByIdMap()) &&
+ isNull(updateRequest.getDeleteQuery()) &&
+ updateRequest.getDocIterator() == null;
+ }
+
+ public String getPath(SolrRequest req) {
+ return req.getPath();
+ }
+
+ public ContentStream getContentStream(UpdateRequest req) throws IOException {
+ return new ContentStreamBase.StringStream(req.getXML());
+ }
+
+ public void write(SolrRequest request, OutputStream os) throws IOException {
+ if (request instanceof UpdateRequest) {
+ UpdateRequest updateRequest = (UpdateRequest) request;
+ BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, UTF_8));
+ updateRequest.writeXML(writer);
+ writer.flush();
+ }
+ }
+
+ public String getUpdateContentType() {
+ return ClientUtils.TEXT_XML;
+
+ }
+
+ public class LazyContentStream implements ContentStream {
+ ContentStream contentStream = null;
+ UpdateRequest req = null;
+
+ public LazyContentStream(UpdateRequest req) {
+ this.req = req;
+ }
+
+ private ContentStream getDelegate() {
+ if (contentStream == null) {
+ try {
+ contentStream = getContentStream(req);
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to write xml into a stream", e);
+ }
+ }
+ return contentStream;
+ }
+
+ @Override
+ public String getName() {
+ return getDelegate().getName();
+ }
+
+ @Override
+ public String getSourceInfo() {
+ return getDelegate().getSourceInfo();
+ }
+
+ @Override
+ public String getContentType() {
+ return getUpdateContentType();
+ }
+
+ @Override
+ public Long getSize() {
+ return getDelegate().getSize();
+ }
+
+ @Override
+ public InputStream getStream() throws IOException {
+ return getDelegate().getStream();
+ }
+
+ @Override
+ public Reader getReader() throws IOException {
+ return getDelegate().getReader();
+ }
+
+ public void writeTo(OutputStream os) throws IOException {
+ write(req, os);
+
+ }
+ }
+
+ protected boolean isNull(List l) {
+ return l == null || l.isEmpty();
+ }
+
+ protected boolean isNull(Map l) {
+ return l == null || l.isEmpty();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/SolrPing.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/SolrPing.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/SolrPing.java
new file mode 100644
index 0000000..00ad41f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/SolrPing.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.ContentStream;
+
+import java.util.Collection;
+
+/**
+ * Verify that there is a working Solr core at the URL of a {@link org.apache.solr.client.solrj.SolrClient}.
+ * To use this class, the solrconfig.xml for the relevant core must include the
+ * request handler for <code>/admin/ping</code>.
+ *
+ * @since solr 1.3
+ */
+public class SolrPing extends SolrRequest<SolrPingResponse> {
+
+ /** serialVersionUID. */
+ private static final long serialVersionUID = 5828246236669090017L;
+
+ /** Request parameters. */
+ private ModifiableSolrParams params;
+
+ /**
+ * Create a new SolrPing object.
+ */
+ public SolrPing() {
+ super(METHOD.GET, CommonParams.PING_HANDLER);
+ params = new ModifiableSolrParams();
+ }
+
+ @Override
+ public Collection<ContentStream> getContentStreams() {
+ return null;
+ }
+
+ @Override
+ protected SolrPingResponse createResponse(SolrClient client) {
+ return new SolrPingResponse();
+ }
+
+ @Override
+ public ModifiableSolrParams getParams() {
+ return params;
+ }
+
+ /**
+ * Remove the action parameter from this request. This will result in the same
+ * behavior as {@code SolrPing#setActionPing()}. For Solr server version 4.0
+ * and later.
+ *
+ * @return this
+ */
+ public SolrPing removeAction() {
+ params.remove(CommonParams.ACTION);
+ return this;
+ }
+
+ /**
+ * Set the action parameter on this request to enable. This will delete the
+ * health-check file for the Solr core. For Solr server version 4.0 and later.
+ *
+ * @return this
+ */
+ public SolrPing setActionDisable() {
+ params.set(CommonParams.ACTION, CommonParams.DISABLE);
+ return this;
+ }
+
+ /**
+ * Set the action parameter on this request to enable. This will create the
+ * health-check file for the Solr core. For Solr server version 4.0 and later.
+ *
+ * @return this
+ */
+ public SolrPing setActionEnable() {
+ params.set(CommonParams.ACTION, CommonParams.ENABLE);
+ return this;
+ }
+
+ /**
+ * Set the action parameter on this request to ping. This is the same as not
+ * including the action at all. For Solr server version 4.0 and later.
+ *
+ * @return this
+ */
+ public SolrPing setActionPing() {
+ params.set(CommonParams.ACTION, CommonParams.PING);
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/UpdateRequest.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/UpdateRequest.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/UpdateRequest.java
new file mode 100644
index 0000000..0e6580c
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/UpdateRequest.java
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.request;
+
+import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.XML;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public class UpdateRequest extends AbstractUpdateRequest {
+
+ public static final String REPFACT = "rf";
+ public static final String MIN_REPFACT = "min_rf";
+ public static final String VER = "ver";
+ public static final String ROUTE = "_route_";
+ public static final String OVERWRITE = "ow";
+ public static final String COMMIT_WITHIN = "cw";
+ private Map<SolrInputDocument,Map<String,Object>> documents = null;
+ private Iterator<SolrInputDocument> docIterator = null;
+ private Map<String,Map<String,Object>> deleteById = null;
+ private List<String> deleteQuery = null;
+
+ public UpdateRequest() {
+ super(METHOD.POST, "/update");
+ }
+
+ public UpdateRequest(String url) {
+ super(METHOD.POST, url);
+ }
+
+ // ---------------------------------------------------------------------------
+ // ---------------------------------------------------------------------------
+
+ /**
+ * clear the pending documents and delete commands
+ */
+ public void clear() {
+ if (documents != null) {
+ documents.clear();
+ }
+ if (deleteById != null) {
+ deleteById.clear();
+ }
+ if (deleteQuery != null) {
+ deleteQuery.clear();
+ }
+ }
+
+ // ---------------------------------------------------------------------------
+ // ---------------------------------------------------------------------------
+
+ public UpdateRequest add(final SolrInputDocument doc) {
+ if (documents == null) {
+ documents = new LinkedHashMap<>();
+ }
+ documents.put(doc, null);
+ return this;
+ }
+
+ public UpdateRequest add(final SolrInputDocument doc, Boolean overwrite) {
+ return add(doc, null, overwrite);
+ }
+
+ public UpdateRequest add(final SolrInputDocument doc, Integer commitWithin) {
+ return add(doc, commitWithin, null);
+ }
+
+ public UpdateRequest add(final SolrInputDocument doc, Integer commitWithin,
+ Boolean overwrite) {
+ if (documents == null) {
+ documents = new LinkedHashMap<>();
+ }
+ Map<String,Object> params = new HashMap<>(2);
+ if (commitWithin != null) params.put(COMMIT_WITHIN, commitWithin);
+ if (overwrite != null) params.put(OVERWRITE, overwrite);
+
+ documents.put(doc, params);
+
+ return this;
+ }
+
+ public UpdateRequest add(final Collection<SolrInputDocument> docs) {
+ if (documents == null) {
+ documents = new LinkedHashMap<>();
+ }
+ for (SolrInputDocument doc : docs) {
+ documents.put(doc, null);
+ }
+ return this;
+ }
+
+ public UpdateRequest deleteById(String id) {
+ if (deleteById == null) {
+ deleteById = new LinkedHashMap<>();
+ }
+ deleteById.put(id, null);
+ return this;
+ }
+
+ public UpdateRequest deleteById(String id, String route) {
+ return deleteById(id, route, null);
+ }
+
+ public UpdateRequest deleteById(String id, String route, Long version) {
+ if (deleteById == null) {
+ deleteById = new LinkedHashMap<>();
+ }
+ Map<String, Object> params = (route == null && version == null) ? null : new HashMap<String, Object>(1);
+ if (version != null)
+ params.put(VER, version);
+ if (route != null)
+ params.put(ROUTE, route);
+ deleteById.put(id, params);
+ return this;
+ }
+
+
+ public UpdateRequest deleteById(List<String> ids) {
+ if (deleteById == null) {
+ deleteById = new LinkedHashMap<>();
+ }
+
+ for (String id : ids) {
+ deleteById.put(id, null);
+ }
+
+ return this;
+ }
+
+ public UpdateRequest deleteById(String id, Long version) {
+ return deleteById(id, null, version);
+ }
+
+ public UpdateRequest deleteByQuery(String q) {
+ if (deleteQuery == null) {
+ deleteQuery = new ArrayList<>();
+ }
+ deleteQuery.add(q);
+ return this;
+ }
+
+ /**
+ * @param router to route updates with
+ * @param col DocCollection for the updates
+ * @param urlMap of the cluster
+ * @param params params to use
+ * @param idField the id field
+ * @return a Map of urls to requests
+ */
+ public Map<String,LBHttpSolrClient.Req> getRoutes(DocRouter router,
+ DocCollection col, Map<String,List<String>> urlMap,
+ ModifiableSolrParams params, String idField) {
+
+ if ((documents == null || documents.size() == 0)
+ && (deleteById == null || deleteById.size() == 0)) {
+ return null;
+ }
+
+ Map<String,LBHttpSolrClient.Req> routes = new HashMap<>();
+ if (documents != null) {
+ Set<Entry<SolrInputDocument,Map<String,Object>>> entries = documents.entrySet();
+ for (Entry<SolrInputDocument,Map<String,Object>> entry : entries) {
+ SolrInputDocument doc = entry.getKey();
+ Object id = doc.getFieldValue(idField);
+ if (id == null) {
+ return null;
+ }
+ Slice slice = router.getTargetSlice(id
+ .toString(), doc, null, null, col);
+ if (slice == null) {
+ return null;
+ }
+ List<String> urls = urlMap.get(slice.getName());
+ String leaderUrl = urls.get(0);
+ LBHttpSolrClient.Req request = (LBHttpSolrClient.Req) routes
+ .get(leaderUrl);
+ if (request == null) {
+ UpdateRequest updateRequest = new UpdateRequest();
+ updateRequest.setMethod(getMethod());
+ updateRequest.setCommitWithin(getCommitWithin());
+ updateRequest.setParams(params);
+ updateRequest.setPath(getPath());
+ request = new LBHttpSolrClient.Req(updateRequest, urls);
+ routes.put(leaderUrl, request);
+ }
+ UpdateRequest urequest = (UpdateRequest) request.getRequest();
+ Map<String,Object> value = entry.getValue();
+ Boolean ow = null;
+ if (value != null) {
+ ow = (Boolean) value.get(OVERWRITE);
+ }
+ if (ow != null) {
+ urequest.add(doc, ow);
+ } else {
+ urequest.add(doc);
+ }
+ }
+ }
+
+ // Route the deleteById's
+
+ if (deleteById != null) {
+
+ Iterator<Map.Entry<String,Map<String,Object>>> entries = deleteById.entrySet()
+ .iterator();
+ while (entries.hasNext()) {
+
+ Map.Entry<String,Map<String,Object>> entry = entries.next();
+
+ String deleteId = entry.getKey();
+ Map<String,Object> map = entry.getValue();
+ Long version = null;
+ if (map != null) {
+ version = (Long) map.get(VER);
+ }
+ Slice slice = router.getTargetSlice(deleteId, null, null, null, col);
+ if (slice == null) {
+ return null;
+ }
+ List<String> urls = urlMap.get(slice.getName());
+ String leaderUrl = urls.get(0);
+ LBHttpSolrClient.Req request = routes.get(leaderUrl);
+ if (request != null) {
+ UpdateRequest urequest = (UpdateRequest) request.getRequest();
+ urequest.deleteById(deleteId, version);
+ } else {
+ UpdateRequest urequest = new UpdateRequest();
+ urequest.setParams(params);
+ urequest.deleteById(deleteId, version);
+ request = new LBHttpSolrClient.Req(urequest, urls);
+ routes.put(leaderUrl, request);
+ }
+ }
+ }
+
+ return routes;
+ }
+
+ public void setDocIterator(Iterator<SolrInputDocument> docIterator) {
+ this.docIterator = docIterator;
+ }
+
+ public void setDeleteQuery(List<String> deleteQuery) {
+ this.deleteQuery = deleteQuery;
+ }
+
+ // --------------------------------------------------------------------------
+ // --------------------------------------------------------------------------
+
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return ClientUtils.toContentStreams(getXML(), ClientUtils.TEXT_XML);
+ }
+
+ public String getXML() throws IOException {
+ StringWriter writer = new StringWriter();
+ writeXML(writer);
+ writer.flush();
+
+ // If action is COMMIT or OPTIMIZE, it is sent with params
+ String xml = writer.toString();
+ // System.out.println( "SEND:"+xml );
+ return (xml.length() > 0) ? xml : null;
+ }
+
+ private List<Map<SolrInputDocument,Map<String,Object>>> getDocLists(Map<SolrInputDocument,Map<String,Object>> documents) {
+ List<Map<SolrInputDocument,Map<String,Object>>> docLists = new ArrayList<>();
+ Map<SolrInputDocument,Map<String,Object>> docList = null;
+ if (this.documents != null) {
+
+ Boolean lastOverwrite = true;
+ Integer lastCommitWithin = -1;
+
+ Set<Entry<SolrInputDocument,Map<String,Object>>> entries = this.documents
+ .entrySet();
+ for (Entry<SolrInputDocument,Map<String,Object>> entry : entries) {
+ Map<String,Object> map = entry.getValue();
+ Boolean overwrite = null;
+ Integer commitWithin = null;
+ if (map != null) {
+ overwrite = (Boolean) entry.getValue().get(OVERWRITE);
+ commitWithin = (Integer) entry.getValue().get(COMMIT_WITHIN);
+ }
+ if (overwrite != lastOverwrite || commitWithin != lastCommitWithin
+ || docLists.size() == 0) {
+ docList = new LinkedHashMap<>();
+ docLists.add(docList);
+ }
+ docList.put(entry.getKey(), entry.getValue());
+ lastCommitWithin = commitWithin;
+ lastOverwrite = overwrite;
+ }
+ }
+
+ if (docIterator != null) {
+ docList = new LinkedHashMap<>();
+ docLists.add(docList);
+ while (docIterator.hasNext()) {
+ SolrInputDocument doc = docIterator.next();
+ if (doc != null) {
+ docList.put(doc, null);
+ }
+ }
+
+ }
+
+ return docLists;
+ }
+
+ /**
+ * @since solr 1.4
+ */
+ public void writeXML(Writer writer) throws IOException {
+ List<Map<SolrInputDocument,Map<String,Object>>> getDocLists = getDocLists(documents);
+
+ for (Map<SolrInputDocument,Map<String,Object>> docs : getDocLists) {
+
+ if ((docs != null && docs.size() > 0)) {
+ Entry<SolrInputDocument,Map<String,Object>> firstDoc = docs.entrySet()
+ .iterator().next();
+ Map<String,Object> map = firstDoc.getValue();
+ Integer cw = null;
+ Boolean ow = null;
+ if (map != null) {
+ cw = (Integer) firstDoc.getValue().get(COMMIT_WITHIN);
+ ow = (Boolean) firstDoc.getValue().get(OVERWRITE);
+ }
+ if (ow == null) ow = true;
+ int commitWithin = (cw != null && cw != -1) ? cw : this.commitWithin;
+ boolean overwrite = ow;
+ if (commitWithin > -1 || overwrite != true) {
+ writer.write("<add commitWithin=\"" + commitWithin + "\" "
+ + "overwrite=\"" + overwrite + "\">");
+ } else {
+ writer.write("<add>");
+ }
+
+ Set<Entry<SolrInputDocument,Map<String,Object>>> entries = docs
+ .entrySet();
+ for (Entry<SolrInputDocument,Map<String,Object>> entry : entries) {
+ ClientUtils.writeXML(entry.getKey(), writer);
+ }
+
+ writer.write("</add>");
+ }
+ }
+
+ // Add the delete commands
+ boolean deleteI = deleteById != null && deleteById.size() > 0;
+ boolean deleteQ = deleteQuery != null && deleteQuery.size() > 0;
+ if (deleteI || deleteQ) {
+ if (commitWithin > 0) {
+ writer.append("<delete commitWithin=\"" + commitWithin + "\">");
+ } else {
+ writer.append("<delete>");
+ }
+ if (deleteI) {
+ for (Map.Entry<String,Map<String,Object>> entry : deleteById.entrySet()) {
+ writer.append("<id");
+ Map<String,Object> map = entry.getValue();
+ if (map != null) {
+ Long version = (Long) map.get(VER);
+ String route = (String)map.get(ROUTE);
+ if (version != null) {
+ writer.append(" version=\"" + version + "\"");
+ }
+
+ if (route != null) {
+ writer.append(" _route_=\"" + route + "\"");
+ }
+ }
+ writer.append(">");
+
+ XML.escapeCharData(entry.getKey(), writer);
+ writer.append("</id>");
+ }
+ }
+ if (deleteQ) {
+ for (String q : deleteQuery) {
+ writer.append("<query>");
+ XML.escapeCharData(q, writer);
+ writer.append("</query>");
+ }
+ }
+ writer.append("</delete>");
+ }
+ }
+
+ // --------------------------------------------------------------------------
+ // --------------------------------------------------------------------------
+
+ // --------------------------------------------------------------------------
+ //
+ // --------------------------------------------------------------------------
+
+ public List<SolrInputDocument> getDocuments() {
+ if (documents == null) return null;
+ List<SolrInputDocument> docs = new ArrayList<>(documents.size());
+ docs.addAll(documents.keySet());
+ return docs;
+ }
+
+ public Map<SolrInputDocument,Map<String,Object>> getDocumentsMap() {
+ return documents;
+ }
+
+ public Iterator<SolrInputDocument> getDocIterator() {
+ return docIterator;
+ }
+
+ public List<String> getDeleteById() {
+ if (deleteById == null) return null;
+ List<String> deletes = new ArrayList<>(deleteById.keySet());
+ return deletes;
+ }
+
+ public Map<String,Map<String,Object>> getDeleteByIdMap() {
+ return deleteById;
+ }
+
+ public List<String> getDeleteQuery() {
+ return deleteQuery;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/package-info.java
new file mode 100644
index 0000000..899a562
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/request/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Convenience classes for dealing with various types of Solr requests.
+ */
+package org.apache.solr.client.solrj.request;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
new file mode 100644
index 0000000..21396a5
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/AnalysisResponseBase.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.common.util.NamedList;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A base class for all analysis responses.
+ *
+ *
+ * @since solr 1.4
+ */
+public class AnalysisResponseBase extends SolrResponseBase {
+
+ /**
+ * Parses the given named list and builds a list of analysis phases form it. Expects a named list of the form:
+ * <br>
+ * <pre><code>
+ * <lst name="index">
+ * <arr name="Tokenizer">
+ * <str name="text">the_text</str>
+ * <str name="rawText">the_raw_text</str> (optional)
+ * <str name="type">the_type</str>
+ * <int name="start">1</str>
+ * <int name="end">3</str>
+ * <int name="position">1</str>
+ * <bool name="match">true | false</bool> (optional)
+ * </arr>
+ * <arr name="Filter1">
+ * <str name="text">the_text</str>
+ * <str name="rawText">the_raw_text</str> (optional)
+ * <str name="type">the_type</str>
+ * <int name="start">1</str>
+ * <int name="end">3</str>
+ * <int name="position">1</str>
+ * <bool name="match">true | false</bool> (optional)
+ * </arr>
+ * ...
+ * </lst>
+ * </code></pre>
+ *
+ * @param phaseNL The names list to parse.
+ *
+ * @return The built analysis phases list.
+ */
+ protected List<AnalysisPhase> buildPhases(NamedList<List<NamedList<Object>>> phaseNL) {
+ List<AnalysisPhase> phases = new ArrayList<>(phaseNL.size());
+ for (Map.Entry<String, List<NamedList<Object>>> phaseEntry : phaseNL) {
+ AnalysisPhase phase = new AnalysisPhase(phaseEntry.getKey());
+ List<NamedList<Object>> tokens = phaseEntry.getValue();
+ for (NamedList<Object> token : tokens) {
+ TokenInfo tokenInfo = buildTokenInfo(token);
+ phase.addTokenInfo(tokenInfo);
+ }
+ phases.add(phase);
+ }
+ return phases;
+ }
+
+ /**
+ * Parses the given named list and builds a token infoform it. Expects a named list of the form:
+ * <br>
+ * <pre><code>
+ * <arr name="Tokenizer">
+ * <str name="text">the_text</str>
+ * <str name="rawText">the_raw_text</str> (optional)
+ * <str name="type">the_type</str>
+ * <int name="start">1</str>
+ * <int name="end">3</str>
+ * <int name="position">1</str>
+ * <bool name="match">true | false</bool> (optional)
+ * </arr>
+ * </code></pre>
+ *
+ * @param tokenNL The named list to parse.
+ *
+ * @return The built token info.
+ */
+ protected TokenInfo buildTokenInfo(NamedList<Object> tokenNL) {
+ String text = (String) tokenNL.get("text");
+ String rawText = (String) tokenNL.get("rawText");
+ String type = (String) tokenNL.get("type");
+ int start = (Integer) tokenNL.get("start");
+ int end = (Integer) tokenNL.get("end");
+ int position = (Integer) tokenNL.get("position");
+ Boolean match = (Boolean) tokenNL.get("match");
+ return new TokenInfo(text, rawText, type, start, end, position, (match == null ? false : match));
+ }
+
+
+ //================================================= Inner Classes ==================================================
+
+ /**
+ * A phase in the analysis process. The phase holds the tokens produced in this phase and the name of the class that
+ * produced them.
+ */
+ public static class AnalysisPhase {
+
+ private final String className;
+ private List<TokenInfo> tokens = new ArrayList<>();
+
+ AnalysisPhase(String className) {
+ this.className = className;
+ }
+
+ /**
+ * The name of the class (analyzer, tokenzier, or filter) that produced the token stream for this phase.
+ *
+ * @return The name of the class that produced the token stream for this phase.
+ */
+ public String getClassName() {
+ return className;
+ }
+
+ private void addTokenInfo(TokenInfo tokenInfo) {
+ tokens.add(tokenInfo);
+ }
+
+ /**
+ * Returns a list of tokens which represent the token stream produced in this phase.
+ *
+ * @return A list of tokens which represent the token stream produced in this phase.
+ */
+ public List<TokenInfo> getTokens() {
+ return tokens;
+ }
+
+ }
+
+ /**
+ * Holds all information of a token as part of an analysis phase.
+ */
+ public static class TokenInfo {
+
+ private final String text;
+ private final String rawText;
+ private final String type;
+ private final int start;
+ private final int end;
+ private final int position;
+ private final boolean match;
+
+ /**
+ * Constructs a new TokenInfo.
+ *
+ * @param text The text of the token
+ * @param rawText The raw text of the token. If the token is stored in the index in a special format (e.g.
+ * dates or padded numbers) this argument should hold this value. If the token is stored as is,
+ * then this value should be {@code null}.
+ * @param type The type fo the token (typically either {@code word} or {@code <ALPHANUM>} though it depends
+ * on the tokenizer/filter used).
+ * @param start The start position of the token in the original text where it was extracted from.
+ * @param end The end position of the token in the original text where it was extracted from.
+ * @param position The position of the token within the token stream.
+ * @param match Indicates whether this token matches one of the the query tokens.
+ */
+ TokenInfo(String text, String rawText, String type, int start, int end, int position, boolean match) {
+ this.text = text;
+ this.rawText = rawText;
+ this.type = type;
+ this.start = start;
+ this.end = end;
+ this.position = position;
+ this.match = match;
+ }
+
+ /**
+ * Returns the text of the token.
+ *
+ * @return The text of the token.
+ */
+ public String getText() {
+ return text;
+ }
+
+ /**
+ * Returns the raw text of the token. If the token is index in a special format (e.g. date or paddded numbers)
+ * it will be returned as the raw text. Returns {@code null} if the token is indexed as is.
+ *
+ * @return Returns the raw text of the token.
+ */
+ public String getRawText() {
+ return rawText;
+ }
+
+ /**
+ * Returns the type of the token. Typically this will be {@code word} or {@code <ALPHANUM>}, but it really
+ * depends on the tokenizer and filters that are used.
+ *
+ * @return The type of the token.
+ */
+ public String getType() {
+ return type;
+ }
+
+ /**
+ * Returns the start position of this token within the text it was originally extracted from.
+ *
+ * @return The start position of this token within the text it was originally extracted from.
+ */
+ public int getStart() {
+ return start;
+ }
+
+ /**
+ * Returns the end position of this token within the text it was originally extracted from.
+ *
+ * @return The end position of this token within the text it was originally extracted from.
+ */
+ public int getEnd() {
+ return end;
+ }
+
+ /**
+ * Returns the position of this token within the produced token stream.
+ *
+ * @return The position of this token within the produced token stream.
+ */
+ public int getPosition() {
+ return position;
+ }
+
+ /**
+ * Returns whether this token matches one of the query tokens (if query analysis is performed).
+ *
+ * @return Whether this token matches one of the query tokens (if query analysis is performed).
+ */
+ public boolean isMatch() {
+ return match;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java
new file mode 100644
index 0000000..8acf2e2
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CollectionAdminResponse.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.solr.common.util.NamedList;
+
+public class CollectionAdminResponse extends SolrResponseBase
+{
+ @SuppressWarnings("unchecked")
+ public NamedList<NamedList<Object>> getCollectionStatus()
+ {
+ return (NamedList<NamedList<Object>>) getResponse().get( "success" );
+ }
+
+ public boolean isSuccess()
+ {
+ return getResponse().get( "success" ) != null;
+ }
+
+ // this messages are typically from individual nodes, since
+ // all the failures at the router are propagated as exceptions
+ @SuppressWarnings("unchecked")
+ public NamedList<String> getErrorMessages()
+ {
+ return (NamedList<String>) getResponse().get( "failure" );
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, NamedList<Integer>> getCollectionCoresStatus()
+ {
+ Map<String, NamedList<Integer>> res = new HashMap<>();
+ NamedList<NamedList<Object>> cols = getCollectionStatus();
+ if( cols != null ) {
+ for (Map.Entry<String, NamedList<Object>> e : cols) {
+ NamedList<Object> item = e.getValue();
+ String core = (String) item.get("core");
+ if (core != null) {
+ res.put(core, (NamedList<Integer>)item.get("responseHeader"));
+ }
+ }
+ }
+
+ return res;
+ }
+
+ @SuppressWarnings("unchecked")
+ public Map<String, NamedList<Integer>> getCollectionNodesStatus()
+ {
+ Map<String, NamedList<Integer>> res = new HashMap<>();
+ NamedList<NamedList<Object>> cols = getCollectionStatus();
+ if( cols != null ) {
+ for (Map.Entry<String,NamedList<Object>> e : cols) {
+ if (e.getKey() != null) {
+ res.put(e.getKey(), (NamedList<Integer>) (e.getValue().get("responseHeader")));
+ }
+ }
+ }
+
+ return res;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CoreAdminResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CoreAdminResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CoreAdminResponse.java
new file mode 100644
index 0000000..0492165
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/CoreAdminResponse.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import java.util.Date;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ *
+ *
+ * @since solr 1.3
+ */
+public class CoreAdminResponse extends SolrResponseBase
+{
+ @SuppressWarnings("unchecked")
+ public NamedList<NamedList<Object>> getCoreStatus()
+ {
+ return (NamedList<NamedList<Object>>) getResponse().get( "status" );
+ }
+
+ public NamedList<Object> getCoreStatus( String core )
+ {
+ return getCoreStatus().get( core );
+ }
+
+ public Date getStartTime( String core )
+ {
+ NamedList<Object> v = getCoreStatus( core );
+ if( v == null ) {
+ return null;
+ }
+ return (Date) v.get( "startTime" );
+ }
+
+ public Long getUptime( String core )
+ {
+ NamedList<Object> v = getCoreStatus( core );
+ if( v == null ) {
+ return null;
+ }
+ return (Long) v.get( "uptime" );
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
new file mode 100644
index 0000000..2f11d78
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/DocumentAnalysisResponse.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.common.util.NamedList;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A response that is returned by processing the {@link org.apache.solr.client.solrj.request.DocumentAnalysisRequest}.
+ * Holds a map of {@link DocumentAnalysis} objects by a document id (unique key).
+ *
+ *
+ * @since solr 1.4
+ */
+public class DocumentAnalysisResponse extends AnalysisResponseBase implements Iterable<Map.Entry<String, DocumentAnalysisResponse.DocumentAnalysis>> {
+
+ private final Map<String, DocumentAnalysis> documentAnalysisByKey = new HashMap<>();
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void setResponse(NamedList<Object> response) {
+ super.setResponse(response);
+
+ @SuppressWarnings("unchecked")
+ NamedList<NamedList<NamedList<Object>>> analysis
+ = (NamedList<NamedList<NamedList<Object>>>) response.get("analysis");
+ for (Map.Entry<String, NamedList<NamedList<Object>>> document : analysis) {
+ DocumentAnalysis documentAnalysis = new DocumentAnalysis(document.getKey());
+ for (Map.Entry<String, NamedList<Object>> fieldEntry : document.getValue()) {
+ FieldAnalysis fieldAnalysis = new FieldAnalysis(fieldEntry.getKey());
+
+ NamedList<Object> field = fieldEntry.getValue();
+
+ @SuppressWarnings("unchecked")
+ NamedList<List<NamedList<Object>>> query
+ = (NamedList<List<NamedList<Object>>>) field.get("query");
+ if (query != null) {
+ List<AnalysisPhase> phases = buildPhases(query);
+ fieldAnalysis.setQueryPhases(phases);
+ }
+
+ @SuppressWarnings("unchecked")
+ NamedList<NamedList<List<NamedList<Object>>>> index
+ = (NamedList<NamedList<List<NamedList<Object>>>>) field.get("index");
+ for (Map.Entry<String, NamedList<List<NamedList<Object>>>> valueEntry : index) {
+ String fieldValue = valueEntry.getKey();
+ NamedList<List<NamedList<Object>>> valueNL = valueEntry.getValue();
+ List<AnalysisPhase> phases = buildPhases(valueNL);
+ fieldAnalysis.setIndexPhases(fieldValue, phases);
+ }
+
+ documentAnalysis.addFieldAnalysis(fieldAnalysis);
+ }
+
+ documentAnalysisByKey.put(documentAnalysis.getDocumentKey(), documentAnalysis);
+ }
+ }
+
+ /**
+ * Returns the number of document analyses in this response.
+ *
+ * @return The number of document analyses in this response.
+ */
+ public int getDocumentAnalysesCount() {
+ return documentAnalysisByKey.size();
+ }
+
+ /**
+ * Returns the document analysis for the document associated with the given unique key (id), {@code null} if no such
+ * association exists.
+ *
+ * @param documentKey The document unique key.
+ *
+ * @return The document analysis for the document associated with the given unique key (id).
+ */
+ public DocumentAnalysis getDocumentAnalysis(String documentKey) {
+ return documentAnalysisByKey.get(documentKey);
+ }
+
+ /**
+ * Returns an iterator over the document analyses map.
+ *
+ * @return An iterator over the document analyses map.
+ */
+ @Override
+ public Iterator<Map.Entry<String, DocumentAnalysis>> iterator() {
+ return documentAnalysisByKey.entrySet().iterator();
+ }
+
+ //================================================= Inner Classes ==================================================
+
+ /**
+ * An analysis process breakdown of a document. Holds a map of field analyses by the field name.
+ */
+ public static class DocumentAnalysis implements Iterable<Map.Entry<String, FieldAnalysis>> {
+
+ private final String documentKey;
+ private Map<String, FieldAnalysis> fieldAnalysisByFieldName = new HashMap<>();
+
+ private DocumentAnalysis(String documentKey) {
+ this.documentKey = documentKey;
+ }
+
+ private void addFieldAnalysis(FieldAnalysis fieldAnalysis) {
+ fieldAnalysisByFieldName.put(fieldAnalysis.getFieldName(), fieldAnalysis);
+ }
+
+ /**
+ * Returns the unique key of the analyzed document.
+ *
+ * @return The unique key of the analyzed document.
+ */
+ public String getDocumentKey() {
+ return documentKey;
+ }
+
+ /**
+ * Returns the number of field analyses for the documents.
+ *
+ * @return The number of field analyses for the documents.
+ */
+ public int getFieldAnalysesCount() {
+ return fieldAnalysisByFieldName.size();
+ }
+
+ public FieldAnalysis getFieldAnalysis(String fieldName) {
+ return fieldAnalysisByFieldName.get(fieldName);
+ }
+
+ /**
+ * Returns an iterator over the field analyses map.
+ *
+ * @return An iterator over the field analyses map.
+ */
+ @Override
+ public Iterator<Map.Entry<String, FieldAnalysis>> iterator() {
+ return fieldAnalysisByFieldName.entrySet().iterator();
+ }
+ }
+
+ /**
+ * An analysis process breakdown for a specific field. Holds a list of query time analysis phases (that is, if a
+ * query analysis was requested in the first place) and a list of index time analysis phases for each field value (a
+ * field can be multi-valued).
+ */
+ public static class FieldAnalysis {
+
+ private final String fieldName;
+ private List<AnalysisPhase> queryPhases;
+ private Map<String, List<AnalysisPhase>> indexPhasesByFieldValue = new HashMap<>();
+
+ private FieldAnalysis(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ public void setQueryPhases(List<AnalysisPhase> queryPhases) {
+ this.queryPhases = queryPhases;
+ }
+
+ public void setIndexPhases(String fieldValue, List<AnalysisPhase> indexPhases) {
+ indexPhasesByFieldValue.put(fieldValue, indexPhases);
+ }
+
+ /**
+ * Returns the field name.
+ *
+ * @return The name of the field.
+ */
+ public String getFieldName() {
+ return fieldName;
+ }
+
+ /**
+ * Returns the number of query time analysis phases or {@code -1} if
+ * this field analysis doesn't hold a query time analysis.
+ *
+ * @return Returns the number of query time analysis phases or {@code -1}
+ * if this field analysis doesn't hold a query time analysis.
+ */
+ public int getQueryPhasesCount() {
+ return queryPhases == null ? -1 : queryPhases.size();
+ }
+
+ /**
+ * Returns the query time analysis phases for the field or {@code null}
+ * if this field doesn't hold a query time analysis.
+ *
+ * @return Returns the query time analysis phases for the field or
+ * {@code null} if this field doesn't hold a query time analysis.
+ */
+ public Iterable<AnalysisPhase> getQueryPhases() {
+ return queryPhases;
+ }
+
+ /**
+ * Returns the number of values the field has.
+ *
+ * @return The number of values the field has.
+ */
+ public int getValueCount() {
+ return indexPhasesByFieldValue.entrySet().size();
+ }
+
+ /**
+ * Returns the number of index time analysis phases the given field value has.
+ *
+ * @param fieldValue The field value.
+ *
+ * @return The number of index time analysis phases the given field value has.
+ */
+ public int getIndexPhasesCount(String fieldValue) {
+ return indexPhasesByFieldValue.get(fieldValue).size();
+ }
+
+ /**
+ * Returns the index time analysis phases for the given field value.
+ *
+ * @param fieldValue The field value.
+ *
+ * @return The index time analysis phases for the given field value.
+ */
+ public Iterable<AnalysisPhase> getIndexPhases(String fieldValue) {
+ return indexPhasesByFieldValue.get(fieldValue);
+ }
+
+ /**
+ * Returns the index time analysis phases for all field values.
+ *
+ * @return Returns the index time analysis phases for all field value.
+ */
+ public Iterable<Map.Entry<String, List<AnalysisPhase>>> getIndexPhasesByFieldValue() {
+ return indexPhasesByFieldValue.entrySet();
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FacetField.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FacetField.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FacetField.java
new file mode 100644
index 0000000..086c999
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/response/FacetField.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.solr.client.solrj.util.ClientUtils;
+
+ /**
+ * A utility class to hold the facet response. It could use the NamedList container,
+ * but for JSTL, it is nice to have something that implements List so it can be iterated
+ *
+ * @since solr 1.3
+ */
+ public class FacetField implements Serializable
+ {
+ public static class Count implements Serializable
+ {
+ private String _name = null;
+ private long _count = 0;
+ // hang onto the FacetField for breadcrumb creation convenience
+ private FacetField _ff = null;
+
+ public Count( FacetField ff, String n, long c )
+ {
+ _name = n;
+ _count = c;
+ _ff = ff;
+ }
+
+ public String getName() {
+ return _name;
+ }
+
+ public void setName( String n )
+ {
+ _name = n;
+ }
+
+ public long getCount() {
+ return _count;
+ }
+
+ public void setCount( long c )
+ {
+ _count = c;
+ }
+
+ public FacetField getFacetField() {
+ return _ff;
+ }
+
+ @Override
+ public String toString()
+ {
+ return _name+" ("+_count+")";
+ }
+
+ public String getAsFilterQuery() {
+ if (_ff.getName().equals("facet_queries")) {
+ return _name;
+ }
+ return
+ ClientUtils.escapeQueryChars( _ff._name ) + ":" +
+ ClientUtils.escapeQueryChars( _name );
+ }
+ }
+
+ private String _name = null;
+ private List<Count> _values = null;
+ private String _gap = null;
+ private Date _end = null;
+
+ public FacetField( final String n )
+ {
+ _name = n;
+ }
+
+ public FacetField(String name, String gap, Date end) {
+ _name = name;
+ _gap = gap;
+ _end = end;
+ }
+
+ /**
+ * Date Gap Facet parameter
+ *
+ * @return the value specified for facet.date.gap
+ */
+ public String getGap() {
+ return _gap;
+ }
+
+ /**
+ * Date End Facet parameter
+ *
+ * @return the value specified for facet.date.end
+ */
+ public Date getEnd() {
+ return _end;
+ }
+
+ /**
+ * Insert at the end of the list
+ */
+ public void add( String name, long cnt )
+ {
+ if( _values == null ) {
+ _values = new ArrayList<>( 30 );
+ }
+ _values.add( new Count( this, name, cnt ) );
+ }
+
+ /**
+ * Insert at the beginning of the list.
+ */
+ public void insert( String name, long cnt )
+ {
+ if( _values == null ) {
+ _values = new ArrayList<>( 30 );
+ }
+ _values.add( 0, new Count( this, name, cnt ) );
+ }
+
+ public String getName() {
+ return _name;
+ }
+
+ public List<Count> getValues() {
+ return _values == null ? Collections.<Count>emptyList() : _values;
+ }
+
+ public int getValueCount()
+ {
+ return _values == null ? 0 : _values.size();
+ }
+
+ public FacetField getLimitingFields(long max)
+ {
+ FacetField ff = new FacetField( _name );
+ if( _values != null ) {
+ ff._values = new ArrayList<>( _values.size() );
+ for( Count c : _values ) {
+ if( c._count < max ) { // !equal to
+ ff._values.add( c );
+ }
+ }
+ }
+ return ff;
+ }
+
+ @Override
+ public String toString()
+ {
+ return _name + ":" + _values;
+ }
+ }
[17/17] incubator-ranger git commit: Updated default property names
for solr audit configuration
Posted by bo...@apache.org.
Updated default property names for solr audit configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/82f71420
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/82f71420
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/82f71420
Branch: refs/heads/master
Commit: 82f714208cdf7d82ccfeae83225a7ddac1720aec
Parents: 40aa090
Author: Don Bosco Durai <bo...@apache.org>
Authored: Tue Mar 17 15:48:51 2015 -0700
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Tue Mar 17 15:48:51 2015 -0700
----------------------------------------------------------------------
hbase-agent/conf/ranger-hbase-audit.xml | 6 +++---
hdfs-agent/conf/ranger-hdfs-audit.xml | 6 +++---
hive-agent/conf/ranger-hive-audit.xml | 6 +++---
knox-agent/conf/ranger-knox-audit.xml | 6 +++---
plugin-yarn/conf/ranger-yarn-audit.xml | 6 +++---
storm-agent/conf/ranger-storm-audit.xml | 6 +++---
6 files changed, 18 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/82f71420/hbase-agent/conf/ranger-hbase-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/ranger-hbase-audit.xml b/hbase-agent/conf/ranger-hbase-audit.xml
index e5bfb89..c5f416e 100644
--- a/hbase-agent/conf/ranger-hbase-audit.xml
+++ b/hbase-agent/conf/ranger-hbase-audit.xml
@@ -212,17 +212,17 @@
<!-- Ranger audit provider configuration -->
<property>
- <name>xasecure.audit.ranger.is.enabled</name>
+ <name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <name>xasecure.audit.solr.async.max.flush.interval.ms</name>
<value>1000</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/82f71420/hdfs-agent/conf/ranger-hdfs-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml
index 09114ad..4c60c02 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit.xml
+++ b/hdfs-agent/conf/ranger-hdfs-audit.xml
@@ -212,17 +212,17 @@
<!-- Ranger audit provider configuration -->
<property>
- <name>xasecure.audit.ranger.is.enabled</name>
+ <name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <name>xasecure.audit.solr.async.max.flush.interval.ms</name>
<value>1000</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/82f71420/hive-agent/conf/ranger-hive-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml
index e753336..867080c 100644
--- a/hive-agent/conf/ranger-hive-audit.xml
+++ b/hive-agent/conf/ranger-hive-audit.xml
@@ -212,17 +212,17 @@
<!-- Ranger audit provider configuration -->
<property>
- <name>xasecure.audit.ranger.is.enabled</name>
+ <name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <name>xasecure.audit.solr.async.max.flush.interval.ms</name>
<value>1000</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/82f71420/knox-agent/conf/ranger-knox-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/ranger-knox-audit.xml b/knox-agent/conf/ranger-knox-audit.xml
index 6f0adb9..ccc2691 100644
--- a/knox-agent/conf/ranger-knox-audit.xml
+++ b/knox-agent/conf/ranger-knox-audit.xml
@@ -212,17 +212,17 @@
<!-- Ranger audit provider configuration -->
<property>
- <name>xasecure.audit.ranger.is.enabled</name>
+ <name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <name>xasecure.audit.solr.async.max.flush.interval.ms</name>
<value>1000</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/82f71420/plugin-yarn/conf/ranger-yarn-audit.xml
----------------------------------------------------------------------
diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml
index f1e9687..c30f963 100644
--- a/plugin-yarn/conf/ranger-yarn-audit.xml
+++ b/plugin-yarn/conf/ranger-yarn-audit.xml
@@ -213,17 +213,17 @@
<!-- Ranger audit provider configuration -->
<property>
- <name>xasecure.audit.ranger.is.enabled</name>
+ <name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <name>xasecure.audit.solr.async.max.flush.interval.ms</name>
<value>1000</value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/82f71420/storm-agent/conf/ranger-storm-audit.xml
----------------------------------------------------------------------
diff --git a/storm-agent/conf/ranger-storm-audit.xml b/storm-agent/conf/ranger-storm-audit.xml
index 3e310d9..6a5ff69 100644
--- a/storm-agent/conf/ranger-storm-audit.xml
+++ b/storm-agent/conf/ranger-storm-audit.xml
@@ -212,17 +212,17 @@
<!-- Ranger audit provider configuration -->
<property>
- <name>xasecure.audit.ranger.is.enabled</name>
+ <name>xasecure.audit.solr.is.enabled</name>
<value>false</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.queue.size</name>
+ <name>xasecure.audit.solr.async.max.queue.size</name>
<value>1</value>
</property>
<property>
- <name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+ <name>xasecure.audit.solr.async.max.flush.interval.ms</name>
<value>1000</value>
</property>
[06/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkStateReader.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkStateReader.java
new file mode 100644
index 0000000..1f9ebbc
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -0,0 +1,925 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required byOCP applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.ByteUtils;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.EventType;
+import org.apache.zookeeper.data.Stat;
+import org.noggit.CharArr;
+import org.noggit.JSONParser;
+import org.noggit.JSONWriter;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+public class ZkStateReader implements Closeable {
+ private static Logger log = LoggerFactory.getLogger(ZkStateReader.class);
+
+ public static final String BASE_URL_PROP = "base_url";
+ public static final String NODE_NAME_PROP = "node_name";
+ public static final String CORE_NODE_NAME_PROP = "core_node_name";
+ public static final String ROLES_PROP = "roles";
+ public static final String STATE_PROP = "state";
+ public static final String CORE_NAME_PROP = "core";
+ public static final String COLLECTION_PROP = "collection";
+ public static final String ELECTION_NODE_PROP = "election_node";
+ public static final String SHARD_ID_PROP = "shard";
+ public static final String REPLICA_PROP = "replica";
+ public static final String SHARD_RANGE_PROP = "shard_range";
+ public static final String SHARD_STATE_PROP = "shard_state";
+ public static final String SHARD_PARENT_PROP = "shard_parent";
+ public static final String NUM_SHARDS_PROP = "numShards";
+ public static final String LEADER_PROP = "leader";
+ public static final String PROPERTY_PROP = "property";
+ public static final String PROPERTY_VALUE_PROP = "property.value";
+ public static final String MAX_AT_ONCE_PROP = "maxAtOnce";
+ public static final String MAX_WAIT_SECONDS_PROP = "maxWaitSeconds";
+ public static final String COLLECTIONS_ZKNODE = "/collections";
+ public static final String LIVE_NODES_ZKNODE = "/live_nodes";
+ public static final String ALIASES = "/aliases.json";
+ public static final String CLUSTER_STATE = "/clusterstate.json";
+ public static final String CLUSTER_PROPS = "/clusterprops.json";
+ public static final String REJOIN_AT_HEAD_PROP = "rejoinAtHead";
+
+ public static final String REPLICATION_FACTOR = "replicationFactor";
+ public static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
+ public static final String AUTO_ADD_REPLICAS = "autoAddReplicas";
+
+ public static final String ROLES = "/roles.json";
+
+ public static final String RECOVERING = "recovering";
+ public static final String RECOVERY_FAILED = "recovery_failed";
+ public static final String ACTIVE = "active";
+ public static final String DOWN = "down";
+ public static final String SYNC = "sync";
+
+ public static final String CONFIGS_ZKNODE = "/configs";
+ public final static String CONFIGNAME_PROP="configName";
+
+ public static final String LEGACY_CLOUD = "legacyCloud";
+
+ public static final String URL_SCHEME = "urlScheme";
+
+ protected volatile ClusterState clusterState;
+
+ private static final long SOLRCLOUD_UPDATE_DELAY = Long.parseLong(System.getProperty("solrcloud.update.delay", "5000"));
+
+ public static final String LEADER_ELECT_ZKNODE = "leader_elect";
+
+ public static final String SHARD_LEADERS_ZKNODE = "leaders";
+ public static final String ELECTION_NODE = "election";
+
+ private final Set<String> watchedCollections = new HashSet<String>();
+
+ /**These are collections which are actively watched by this instance .
+ *
+ */
+ private Map<String , DocCollection> watchedCollectionStates = new ConcurrentHashMap<String, DocCollection>();
+
+ private final ZkConfigManager configManager;
+
+
+ //
+ // convenience methods... should these go somewhere else?
+ //
+ public static byte[] toJSON(Object o) {
+ CharArr out = new CharArr();
+ new JSONWriter(out, 2).write(o); // indentation by default
+ return toUTF8(out);
+ }
+
+ public static byte[] toUTF8(CharArr out) {
+ byte[] arr = new byte[out.size() << 2]; // is 4x the real worst-case upper-bound?
+ int nBytes = ByteUtils.UTF16toUTF8(out, 0, out.size(), arr, 0);
+ return Arrays.copyOf(arr, nBytes);
+ }
+
+ public static Object fromJSON(byte[] utf8) {
+ // convert directly from bytes to chars
+ // and parse directly from that instead of going through
+ // intermediate strings or readers
+ CharArr chars = new CharArr();
+ ByteUtils.UTF8toUTF16(utf8, 0, utf8.length, chars);
+ JSONParser parser = new JSONParser(chars.getArray(), chars.getStart(), chars.length());
+ try {
+ return ObjectBuilder.getVal(parser);
+ } catch (IOException e) {
+ throw new RuntimeException(e); // should never happen w/o using real IO
+ }
+ }
+
+ /**
+ * Returns config set name for collection.
+ *
+ * @param collection to return config set name for
+ */
+ public String readConfigName(String collection) {
+
+ String configName = null;
+
+ String path = COLLECTIONS_ZKNODE + "/" + collection;
+ if (log.isInfoEnabled()) {
+ log.info("Load collection config from:" + path);
+ }
+
+ try {
+ byte[] data = zkClient.getData(path, null, null, true);
+
+ if(data != null) {
+ ZkNodeProps props = ZkNodeProps.load(data);
+ configName = props.getStr(CONFIGNAME_PROP);
+ }
+
+ if (configName != null) {
+ if (!zkClient.exists(CONFIGS_ZKNODE + "/" + configName, true)) {
+ log.error("Specified config does not exist in ZooKeeper:" + configName);
+ throw new ZooKeeperException(ErrorCode.SERVER_ERROR,
+ "Specified config does not exist in ZooKeeper:" + configName);
+ } else if (log.isInfoEnabled()) {
+ log.info("path={} {}={} specified config exists in ZooKeeper",
+ new Object[] {path, CONFIGNAME_PROP, configName});
+ }
+ } else {
+ throw new ZooKeeperException(ErrorCode.INVALID_STATE, "No config data found at path: " + path);
+ }
+ }
+ catch (KeeperException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Error loading config name for collection " + collection, e);
+ }
+ catch (InterruptedException e) {
+ Thread.interrupted();
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Error loading config name for collection " + collection, e);
+ }
+
+ return configName;
+ }
+
+
+ private static class ZKTF implements ThreadFactory {
+ private static ThreadGroup tg = new ThreadGroup("ZkStateReader");
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread td = new Thread(tg, r);
+ td.setDaemon(true);
+ return td;
+ }
+ }
+ private ScheduledExecutorService updateCloudExecutor = Executors.newScheduledThreadPool(1, new ZKTF());
+
+ private boolean clusterStateUpdateScheduled;
+
+ private final SolrZkClient zkClient;
+
+ private final boolean closeClient;
+
+ private final ZkCmdExecutor cmdExecutor;
+
+ private volatile Aliases aliases = new Aliases();
+
+ private volatile boolean closed = false;
+
+ public ZkStateReader(SolrZkClient zkClient) {
+ this.zkClient = zkClient;
+ this.cmdExecutor = new ZkCmdExecutor(zkClient.getZkClientTimeout());
+ this.configManager = new ZkConfigManager(zkClient);
+ this.closeClient = false;
+ }
+
+ public ZkStateReader(String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout) {
+ this.zkClient = new SolrZkClient(zkServerAddress, zkClientTimeout, zkClientConnectTimeout,
+ // on reconnect, reload cloud info
+ new OnReconnect() {
+ @Override
+ public void command() {
+ try {
+ ZkStateReader.this.createClusterStateWatchersAndUpdate();
+ } catch (KeeperException e) {
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ }
+ }
+ });
+ this.cmdExecutor = new ZkCmdExecutor(zkClientTimeout);
+ this.configManager = new ZkConfigManager(zkClient);
+ this.closeClient = true;
+ }
+
+ public ZkConfigManager getConfigManager() {
+ return configManager;
+ }
+
+ // load and publish a new CollectionInfo
+ public void updateClusterState(boolean immediate) throws KeeperException, InterruptedException {
+ updateClusterState(immediate, false);
+ }
+
+ // load and publish a new CollectionInfo
+ public void updateLiveNodes() throws KeeperException, InterruptedException {
+ updateClusterState(true, true);
+ }
+
+ public Aliases getAliases() {
+ return aliases;
+ }
+
+ public Integer compareStateVersions(String coll, int version) {
+ DocCollection collection = clusterState.getCollectionOrNull(coll);
+ if (collection == null) return null;
+ if (collection.getZNodeVersion() < version) {
+ log.debug("server older than client {}<{}", collection.getZNodeVersion(), version);
+ DocCollection nu = getCollectionLive(this, coll);
+ if (nu == null) return -1 ;
+ if (nu.getZNodeVersion() > collection.getZNodeVersion()) {
+ updateWatchedCollection(nu);
+ collection = nu;
+ }
+ }
+
+ if (collection.getZNodeVersion() == version) {
+ return null;
+ }
+
+ log.debug("wrong version from client {}!={} ", version, collection.getZNodeVersion());
+
+ return collection.getZNodeVersion();
+ }
+
+ public synchronized void createClusterStateWatchersAndUpdate() throws KeeperException,
+ InterruptedException {
+ // We need to fetch the current cluster state and the set of live nodes
+
+ synchronized (getUpdateLock()) {
+ cmdExecutor.ensureExists(CLUSTER_STATE, zkClient);
+ cmdExecutor.ensureExists(ALIASES, zkClient);
+
+ log.info("Updating cluster state from ZooKeeper... ");
+
+ zkClient.exists(CLUSTER_STATE, new Watcher() {
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events,
+ // and do not remove the watcher
+ if (EventType.None.equals(event.getType())) {
+ return;
+ }
+ log.info("A cluster state change: {}, has occurred - updating... (live nodes size: {})", (event) , ZkStateReader.this.clusterState == null ? 0 : ZkStateReader.this.clusterState.getLiveNodes().size());
+ try {
+
+ // delayed approach
+ // ZkStateReader.this.updateClusterState(false, false);
+ synchronized (ZkStateReader.this.getUpdateLock()) {
+ // remake watch
+ final Watcher thisWatch = this;
+ Set<String> ln = ZkStateReader.this.clusterState.getLiveNodes();
+ // update volatile
+ ZkStateReader.this.clusterState = constructState(ln, thisWatch);
+ }
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED
+ || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+ return;
+ }
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR,
+ "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ return;
+ }
+ }
+
+ }, true);
+ }
+
+
+ synchronized (ZkStateReader.this.getUpdateLock()) {
+ List<String> liveNodes = zkClient.getChildren(LIVE_NODES_ZKNODE,
+ new Watcher() {
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events,
+ // and do not remove the watcher
+ if (EventType.None.equals(event.getType())) {
+ return;
+ }
+ try {
+ // delayed approach
+ // ZkStateReader.this.updateClusterState(false, true);
+ synchronized (ZkStateReader.this.getUpdateLock()) {
+ List<String> liveNodes = zkClient.getChildren(
+ LIVE_NODES_ZKNODE, this, true);
+ log.debug("Updating live nodes... ({})", liveNodes.size());
+ Set<String> liveNodesSet = new HashSet<>();
+ liveNodesSet.addAll(liveNodes);
+
+ ClusterState clusterState = ZkStateReader.this.clusterState;
+
+ clusterState.setLiveNodes(liveNodesSet);
+ }
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED
+ || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+ return;
+ }
+ log.error("", e);
+ throw new ZooKeeperException(
+ SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ return;
+ }
+ }
+
+ }, true);
+
+ Set<String> liveNodeSet = new HashSet<>();
+ liveNodeSet.addAll(liveNodes);
+ this.clusterState = constructState(liveNodeSet, null);
+
+ zkClient.exists(ALIASES,
+ new Watcher() {
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events,
+ // and do not remove the watcher
+ if (EventType.None.equals(event.getType())) {
+ return;
+ }
+ try {
+ synchronized (ZkStateReader.this.getUpdateLock()) {
+ log.info("Updating aliases... ");
+
+ // remake watch
+ final Watcher thisWatch = this;
+ Stat stat = new Stat();
+ byte[] data = zkClient.getData(ALIASES, thisWatch, stat ,
+ true);
+
+ Aliases aliases = ClusterState.load(data);
+
+ ZkStateReader.this.aliases = aliases;
+ }
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED
+ || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+ return;
+ }
+ log.error("", e);
+ throw new ZooKeeperException(
+ SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ return;
+ }
+ }
+
+ }, true);
+ }
+ updateAliases();
+ //on reconnect of SolrZkClient re-add watchers for the watched external collections
+ synchronized (this) {
+ for (String watchedCollection : watchedCollections) {
+ addZkWatch(watchedCollection);
+ }
+ }
+ }
+
+ private ClusterState constructState(Set<String> ln, Watcher watcher)
+ throws KeeperException, InterruptedException {
+ Stat stat = new Stat();
+ byte[] data = zkClient.getData(CLUSTER_STATE, watcher, stat, true);
+ ClusterState loadedData = ClusterState.load(stat.getVersion(), data, ln,
+ CLUSTER_STATE);
+ Map<String,ClusterState.CollectionRef> result = new LinkedHashMap<>();
+ result.putAll(loadedData.getCollectionStates());// first load all
+ // collections in
+ // clusterstate.json
+ for (String s : getIndividualColls()) {
+ synchronized (this) {
+ if (watchedCollections.contains(s)) {
+ DocCollection live = getCollectionLive(this, s);
+ if (live != null) {
+ watchedCollectionStates.put(s, live);
+ // if it is a watched collection, add too
+ result.put(s, new ClusterState.CollectionRef(live));
+ }
+ } else {
+ // if it is not collection, then just create a reference which can fetch
+ // the collection object just in time from ZK
+ // this is also cheap (lazy loaded) so we put it inside the synchronized
+ // block although it is not required
+ final String collName = s;
+ result.put(s, new ClusterState.CollectionRef(null) {
+ @Override
+ public DocCollection get() {
+ return getCollectionLive(ZkStateReader.this, collName);
+ }
+
+ @Override
+ public boolean isLazilyLoaded() { return true; }
+ });
+ }
+ }
+ }
+ return new ClusterState(ln, result, stat.getVersion());
+ }
+
+
+ private Set<String> getIndividualColls() throws KeeperException, InterruptedException {
+ List<String> children = null;
+ try {
+ children = zkClient.getChildren(COLLECTIONS_ZKNODE, null, true);
+ } catch (KeeperException.NoNodeException e) {
+ log.warn("Error fetching collection names");
+
+ return new HashSet<>();
+ }
+ if (children == null || children.isEmpty()) return new HashSet<>();
+ HashSet<String> result = new HashSet<>(children.size());
+
+ for (String c : children) {
+ try {
+ if (zkClient.exists(getCollectionPath(c), true)) {
+ result.add(c);
+ }
+ } catch (Exception e) {
+ log.warn("Error reading collections nodes", e);
+ }
+ }
+ return result;
+ }
+
+ // load and publish a new CollectionInfo
+ private synchronized void updateClusterState(boolean immediate,
+ final boolean onlyLiveNodes) throws KeeperException,
+ InterruptedException {
+ // build immutable CloudInfo
+
+ if (immediate) {
+ ClusterState clusterState;
+ synchronized (getUpdateLock()) {
+ List<String> liveNodes = zkClient.getChildren(LIVE_NODES_ZKNODE, null,
+ true);
+ Set<String> liveNodesSet = new HashSet<>();
+ liveNodesSet.addAll(liveNodes);
+
+ if (!onlyLiveNodes) {
+ log.debug("Updating cloud state from ZooKeeper... ");
+
+ clusterState = constructState(liveNodesSet,null);
+ } else {
+ log.debug("Updating live nodes from ZooKeeper... ({})", liveNodesSet.size());
+ clusterState = this.clusterState;
+ clusterState.setLiveNodes(liveNodesSet);
+ }
+ this.clusterState = clusterState;
+ }
+ synchronized (ZkStateReader.this) {
+ for (String watchedCollection : watchedCollections) {
+ DocCollection live = getCollectionLive(this, watchedCollection);
+ if (live != null) {
+ updateWatchedCollection(live);
+ }
+ }
+ }
+
+ } else {
+ if (clusterStateUpdateScheduled) {
+ log.debug("Cloud state update for ZooKeeper already scheduled");
+ return;
+ }
+ log.debug("Scheduling cloud state update from ZooKeeper...");
+ clusterStateUpdateScheduled = true;
+ updateCloudExecutor.schedule(new Runnable() {
+
+ @Override
+ public void run() {
+ log.debug("Updating cluster state from ZooKeeper...");
+ synchronized (getUpdateLock()) {
+ clusterStateUpdateScheduled = false;
+ ClusterState clusterState;
+ try {
+ List<String> liveNodes = zkClient.getChildren(LIVE_NODES_ZKNODE,
+ null, true);
+ Set<String> liveNodesSet = new HashSet<>();
+ liveNodesSet.addAll(liveNodes);
+
+ if (!onlyLiveNodes) {
+ log.debug("Updating cloud state from ZooKeeper... ");
+
+ clusterState = constructState(liveNodesSet,null);
+ } else {
+ log.debug("Updating live nodes from ZooKeeper... ");
+ clusterState = ZkStateReader.this.clusterState;
+ clusterState.setLiveNodes(liveNodesSet);
+ }
+
+ ZkStateReader.this.clusterState = clusterState;
+
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED
+ || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+ return;
+ }
+ log.error("", e);
+ throw new ZooKeeperException(
+ SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(
+ SolrException.ErrorCode.SERVER_ERROR, "", e);
+ }
+ // update volatile
+ ZkStateReader.this.clusterState = clusterState;
+
+ synchronized (ZkStateReader.this) {
+ for (String watchedCollection : watchedCollections) {
+ DocCollection live = getCollectionLive(ZkStateReader.this, watchedCollection);
+ assert live != null;
+ if (live != null) {
+ updateWatchedCollection(live);
+ }
+ }
+ }
+ }
+ }
+ }, SOLRCLOUD_UPDATE_DELAY, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ /**
+ * @return information about the cluster from ZooKeeper
+ */
+ public ClusterState getClusterState() {
+ return clusterState;
+ }
+
+ public Object getUpdateLock() {
+ return this;
+ }
+
+ public void close() {
+ this.closed = true;
+ if (closeClient) {
+ zkClient.close();
+ }
+ }
+
+ abstract class RunnableWatcher implements Runnable {
+ Watcher watcher;
+ public RunnableWatcher(Watcher watcher){
+ this.watcher = watcher;
+ }
+
+ }
+
+ public String getLeaderUrl(String collection, String shard, int timeout)
+ throws InterruptedException, KeeperException {
+ ZkCoreNodeProps props = new ZkCoreNodeProps(getLeaderRetry(collection,
+ shard, timeout));
+ return props.getCoreUrl();
+ }
+
+ /**
+ * Get shard leader properties, with retry if none exist.
+ */
+ public Replica getLeaderRetry(String collection, String shard) throws InterruptedException {
+ return getLeaderRetry(collection, shard, 4000);
+ }
+
+ /**
+ * Get shard leader properties, with retry if none exist.
+ */
+ public Replica getLeaderRetry(String collection, String shard, int timeout) throws InterruptedException {
+ long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS);
+ while (System.nanoTime() < timeoutAt && !closed) {
+ if (clusterState != null) {
+ Replica replica = clusterState.getLeader(collection, shard);
+ if (replica != null && getClusterState().liveNodesContain(replica.getNodeName())) {
+ return replica;
+ }
+ }
+ Thread.sleep(50);
+ }
+ throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "No registered leader was found after waiting for "
+ + timeout + "ms " + ", collection: " + collection + " slice: " + shard);
+ }
+
+ /**
+ * Get path where shard leader properties live in zookeeper.
+ */
+ public static String getShardLeadersPath(String collection, String shardId) {
+ return COLLECTIONS_ZKNODE + "/" + collection + "/"
+ + SHARD_LEADERS_ZKNODE + (shardId != null ? ("/" + shardId)
+ : "");
+ }
+
+ /**
+ * Get path where shard leader elections ephemeral nodes are.
+ */
+ public static String getShardLeadersElectPath(String collection, String shardId) {
+ return COLLECTIONS_ZKNODE + "/" + collection + "/"
+ + LEADER_ELECT_ZKNODE + (shardId != null ? ("/" + shardId + "/" + ELECTION_NODE)
+ : "");
+ }
+
+
+ public List<ZkCoreNodeProps> getReplicaProps(String collection,
+ String shardId, String thisCoreNodeName) {
+ return getReplicaProps(collection, shardId, thisCoreNodeName, null);
+ }
+
+ public List<ZkCoreNodeProps> getReplicaProps(String collection,
+ String shardId, String thisCoreNodeName, String mustMatchStateFilter) {
+ return getReplicaProps(collection, shardId, thisCoreNodeName, mustMatchStateFilter, null);
+ }
+
+ public List<ZkCoreNodeProps> getReplicaProps(String collection,
+ String shardId, String thisCoreNodeName, String mustMatchStateFilter, String mustNotMatchStateFilter) {
+ assert thisCoreNodeName != null;
+ ClusterState clusterState = this.clusterState;
+ if (clusterState == null) {
+ return null;
+ }
+ Map<String,Slice> slices = clusterState.getSlicesMap(collection);
+ if (slices == null) {
+ throw new ZooKeeperException(ErrorCode.BAD_REQUEST,
+ "Could not find collection in zk: " + collection + " "
+ + clusterState.getCollections());
+ }
+
+ Slice replicas = slices.get(shardId);
+ if (replicas == null) {
+ throw new ZooKeeperException(ErrorCode.BAD_REQUEST, "Could not find shardId in zk: " + shardId);
+ }
+
+ Map<String,Replica> shardMap = replicas.getReplicasMap();
+ List<ZkCoreNodeProps> nodes = new ArrayList<>(shardMap.size());
+ for (Entry<String,Replica> entry : shardMap.entrySet()) {
+ ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(entry.getValue());
+
+ String coreNodeName = entry.getValue().getName();
+
+ if (clusterState.liveNodesContain(nodeProps.getNodeName()) && !coreNodeName.equals(thisCoreNodeName)) {
+ if (mustMatchStateFilter == null || mustMatchStateFilter.equals(nodeProps.getState())) {
+ if (mustNotMatchStateFilter == null || !mustNotMatchStateFilter.equals(nodeProps.getState())) {
+ nodes.add(nodeProps);
+ }
+ }
+ }
+ }
+ if (nodes.size() == 0) {
+ // no replicas
+ return null;
+ }
+
+ return nodes;
+ }
+
+ public SolrZkClient getZkClient() {
+ return zkClient;
+ }
+
+ public void updateAliases() throws KeeperException, InterruptedException {
+ byte[] data = zkClient.getData(ALIASES, null, null, true);
+
+ Aliases aliases = ClusterState.load(data);
+
+ ZkStateReader.this.aliases = aliases;
+ }
+ public Map getClusterProps(){
+ Map result = null;
+ try {
+ if(getZkClient().exists(ZkStateReader.CLUSTER_PROPS, true)){
+ result = (Map) ZkStateReader.fromJSON(getZkClient().getData(ZkStateReader.CLUSTER_PROPS, null, new Stat(), true)) ;
+ } else {
+ result= new LinkedHashMap();
+ }
+ return result;
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading cluster properties",e) ;
+ }
+ }
+
+ /**
+ * Returns the baseURL corresponding to a given node's nodeName --
+ * NOTE: does not (currently) imply that the nodeName (or resulting
+ * baseURL) exists in the cluster.
+ * @lucene.experimental
+ */
+ public String getBaseUrlForNodeName(final String nodeName) {
+ final int _offset = nodeName.indexOf("_");
+ if (_offset < 0) {
+ throw new IllegalArgumentException("nodeName does not contain expected '_' seperator: " + nodeName);
+ }
+ final String hostAndPort = nodeName.substring(0,_offset);
+ try {
+ final String path = URLDecoder.decode(nodeName.substring(1+_offset), "UTF-8");
+ String urlScheme = (String) getClusterProps().get(URL_SCHEME);
+ if(urlScheme == null) {
+ urlScheme = "http";
+ }
+ return urlScheme + "://" + hostAndPort + (path.isEmpty() ? "" : ("/" + path));
+ } catch (UnsupportedEncodingException e) {
+ throw new IllegalStateException("JVM Does not seem to support UTF-8", e);
+ }
+ }
+
+ public static DocCollection getCollectionLive(ZkStateReader zkStateReader,
+ String coll) {
+ String collectionPath = getCollectionPath(coll);
+ try {
+ if (!zkStateReader.getZkClient().exists(collectionPath, true)) return null;
+ Stat stat = new Stat();
+ byte[] data = zkStateReader.getZkClient().getData(collectionPath, null, stat, true);
+ ClusterState state = ClusterState.load(stat.getVersion(), data,
+ Collections.<String> emptySet(), collectionPath);
+ ClusterState.CollectionRef collectionRef = state.getCollectionStates().get(coll);
+ return collectionRef == null ? null : collectionRef.get();
+ } catch (KeeperException.NoNodeException e) {
+ log.warn("No node available : " + collectionPath, e);
+ return null;
+ } catch (KeeperException e) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Could not load collection from ZK:" + coll, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Could not load collection from ZK:" + coll, e);
+ }
+ }
+
+ public static String getCollectionPath(String coll) {
+ return COLLECTIONS_ZKNODE+"/"+coll + "/state.json";
+ }
+
+ public void addCollectionWatch(String coll) throws KeeperException, InterruptedException {
+ synchronized (this) {
+ if (watchedCollections.contains(coll)) return;
+ else {
+ watchedCollections.add(coll);
+ }
+ addZkWatch(coll);
+ }
+ }
+
+ private void addZkWatch(final String coll) throws KeeperException,
+ InterruptedException {
+ log.info("addZkWatch {}", coll);
+ final String fullpath = getCollectionPath(coll);
+ synchronized (getUpdateLock()) {
+
+ cmdExecutor.ensureExists(fullpath, zkClient);
+ log.info("Updating collection state at {} from ZooKeeper... ", fullpath);
+
+ Watcher watcher = new Watcher() {
+
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events,
+ // and do not remove the watcher
+ if (EventType.None.equals(event.getType())) {
+ return;
+ }
+ log.info("A cluster state change: {} for collection {} has occurred - updating... (live nodes size: {})",
+ (event), coll, ZkStateReader.this.clusterState == null ? 0
+ : ZkStateReader.this.clusterState.getLiveNodes().size());
+ try {
+
+ // delayed approach
+ // ZkStateReader.this.updateClusterState(false, false);
+ synchronized (ZkStateReader.this.getUpdateLock()) {
+ if (!watchedCollections.contains(coll)) {
+ log.info("Unwatched collection {}", coll);
+ return;
+ }
+ // remake watch
+ final Watcher thisWatch = this;
+ Stat stat = new Stat();
+ byte[] data = zkClient.getData(fullpath, thisWatch, stat, true);
+
+ if (data == null || data.length == 0) {
+ log.warn("No value set for collection state : {}", coll);
+ return;
+
+ }
+ ClusterState clusterState = ClusterState.load(stat.getVersion(),
+ data, Collections.<String> emptySet(), fullpath);
+ // update volatile
+
+ DocCollection newState = clusterState.getCollectionStates()
+ .get(coll).get();
+ updateWatchedCollection(newState);
+
+ }
+ } catch (KeeperException e) {
+ if (e.code() == KeeperException.Code.SESSIONEXPIRED
+ || e.code() == KeeperException.Code.CONNECTIONLOSS) {
+ log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK");
+ return;
+ }
+ log.error("Unwatched collection :" + coll, e);
+ throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "", e);
+
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.error("Unwatched collection :" + coll, e);
+ return;
+ }
+ }
+
+ };
+ zkClient.exists(fullpath, watcher, true);
+ }
+ DocCollection collection = getCollectionLive(this, coll);
+ if (collection != null) {
+ updateWatchedCollection(collection);
+ }
+ }
+
+ private void updateWatchedCollection(DocCollection newState) {
+ watchedCollectionStates.put(newState.getName(), newState);
+ log.info("Updating data for {} to ver {} ", newState.getName(),
+ newState.getZNodeVersion());
+
+ this.clusterState = clusterState.copyWith(newState.getName(), newState);
+ }
+
+ /** This is not a public API. Only used by ZkController */
+ public void removeZKWatch(final String coll) {
+ synchronized (this) {
+ watchedCollections.remove(coll);
+ watchedCollectionStates.remove(coll);
+ try {
+ updateClusterState(true);
+ } catch (KeeperException e) {
+ log.error("Error updating state",e);
+ } catch (InterruptedException e) {
+ log.error("Error updating state",e);
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZooKeeperException.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZooKeeperException.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZooKeeperException.java
new file mode 100644
index 0000000..c712dea
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZooKeeperException.java
@@ -0,0 +1,33 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.solr.common.SolrException;
+
+public class ZooKeeperException extends SolrException {
+
+ public ZooKeeperException(ErrorCode code, String msg, Throwable th) {
+ super(code, msg, th);
+ }
+
+ public ZooKeeperException(ErrorCode code, String msg) {
+ super(code, msg);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/package-info.java
new file mode 100644
index 0000000..04ef67c
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common Solr Cloud and ZooKeeper related classes reused on both clients & server.
+ */
+package org.apache.solr.common.cloud;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/luke/FieldFlag.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/luke/FieldFlag.java b/ranger_solrj/src/main/java/org/apache/solr/common/luke/FieldFlag.java
new file mode 100644
index 0000000..720c0b6
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/luke/FieldFlag.java
@@ -0,0 +1,70 @@
+package org.apache.solr.common.luke;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ *
+ * @since solr 1.3
+ */
+public enum FieldFlag {
+ INDEXED('I', "Indexed"),
+ TOKENIZED('T', "Tokenized"),
+ STORED('S', "Stored"),
+ DOC_VALUES('D', "DocValues"),
+ MULTI_VALUED('M', "Multivalued"),
+ TERM_VECTOR_STORED('V', "TermVector Stored"),
+ TERM_VECTOR_OFFSET('o', "Store Offset With TermVector"),
+ TERM_VECTOR_POSITION('p', "Store Position With TermVector"),
+ OMIT_NORMS('O', "Omit Norms"),
+ OMIT_TF('F', "Omit Term Frequencies & Positions"),
+ OMIT_POSITIONS('P', "Omit Positions"),
+ STORE_OFFSETS_WITH_POSITIONS('H', "Store Offsets with Positions"),
+ LAZY('L', "Lazy"),
+ BINARY('B', "Binary"),
+ SORT_MISSING_FIRST('f', "Sort Missing First"),
+ SORT_MISSING_LAST('l', "Sort Missing Last");
+
+ private final char abbreviation;
+ private final String display;
+
+ FieldFlag(char abbreviation, String display) {
+ this.abbreviation = abbreviation;
+ this.display = display;
+ this.display.intern();//QUESTION: Need we bother here?
+ }
+
+ public static FieldFlag getFlag(char abbrev){
+ FieldFlag result = null;
+ FieldFlag [] vals = FieldFlag.values();
+ for (int i = 0; i < vals.length; i++) {
+ if (vals[i].getAbbreviation() == abbrev){
+ result = vals[i];
+ break;
+ }
+ }
+ return result;
+ }
+
+ public char getAbbreviation() {
+ return abbreviation;
+ }
+
+ public String getDisplay() {
+ return display;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/luke/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/luke/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/common/luke/package-info.java
new file mode 100644
index 0000000..e710d39
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/luke/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common constants used by the <code>LukeRequestHandler</code>.
+ */
+package org.apache.solr.common.luke;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/common/package-info.java
new file mode 100644
index 0000000..690a83c
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Common classes reused on both clients & server for dealing with {@link org.apache.solr.common.SolrInputDocument documents to be indexed} and {@link org.apache.solr.common.SolrDocumentList result documents}.
+ */
+package org.apache.solr.common;
+
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/AnalysisParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/AnalysisParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/AnalysisParams.java
new file mode 100644
index 0000000..36c276f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/AnalysisParams.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * Defines the request parameters used by all analysis request handlers.
+ *
+ *
+ * @since solr 1.4
+ */
+public interface AnalysisParams {
+
+ /**
+ * The prefix for all parameters.
+ */
+ static final String PREFIX = "analysis";
+
+ /**
+ * Holds the query to be analyzed.
+ */
+ static final String QUERY = PREFIX + ".query";
+
+ /**
+ * Set to {@code true} to indicate that the index tokens that match query tokens should be marked as "mateched".
+ */
+ static final String SHOW_MATCH = PREFIX + ".showmatch";
+
+
+ //===================================== FieldAnalysisRequestHandler Params =========================================
+
+ /**
+ * Holds the value of the field which should be analyzed.
+ */
+ static final String FIELD_NAME = PREFIX + ".fieldname";
+
+ /**
+ * Holds a comma-separated list of field types that the analysis should be peformed for.
+ */
+ static final String FIELD_TYPE = PREFIX + ".fieldtype";
+
+ /**
+ * Hodls a comma-separated list of field named that the analysis should be performed for.
+ */
+ static final String FIELD_VALUE = PREFIX + ".fieldvalue";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/AppendedSolrParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/AppendedSolrParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/AppendedSolrParams.java
new file mode 100644
index 0000000..4c9cc2e
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/AppendedSolrParams.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * SolrParams wrapper which acts similar to DefaultSolrParams except that
+ * it "appends" the values of multi-value params from both sub instances, so
+ * that all of the values are returned.
+ */
+public class AppendedSolrParams extends DefaultSolrParams {
+
+ public static AppendedSolrParams wrapAppended(SolrParams params, SolrParams extra) {
+ return new AppendedSolrParams(params, extra);
+ }
+
+ private AppendedSolrParams(SolrParams main, SolrParams extra) {
+ super(main, extra);
+ }
+
+ @Override
+ public String[] getParams(String param) {
+ String[] main = params.getParams(param);
+ String[] extra = defaults.getParams(param);
+ if (null == extra || 0 == extra.length) {
+ return main;
+ }
+ if (null == main || 0 == main.length) {
+ return extra;
+ }
+ String[] result = new String[main.length + extra.length];
+ System.arraycopy(main,0,result,0,main.length);
+ System.arraycopy(extra,0,result,main.length,extra.length);
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "{main("+params+"),extra("+defaults+")}";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/CollectionParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/CollectionParams.java
new file mode 100644
index 0000000..be8906e
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/CollectionParams.java
@@ -0,0 +1,74 @@
+package org.apache.solr.common.params;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Locale;
+
+public interface CollectionParams
+{
+ /** What action **/
+ public final static String ACTION = "action";
+ public final static String NAME = "name";
+
+
+
+ public enum CollectionAction {
+ CREATE,
+ DELETE,
+ RELOAD,
+ SYNCSHARD,
+ CREATEALIAS,
+ DELETEALIAS,
+ SPLITSHARD,
+ DELETESHARD,
+ CREATESHARD,
+ DELETEREPLICA,
+ MIGRATE,
+ ADDROLE,
+ REMOVEROLE,
+ CLUSTERPROP,
+ REQUESTSTATUS,
+ ADDREPLICA,
+ OVERSEERSTATUS,
+ LIST,
+ CLUSTERSTATUS,
+ ADDREPLICAPROP,
+ DELETEREPLICAPROP,
+ BALANCESHARDUNIQUE,
+ REBALANCELEADERS;
+
+ public static CollectionAction get( String p )
+ {
+ if( p != null ) {
+ try {
+ return CollectionAction.valueOf( p.toUpperCase(Locale.ROOT) );
+ }
+ catch( Exception ex ) {}
+ }
+ return null;
+ }
+ public boolean isEqual(String s){
+ if(s == null) return false;
+ return toString().equals(s.toUpperCase(Locale.ROOT));
+ }
+ public String toLower(){
+ return toString().toLowerCase(Locale.ROOT);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/CommonParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/CommonParams.java
new file mode 100644
index 0000000..699059a
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/CommonParams.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import java.util.Locale;
+
+
+/**
+ * Parameters used across many handlers
+ */
+public interface CommonParams {
+
+ /**
+ * Override for the concept of "NOW" to be used throughout this request,
+ * expressed as milliseconds since epoch. This is primarily used in
+ * distributed search to ensure consistent time values are used across
+ * multiple sub-requests.
+ */
+ public static final String NOW = "NOW";
+
+ /**
+ * Specifies the TimeZone used by the client for the purposes of
+ * any DateMath rounding that may take place when executing the request
+ */
+ public static final String TZ = "TZ";
+
+ /** the Request Handler (formerly known as the Query Type) - which Request Handler should handle the request */
+ public static final String QT ="qt";
+
+ /** the response writer type - the format of the response */
+ public static final String WT ="wt";
+
+ /** query string */
+ public static final String Q ="q";
+
+ /** rank query */
+ public static final String RQ ="rq";
+
+ /** distrib string */
+ public static final String DISTRIB = "distrib";
+
+ /** sort order */
+ public static final String SORT ="sort";
+
+ /** Lucene query string(s) for filtering the results without affecting scoring */
+ public static final String FQ ="fq";
+
+ /** zero based offset of matching documents to retrieve */
+ public static final String START ="start";
+
+ /** number of documents to return starting at "start" */
+ public static final String ROWS ="rows";
+
+ // SOLR-4228 start
+ /** handler value for SolrPing */
+ public static final String PING_HANDLER = "/admin/ping";
+
+ /** "action" parameter for SolrPing */
+ public static final String ACTION = "action";
+
+ /** "disable" value for SolrPing action */
+ public static final String DISABLE = "disable";
+
+ /** "enable" value for SolrPing action */
+ public static final String ENABLE = "enable";
+
+ /** "ping" value for SolrPing action */
+ public static final String PING = "ping";
+ // SOLR-4228 end
+
+ /** stylesheet to apply to XML results */
+ public static final String XSL ="xsl";
+
+ /** version parameter to check request-response compatibility */
+ public static final String VERSION ="version";
+
+ /** query and init param for field list */
+ public static final String FL = "fl";
+
+ /** default query field */
+ public static final String DF = "df";
+
+ /** Transformer param -- used with XSLT */
+ public static final String TR = "tr";
+
+ /** whether to include debug data for all components pieces, including doing explains*/
+ public static final String DEBUG_QUERY = "debugQuery";
+
+ /**
+ * Whether to provide debug info for specific items.
+ *
+ * @see #DEBUG_QUERY
+ */
+ public static final String DEBUG = "debug";
+
+ /**
+ * {@link #DEBUG} value indicating an interest in debug output related to timing
+ */
+ public static final String TIMING = "timing";
+ /**
+ * {@link #DEBUG} value indicating an interest in debug output related to the results (explains)
+ */
+ public static final String RESULTS = "results";
+ /**
+ * {@link #DEBUG} value indicating an interest in debug output related to the Query (parsing, etc.)
+ */
+ public static final String QUERY = "query";
+ /**
+ * {@link #DEBUG} value indicating an interest in debug output related to the distributed tracking
+ */
+ public static final String TRACK = "track";
+ /**
+ * boolean indicating whether score explanations should structured (true),
+ * or plain text (false)
+ */
+ public static final String EXPLAIN_STRUCT = "debug.explain.structured";
+
+ /** another query to explain against */
+ public static final String EXPLAIN_OTHER = "explainOther";
+
+
+ /** If the content stream should come from a URL (using URLConnection) */
+ public static final String STREAM_URL = "stream.url";
+
+ /** If the content stream should come from a File (using FileReader) */
+ public static final String STREAM_FILE = "stream.file";
+
+ /** If the content stream should come directly from a field */
+ public static final String STREAM_BODY = "stream.body";
+
+ /**
+ * Explicitly set the content type for the input stream
+ * If multiple streams are specified, the explicit contentType
+ * will be used for all of them.
+ */
+ public static final String STREAM_CONTENTTYPE = "stream.contentType";
+
+ /**
+ * Timeout value in milliseconds. If not set, or the value is >= 0, there is no timeout.
+ */
+ public static final String TIME_ALLOWED = "timeAllowed";
+
+ /** 'true' if the header should include the handler name */
+ public static final String HEADER_ECHO_HANDLER = "echoHandler";
+
+ /** include the parameters in the header **/
+ public static final String HEADER_ECHO_PARAMS = "echoParams";
+
+ /** include header in the response */
+ public static final String OMIT_HEADER = "omitHeader";
+
+ /** valid values for: <code>echoParams</code> */
+ public enum EchoParamStyle {
+ EXPLICIT,
+ ALL,
+ NONE;
+
+ public static EchoParamStyle get( String v ) {
+ if( v != null ) {
+ v = v.toUpperCase(Locale.ROOT);
+ if( v.equals( "EXPLICIT" ) ) {
+ return EXPLICIT;
+ }
+ if( v.equals( "ALL") ) {
+ return ALL;
+ }
+ if( v.equals( "NONE") ) { // the same as nothing...
+ return NONE;
+ }
+ }
+ return null;
+ }
+ };
+
+ /** which parameters to log (if not supplied all parameters will be logged) **/
+ public static final String LOG_PARAMS_LIST = "logParamsList";
+
+ public static final String EXCLUDE = "ex";
+ public static final String TAG = "tag";
+ public static final String TERMS = "terms";
+ public static final String OUTPUT_KEY = "key";
+ public static final String FIELD = "f";
+ public static final String VALUE = "v";
+ public static final String THREADS = "threads";
+ public static final String TRUE = Boolean.TRUE.toString();
+ public static final String FALSE = Boolean.FALSE.toString();
+
+ /** Used as a local parameter on queries. cache=false means don't check any query or filter caches.
+ * cache=true is the default.
+ */
+ public static final String CACHE = "cache";
+
+ /** Used as a local param on filter queries in conjunction with cache=false. Filters are checked in order, from
+ * smallest cost to largest. If cost>=100 and the query implements PostFilter, then that interface will be used to do post query filtering.
+ */
+ public static final String COST = "cost";
+
+ /**
+ * Request ID parameter added to the request when using debug=track
+ */
+ public static final String REQUEST_ID = "rid";
+
+ /**
+ * Request Purpose parameter added to each internal shard request when using debug=track
+ */
+ public static final String REQUEST_PURPOSE = "requestPurpose";
+
+ /**
+ * When querying a node, prefer local node's cores for distributed queries.
+ */
+ public static final String PREFER_LOCAL_SHARDS = "preferLocalShards";
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/CoreAdminParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/CoreAdminParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/CoreAdminParams.java
new file mode 100644
index 0000000..f673c6c
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/CoreAdminParams.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import java.util.Locale;
+
+/**
+ * @since solr 1.3
+ */
+public abstract class CoreAdminParams
+{
+ /** What Core are we talking about **/
+ public final static String CORE = "core";
+
+ /** Should the STATUS request include index info **/
+ public final static String INDEX_INFO = "indexInfo";
+
+ /** Persistent -- should it save the cores state? **/
+ public final static String PERSISTENT = "persistent";
+
+ /** If you rename something, what is the new name **/
+ public final static String NAME = "name";
+
+ /** Core data directory **/
+ public final static String DATA_DIR = "dataDir";
+
+ /** Core updatelog directory **/
+ public final static String ULOG_DIR = "ulogDir";
+
+ /** Name of the other core in actions involving 2 cores **/
+ public final static String OTHER = "other";
+
+ /** What action **/
+ public final static String ACTION = "action";
+
+ /** If you specify a schema, what is its name **/
+ public final static String SCHEMA = "schema";
+
+ /** If you specify a configset, what is its name **/
+ public final static String CONFIGSET = "configSet";
+
+ /** If you specify a config, what is its name **/
+ public final static String CONFIG = "config";
+
+ /** Specifies a core instance dir. */
+ public final static String INSTANCE_DIR = "instanceDir";
+
+ /** If you specify a file, what is its name **/
+ public final static String FILE = "file";
+
+ /** If you merge indexes, what are the index directories.
+ * The directories are specified by multiple indexDir parameters. */
+ public final static String INDEX_DIR = "indexDir";
+
+ /** If you merge indexes, what is the source core's name
+ * More than one source core can be specified by multiple srcCore parameters */
+ public final static String SRC_CORE = "srcCore";
+
+ /** The collection name in solr cloud */
+ public final static String COLLECTION = "collection";
+
+ /** The shard id in solr cloud */
+ public final static String SHARD = "shard";
+
+ /** The shard range in solr cloud */
+ public final static String SHARD_RANGE = "shard.range";
+
+ /** The shard range in solr cloud */
+ public final static String SHARD_STATE = "shard.state";
+
+ /** The parent shard if applicable */
+ public final static String SHARD_PARENT = "shard.parent";
+
+ /** The target core to which a split index should be written to
+ * Multiple targetCores can be specified by multiple targetCore parameters */
+ public final static String TARGET_CORE = "targetCore";
+
+ /** The hash ranges to be used to split a shard or an index */
+ public final static String RANGES = "ranges";
+
+ public static final String ROLES = "roles";
+
+ public static final String REQUESTID = "requestid";
+
+ public static final String CORE_NODE_NAME = "coreNodeName";
+
+ /** Prefix for core property name=value pair **/
+ public final static String PROPERTY_PREFIX = "property.";
+
+ /** If you unload a core, delete the index too */
+ public final static String DELETE_INDEX = "deleteIndex";
+
+ public static final String DELETE_DATA_DIR = "deleteDataDir";
+
+ public static final String DELETE_INSTANCE_DIR = "deleteInstanceDir";
+
+ public static final String LOAD_ON_STARTUP = "loadOnStartup";
+
+ public static final String TRANSIENT = "transient";
+
+ public enum CoreAdminAction {
+ STATUS,
+ LOAD,
+ UNLOAD,
+ RELOAD,
+ CREATE,
+ PERSIST,
+ SWAP,
+ RENAME,
+ MERGEINDEXES,
+ SPLIT,
+ PREPRECOVERY,
+ REQUESTRECOVERY,
+ REQUESTSYNCSHARD,
+ CREATEALIAS,
+ DELETEALIAS,
+ REQUESTBUFFERUPDATES,
+ REQUESTAPPLYUPDATES,
+ LOAD_ON_STARTUP,
+ TRANSIENT,
+ OVERSEEROP,
+ REQUESTSTATUS,
+ REJOINLEADERELECTION;
+
+ public static CoreAdminAction get( String p )
+ {
+ if( p != null ) {
+ try {
+ return CoreAdminAction.valueOf( p.toUpperCase(Locale.ROOT) );
+ }
+ catch( Exception ex ) {}
+ }
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/CursorMarkParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/CursorMarkParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/CursorMarkParams.java
new file mode 100644
index 0000000..c20cfd9
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/CursorMarkParams.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * Parameters and constants used when dealing with cursor based requests across
+ * large sorted result sets.
+ */
+public interface CursorMarkParams {
+
+ /**
+ * Param clients should specify indicating that they want a cursor based search.
+ * The value specified must either be {@link #CURSOR_MARK_START} indicating the
+ * first page of results, or a value returned by a previous search via the
+ * {@link #CURSOR_MARK_NEXT} key.
+ */
+ public static final String CURSOR_MARK_PARAM = "cursorMark";
+
+ /**
+ * Key used in Solr response to inform the client what the "next"
+ * {@link #CURSOR_MARK_PARAM} value should be to continue pagination
+ */
+ public static final String CURSOR_MARK_NEXT = "nextCursorMark";
+
+ /**
+ * Special value for {@link #CURSOR_MARK_PARAM} indicating that cursor functionality
+ * should be used, and a new cursor value should be computed afte the last result,
+ * but that currently the "first page" of results is being requested
+ */
+ public static final String CURSOR_MARK_START = "*";
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/DefaultSolrParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/DefaultSolrParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/DefaultSolrParams.java
new file mode 100644
index 0000000..1f308cd
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/DefaultSolrParams.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+
+/**
+ *
+ */
+public class DefaultSolrParams extends SolrParams {
+
+ protected final SolrParams params;
+ protected final SolrParams defaults;
+
+ protected DefaultSolrParams(SolrParams params, SolrParams defaults) {
+ assert params != null && defaults != null;
+ this.params = params;
+ this.defaults = defaults;
+ }
+
+ @Override
+ public String get(String param) {
+ String val = params.get(param);
+ return val!=null ? val : defaults.get(param);
+ }
+
+ @Override
+ public String[] getParams(String param) {
+ String[] vals = params.getParams(param);
+ return vals!=null ? vals : defaults.getParams(param);
+ }
+
+ @Override
+ public Iterator<String> getParameterNamesIterator() {
+ // We need to compute the set of all param names in advance
+ // So we don't wind up with an iterator that returns the same
+ // String more then once (SOLR-6780)
+ LinkedHashSet<String> allKeys = new LinkedHashSet<>();
+ for (SolrParams p : new SolrParams [] {params, defaults}) {
+ Iterator<String> localKeys = p.getParameterNamesIterator();
+ while (localKeys.hasNext()) {
+ allKeys.add(localKeys.next());
+ }
+ }
+ return allKeys.iterator();
+ }
+
+ @Override
+ public String toString() {
+ return "{params("+params+"),defaults("+defaults+")}";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/DisMaxParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/DisMaxParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/DisMaxParams.java
new file mode 100644
index 0000000..c2c268e
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/DisMaxParams.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+
+
+/**
+ * A collection of params used in DisMaxRequestHandler,
+ * both for Plugin initialization and for Requests.
+ */
+public interface DisMaxParams {
+
+ /** query and init param for tiebreaker value */
+ public static String TIE = "tie";
+
+ /** query and init param for query fields */
+ public static String QF = "qf";
+
+ /** query and init param for phrase boost fields */
+ public static String PF = "pf";
+
+ /** query and init param for bigram phrase boost fields */
+ public static String PF2 = "pf2";
+
+ /** query and init param for trigram phrase boost fields */
+ public static String PF3 = "pf3";
+
+ /** query and init param for MinShouldMatch specification */
+ public static String MM = "mm";
+
+ /**
+ * query and init param for Phrase Slop value in phrase
+ * boost query (in pf fields)
+ */
+ public static String PS = "ps";
+
+ /** default phrase slop for bigram phrases (pf2) */
+ public static String PS2 = "ps2";
+
+ /** default phrase slop for bigram phrases (pf3) */
+ public static String PS3 = "ps3";
+
+ /**
+ * query and init param for phrase Slop value in phrases
+ * explicitly included in the user's query string ( in qf fields)
+ */
+ public static String QS = "qs";
+
+ /** query and init param for boosting query */
+ public static String BQ = "bq";
+
+ /** query and init param for boosting functions */
+ public static String BF = "bf";
+
+ /**
+ * Alternate query (expressed in Solr QuerySyntax)
+ * to use if main query (q) is empty
+ */
+ public static String ALTQ = "q.alt";
+
+ /** query and init param for field list */
+ public static String GEN = "gen";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/EventParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/EventParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/EventParams.java
new file mode 100644
index 0000000..f5d43c0
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/EventParams.java
@@ -0,0 +1,29 @@
+package org.apache.solr.common.params;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/**
+ *
+ *
+ **/
+public interface EventParams {
+ /** Event param for things like newSearcher, firstSearcher**/
+ public static final String EVENT = "event";
+ public static final String NEW_SEARCHER = "newSearcher";
+ public static final String FIRST_SEARCHER = "firstSearcher";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/ExpandParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/ExpandParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/ExpandParams.java
new file mode 100644
index 0000000..a8f0cf7
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/ExpandParams.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * Expand parameters
+ */
+public interface ExpandParams {
+
+ public static final String EXPAND = "expand";
+ public static final String EXPAND_SORT = EXPAND + ".sort";
+ public static final String EXPAND_ROWS = EXPAND + ".rows";
+ public static final String EXPAND_FIELD = EXPAND + ".field";
+ public static final String EXPAND_Q = EXPAND + ".q";
+ public static final String EXPAND_FQ = EXPAND + ".fq";
+}
+
[04/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/UpdateParams.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/UpdateParams.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/UpdateParams.java
new file mode 100644
index 0000000..cf965a9
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/UpdateParams.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.params;
+
+/**
+ * A collection of standard params used by Update handlers
+ *
+ *
+ * @since solr 1.2
+ */
+public interface UpdateParams
+{
+
+ /** Open up a new searcher as part of a commit */
+ public static String OPEN_SEARCHER = "openSearcher";
+
+ /** wait for the searcher to be registered/visible */
+ public static String WAIT_SEARCHER = "waitSearcher";
+
+ public static String SOFT_COMMIT = "softCommit";
+
+ /** overwrite indexing fields */
+ public static String OVERWRITE = "overwrite";
+
+ /** Commit everything after the command completes */
+ public static String COMMIT = "commit";
+
+ /** Commit within a certain time period (in ms) */
+ public static String COMMIT_WITHIN = "commitWithin";
+
+ /** Optimize the index and commit everything after the command completes */
+ public static String OPTIMIZE = "optimize";
+
+ /** expert: calls IndexWriter.prepareCommit */
+ public static String PREPARE_COMMIT = "prepareCommit";
+
+ /** Rollback update commands */
+ public static String ROLLBACK = "rollback";
+
+ public static String COLLECTION = "collection";
+
+ /** Select the update processor chain to use. A RequestHandler may or may not respect this parameter */
+ public static final String UPDATE_CHAIN = "update.chain";
+
+ /** Override the content type used for UpdateLoader **/
+ public static final String ASSUME_CONTENT_TYPE = "update.contentType";
+
+ /**
+ * If optimizing, set the maximum number of segments left in the index after optimization. 1 is the default (and is equivalent to calling IndexWriter.optimize() in Lucene).
+ */
+ public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
+
+ public static final String EXPUNGE_DELETES = "expungeDeletes";
+
+ /** Return versions of updates? */
+ public static final String VERSIONS = "versions";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/params/package-info.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/params/package-info.java b/ranger_solrj/src/main/java/org/apache/solr/common/params/package-info.java
new file mode 100644
index 0000000..94622cc
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/params/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Parameter constants and enumerations.
+ */
+package org.apache.solr.common.params;
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/Base64.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/Base64.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/Base64.java
new file mode 100644
index 0000000..8cc3381
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/Base64.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.common.util;
+
+/**
+ * Static methods for translating Base64 encoded strings to byte arrays
+ * and vice-versa.
+ */
+
+public class Base64 {
+ /**
+ * This array is a lookup table that translates 6-bit positive integer
+ * index values into their "Base64 Alphabet" equivalents as specified
+ * in Table 1 of RFC 2045.
+ */
+ private static final char intToBase64[] = {
+ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+ 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/'
+ };
+
+ /**
+ * This array is a lookup table that translates unicode characters
+ * drawn from the "Base64 Alphabet" (as specified in Table 1 of RFC 2045)
+ * into their 6-bit positive integer equivalents. Characters that
+ * are not in the Base64 alphabet but fall within the bounds of the
+ * array are translated to -1.
+ */
+ private static final byte base64ToInt[] = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, 52, 53, 54,
+ 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4,
+ 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
+ 24, 25, -1, -1, -1, -1, -1, -1, 26, 27, 28, 29, 30, 31, 32, 33, 34,
+ 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51
+ };
+
+ public static String byteArrayToBase64(byte[] a, int offset, int len) {
+ int aLen = len;
+ int numFullGroups = aLen / 3;
+ int numBytesInPartialGroup = aLen - 3 * numFullGroups;
+ int resultLen = 4 * ((aLen + 2) / 3);
+ StringBuilder result = new StringBuilder(resultLen);
+ char[] intToAlpha = intToBase64;
+
+ // Translate all full groups from byte array elements to Base64
+ int inCursor = offset;
+ for (int i = 0; i < numFullGroups; i++) {
+ int byte0 = a[inCursor++] & 0xff;
+ int byte1 = a[inCursor++] & 0xff;
+ int byte2 = a[inCursor++] & 0xff;
+ result.append(intToAlpha[byte0 >> 2]);
+ result.append(intToAlpha[(byte0 << 4) & 0x3f | (byte1 >> 4)]);
+ result.append(intToAlpha[(byte1 << 2) & 0x3f | (byte2 >> 6)]);
+ result.append(intToAlpha[byte2 & 0x3f]);
+ }
+
+ // Translate partial group if present
+ if (numBytesInPartialGroup != 0) {
+ int byte0 = a[inCursor++] & 0xff;
+ result.append(intToAlpha[byte0 >> 2]);
+ if (numBytesInPartialGroup == 1) {
+ result.append(intToAlpha[(byte0 << 4) & 0x3f]);
+ result.append("==");
+ } else {
+ // assert numBytesInPartialGroup == 2;
+ int byte1 = a[inCursor++] & 0xff;
+ result.append(intToAlpha[(byte0 << 4) & 0x3f | (byte1 >> 4)]);
+ result.append(intToAlpha[(byte1 << 2) & 0x3f]);
+ result.append('=');
+ }
+ }
+ return result.toString();
+ }
+
+ public static byte[] base64ToByteArray(String s) {
+ byte[] alphaToInt = base64ToInt;
+ int sLen = s.length();
+ int numGroups = sLen / 4;
+ if (4 * numGroups != sLen)
+ throw new IllegalArgumentException(
+ "String length must be a multiple of four.");
+ int missingBytesInLastGroup = 0;
+ int numFullGroups = numGroups;
+ if (sLen != 0) {
+ if (s.charAt(sLen - 1) == '=') {
+ missingBytesInLastGroup++;
+ numFullGroups--;
+ }
+ if (s.charAt(sLen - 2) == '=')
+ missingBytesInLastGroup++;
+ }
+ byte[] result = new byte[3 * numGroups - missingBytesInLastGroup];
+
+ // Translate all full groups from base64 to byte array elements
+ int inCursor = 0, outCursor = 0;
+ for (int i = 0; i < numFullGroups; i++) {
+ int ch0 = base64toInt(s.charAt(inCursor++), alphaToInt);
+ int ch1 = base64toInt(s.charAt(inCursor++), alphaToInt);
+ int ch2 = base64toInt(s.charAt(inCursor++), alphaToInt);
+ int ch3 = base64toInt(s.charAt(inCursor++), alphaToInt);
+ result[outCursor++] = (byte) ((ch0 << 2) | (ch1 >> 4));
+ result[outCursor++] = (byte) ((ch1 << 4) | (ch2 >> 2));
+ result[outCursor++] = (byte) ((ch2 << 6) | ch3);
+ }
+
+ // Translate partial group, if present
+ if (missingBytesInLastGroup != 0) {
+ int ch0 = base64toInt(s.charAt(inCursor++), alphaToInt);
+ int ch1 = base64toInt(s.charAt(inCursor++), alphaToInt);
+ result[outCursor++] = (byte) ((ch0 << 2) | (ch1 >> 4));
+
+ if (missingBytesInLastGroup == 1) {
+ int ch2 = base64toInt(s.charAt(inCursor++), alphaToInt);
+ result[outCursor++] = (byte) ((ch1 << 4) | (ch2 >> 2));
+ }
+ }
+ // assert inCursor == s.length()-missingBytesInLastGroup;
+ // assert outCursor == result.length;
+ return result;
+ }
+
+ /**
+ * Translates the specified character, which is assumed to be in the
+ * "Base 64 Alphabet" into its equivalent 6-bit positive integer.
+ *
+ * @throw IllegalArgumentException or ArrayOutOfBoundsException if
+ * c is not in the Base64 Alphabet.
+ */
+ private static int base64toInt(char c, byte[] alphaToInt) {
+ int result = alphaToInt[c];
+ if (result < 0)
+ throw new IllegalArgumentException("Illegal character " + c);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/ByteUtils.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/ByteUtils.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/ByteUtils.java
new file mode 100644
index 0000000..9ce607e
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/ByteUtils.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import org.noggit.CharArr;
+
+public class ByteUtils {
+
+ /** Converts utf8 to utf16 and returns the number of 16 bit Java chars written.
+ * Full characters are read, even if this reads past the length passed (and can result in
+ * an ArrayOutOfBoundsException if invalid UTF8 is passed). Explicit checks for valid UTF8 are not performed.
+ * The char[] out should probably have enough room to hold the worst case of each byte becoming a Java char.
+ */
+ public static int UTF8toUTF16(byte[] utf8, int offset, int len, char[] out, int out_offset) {
+ int out_start = out_offset;
+ final int limit = offset + len;
+ while (offset < limit) {
+ int b = utf8[offset++]&0xff;
+
+ if (b < 0xc0) {
+ assert b < 0x80;
+ out[out_offset++] = (char)b;
+ } else if (b < 0xe0) {
+ out[out_offset++] = (char)(((b&0x1f)<<6) + (utf8[offset++]&0x3f));
+ } else if (b < 0xf0) {
+ out[out_offset++] = (char)(((b&0xf)<<12) + ((utf8[offset]&0x3f)<<6) + (utf8[offset+1]&0x3f));
+ offset += 2;
+ } else {
+ assert b < 0xf8;
+ int ch = ((b&0x7)<<18) + ((utf8[offset]&0x3f)<<12) + ((utf8[offset+1]&0x3f)<<6) + (utf8[offset+2]&0x3f);
+ offset += 3;
+ if (ch < 0xffff) {
+ out[out_offset++] = (char)ch;
+ } else {
+ int chHalf = ch - 0x0010000;
+ out[out_offset++] = (char) ((chHalf >> 10) + 0xD800);
+ out[out_offset++] = (char) ((chHalf & 0x3FFL) + 0xDC00);
+ }
+ }
+ }
+
+ return out_offset - out_start;
+ }
+
+ /** Convert UTF8 bytes into UTF16 characters. */
+ public static void UTF8toUTF16(byte[] utf8, int offset, int len, CharArr out) {
+ // TODO: do in chunks if the input is large
+ out.reserve(len);
+ int n = UTF8toUTF16(utf8, offset, len, out.getArray(), out.getEnd());
+ out.setEnd(out.getEnd() + n);
+ }
+
+ /** Convert UTF8 bytes into a String */
+ public static String UTF8toUTF16(byte[] utf8, int offset, int len) {
+ char[] out = new char[len];
+ int n = UTF8toUTF16(utf8, offset, len, out, 0);
+ return new String(out,0,n);
+ }
+
+
+
+ /** Writes UTF8 into the byte array, starting at offset. The caller should ensure that
+ * there is enough space for the worst-case scenario.
+ * @return the number of bytes written
+ */
+ public static int UTF16toUTF8(CharSequence s, int offset, int len, byte[] result, int resultOffset) {
+ final int end = offset + len;
+
+ int upto = resultOffset;
+ for(int i=offset;i<end;i++) {
+ final int code = (int) s.charAt(i);
+
+ if (code < 0x80)
+ result[upto++] = (byte) code;
+ else if (code < 0x800) {
+ result[upto++] = (byte) (0xC0 | (code >> 6));
+ result[upto++] = (byte)(0x80 | (code & 0x3F));
+ } else if (code < 0xD800 || code > 0xDFFF) {
+ result[upto++] = (byte)(0xE0 | (code >> 12));
+ result[upto++] = (byte)(0x80 | ((code >> 6) & 0x3F));
+ result[upto++] = (byte)(0x80 | (code & 0x3F));
+ } else {
+ // surrogate pair
+ // confirm valid high surrogate
+ if (code < 0xDC00 && (i < end-1)) {
+ int utf32 = (int) s.charAt(i+1);
+ // confirm valid low surrogate and write pair
+ if (utf32 >= 0xDC00 && utf32 <= 0xDFFF) {
+ utf32 = ((code - 0xD7C0) << 10) + (utf32 & 0x3FF);
+ i++;
+ result[upto++] = (byte)(0xF0 | (utf32 >> 18));
+ result[upto++] = (byte)(0x80 | ((utf32 >> 12) & 0x3F));
+ result[upto++] = (byte)(0x80 | ((utf32 >> 6) & 0x3F));
+ result[upto++] = (byte)(0x80 | (utf32 & 0x3F));
+ continue;
+ }
+ }
+ // replace unpaired surrogate or out-of-order low surrogate
+ // with substitution character
+ result[upto++] = (byte) 0xEF;
+ result[upto++] = (byte) 0xBF;
+ result[upto++] = (byte) 0xBD;
+ }
+ }
+
+ return upto - resultOffset;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStream.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStream.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStream.java
new file mode 100644
index 0000000..b455d80
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStream.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+
+/**
+ *
+ * @since solr 1.2
+ */
+public interface ContentStream {
+ String getName();
+ String getSourceInfo();
+ String getContentType();
+
+ /**
+ * @return the stream size or <code>null</code> if not known
+ */
+ Long getSize(); // size if we know it, otherwise null
+
+ /**
+ * Get an open stream. You are responsible for closing it. Consider using
+ * something like:
+ * <pre>
+ * InputStream stream = stream.getStream();
+ * try {
+ * // use the stream...
+ * }
+ * finally {
+ * IOUtils.closeQuietly(stream);
+ * }
+ * </pre>
+ *
+ * Only the first call to <code>getStream()</code> or <code>getReader()</code>
+ * is guaranteed to work. The runtime behavior for additional calls is undefined.
+ *
+ * Note: you must call <code>getStream()</code> or <code>getReader()</code> before
+ * the attributes (name, contentType, etc) are guaranteed to be set. Streams may be
+ * lazy loaded only when this method is called.
+ */
+ InputStream getStream() throws IOException;
+
+ /**
+ * Get an open stream. You are responsible for closing it. Consider using
+ * something like:
+ * <pre>
+ * Reader reader = stream.getReader();
+ * try {
+ * // use the reader...
+ * }
+ * finally {
+ * IOUtils.closeQuietly(reader);
+ * }
+ * </pre>
+ *
+ * Only the first call to <code>getStream()</code> or <code>getReader()</code>
+ * is guaranteed to work. The runtime behavior for additional calls is undefined.
+ *
+ * Note: you must call <code>getStream()</code> or <code>getReader()</code> before
+ * the attributes (name, contentType, etc) are guaranteed to be set. Streams may be
+ * lazy loaded only when this method is called.
+ */
+ Reader getReader() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStreamBase.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStreamBase.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStreamBase.java
new file mode 100644
index 0000000..34238b5
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/ContentStreamBase.java
@@ -0,0 +1,260 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.util.Locale;
+
+/**
+ * Three concrete implementations for ContentStream - one for File/URL/String
+ *
+ *
+ * @since solr 1.2
+ */
+public abstract class ContentStreamBase implements ContentStream
+{
+ public static final String DEFAULT_CHARSET = StandardCharsets.UTF_8.name();
+
+ protected String name;
+ protected String sourceInfo;
+ protected String contentType;
+ protected Long size;
+
+ //---------------------------------------------------------------------
+ //---------------------------------------------------------------------
+
+ public static String getCharsetFromContentType( String contentType )
+ {
+ if( contentType != null ) {
+ int idx = contentType.toLowerCase(Locale.ROOT).indexOf( "charset=" );
+ if( idx > 0 ) {
+ return contentType.substring( idx + "charset=".length() ).trim();
+ }
+ }
+ return null;
+ }
+
+ //------------------------------------------------------------------------
+ //------------------------------------------------------------------------
+
+ /**
+ * Construct a <code>ContentStream</code> from a <code>URL</code>
+ *
+ * This uses a <code>URLConnection</code> to get the content stream
+ * @see URLConnection
+ */
+ public static class URLStream extends ContentStreamBase
+ {
+ private final URL url;
+
+ public URLStream( URL url ) {
+ this.url = url;
+ sourceInfo = "url";
+ }
+
+ @Override
+ public InputStream getStream() throws IOException {
+ URLConnection conn = this.url.openConnection();
+
+ contentType = conn.getContentType();
+ name = url.toExternalForm();
+ size = new Long( conn.getContentLength() );
+ return conn.getInputStream();
+ }
+ }
+
+ /**
+ * Construct a <code>ContentStream</code> from a <code>File</code>
+ */
+ public static class FileStream extends ContentStreamBase
+ {
+ private final File file;
+
+ public FileStream( File f ) {
+ file = f;
+
+ contentType = null; // ??
+ name = file.getName();
+ size = file.length();
+ sourceInfo = file.toURI().toString();
+ }
+
+ @Override
+ public String getContentType() {
+ if(contentType==null) {
+ InputStream stream = null;
+ try {
+ stream = new FileInputStream(file);
+ char first = (char)stream.read();
+ if(first == '<') {
+ return "application/xml";
+ }
+ if(first == '{') {
+ return "application/json";
+ }
+ } catch(Exception ex) {
+ } finally {
+ if (stream != null) try {
+ stream.close();
+ } catch (IOException ioe) {}
+ }
+ }
+ return contentType;
+ }
+
+ @Override
+ public InputStream getStream() throws IOException {
+ return new FileInputStream( file );
+ }
+ }
+
+
+ /**
+ * Construct a <code>ContentStream</code> from a <code>String</code>
+ */
+ public static class StringStream extends ContentStreamBase
+ {
+ private final String str;
+
+ public StringStream( String str ) {
+ this.str = str;
+
+ contentType = null;
+ name = null;
+ size = new Long( str.length() );
+ sourceInfo = "string";
+ }
+
+ @Override
+ public String getContentType() {
+ if(contentType==null && str.length() > 0) {
+ char first = str.charAt(0);
+ if(first == '<') {
+ return "application/xml";
+ }
+ if(first == '{') {
+ return "application/json";
+ }
+ // find a comma? for CSV?
+ }
+ return contentType;
+ }
+
+ @Override
+ public InputStream getStream() throws IOException {
+ return new ByteArrayInputStream( str.getBytes(DEFAULT_CHARSET) );
+ }
+
+ /**
+ * If an charset is defined (by the contentType) use that, otherwise
+ * use a StringReader
+ */
+ @Override
+ public Reader getReader() throws IOException {
+ String charset = getCharsetFromContentType( contentType );
+ return charset == null
+ ? new StringReader( str )
+ : new InputStreamReader( getStream(), charset );
+ }
+ }
+
+ /**
+ * Base reader implementation. If the contentType declares a
+ * charset use it, otherwise use "utf-8".
+ */
+ @Override
+ public Reader getReader() throws IOException {
+ String charset = getCharsetFromContentType( getContentType() );
+ return charset == null
+ ? new InputStreamReader( getStream(), DEFAULT_CHARSET )
+ : new InputStreamReader( getStream(), charset );
+ }
+
+ //------------------------------------------------------------------
+ // Getters / Setters for overrideable attributes
+ //------------------------------------------------------------------
+
+ @Override
+ public String getContentType() {
+ return contentType;
+ }
+
+ public void setContentType(String contentType) {
+ this.contentType = contentType;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public Long getSize() {
+ return size;
+ }
+
+ public void setSize(Long size) {
+ this.size = size;
+ }
+
+ @Override
+ public String getSourceInfo() {
+ return sourceInfo;
+ }
+
+ public void setSourceInfo(String sourceInfo) {
+ this.sourceInfo = sourceInfo;
+ }
+
+ /**
+ * Construct a <code>ContentStream</code> from a <code>File</code>
+ */
+ public static class ByteArrayStream extends ContentStreamBase
+ {
+ private final byte[] bytes;
+
+ public ByteArrayStream( byte[] bytes, String source ) {
+ this.bytes = bytes;
+
+ this.contentType = null;
+ name = source;
+ size = new Long(bytes.length);
+ sourceInfo = source;
+ }
+
+
+ @Override
+ public InputStream getStream() throws IOException {
+ return new ByteArrayInputStream( bytes );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/DataInputInputStream.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/DataInputInputStream.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/DataInputInputStream.java
new file mode 100644
index 0000000..d412f40
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/DataInputInputStream.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.DataInput;
+import java.io.InputStream;
+
+/**
+ * An abstract DataInput that extends InputStream
+ */
+public abstract class DataInputInputStream extends InputStream implements DataInput {
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/DateUtil.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/DateUtil.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/DateUtil.java
new file mode 100644
index 0000000..49b7fc4
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/DateUtil.java
@@ -0,0 +1,260 @@
+package org.apache.solr.common.util;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.TimeZone;
+
+
+/**
+ * This class has some code from HttpClient DateUtil.
+ */
+public class DateUtil {
+ //start HttpClient
+ /**
+ * Date format pattern used to parse HTTP date headers in RFC 1123 format.
+ */
+ public static final String PATTERN_RFC1123 = "EEE, dd MMM yyyy HH:mm:ss zzz";
+
+ /**
+ * Date format pattern used to parse HTTP date headers in RFC 1036 format.
+ */
+ public static final String PATTERN_RFC1036 = "EEEE, dd-MMM-yy HH:mm:ss zzz";
+
+ /**
+ * Date format pattern used to parse HTTP date headers in ANSI C
+ * <code>asctime()</code> format.
+ */
+ public static final String PATTERN_ASCTIME = "EEE MMM d HH:mm:ss yyyy";
+ //These are included for back compat
+ private static final Collection<String> DEFAULT_HTTP_CLIENT_PATTERNS = Arrays.asList(
+ PATTERN_ASCTIME, PATTERN_RFC1036, PATTERN_RFC1123);
+
+ private static final Date DEFAULT_TWO_DIGIT_YEAR_START;
+
+ static {
+ Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
+ calendar.set(2000, Calendar.JANUARY, 1, 0, 0);
+ DEFAULT_TWO_DIGIT_YEAR_START = calendar.getTime();
+ }
+
+ private static final TimeZone GMT = TimeZone.getTimeZone("GMT");
+
+ //end HttpClient
+
+ //---------------------------------------------------------------------------------------
+
+ /**
+ * A suite of default date formats that can be parsed, and thus transformed to the Solr specific format
+ */
+ public static final Collection<String> DEFAULT_DATE_FORMATS = new ArrayList<>();
+
+ static {
+ DEFAULT_DATE_FORMATS.add("yyyy-MM-dd'T'HH:mm:ss'Z'");
+ DEFAULT_DATE_FORMATS.add("yyyy-MM-dd'T'HH:mm:ss");
+ DEFAULT_DATE_FORMATS.add("yyyy-MM-dd");
+ DEFAULT_DATE_FORMATS.add("yyyy-MM-dd hh:mm:ss");
+ DEFAULT_DATE_FORMATS.add("yyyy-MM-dd HH:mm:ss");
+ DEFAULT_DATE_FORMATS.add("EEE MMM d hh:mm:ss z yyyy");
+ DEFAULT_DATE_FORMATS.addAll(DEFAULT_HTTP_CLIENT_PATTERNS);
+ }
+
+ /**
+ * Returns a formatter that can be use by the current thread if needed to
+ * convert Date objects to the Internal representation.
+ *
+ * @param d The input date to parse
+ * @return The parsed {@link java.util.Date}
+ * @throws java.text.ParseException If the input can't be parsed
+ */
+ public static Date parseDate(String d) throws ParseException {
+ return parseDate(d, DEFAULT_DATE_FORMATS);
+ }
+
+ public static Date parseDate(String d, Collection<String> fmts) throws ParseException {
+ // 2007-04-26T08:05:04Z
+ if (d.endsWith("Z") && d.length() > 20) {
+ return getThreadLocalDateFormat().parse(d);
+ }
+ return parseDate(d, fmts, null);
+ }
+
+ /**
+ * Slightly modified from org.apache.commons.httpclient.util.DateUtil.parseDate
+ * <p>
+ * Parses the date value using the given date formats.
+ *
+ * @param dateValue the date value to parse
+ * @param dateFormats the date formats to use
+ * @param startDate During parsing, two digit years will be placed in the range
+ * <code>startDate</code> to <code>startDate + 100 years</code>. This value may
+ * be <code>null</code>. When <code>null</code> is given as a parameter, year
+ * <code>2000</code> will be used.
+ * @return the parsed date
+ * @throws ParseException if none of the dataFormats could parse the dateValue
+ */
+ public static Date parseDate(
+ String dateValue,
+ Collection<String> dateFormats,
+ Date startDate
+ ) throws ParseException {
+
+ if (dateValue == null) {
+ throw new IllegalArgumentException("dateValue is null");
+ }
+ if (dateFormats == null) {
+ dateFormats = DEFAULT_HTTP_CLIENT_PATTERNS;
+ }
+ if (startDate == null) {
+ startDate = DEFAULT_TWO_DIGIT_YEAR_START;
+ }
+ // trim single quotes around date if present
+ // see issue #5279
+ if (dateValue.length() > 1
+ && dateValue.startsWith("'")
+ && dateValue.endsWith("'")
+ ) {
+ dateValue = dateValue.substring(1, dateValue.length() - 1);
+ }
+
+ SimpleDateFormat dateParser = null;
+ Iterator formatIter = dateFormats.iterator();
+
+ while (formatIter.hasNext()) {
+ String format = (String) formatIter.next();
+ if (dateParser == null) {
+ dateParser = new SimpleDateFormat(format, Locale.ROOT);
+ dateParser.setTimeZone(GMT);
+ dateParser.set2DigitYearStart(startDate);
+ } else {
+ dateParser.applyPattern(format);
+ }
+ try {
+ return dateParser.parse(dateValue);
+ } catch (ParseException pe) {
+ // ignore this exception, we will try the next format
+ }
+ }
+
+ // we were unable to parse the date
+ throw new ParseException("Unable to parse the date " + dateValue, 0);
+ }
+
+
+ /**
+ * Returns a formatter that can be use by the current thread if needed to
+ * convert Date objects to the Internal representation.
+ *
+ * @return The {@link java.text.DateFormat} for the current thread
+ */
+ public static DateFormat getThreadLocalDateFormat() {
+ return fmtThreadLocal.get();
+ }
+
+ public static TimeZone UTC = TimeZone.getTimeZone("UTC");
+ private static ThreadLocalDateFormat fmtThreadLocal = new ThreadLocalDateFormat();
+
+ private static class ThreadLocalDateFormat extends ThreadLocal<DateFormat> {
+ DateFormat proto;
+
+ public ThreadLocalDateFormat() {
+ super();
+ //2007-04-26T08:05:04Z
+ SimpleDateFormat tmp = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT);
+ tmp.setTimeZone(UTC);
+ proto = tmp;
+ }
+
+ @Override
+ protected DateFormat initialValue() {
+ return (DateFormat) proto.clone();
+ }
+ }
+
+ /** Formats the date and returns the calendar instance that was used (which may be reused) */
+ public static Calendar formatDate(Date date, Calendar cal, Appendable out) throws IOException {
+ // using a stringBuilder for numbers can be nice since
+ // a temporary string isn't used (it's added directly to the
+ // builder's buffer.
+
+ StringBuilder sb = out instanceof StringBuilder ? (StringBuilder)out : new StringBuilder();
+ if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.ROOT);
+ cal.setTime(date);
+
+ int i = cal.get(Calendar.YEAR);
+ sb.append(i);
+ sb.append('-');
+ i = cal.get(Calendar.MONTH) + 1; // 0 based, so add 1
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append('-');
+ i=cal.get(Calendar.DAY_OF_MONTH);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append('T');
+ i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append(':');
+ i=cal.get(Calendar.MINUTE);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append(':');
+ i=cal.get(Calendar.SECOND);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ i=cal.get(Calendar.MILLISECOND);
+ if (i != 0) {
+ sb.append('.');
+ if (i<100) sb.append('0');
+ if (i<10) sb.append('0');
+ sb.append(i);
+
+ // handle canonical format specifying fractional
+ // seconds shall not end in '0'. Given the slowness of
+ // integer div/mod, simply checking the last character
+ // is probably the fastest way to check.
+ int lastIdx = sb.length()-1;
+ if (sb.charAt(lastIdx)=='0') {
+ lastIdx--;
+ if (sb.charAt(lastIdx)=='0') {
+ lastIdx--;
+ }
+ sb.setLength(lastIdx+1);
+ }
+
+ }
+ sb.append('Z');
+
+ if (out != sb)
+ out.append(sb);
+
+ return cal;
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/ExecutorUtil.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/ExecutorUtil.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/ExecutorUtil.java
new file mode 100644
index 0000000..0c058c9
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/ExecutorUtil.java
@@ -0,0 +1,64 @@
+package org.apache.solr.common.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class ExecutorUtil {
+ public static Logger log = LoggerFactory.getLogger(ExecutorUtil.class);
+
+ public static void shutdownNowAndAwaitTermination(ExecutorService pool) {
+ pool.shutdown(); // Disable new tasks from being submitted
+ pool.shutdownNow(); // Cancel currently executing tasks
+ boolean shutdown = false;
+ while (!shutdown) {
+ try {
+ // Wait a while for existing tasks to terminate
+ shutdown = pool.awaitTermination(5, TimeUnit.SECONDS);
+ } catch (InterruptedException ie) {
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ if (!shutdown) {
+ pool.shutdownNow(); // Cancel currently executing tasks
+ }
+ }
+ }
+
+ public static void shutdownAndAwaitTermination(ExecutorService pool) {
+ pool.shutdown(); // Disable new tasks from being submitted
+ boolean shutdown = false;
+ while (!shutdown) {
+ try {
+ // Wait a while for existing tasks to terminate
+ shutdown = pool.awaitTermination(60, TimeUnit.SECONDS);
+ } catch (InterruptedException ie) {
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ if (!shutdown) {
+ pool.shutdownNow(); // Cancel currently executing tasks
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/FastInputStream.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/FastInputStream.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/FastInputStream.java
new file mode 100644
index 0000000..8a2ecee
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/FastInputStream.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.*;
+
+/** Single threaded buffered InputStream
+ * Internal Solr use only, subject to change.
+ */
+public class FastInputStream extends DataInputInputStream {
+ protected final InputStream in;
+ protected final byte[] buf;
+ protected int pos;
+ protected int end;
+ protected long readFromStream; // number of bytes read from the underlying inputstream
+
+ public FastInputStream(InputStream in) {
+ // use default BUFSIZE of BufferedOutputStream so if we wrap that
+ // it won't cause double buffering.
+ this(in, new byte[8192], 0, 0);
+ }
+
+ public FastInputStream(InputStream in, byte[] tempBuffer, int start, int end) {
+ this.in = in;
+ this.buf = tempBuffer;
+ this.pos = start;
+ this.end = end;
+ }
+
+
+ public static FastInputStream wrap(InputStream in) {
+ return (in instanceof FastInputStream) ? (FastInputStream)in : new FastInputStream(in);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (pos >= end) {
+ refill();
+ if (pos >= end) return -1;
+ }
+ return buf[pos++] & 0xff;
+ }
+
+ public int peek() throws IOException {
+ if (pos >= end) {
+ refill();
+ if (pos >= end) return -1;
+ }
+ return buf[pos] & 0xff;
+ }
+
+
+ @Override
+ public int readUnsignedByte() throws IOException {
+ if (pos >= end) {
+ refill();
+ if (pos >= end) {
+ throw new EOFException();
+ }
+ }
+ return buf[pos++] & 0xff;
+ }
+
+ public int readWrappedStream(byte[] target, int offset, int len) throws IOException {
+ return in.read(target, offset, len);
+ }
+
+ public long position() {
+ return readFromStream - (end - pos);
+ }
+
+ public void refill() throws IOException {
+ // this will set end to -1 at EOF
+ end = readWrappedStream(buf, 0, buf.length);
+ if (end > 0) readFromStream += end;
+ pos = 0;
+ }
+
+ @Override
+ public int available() throws IOException {
+ return end - pos;
+ }
+
+ @Override
+ public int read(byte b[], int off, int len) throws IOException {
+ int r=0; // number of bytes we have read
+
+ // first read from our buffer;
+ if (end-pos > 0) {
+ r = Math.min(end-pos, len);
+ System.arraycopy(buf, pos, b, off, r);
+ pos += r;
+ }
+
+ if (r == len) return r;
+
+ // amount left to read is >= buffer size
+ if (len-r >= buf.length) {
+ int ret = readWrappedStream(b, off+r, len-r);
+ if (ret >= 0) {
+ readFromStream += ret;
+ r += ret;
+ return r;
+ } else {
+ // negative return code
+ return r > 0 ? r : -1;
+ }
+ }
+
+ refill();
+
+ // read rest from our buffer
+ if (end-pos > 0) {
+ int toRead = Math.min(end-pos, len-r);
+ System.arraycopy(buf, pos, b, off+r, toRead);
+ pos += toRead;
+ r += toRead;
+ return r;
+ }
+
+ return r > 0 ? r : -1;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ @Override
+ public void readFully(byte b[]) throws IOException {
+ readFully(b, 0, b.length);
+ }
+
+ @Override
+ public void readFully(byte b[], int off, int len) throws IOException {
+ while (len>0) {
+ int ret = read(b, off, len);
+ if (ret==-1) {
+ throw new EOFException();
+ }
+ off += ret;
+ len -= ret;
+ }
+ }
+
+ @Override
+ public int skipBytes(int n) throws IOException {
+ if (end-pos >= n) {
+ pos += n;
+ return n;
+ }
+
+ if (end-pos<0) return -1;
+
+ int r = end-pos;
+ pos = end;
+
+ while (r < n) {
+ refill();
+ if (end-pos <= 0) return r;
+ int toRead = Math.min(end-pos, n-r);
+ r += toRead;
+ pos += toRead;
+ }
+
+ return r;
+ }
+
+ @Override
+ public boolean readBoolean() throws IOException {
+ return readByte()==1;
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ if (pos >= end) {
+ refill();
+ if (pos >= end) throw new EOFException();
+ }
+ return buf[pos++];
+ }
+
+
+ @Override
+ public short readShort() throws IOException {
+ return (short)((readUnsignedByte() << 8) | readUnsignedByte());
+ }
+
+ @Override
+ public int readUnsignedShort() throws IOException {
+ return (readUnsignedByte() << 8) | readUnsignedByte();
+ }
+
+ @Override
+ public char readChar() throws IOException {
+ return (char)((readUnsignedByte() << 8) | readUnsignedByte());
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ return ((readUnsignedByte() << 24)
+ |(readUnsignedByte() << 16)
+ |(readUnsignedByte() << 8)
+ | readUnsignedByte());
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ return (((long)readUnsignedByte()) << 56)
+ | (((long)readUnsignedByte()) << 48)
+ | (((long)readUnsignedByte()) << 40)
+ | (((long)readUnsignedByte()) << 32)
+ | (((long)readUnsignedByte()) << 24)
+ | (readUnsignedByte() << 16)
+ | (readUnsignedByte() << 8)
+ | (readUnsignedByte());
+ }
+
+ @Override
+ public float readFloat() throws IOException {
+ return Float.intBitsToFloat(readInt());
+ }
+
+ @Override
+ public double readDouble() throws IOException {
+ return Double.longBitsToDouble(readLong());
+ }
+
+ @Override
+ public String readLine() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String readUTF() throws IOException {
+ return new DataInputStream(this).readUTF();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/FastOutputStream.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/FastOutputStream.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/FastOutputStream.java
new file mode 100644
index 0000000..09a7441
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/FastOutputStream.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.io.*;
+
+/** Single threaded buffered OutputStream
+ * Internal Solr use only, subject to change.
+ */
+public class FastOutputStream extends OutputStream implements DataOutput {
+ protected final OutputStream out;
+ protected byte[] buf;
+ protected long written; // how many bytes written to the underlying stream
+ protected int pos;
+
+ public FastOutputStream(OutputStream w) {
+ // use default BUFSIZE of BufferedOutputStream so if we wrap that
+ // it won't cause double buffering.
+ this(w, new byte[8192], 0);
+ }
+
+ public FastOutputStream(OutputStream sink, byte[] tempBuffer, int start) {
+ this.out = sink;
+ this.buf = tempBuffer;
+ this.pos = start;
+ }
+
+
+ public static FastOutputStream wrap(OutputStream sink) {
+ return (sink instanceof FastOutputStream) ? (FastOutputStream)sink : new FastOutputStream(sink);
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ write((byte)b);
+ }
+
+ @Override
+ public void write(byte b[]) throws IOException {
+ write(b,0,b.length);
+ }
+
+ public void write(byte b) throws IOException {
+ if (pos >= buf.length) {
+ written += pos;
+ flush(buf, 0, buf.length);
+ pos=0;
+ }
+ buf[pos++] = b;
+ }
+
+ @Override
+ public void write(byte arr[], int off, int len) throws IOException {
+
+ for(;;) {
+ int space = buf.length - pos;
+
+ if (len <= space) {
+ System.arraycopy(arr, off, buf, pos, len);
+ pos += len;
+ return;
+ } else if (len > buf.length) {
+ if (pos>0) {
+ flush(buf,0,pos); // flush
+ written += pos;
+ pos=0;
+ }
+ // don't buffer, just write to sink
+ flush(arr, off, len);
+ written += len;
+ return;
+ }
+
+ // buffer is too big to fit in the free space, but
+ // not big enough to warrant writing on its own.
+ // write whatever we can fit, then flush and iterate.
+
+ System.arraycopy(arr, off, buf, pos, space);
+ written += buf.length; // important to do this first, since buf.length can change after a flush!
+ flush(buf, 0, buf.length);
+ pos = 0;
+ off += space;
+ len -= space;
+ }
+ }
+
+
+ /** reserve at least len bytes at the end of the buffer.
+ * Invalid if len > buffer.length
+ */
+ public void reserve(int len) throws IOException {
+ if (len > (buf.length - pos))
+ flushBuffer();
+ }
+
+ ////////////////// DataOutput methods ///////////////////
+ @Override
+ public void writeBoolean(boolean v) throws IOException {
+ write(v ? 1:0);
+ }
+
+ @Override
+ public void writeByte(int v) throws IOException {
+ write((byte)v);
+ }
+
+ @Override
+ public void writeShort(int v) throws IOException {
+ write((byte)(v >>> 8));
+ write((byte)v);
+ }
+
+ @Override
+ public void writeChar(int v) throws IOException {
+ writeShort(v);
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ reserve(4);
+ buf[pos] = (byte)(v>>>24);
+ buf[pos+1] = (byte)(v>>>16);
+ buf[pos+2] = (byte)(v>>>8);
+ buf[pos+3] = (byte)(v);
+ pos+=4;
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ reserve(8);
+ buf[pos] = (byte)(v>>>56);
+ buf[pos+1] = (byte)(v>>>48);
+ buf[pos+2] = (byte)(v>>>40);
+ buf[pos+3] = (byte)(v>>>32);
+ buf[pos+4] = (byte)(v>>>24);
+ buf[pos+5] = (byte)(v>>>16);
+ buf[pos+6] = (byte)(v>>>8);
+ buf[pos+7] = (byte)(v);
+ pos+=8;
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ writeInt(Float.floatToRawIntBits(v));
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ writeLong(Double.doubleToRawLongBits(v));
+ }
+
+ @Override
+ public void writeBytes(String s) throws IOException {
+ // non-optimized version, but this shouldn't be used anyway
+ for (int i=0; i<s.length(); i++)
+ write((byte)s.charAt(i));
+ }
+
+ @Override
+ public void writeChars(String s) throws IOException {
+ // non-optimized version
+ for (int i=0; i<s.length(); i++)
+ writeChar(s.charAt(i));
+ }
+
+ @Override
+ public void writeUTF(String s) throws IOException {
+ // non-optimized version, but this shouldn't be used anyway
+ DataOutputStream daos = new DataOutputStream(this);
+ daos.writeUTF(s);
+ }
+
+
+ @Override
+ public void flush() throws IOException {
+ flushBuffer();
+ if (out != null) out.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ flushBuffer();
+ if (out != null) out.close();
+ }
+
+ /** Only flushes the buffer of the FastOutputStream, not that of the
+ * underlying stream.
+ */
+ public void flushBuffer() throws IOException {
+ if (pos > 0) {
+ written += pos;
+ flush(buf, 0, pos);
+ pos=0;
+ }
+ }
+
+ /** All writes to the sink will go through this method */
+ public void flush(byte[] buf, int offset, int len) throws IOException {
+ out.write(buf, offset, len);
+ }
+
+ public long size() {
+ return written + pos;
+ }
+
+ /** Returns the number of bytes actually written to the underlying OutputStream, not including
+ * anything currently buffered by this class itself.
+ */
+ public long written() {
+ return written;
+ }
+
+ /** Resets the count returned by written() */
+ public void setWritten(long written) {
+ this.written = written;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/Hash.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/Hash.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/Hash.java
new file mode 100644
index 0000000..81b5232
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/Hash.java
@@ -0,0 +1,422 @@
+package org.apache.solr.common.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <p>Fast, well distributed, cross-platform hash functions.
+ * </p>
+ *
+ * <p>Development background: I was surprised to discovered that there isn't a good cross-platform hash function defined for strings. MD5, SHA, FVN, etc, all define hash functions over bytes, meaning that it's under-specified for strings.
+ * </p>
+ *
+ * <p>So I set out to create a standard 32 bit string hash that would be well defined for implementation in all languages, have very high performance, and have very good hash properties such as distribution. After evaluating all the options, I settled on using Bob Jenkins' lookup3 as a base. It's a well studied and very fast hash function, and the hashword variant can work with 32 bits at a time (perfect for hashing unicode code points). It's also even faster on the latest JVMs which can translate pairs of shifts into native rotate instructions.
+ * </p>
+ * <p>The only problem with using lookup3 hashword is that it includes a length in the initial value. This would suck some performance out since directly hashing a UTF8 or UTF16 string (Java) would require a pre-scan to get the actual number of unicode code points. The solution was to simply remove the length factor, which is equivalent to biasing initVal by -(numCodePoints*4). This slightly modified lookup3 I define as lookup3ycs.
+ * </p>
+ * <p>So the definition of the cross-platform string hash lookup3ycs is as follows:
+ * </p>
+ * <p>The hash value of a character sequence (a string) is defined to be the hash of its unicode code points, according to lookup3 hashword, with the initval biased by -(length*4).
+ * </p>
+ *<p>So by definition
+ *</p>
+ * <pre>
+ * lookup3ycs(k,offset,length,initval) == lookup3(k,offset,length,initval-(length*4))
+ *
+ * AND
+ *
+ * lookup3ycs(k,offset,length,initval+(length*4)) == lookup3(k,offset,length,initval)
+ * </pre>
+ * <p>An obvious advantage of this relationship is that you can use lookup3 if you don't have an implementation of lookup3ycs.
+ * </p>
+ */
+public class Hash {
+ /**
+ * A Java implementation of hashword from lookup3.c by Bob Jenkins
+ * (<a href="http://burtleburtle.net/bob/c/lookup3.c">original source</a>).
+ *
+ * @param k the key to hash
+ * @param offset offset of the start of the key
+ * @param length length of the key
+ * @param initval initial value to fold into the hash
+ * @return the 32 bit hash code
+ */
+ @SuppressWarnings("fallthrough")
+ public static int lookup3(int[] k, int offset, int length, int initval) {
+ int a,b,c;
+ a = b = c = 0xdeadbeef + (length<<2) + initval;
+
+ int i=offset;
+ while (length > 3)
+ {
+ a += k[i];
+ b += k[i+1];
+ c += k[i+2];
+
+ // mix(a,b,c)... Java needs "out" parameters!!!
+ // Note: recent JVMs (Sun JDK6) turn pairs of shifts (needed to do a rotate)
+ // into real x86 rotate instructions.
+ {
+ a -= c; a ^= (c<<4)|(c>>>-4); c += b;
+ b -= a; b ^= (a<<6)|(a>>>-6); a += c;
+ c -= b; c ^= (b<<8)|(b>>>-8); b += a;
+ a -= c; a ^= (c<<16)|(c>>>-16); c += b;
+ b -= a; b ^= (a<<19)|(a>>>-19); a += c;
+ c -= b; c ^= (b<<4)|(b>>>-4); b += a;
+ }
+
+ length -= 3;
+ i += 3;
+ }
+
+ switch(length) {
+ case 3 : c+=k[i+2]; // fall through
+ case 2 : b+=k[i+1]; // fall through
+ case 1 : a+=k[i+0]; // fall through
+ // final(a,b,c);
+ {
+ c ^= b; c -= (b<<14)|(b>>>-14);
+ a ^= c; a -= (c<<11)|(c>>>-11);
+ b ^= a; b -= (a<<25)|(a>>>-25);
+ c ^= b; c -= (b<<16)|(b>>>-16);
+ a ^= c; a -= (c<<4)|(c>>>-4);
+ b ^= a; b -= (a<<14)|(a>>>-14);
+ c ^= b; c -= (b<<24)|(b>>>-24);
+ }
+ case 0:
+ break;
+ }
+ return c;
+ }
+
+
+ /**
+ * Identical to lookup3, except initval is biased by -(length<<2).
+ * This is equivalent to leaving out the length factor in the initial state.
+ * {@code lookup3ycs(k,offset,length,initval) == lookup3(k,offset,length,initval-(length<<2))}
+ * and
+ * {@code lookup3ycs(k,offset,length,initval+(length<<2)) == lookup3(k,offset,length,initval)}
+ */
+ public static int lookup3ycs(int[] k, int offset, int length, int initval) {
+ return lookup3(k, offset, length, initval-(length<<2));
+ }
+
+
+ /**
+ * <p>The hash value of a character sequence is defined to be the hash of
+ * its unicode code points, according to {@link #lookup3ycs(int[] k, int offset, int length, int initval)}
+ * </p>
+ * <p>If you know the number of code points in the {@code CharSequence}, you can
+ * generate the same hash as the original lookup3
+ * via {@code lookup3ycs(s, start, end, initval+(numCodePoints<<2))}
+ */
+ public static int lookup3ycs(CharSequence s, int start, int end, int initval) {
+ int a,b,c;
+ a = b = c = 0xdeadbeef + initval;
+ // only difference from lookup3 is that "+ (length<<2)" is missing
+ // since we don't know the number of code points to start with,
+ // and don't want to have to pre-scan the string to find out.
+
+ int i=start;
+ boolean mixed=true; // have the 3 state variables been adequately mixed?
+ for(;;) {
+ if (i>= end) break;
+ mixed=false;
+ char ch;
+ ch = s.charAt(i++);
+ a += Character.isHighSurrogate(ch) && i< end ? Character.toCodePoint(ch, s.charAt(i++)) : ch;
+ if (i>= end) break;
+ ch = s.charAt(i++);
+ b += Character.isHighSurrogate(ch) && i< end ? Character.toCodePoint(ch, s.charAt(i++)) : ch;
+ if (i>= end) break;
+ ch = s.charAt(i++);
+ c += Character.isHighSurrogate(ch) && i< end ? Character.toCodePoint(ch, s.charAt(i++)) : ch;
+ if (i>= end) break;
+
+ // mix(a,b,c)... Java needs "out" parameters!!!
+ // Note: recent JVMs (Sun JDK6) turn pairs of shifts (needed to do a rotate)
+ // into real x86 rotate instructions.
+ {
+ a -= c; a ^= (c<<4)|(c>>>-4); c += b;
+ b -= a; b ^= (a<<6)|(a>>>-6); a += c;
+ c -= b; c ^= (b<<8)|(b>>>-8); b += a;
+ a -= c; a ^= (c<<16)|(c>>>-16); c += b;
+ b -= a; b ^= (a<<19)|(a>>>-19); a += c;
+ c -= b; c ^= (b<<4)|(b>>>-4); b += a;
+ }
+ mixed=true;
+ }
+
+
+ if (!mixed) {
+ // final(a,b,c)
+ c ^= b; c -= (b<<14)|(b>>>-14);
+ a ^= c; a -= (c<<11)|(c>>>-11);
+ b ^= a; b -= (a<<25)|(a>>>-25);
+ c ^= b; c -= (b<<16)|(b>>>-16);
+ a ^= c; a -= (c<<4)|(c>>>-4);
+ b ^= a; b -= (a<<14)|(a>>>-14);
+ c ^= b; c -= (b<<24)|(b>>>-24);
+ }
+
+ return c;
+ }
+
+
+ /**<p>This is the 64 bit version of lookup3ycs, corresponding to Bob Jenkin's
+ * lookup3 hashlittle2 with initval biased by -(numCodePoints<<2). It is equivalent
+ * to lookup3ycs in that if the high bits of initval==0, then the low bits of the
+ * result will be the same as lookup3ycs.
+ * </p>
+ */
+ public static long lookup3ycs64(CharSequence s, int start, int end, long initval) {
+ int a,b,c;
+ a = b = c = 0xdeadbeef + (int)initval;
+ c += (int)(initval>>>32);
+ // only difference from lookup3 is that "+ (length<<2)" is missing
+ // since we don't know the number of code points to start with,
+ // and don't want to have to pre-scan the string to find out.
+
+ int i=start;
+ boolean mixed=true; // have the 3 state variables been adequately mixed?
+ for(;;) {
+ if (i>= end) break;
+ mixed=false;
+ char ch;
+ ch = s.charAt(i++);
+ a += Character.isHighSurrogate(ch) && i< end ? Character.toCodePoint(ch, s.charAt(i++)) : ch;
+ if (i>= end) break;
+ ch = s.charAt(i++);
+ b += Character.isHighSurrogate(ch) && i< end ? Character.toCodePoint(ch, s.charAt(i++)) : ch;
+ if (i>= end) break;
+ ch = s.charAt(i++);
+ c += Character.isHighSurrogate(ch) && i< end ? Character.toCodePoint(ch, s.charAt(i++)) : ch;
+ if (i>= end) break;
+
+ // mix(a,b,c)... Java needs "out" parameters!!!
+ // Note: recent JVMs (Sun JDK6) turn pairs of shifts (needed to do a rotate)
+ // into real x86 rotate instructions.
+ {
+ a -= c; a ^= (c<<4)|(c>>>-4); c += b;
+ b -= a; b ^= (a<<6)|(a>>>-6); a += c;
+ c -= b; c ^= (b<<8)|(b>>>-8); b += a;
+ a -= c; a ^= (c<<16)|(c>>>-16); c += b;
+ b -= a; b ^= (a<<19)|(a>>>-19); a += c;
+ c -= b; c ^= (b<<4)|(b>>>-4); b += a;
+ }
+ mixed=true;
+ }
+
+
+ if (!mixed) {
+ // final(a,b,c)
+ c ^= b; c -= (b<<14)|(b>>>-14);
+ a ^= c; a -= (c<<11)|(c>>>-11);
+ b ^= a; b -= (a<<25)|(a>>>-25);
+ c ^= b; c -= (b<<16)|(b>>>-16);
+ a ^= c; a -= (c<<4)|(c>>>-4);
+ b ^= a; b -= (a<<14)|(a>>>-14);
+ c ^= b; c -= (b<<24)|(b>>>-24);
+ }
+
+ return c + (((long)b) << 32);
+ }
+
+
+ /** Returns the MurmurHash3_x86_32 hash.
+ * Original source/tests at https://github.com/yonik/java_util/
+ */
+ public static int murmurhash3_x86_32(byte[] data, int offset, int len, int seed) {
+
+ final int c1 = 0xcc9e2d51;
+ final int c2 = 0x1b873593;
+
+ int h1 = seed;
+ int roundedEnd = offset + (len & 0xfffffffc); // round down to 4 byte block
+
+ for (int i=offset; i<roundedEnd; i+=4) {
+ // little endian load order
+ int k1 = (data[i] & 0xff) | ((data[i+1] & 0xff) << 8) | ((data[i+2] & 0xff) << 16) | (data[i+3] << 24);
+ k1 *= c1;
+ k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15);
+ k1 *= c2;
+
+ h1 ^= k1;
+ h1 = (h1 << 13) | (h1 >>> 19); // ROTL32(h1,13);
+ h1 = h1*5+0xe6546b64;
+ }
+
+ // tail
+ int k1 = 0;
+
+ switch(len & 0x03) {
+ case 3:
+ k1 = (data[roundedEnd + 2] & 0xff) << 16;
+ // fallthrough
+ case 2:
+ k1 |= (data[roundedEnd + 1] & 0xff) << 8;
+ // fallthrough
+ case 1:
+ k1 |= (data[roundedEnd] & 0xff);
+ k1 *= c1;
+ k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15);
+ k1 *= c2;
+ h1 ^= k1;
+ }
+
+ // finalization
+ h1 ^= len;
+
+ // fmix(h1);
+ h1 ^= h1 >>> 16;
+ h1 *= 0x85ebca6b;
+ h1 ^= h1 >>> 13;
+ h1 *= 0xc2b2ae35;
+ h1 ^= h1 >>> 16;
+
+ return h1;
+ }
+
+
+
+ /** Returns the MurmurHash3_x86_32 hash of the UTF-8 bytes of the String without actually encoding
+ * the string to a temporary buffer. This is more than 2x faster than hashing the result
+ * of String.getBytes().
+ */
+ public static int murmurhash3_x86_32(CharSequence data, int offset, int len, int seed) {
+
+ final int c1 = 0xcc9e2d51;
+ final int c2 = 0x1b873593;
+
+ int h1 = seed;
+
+ int pos = offset;
+ int end = offset + len;
+ int k1 = 0;
+ int k2 = 0;
+ int shift = 0;
+ int bits = 0;
+ int nBytes = 0; // length in UTF8 bytes
+
+
+ while (pos < end) {
+ int code = data.charAt(pos++);
+ if (code < 0x80) {
+ k2 = code;
+ bits = 8;
+
+ /***
+ // optimized ascii implementation (currently slower!!! code size?)
+ if (shift == 24) {
+ k1 = k1 | (code << 24);
+
+ k1 *= c1;
+ k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15);
+ k1 *= c2;
+
+ h1 ^= k1;
+ h1 = (h1 << 13) | (h1 >>> 19); // ROTL32(h1,13);
+ h1 = h1*5+0xe6546b64;
+
+ shift = 0;
+ nBytes += 4;
+ k1 = 0;
+ } else {
+ k1 |= code << shift;
+ shift += 8;
+ }
+ continue;
+ ***/
+
+ }
+ else if (code < 0x800) {
+ k2 = (0xC0 | (code >> 6))
+ | ((0x80 | (code & 0x3F)) << 8);
+ bits = 16;
+ }
+ else if (code < 0xD800 || code > 0xDFFF || pos>=end) {
+ // we check for pos>=end to encode an unpaired surrogate as 3 bytes.
+ k2 = (0xE0 | (code >> 12))
+ | ((0x80 | ((code >> 6) & 0x3F)) << 8)
+ | ((0x80 | (code & 0x3F)) << 16);
+ bits = 24;
+ } else {
+ // surrogate pair
+ // int utf32 = pos < end ? (int) data.charAt(pos++) : 0;
+ int utf32 = (int) data.charAt(pos++);
+ utf32 = ((code - 0xD7C0) << 10) + (utf32 & 0x3FF);
+ k2 = (0xff & (0xF0 | (utf32 >> 18)))
+ | ((0x80 | ((utf32 >> 12) & 0x3F))) << 8
+ | ((0x80 | ((utf32 >> 6) & 0x3F))) << 16
+ | (0x80 | (utf32 & 0x3F)) << 24;
+ bits = 32;
+ }
+
+
+ k1 |= k2 << shift;
+
+ // int used_bits = 32 - shift; // how many bits of k2 were used in k1.
+ // int unused_bits = bits - used_bits; // (bits-(32-shift)) == bits+shift-32 == bits-newshift
+
+ shift += bits;
+ if (shift >= 32) {
+ // mix after we have a complete word
+
+ k1 *= c1;
+ k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15);
+ k1 *= c2;
+
+ h1 ^= k1;
+ h1 = (h1 << 13) | (h1 >>> 19); // ROTL32(h1,13);
+ h1 = h1*5+0xe6546b64;
+
+ shift -= 32;
+ // unfortunately, java won't let you shift 32 bits off, so we need to check for 0
+ if (shift != 0) {
+ k1 = k2 >>> (bits-shift); // bits used == bits - newshift
+ } else {
+ k1 = 0;
+ }
+ nBytes += 4;
+ }
+
+ } // inner
+
+ // handle tail
+ if (shift > 0) {
+ nBytes += shift >> 3;
+ k1 *= c1;
+ k1 = (k1 << 15) | (k1 >>> 17); // ROTL32(k1,15);
+ k1 *= c2;
+ h1 ^= k1;
+ }
+
+ // finalization
+ h1 ^= nBytes;
+
+ // fmix(h1);
+ h1 ^= h1 >>> 16;
+ h1 *= 0x85ebca6b;
+ h1 ^= h1 >>> 13;
+ h1 *= 0xc2b2ae35;
+ h1 ^= h1 >>> 16;
+
+ return h1;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/IOUtils.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/IOUtils.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/IOUtils.java
new file mode 100644
index 0000000..cbf1dcb
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/IOUtils.java
@@ -0,0 +1,37 @@
+package org.apache.solr.common.util;
+
+import java.io.Closeable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class IOUtils {
+ public static Logger LOG = LoggerFactory.getLogger(IOUtils.class);
+
+ public static void closeQuietly(Closeable closeable) {
+ try {
+ if (closeable != null) {
+ closeable.close();
+ }
+ } catch (Exception e) {
+ LOG.error("Error while closing", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/util/IteratorChain.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/util/IteratorChain.java b/ranger_solrj/src/main/java/org/apache/solr/common/util/IteratorChain.java
new file mode 100644
index 0000000..8a0bb1d
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/util/IteratorChain.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.util;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/** Chain several Iterators, so that this iterates
+ * over all of them in sequence.
+ *
+ * @deprecated This class is no longer used by Solr, and may be removed in future versions
+ */
+@Deprecated
+public class IteratorChain<E> implements Iterator<E> {
+
+ private final List<Iterator<E>> iterators = new ArrayList<>();
+ private Iterator<Iterator<E>> itit;
+ private Iterator<E> current;
+
+ public void addIterator(Iterator<E> it) {
+ if(itit!=null) throw new RuntimeException("all Iterators must be added before calling hasNext()");
+ iterators.add(it);
+ }
+
+ @Override
+ public boolean hasNext() {
+ if(itit==null) itit = iterators.iterator();
+ return recursiveHasNext();
+ }
+
+ /** test if current iterator hasNext(), and if not try the next
+ * one in sequence, recursively
+ */
+ private boolean recursiveHasNext() {
+ // return false if we have no more iterators
+ if(current==null) {
+ if(itit.hasNext()) {
+ current=itit.next();
+ } else {
+ return false;
+ }
+ }
+
+ boolean result = current.hasNext();
+ if(!result) {
+ current = null;
+ result = recursiveHasNext();
+ }
+
+ return result;
+ }
+
+ /** hasNext() must ALWAYS be called before calling this
+ * otherwise it's a bit hard to keep track of what's happening
+ */
+ @Override
+ public E next() {
+ if(current==null) {
+ throw new RuntimeException("For an IteratorChain, hasNext() MUST be called before calling next()");
+ }
+ return current.next();
+ }
+
+ @Override
+ public void remove() {
+ // we just need this class
+ // to iterate in readonly mode
+ throw new UnsupportedOperationException();
+ }
+
+}
[07/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZkClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZkClient.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZkClient.java
new file mode 100644
index 0000000..f14aedd
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZkClient.java
@@ -0,0 +1,736 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import org.apache.commons.io.FileUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.cloud.ZkClientConnectionStrategy.ZkUpdate;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+import org.apache.zookeeper.KeeperException.NotEmptyException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ *
+ * All Solr ZooKeeper interactions should go through this class rather than
+ * ZooKeeper. This class handles synchronous connects and reconnections.
+ *
+ */
+public class SolrZkClient implements Closeable {
+
+ static final String NEWL = System.getProperty("line.separator");
+
+ static final int DEFAULT_CLIENT_CONNECT_TIMEOUT = 30000;
+
+ private static final Logger log = LoggerFactory
+ .getLogger(SolrZkClient.class);
+
+ private ConnectionManager connManager;
+
+ private volatile SolrZooKeeper keeper;
+
+ private ZkCmdExecutor zkCmdExecutor;
+
+ private final ExecutorService zkCallbackExecutor = Executors.newCachedThreadPool(new SolrjNamedThreadFactory("zkCallback"));
+
+ private volatile boolean isClosed = false;
+ private ZkClientConnectionStrategy zkClientConnectionStrategy;
+ private int zkClientTimeout;
+ private ZkACLProvider zkACLProvider;
+ private String zkServerAddress;
+
+ public int getZkClientTimeout() {
+ return zkClientTimeout;
+ }
+
+ // expert: for tests
+ public SolrZkClient() {
+
+ }
+
+ public SolrZkClient(String zkServerAddress, int zkClientTimeout) {
+ this(zkServerAddress, zkClientTimeout, new DefaultConnectionStrategy(), null);
+ }
+
+ public SolrZkClient(String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout) {
+ this(zkServerAddress, zkClientTimeout, zkClientConnectTimeout, new DefaultConnectionStrategy(), null);
+ }
+
+ public SolrZkClient(String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, OnReconnect onReonnect) {
+ this(zkServerAddress, zkClientTimeout, zkClientConnectTimeout, new DefaultConnectionStrategy(), onReonnect);
+ }
+
+ public SolrZkClient(String zkServerAddress, int zkClientTimeout,
+ ZkClientConnectionStrategy strat, final OnReconnect onReconnect) {
+ this(zkServerAddress, zkClientTimeout, DEFAULT_CLIENT_CONNECT_TIMEOUT, strat, onReconnect);
+ }
+
+ public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout,
+ ZkClientConnectionStrategy strat, final OnReconnect onReconnect) {
+ this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, null, null);
+ }
+
+ public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout,
+ ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect) {
+ this(zkServerAddress, zkClientTimeout, clientConnectTimeout, strat, onReconnect, beforeReconnect, null);
+ }
+
+ public SolrZkClient(String zkServerAddress, int zkClientTimeout, int clientConnectTimeout,
+ ZkClientConnectionStrategy strat, final OnReconnect onReconnect, BeforeReconnect beforeReconnect, ZkACLProvider zkACLProvider) {
+ this.zkClientConnectionStrategy = strat;
+ this.zkServerAddress = zkServerAddress;
+
+ if (strat == null) {
+ strat = new DefaultConnectionStrategy();
+ }
+
+ if (!strat.hasZkCredentialsToAddAutomatically()) {
+ ZkCredentialsProvider zkCredentialsToAddAutomatically = createZkCredentialsToAddAutomatically();
+ strat.setZkCredentialsToAddAutomatically(zkCredentialsToAddAutomatically);
+ }
+
+ this.zkClientTimeout = zkClientTimeout;
+ // we must retry at least as long as the session timeout
+ zkCmdExecutor = new ZkCmdExecutor(zkClientTimeout);
+ connManager = new ConnectionManager("ZooKeeperConnection Watcher:"
+ + zkServerAddress, this, zkServerAddress, strat, onReconnect, beforeReconnect);
+
+ try {
+ strat.connect(zkServerAddress, zkClientTimeout, wrapWatcher(connManager),
+ new ZkUpdate() {
+ @Override
+ public void update(SolrZooKeeper zooKeeper) {
+ SolrZooKeeper oldKeeper = keeper;
+ keeper = zooKeeper;
+ try {
+ closeKeeper(oldKeeper);
+ } finally {
+ if (isClosed) {
+ // we may have been closed
+ closeKeeper(SolrZkClient.this.keeper);
+ }
+ }
+ }
+ });
+ } catch (Exception e) {
+ connManager.close();
+ if (keeper != null) {
+ try {
+ keeper.close();
+ } catch (InterruptedException e1) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+
+ try {
+ connManager.waitForConnected(clientConnectTimeout);
+ } catch (Exception e) {
+ connManager.close();
+ try {
+ keeper.close();
+ } catch (InterruptedException e1) {
+ Thread.currentThread().interrupt();
+ }
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ assert ObjectReleaseTracker.track(this);
+ if (zkACLProvider == null) {
+ this.zkACLProvider = createZkACLProvider();
+ } else {
+ this.zkACLProvider = zkACLProvider;
+ }
+ }
+
+ public ConnectionManager getConnectionManager() {
+ return connManager;
+ }
+
+ public ZkClientConnectionStrategy getZkClientConnectionStrategy() {
+ return zkClientConnectionStrategy;
+ }
+
+ public static final String ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME = "zkCredentialsProvider";
+ protected ZkCredentialsProvider createZkCredentialsToAddAutomatically() {
+ String zkCredentialsProviderClassName = System.getProperty(ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
+ if (!StringUtils.isEmpty(zkCredentialsProviderClassName)) {
+ try {
+ log.info("Using ZkCredentialsProvider: " + zkCredentialsProviderClassName);
+ return (ZkCredentialsProvider)Class.forName(zkCredentialsProviderClassName).getConstructor().newInstance();
+ } catch (Throwable t) {
+ // just ignore - go default
+ log.warn("VM param zkCredentialsProvider does not point to a class implementing ZkCredentialsProvider and with a non-arg constructor", t);
+ }
+ }
+ log.info("Using default ZkCredentialsProvider");
+ return new DefaultZkCredentialsProvider();
+ }
+
+ public static final String ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME = "zkACLProvider";
+ protected ZkACLProvider createZkACLProvider() {
+ String zkACLProviderClassName = System.getProperty(ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME);
+ if (!StringUtils.isEmpty(zkACLProviderClassName)) {
+ try {
+ log.info("Using ZkACLProvider: " + zkACLProviderClassName);
+ return (ZkACLProvider)Class.forName(zkACLProviderClassName).getConstructor().newInstance();
+ } catch (Throwable t) {
+ // just ignore - go default
+ log.warn("VM param zkACLProvider does not point to a class implementing ZkACLProvider and with a non-arg constructor", t);
+ }
+ }
+ log.info("Using default ZkACLProvider");
+ return new DefaultZkACLProvider();
+ }
+
+ /**
+ * Returns true if client is connected
+ */
+ public boolean isConnected() {
+ return keeper != null && keeper.getState() == ZooKeeper.States.CONNECTED;
+ }
+
+ public void delete(final String path, final int version, boolean retryOnConnLoss)
+ throws InterruptedException, KeeperException {
+ if (retryOnConnLoss) {
+ zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public Stat execute() throws KeeperException, InterruptedException {
+ keeper.delete(path, version);
+ return null;
+ }
+ });
+ } else {
+ keeper.delete(path, version);
+ }
+ }
+
+ private Watcher wrapWatcher (final Watcher watcher) {
+ if (watcher == null) return watcher;
+
+ // wrap the watcher so that it doesn't fire off ZK's event queue
+ return new Watcher() {
+ @Override
+ public void process(final WatchedEvent event) {
+ log.debug("Submitting job to respond to event " + event);
+ zkCallbackExecutor.submit(new Runnable () {
+ @Override
+ public void run () {
+ watcher.process(event);
+ }
+ });
+ }
+ };
+ }
+
+ /**
+ * Return the stat of the node of the given path. Return null if no such a
+ * node exists.
+ * <p>
+ * If the watch is non-null and the call is successful (no exception is thrown),
+ * a watch will be left on the node with the given path. The watch will be
+ * triggered by a successful operation that creates/delete the node or sets
+ * the data on the node.
+ *
+ * @param path the node path
+ * @param watcher explicit watcher
+ * @return the stat of the node of the given path; return null if no such a
+ * node exists.
+ * @throws KeeperException If the server signals an error
+ * @throws InterruptedException If the server transaction is interrupted.
+ * @throws IllegalArgumentException if an invalid path is specified
+ */
+ public Stat exists(final String path, final Watcher watcher, boolean retryOnConnLoss)
+ throws KeeperException, InterruptedException {
+ if (retryOnConnLoss) {
+ return zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public Stat execute() throws KeeperException, InterruptedException {
+ return keeper.exists(path, wrapWatcher(watcher));
+ }
+ });
+ } else {
+ return keeper.exists(path, wrapWatcher(watcher));
+ }
+ }
+
+ /**
+ * Returns true if path exists
+ */
+ public Boolean exists(final String path, boolean retryOnConnLoss)
+ throws KeeperException, InterruptedException {
+ if (retryOnConnLoss) {
+ return zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public Boolean execute() throws KeeperException, InterruptedException {
+ return keeper.exists(path, null) != null;
+ }
+ });
+ } else {
+ return keeper.exists(path, null) != null;
+ }
+ }
+
+ /**
+ * Returns children of the node at the path
+ */
+ public List<String> getChildren(final String path, final Watcher watcher, boolean retryOnConnLoss)
+ throws KeeperException, InterruptedException {
+ if (retryOnConnLoss) {
+ return zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public List<String> execute() throws KeeperException, InterruptedException {
+ return keeper.getChildren(path, wrapWatcher(watcher));
+ }
+ });
+ } else {
+ return keeper.getChildren(path, wrapWatcher(watcher));
+ }
+ }
+
+ /**
+ * Returns node's data
+ */
+ public byte[] getData(final String path, final Watcher watcher, final Stat stat, boolean retryOnConnLoss)
+ throws KeeperException, InterruptedException {
+ if (retryOnConnLoss) {
+ return zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public byte[] execute() throws KeeperException, InterruptedException {
+ return keeper.getData(path, wrapWatcher(watcher), stat);
+ }
+ });
+ } else {
+ return keeper.getData(path, wrapWatcher(watcher), stat);
+ }
+ }
+
+ /**
+ * Returns node's state
+ */
+ public Stat setData(final String path, final byte data[], final int version, boolean retryOnConnLoss)
+ throws KeeperException, InterruptedException {
+ if (retryOnConnLoss) {
+ return zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public Stat execute() throws KeeperException, InterruptedException {
+ return keeper.setData(path, data, version);
+ }
+ });
+ } else {
+ return keeper.setData(path, data, version);
+ }
+ }
+
+ /**
+ * Returns path of created node
+ */
+ public String create(final String path, final byte[] data,
+ final CreateMode createMode, boolean retryOnConnLoss) throws KeeperException,
+ InterruptedException {
+ if (retryOnConnLoss) {
+ return zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public String execute() throws KeeperException, InterruptedException {
+ return keeper.create(path, data, zkACLProvider.getACLsToAdd(path),
+ createMode);
+ }
+ });
+ } else {
+ List<ACL> acls = zkACLProvider.getACLsToAdd(path);
+ return keeper.create(path, data, acls, createMode);
+ }
+ }
+
+ /**
+ * Creates the path in ZooKeeper, creating each node as necessary.
+ *
+ * e.g. If <code>path=/solr/group/node</code> and none of the nodes, solr,
+ * group, node exist, each will be created.
+ */
+ public void makePath(String path, boolean retryOnConnLoss) throws KeeperException,
+ InterruptedException {
+ makePath(path, null, CreateMode.PERSISTENT, retryOnConnLoss);
+ }
+
+ public void makePath(String path, boolean failOnExists, boolean retryOnConnLoss) throws KeeperException,
+ InterruptedException {
+ makePath(path, null, CreateMode.PERSISTENT, null, failOnExists, retryOnConnLoss);
+ }
+
+ public void makePath(String path, File file, boolean failOnExists, boolean retryOnConnLoss)
+ throws IOException, KeeperException, InterruptedException {
+ makePath(path, FileUtils.readFileToByteArray(file),
+ CreateMode.PERSISTENT, null, failOnExists, retryOnConnLoss);
+ }
+
+ public void makePath(String path, File file, boolean retryOnConnLoss) throws IOException,
+ KeeperException, InterruptedException {
+ makePath(path, FileUtils.readFileToByteArray(file), retryOnConnLoss);
+ }
+
+ public void makePath(String path, CreateMode createMode, boolean retryOnConnLoss) throws KeeperException,
+ InterruptedException {
+ makePath(path, null, createMode, retryOnConnLoss);
+ }
+
+ /**
+ * Creates the path in ZooKeeper, creating each node as necessary.
+ *
+ * @param data to set on the last zkNode
+ */
+ public void makePath(String path, byte[] data, boolean retryOnConnLoss) throws KeeperException,
+ InterruptedException {
+ makePath(path, data, CreateMode.PERSISTENT, retryOnConnLoss);
+ }
+
+ /**
+ * Creates the path in ZooKeeper, creating each node as necessary.
+ *
+ * e.g. If <code>path=/solr/group/node</code> and none of the nodes, solr,
+ * group, node exist, each will be created.
+ *
+ * @param data to set on the last zkNode
+ */
+ public void makePath(String path, byte[] data, CreateMode createMode, boolean retryOnConnLoss)
+ throws KeeperException, InterruptedException {
+ makePath(path, data, createMode, null, retryOnConnLoss);
+ }
+
+ /**
+ * Creates the path in ZooKeeper, creating each node as necessary.
+ *
+ * e.g. If <code>path=/solr/group/node</code> and none of the nodes, solr,
+ * group, node exist, each will be created.
+ *
+ * @param data to set on the last zkNode
+ */
+ public void makePath(String path, byte[] data, CreateMode createMode,
+ Watcher watcher, boolean retryOnConnLoss) throws KeeperException, InterruptedException {
+ makePath(path, data, createMode, watcher, true, retryOnConnLoss);
+ }
+
+
+
+ /**
+ * Creates the path in ZooKeeper, creating each node as necessary.
+ *
+ * e.g. If <code>path=/solr/group/node</code> and none of the nodes, solr,
+ * group, node exist, each will be created.
+ *
+ * Note: retryOnConnLoss is only respected for the final node - nodes
+ * before that are always retried on connection loss.
+ */
+ public void makePath(String path, byte[] data, CreateMode createMode,
+ Watcher watcher, boolean failOnExists, boolean retryOnConnLoss) throws KeeperException, InterruptedException {
+ if (log.isInfoEnabled()) {
+ log.info("makePath: " + path);
+ }
+ boolean retry = true;
+
+ if (path.startsWith("/")) {
+ path = path.substring(1, path.length());
+ }
+ String[] paths = path.split("/");
+ StringBuilder sbPath = new StringBuilder();
+ for (int i = 0; i < paths.length; i++) {
+ byte[] bytes = null;
+ String pathPiece = paths[i];
+ sbPath.append("/" + pathPiece);
+ final String currentPath = sbPath.toString();
+ Object exists = exists(currentPath, watcher, retryOnConnLoss);
+ if (exists == null || ((i == paths.length -1) && failOnExists)) {
+ CreateMode mode = CreateMode.PERSISTENT;
+ if (i == paths.length - 1) {
+ mode = createMode;
+ bytes = data;
+ if (!retryOnConnLoss) retry = false;
+ }
+ try {
+ if (retry) {
+ final CreateMode finalMode = mode;
+ final byte[] finalBytes = bytes;
+ zkCmdExecutor.retryOperation(new ZkOperation() {
+ @Override
+ public Object execute() throws KeeperException, InterruptedException {
+ keeper.create(currentPath, finalBytes, zkACLProvider.getACLsToAdd(currentPath), finalMode);
+ return null;
+ }
+ });
+ } else {
+ keeper.create(currentPath, bytes, zkACLProvider.getACLsToAdd(currentPath), mode);
+ }
+ } catch (NodeExistsException e) {
+
+ if (!failOnExists) {
+ // TODO: version ? for now, don't worry about race
+ setData(currentPath, data, -1, retryOnConnLoss);
+ // set new watch
+ exists(currentPath, watcher, retryOnConnLoss);
+ return;
+ }
+
+ // ignore unless it's the last node in the path
+ if (i == paths.length - 1) {
+ throw e;
+ }
+ }
+ if(i == paths.length -1) {
+ // set new watch
+ exists(currentPath, watcher, retryOnConnLoss);
+ }
+ } else if (i == paths.length - 1) {
+ // TODO: version ? for now, don't worry about race
+ setData(currentPath, data, -1, retryOnConnLoss);
+ // set new watch
+ exists(currentPath, watcher, retryOnConnLoss);
+ }
+ }
+ }
+
+ public void makePath(String zkPath, CreateMode createMode, Watcher watcher, boolean retryOnConnLoss)
+ throws KeeperException, InterruptedException {
+ makePath(zkPath, null, createMode, watcher, retryOnConnLoss);
+ }
+
+ /**
+ * Write data to ZooKeeper.
+ */
+ public Stat setData(String path, byte[] data, boolean retryOnConnLoss) throws KeeperException,
+ InterruptedException {
+ return setData(path, data, -1, retryOnConnLoss);
+ }
+
+ /**
+ * Write file to ZooKeeper - default system encoding used.
+ *
+ * @param path path to upload file to e.g. /solr/conf/solrconfig.xml
+ * @param file path to file to be uploaded
+ */
+ public Stat setData(String path, File file, boolean retryOnConnLoss) throws IOException,
+ KeeperException, InterruptedException {
+ if (log.isInfoEnabled()) {
+ log.info("Write to ZooKeepeer " + file.getAbsolutePath() + " to " + path);
+ }
+
+ byte[] data = FileUtils.readFileToByteArray(file);
+ return setData(path, data, retryOnConnLoss);
+ }
+
+ /**
+ * Fills string with printout of current ZooKeeper layout.
+ */
+ public void printLayout(String path, int indent, StringBuilder string)
+ throws KeeperException, InterruptedException {
+ byte[] data = getData(path, null, null, true);
+ List<String> children = getChildren(path, null, true);
+ StringBuilder dent = new StringBuilder();
+ for (int i = 0; i < indent; i++) {
+ dent.append(" ");
+ }
+ string.append(dent + path + " (" + children.size() + ")" + NEWL);
+ if (data != null) {
+ String dataString = new String(data, StandardCharsets.UTF_8);
+ if ((!path.endsWith(".txt") && !path.endsWith(".xml")) || path.endsWith(ZkStateReader.CLUSTER_STATE)) {
+ if (path.endsWith(".xml")) {
+ // this is the cluster state in xml format - lets pretty print
+ dataString = prettyPrint(dataString);
+ }
+
+ string.append(dent + "DATA:\n" + dent + " "
+ + dataString.replaceAll("\n", "\n" + dent + " ") + NEWL);
+ } else {
+ string.append(dent + "DATA: ...supressed..." + NEWL);
+ }
+ }
+
+ for (String child : children) {
+ if (!child.equals("quota")) {
+ try {
+ printLayout(path + (path.equals("/") ? "" : "/") + child, indent + 1,
+ string);
+ } catch (NoNodeException e) {
+ // must have gone away
+ }
+ }
+ }
+
+ }
+
+ /**
+ * Prints current ZooKeeper layout to stdout.
+ */
+ public void printLayoutToStdOut() throws KeeperException,
+ InterruptedException {
+ StringBuilder sb = new StringBuilder();
+ printLayout("/", 0, sb);
+ System.out.println(sb.toString());
+ }
+
+ public static String prettyPrint(String input, int indent) {
+ try {
+ Source xmlInput = new StreamSource(new StringReader(input));
+ StringWriter stringWriter = new StringWriter();
+ StreamResult xmlOutput = new StreamResult(stringWriter);
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ transformerFactory.setAttribute("indent-number", indent);
+ Transformer transformer = transformerFactory.newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.transform(xmlInput, xmlOutput);
+ return xmlOutput.getWriter().toString();
+ } catch (Exception e) {
+ throw new RuntimeException("Problem pretty printing XML", e);
+ }
+ }
+
+ private static String prettyPrint(String input) {
+ return prettyPrint(input, 2);
+ }
+
+ public void close() {
+ if (isClosed) return; // it's okay if we over close - same as solrcore
+ isClosed = true;
+ try {
+ closeKeeper(keeper);
+ } finally {
+ connManager.close();
+ closeCallbackExecutor();
+ }
+ assert ObjectReleaseTracker.release(this);
+ }
+
+ public boolean isClosed() {
+ return isClosed;
+ }
+
+ /**
+ * Allows package private classes to update volatile ZooKeeper.
+ */
+ void updateKeeper(SolrZooKeeper keeper) throws InterruptedException {
+ SolrZooKeeper oldKeeper = this.keeper;
+ this.keeper = keeper;
+ if (oldKeeper != null) {
+ oldKeeper.close();
+ }
+ // we might have been closed already
+ if (isClosed) this.keeper.close();
+ }
+
+ public SolrZooKeeper getSolrZooKeeper() {
+ return keeper;
+ }
+
+ private void closeKeeper(SolrZooKeeper keeper) {
+ if (keeper != null) {
+ try {
+ keeper.close();
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.error("", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "",
+ e);
+ }
+ }
+ }
+
+ private void closeCallbackExecutor() {
+ try {
+ ExecutorUtil.shutdownAndAwaitTermination(zkCallbackExecutor);
+ } catch (Exception e) {
+ SolrException.log(log, e);
+ }
+ }
+
+ // yeah, it's recursive :(
+ public void clean(String path) throws InterruptedException, KeeperException {
+ List<String> children;
+ try {
+ children = getChildren(path, null, true);
+ } catch (NoNodeException r) {
+ return;
+ }
+ for (String string : children) {
+ // we can't clean the built-in zookeeper node
+ if (path.equals("/") && string.equals("zookeeper")) continue;
+ if (path.equals("/")) {
+ clean(path + string);
+ } else {
+ clean(path + "/" + string);
+ }
+ }
+ try {
+ if (!path.equals("/")) {
+ try {
+ delete(path, -1, true);
+ } catch (NotEmptyException e) {
+ clean(path);
+ }
+ }
+ } catch (NoNodeException r) {
+ return;
+ }
+ }
+
+ /**
+ * Validates if zkHost contains a chroot. See http://zookeeper.apache.org/doc/r3.2.2/zookeeperProgrammers.html#ch_zkSessions
+ */
+ public static boolean containsChroot(String zkHost) {
+ return zkHost.contains("/");
+ }
+
+ /**
+ * Check to see if a Throwable is an InterruptedException, and if it is, set the thread interrupt flag
+ * @param e the Throwable
+ * @return the Throwable
+ */
+ public static Throwable checkInterrupted(Throwable e) {
+ if (e instanceof InterruptedException)
+ Thread.interrupted();
+ return e;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZooKeeper.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZooKeeper.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZooKeeper.java
new file mode 100644
index 0000000..35ad8bf
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/SolrZooKeeper.java
@@ -0,0 +1,103 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.SocketAddress;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+
+import org.apache.zookeeper.ClientCnxn;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+
+// we use this class to expose nasty stuff for tests
+public class SolrZooKeeper extends ZooKeeper {
+ final Set<Thread> spawnedThreads = new CopyOnWriteArraySet<>();
+
+ // for test debug
+ //static Map<SolrZooKeeper,Exception> clients = new ConcurrentHashMap<SolrZooKeeper,Exception>();
+
+ public SolrZooKeeper(String connectString, int sessionTimeout,
+ Watcher watcher) throws IOException {
+ super(connectString, sessionTimeout, watcher);
+ //clients.put(this, new RuntimeException());
+ }
+
+ public ClientCnxn getConnection() {
+ return cnxn;
+ }
+
+ public SocketAddress getSocketAddress() {
+ return testableLocalSocketAddress();
+ }
+
+ public void closeCnxn() {
+ final Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ final ClientCnxn cnxn = getConnection();
+ synchronized (cnxn) {
+ try {
+ final Field sendThreadFld = cnxn.getClass().getDeclaredField("sendThread");
+ sendThreadFld.setAccessible(true);
+ Object sendThread = sendThreadFld.get(cnxn);
+ if (sendThread != null) {
+ Method method = sendThread.getClass().getDeclaredMethod("testableCloseSocket");
+ method.setAccessible(true);
+ try {
+ method.invoke(sendThread);
+ } catch (InvocationTargetException e) {
+ // is fine
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Closing Zookeeper send channel failed.", e);
+ }
+ }
+ } finally {
+ spawnedThreads.remove(this);
+ }
+ }
+ };
+ spawnedThreads.add(t);
+ t.start();
+ }
+
+ @Override
+ public synchronized void close() throws InterruptedException {
+ for (Thread t : spawnedThreads) {
+ if (t.isAlive()) t.interrupt();
+ }
+ super.close();
+ }
+
+// public static void assertCloses() {
+// if (clients.size() > 0) {
+// Iterator<Exception> stacktraces = clients.values().iterator();
+// Exception cause = null;
+// cause = stacktraces.next();
+// throw new RuntimeException("Found a bad one!", cause);
+// }
+// }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java
new file mode 100644
index 0000000..0b9ae1d
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsAllAndReadonlyDigestZkACLProvider.java
@@ -0,0 +1,89 @@
+package org.apache.solr.common.cloud;
+
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.solr.common.StringUtils;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Id;
+import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class VMParamsAllAndReadonlyDigestZkACLProvider extends DefaultZkACLProvider {
+
+ public static final String DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME = "zkDigestReadonlyUsername";
+ public static final String DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME = "zkDigestReadonlyPassword";
+
+ final String zkDigestAllUsernameVMParamName;
+ final String zkDigestAllPasswordVMParamName;
+ final String zkDigestReadonlyUsernameVMParamName;
+ final String zkDigestReadonlyPasswordVMParamName;
+
+ public VMParamsAllAndReadonlyDigestZkACLProvider() {
+ this(
+ VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME,
+ VMParamsSingleSetCredentialsDigestZkCredentialsProvider.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME,
+ DEFAULT_DIGEST_READONLY_USERNAME_VM_PARAM_NAME,
+ DEFAULT_DIGEST_READONLY_PASSWORD_VM_PARAM_NAME
+ );
+ }
+
+ public VMParamsAllAndReadonlyDigestZkACLProvider(String zkDigestAllUsernameVMParamName, String zkDigestAllPasswordVMParamName,
+ String zkDigestReadonlyUsernameVMParamName, String zkDigestReadonlyPasswordVMParamName) {
+ this.zkDigestAllUsernameVMParamName = zkDigestAllUsernameVMParamName;
+ this.zkDigestAllPasswordVMParamName = zkDigestAllPasswordVMParamName;
+ this.zkDigestReadonlyUsernameVMParamName = zkDigestReadonlyUsernameVMParamName;
+ this.zkDigestReadonlyPasswordVMParamName = zkDigestReadonlyPasswordVMParamName;
+ }
+
+
+ @Override
+ protected List<ACL> createGlobalACLsToAdd() {
+ try {
+ List<ACL> result = new ArrayList<ACL>();
+
+ // Not to have to provide too much credentials and ACL information to the process it is assumed that you want "ALL"-acls
+ // added to the user you are using to connect to ZK (if you are using VMParamsSingleSetCredentialsDigestZkCredentialsProvider)
+ String digestAllUsername = System.getProperty(zkDigestAllUsernameVMParamName);
+ String digestAllPassword = System.getProperty(zkDigestAllPasswordVMParamName);
+ if (!StringUtils.isEmpty(digestAllUsername) && !StringUtils.isEmpty(digestAllPassword)) {
+ result.add(new ACL(ZooDefs.Perms.ALL, new Id("digest", DigestAuthenticationProvider.generateDigest(digestAllUsername + ":" + digestAllPassword))));
+ }
+
+ // Besides that support for adding additional "READONLY"-acls for another user
+ String digestReadonlyUsername = System.getProperty(zkDigestReadonlyUsernameVMParamName);
+ String digestReadonlyPassword = System.getProperty(zkDigestReadonlyPasswordVMParamName);
+ if (!StringUtils.isEmpty(digestReadonlyUsername) && !StringUtils.isEmpty(digestReadonlyPassword)) {
+ result.add(new ACL(ZooDefs.Perms.READ, new Id("digest", DigestAuthenticationProvider.generateDigest(digestReadonlyUsername + ":" + digestReadonlyPassword))));
+ }
+
+ if (result.isEmpty()) {
+ result = super.createGlobalACLsToAdd();
+ }
+
+ return result;
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java
new file mode 100644
index 0000000..1e575fd
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/VMParamsSingleSetCredentialsDigestZkCredentialsProvider.java
@@ -0,0 +1,60 @@
+package org.apache.solr.common.cloud;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.solr.common.StringUtils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class VMParamsSingleSetCredentialsDigestZkCredentialsProvider extends DefaultZkCredentialsProvider {
+
+ public static final String DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME = "zkDigestUsername";
+ public static final String DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME = "zkDigestPassword";
+
+ final String zkDigestUsernameVMParamName;
+ final String zkDigestPasswordVMParamName;
+
+ public VMParamsSingleSetCredentialsDigestZkCredentialsProvider() {
+ this(DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME);
+ }
+
+ public VMParamsSingleSetCredentialsDigestZkCredentialsProvider(String zkDigestUsernameVMParamName, String zkDigestPasswordVMParamName) {
+ this.zkDigestUsernameVMParamName = zkDigestUsernameVMParamName;
+ this.zkDigestPasswordVMParamName = zkDigestPasswordVMParamName;
+ }
+
+ @Override
+ protected Collection<ZkCredentials> createCredentials() {
+ List<ZkCredentials> result = new ArrayList<ZkCredentials>();
+ String digestUsername = System.getProperty(zkDigestUsernameVMParamName);
+ String digestPassword = System.getProperty(zkDigestPasswordVMParamName);
+ if (!StringUtils.isEmpty(digestUsername) && !StringUtils.isEmpty(digestPassword)) {
+ try {
+ result.add(new ZkCredentials("digest", (digestUsername + ":" + digestPassword).getBytes("UTF-8")));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return result;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkACLProvider.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkACLProvider.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkACLProvider.java
new file mode 100644
index 0000000..03149b3
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkACLProvider.java
@@ -0,0 +1,28 @@
+package org.apache.solr.common.cloud;
+
+import java.util.List;
+
+import org.apache.zookeeper.data.ACL;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public interface ZkACLProvider {
+
+ List<ACL> getACLsToAdd(String zNodePath);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java
new file mode 100644
index 0000000..5f4baa5
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkClientConnectionStrategy.java
@@ -0,0 +1,113 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkCredentialsProvider.ZkCredentials;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public abstract class ZkClientConnectionStrategy {
+ private static Logger log = LoggerFactory.getLogger(ZkClientConnectionStrategy.class);
+
+ private volatile ZkCredentialsProvider zkCredentialsToAddAutomatically;
+ private volatile boolean zkCredentialsToAddAutomaticallyUsed;
+
+ private List<DisconnectedListener> disconnectedListeners = new ArrayList<>();
+ private List<ConnectedListener> connectedListeners = new ArrayList<>();
+
+ public abstract void connect(String zkServerAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException;
+ public abstract void reconnect(String serverAddress, int zkClientTimeout, Watcher watcher, ZkUpdate updater) throws IOException, InterruptedException, TimeoutException;
+
+ public ZkClientConnectionStrategy() {
+ zkCredentialsToAddAutomaticallyUsed = false;
+ }
+
+ public synchronized void disconnected() {
+ for (DisconnectedListener listener : disconnectedListeners) {
+ try {
+ listener.disconnected();
+ } catch (Exception e) {
+ SolrException.log(log, "", e);
+ }
+ }
+ }
+
+ public synchronized void connected() {
+ for (ConnectedListener listener : connectedListeners) {
+ try {
+ listener.connected();
+ } catch (Exception e) {
+ SolrException.log(log, "", e);
+ }
+ }
+ }
+
+ public interface DisconnectedListener {
+ public void disconnected();
+ };
+
+ public interface ConnectedListener {
+ public void connected();
+ };
+
+
+ public synchronized void addDisconnectedListener(DisconnectedListener listener) {
+ disconnectedListeners.add(listener);
+ }
+
+ public synchronized void addConnectedListener(ConnectedListener listener) {
+ connectedListeners.add(listener);
+ }
+
+ public static abstract class ZkUpdate {
+ public abstract void update(SolrZooKeeper zooKeeper) throws InterruptedException, TimeoutException, IOException;
+ }
+
+ public void setZkCredentialsToAddAutomatically(ZkCredentialsProvider zkCredentialsToAddAutomatically) {
+ if (zkCredentialsToAddAutomaticallyUsed || (zkCredentialsToAddAutomatically == null))
+ throw new RuntimeException("Cannot change zkCredentialsToAddAutomatically after it has been (connect or reconnect was called) used or to null");
+ this.zkCredentialsToAddAutomatically = zkCredentialsToAddAutomatically;
+ }
+
+ public boolean hasZkCredentialsToAddAutomatically() {
+ return zkCredentialsToAddAutomatically != null;
+ }
+
+ protected SolrZooKeeper createSolrZooKeeper(final String serverAddress, final int zkClientTimeout,
+ final Watcher watcher) throws IOException {
+ SolrZooKeeper result = new SolrZooKeeper(serverAddress, zkClientTimeout, watcher);
+
+ zkCredentialsToAddAutomaticallyUsed = true;
+ for (ZkCredentials zkCredentials : zkCredentialsToAddAutomatically.getCredentials()) {
+ result.addAuthInfo(zkCredentials.getScheme(), zkCredentials.getAuth());
+ }
+
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCmdExecutor.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
new file mode 100644
index 0000000..d77ad06
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCmdExecutor.java
@@ -0,0 +1,111 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NodeExistsException;
+
+
+public class ZkCmdExecutor {
+ private long retryDelay = 1500L; // 1 second would match timeout, so 500 ms over for padding
+ private int retryCount;
+ private double timeouts;
+
+ /**
+ * TODO: At this point, this should probably take a SolrZkClient in
+ * its constructor.
+ *
+ * @param timeoutms
+ * the client timeout for the ZooKeeper clients that will be used
+ * with this class.
+ */
+ public ZkCmdExecutor(int timeoutms) {
+ timeouts = timeoutms / 1000.0;
+ this.retryCount = Math.round(0.5f * ((float)Math.sqrt(8.0f * timeouts + 1.0f) - 1.0f)) + 1;
+ }
+
+ public long getRetryDelay() {
+ return retryDelay;
+ }
+
+ public void setRetryDelay(long retryDelay) {
+ this.retryDelay = retryDelay;
+ }
+
+
+ /**
+ * Perform the given operation, retrying if the connection fails
+ */
+ @SuppressWarnings("unchecked")
+ public <T> T retryOperation(ZkOperation operation)
+ throws KeeperException, InterruptedException {
+ KeeperException exception = null;
+ for (int i = 0; i < retryCount; i++) {
+ try {
+ return (T) operation.execute();
+ } catch (KeeperException.ConnectionLossException e) {
+ if (exception == null) {
+ exception = e;
+ }
+ if (Thread.currentThread().isInterrupted()) {
+ Thread.currentThread().interrupt();
+ throw new InterruptedException();
+ }
+ if (Thread.currentThread() instanceof ClosableThread) {
+ if (((ClosableThread) Thread.currentThread()).isClosed()) {
+ throw exception;
+ }
+ }
+ if (i != retryCount -1) {
+ retryDelay(i);
+ }
+ }
+ }
+ throw exception;
+ }
+
+ public void ensureExists(String path, final SolrZkClient zkClient) throws KeeperException, InterruptedException {
+ ensureExists(path, null, CreateMode.PERSISTENT, zkClient);
+ }
+
+ public void ensureExists(final String path, final byte[] data,
+ CreateMode createMode, final SolrZkClient zkClient) throws KeeperException, InterruptedException {
+
+ if (zkClient.exists(path, true)) {
+ return;
+ }
+ try {
+ zkClient.makePath(path, data, true);
+ } catch (NodeExistsException e) {
+ // it's okay if another beats us creating the node
+ }
+
+ }
+
+ /**
+ * Performs a retry delay if this is not the first attempt
+ *
+ * @param attemptCount
+ * the number of the attempts performed so far
+ */
+ protected void retryDelay(int attemptCount) throws InterruptedException {
+ Thread.sleep((attemptCount + 1) * retryDelay);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkConfigManager.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkConfigManager.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkConfigManager.java
new file mode 100644
index 0000000..a3a8060
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkConfigManager.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common.cloud;
+
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Class that manages named configs in Zookeeper
+ */
+public class ZkConfigManager {
+
+ private static final Logger logger = LoggerFactory.getLogger(ZkConfigManager.class);
+
+ /** ZkNode where named configs are stored */
+ public static final String CONFIGS_ZKNODE = "/configs";
+
+ private final SolrZkClient zkClient;
+
+ /**
+ * Creates a new ZkConfigManager
+ * @param zkClient the {@link SolrZkClient} to use
+ */
+ public ZkConfigManager(SolrZkClient zkClient) {
+ this.zkClient = zkClient;
+ }
+
+ private void uploadToZK(final Path rootPath, final String zkPath) throws IOException {
+
+ if (!Files.exists(rootPath))
+ throw new IOException("Path " + rootPath + " does not exist");
+
+ Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>(){
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+ String filename = file.getFileName().toString();
+ if (filename.startsWith("."))
+ return FileVisitResult.CONTINUE;
+ String zkNode = createZkNodeName(zkPath, rootPath, file);
+ try {
+ zkClient.makePath(zkNode, file.toFile(), false, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error uploading file " + file.toString() + " to zookeeper path " + zkNode,
+ SolrZkClient.checkInterrupted(e));
+ }
+ return FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+ return (dir.getFileName().toString().startsWith(".")) ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ private static String createZkNodeName(String zkRoot, Path root, Path file) {
+ String relativePath = root.relativize(file).toString();
+ // Windows shenanigans
+ String separator = root.getFileSystem().getSeparator();
+ if ("\\".equals(separator))
+ relativePath = relativePath.replaceAll("\\\\", "/");
+ return zkRoot + "/" + relativePath;
+ }
+
+ private void downloadFromZK(String zkPath, Path dir) throws IOException {
+ try {
+ List<String> files = zkClient.getChildren(zkPath, null, true);
+ Files.createDirectories(dir);
+ for (String file : files) {
+ List<String> children = zkClient.getChildren(zkPath + "/" + file, null, true);
+ if (children.size() == 0) {
+ byte[] data = zkClient.getData(zkPath + "/" + file, null, null, true);
+ Path filename = dir.resolve(file);
+ logger.info("Writing file {}", filename);
+ Files.write(filename, data);
+ } else {
+ downloadFromZK(zkPath + "/" + file, dir.resolve(file));
+ }
+ }
+ }
+ catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error downloading files from zookeeper path " + zkPath + " to " + dir.toString(),
+ SolrZkClient.checkInterrupted(e));
+ }
+ }
+
+ /**
+ * Upload files from a given path to a config in Zookeeper
+ * @param dir {@link java.nio.file.Path} to the files
+ * @param configName the name to give the config
+ * @throws IOException
+ * if an I/O error occurs or the path does not exist
+ */
+ public void uploadConfigDir(Path dir, String configName) throws IOException {
+ uploadToZK(dir, CONFIGS_ZKNODE + "/" + configName);
+ }
+
+ /**
+ * Download a config from Zookeeper and write it to the filesystem
+ * @param configName the config to download
+ * @param dir the {@link Path} to write files under
+ * @throws IOException
+ * if an I/O error occurs or the config does not exist
+ */
+ public void downloadConfigDir(String configName, Path dir) throws IOException {
+ downloadFromZK(CONFIGS_ZKNODE + "/" + configName, dir);
+ }
+
+ public List<String> listConfigs() throws IOException {
+ try {
+ return zkClient.getChildren(ZkConfigManager.CONFIGS_ZKNODE, null, true);
+ }
+ catch (KeeperException.NoNodeException e) {
+ return Collections.emptyList();
+ }
+ catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error listing configs", SolrZkClient.checkInterrupted(e));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
new file mode 100644
index 0000000..131d330
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
@@ -0,0 +1,74 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+public class ZkCoreNodeProps {
+ private ZkNodeProps nodeProps;
+
+ public ZkCoreNodeProps(ZkNodeProps nodeProps) {
+ this.nodeProps = nodeProps;
+ }
+
+ public String getCoreUrl() {
+ return getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), nodeProps.getStr(ZkStateReader.CORE_NAME_PROP));
+ }
+
+ public String getNodeName() {
+ return nodeProps.getStr(ZkStateReader.NODE_NAME_PROP);
+ }
+
+ public String getState() {
+ return nodeProps.getStr(ZkStateReader.STATE_PROP);
+ }
+
+ public String getBaseUrl() {
+ return nodeProps.getStr(ZkStateReader.BASE_URL_PROP);
+ }
+
+ public String getCoreName() {
+ return nodeProps.getStr(ZkStateReader.CORE_NAME_PROP);
+ }
+
+ public static String getCoreUrl(ZkNodeProps nodeProps) {
+ return getCoreUrl(nodeProps.getStr(ZkStateReader.BASE_URL_PROP), nodeProps.getStr(ZkStateReader.CORE_NAME_PROP));
+ }
+
+ public static String getCoreUrl(String baseUrl, String coreName) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(baseUrl);
+ if (!baseUrl.endsWith("/")) sb.append("/");
+ sb.append(coreName);
+ if (!(sb.substring(sb.length() - 1).equals("/"))) sb.append("/");
+ return sb.toString();
+ }
+
+ @Override
+ public String toString() {
+ return nodeProps.toString();
+ }
+
+ public ZkNodeProps getNodeProps() {
+ return nodeProps;
+ }
+
+ public boolean isLeader() {
+ return nodeProps.containsKey(ZkStateReader.LEADER_PROP);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java
new file mode 100644
index 0000000..b4ab6d8
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkCredentialsProvider.java
@@ -0,0 +1,45 @@
+package org.apache.solr.common.cloud;
+
+import java.util.Collection;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public interface ZkCredentialsProvider {
+
+ public class ZkCredentials {
+ String scheme;
+ byte[] auth;
+
+ public ZkCredentials(String scheme, byte[] auth) {
+ super();
+ this.scheme = scheme;
+ this.auth = auth;
+ }
+
+ String getScheme() {
+ return scheme;
+ }
+
+ byte[] getAuth() {
+ return auth;
+ }
+ }
+
+ Collection<ZkCredentials> getCredentials();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkNodeProps.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkNodeProps.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkNodeProps.java
new file mode 100644
index 0000000..5ddfa24
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkNodeProps.java
@@ -0,0 +1,154 @@
+package org.apache.solr.common.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.noggit.JSONUtil;
+import org.noggit.JSONWriter;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * ZkNodeProps contains generic immutable properties.
+ */
+public class ZkNodeProps implements JSONWriter.Writable {
+
+ protected final Map<String,Object> propMap;
+
+ /**
+ * Construct ZKNodeProps from map.
+ */
+ public ZkNodeProps(Map<String,Object> propMap) {
+ this.propMap = propMap;
+ // TODO: store an unmodifiable map, but in a way that guarantees not to wrap more than once.
+ // Always wrapping introduces a memory leak.
+ }
+
+
+ /**
+ * Constructor that populates the from array of Strings in form key1, value1,
+ * key2, value2, ..., keyN, valueN
+ */
+ public ZkNodeProps(String... keyVals) {
+ this( makeMap((Object[])keyVals) );
+ }
+
+ public static ZkNodeProps fromKeyVals(Object... keyVals) {
+ return new ZkNodeProps( makeMap(keyVals) );
+ }
+
+ public static Map<String,Object> makeMap(Object... keyVals) {
+ if ((keyVals.length & 0x01) != 0) {
+ throw new IllegalArgumentException("arguments should be key,value");
+ }
+ Map<String,Object> propMap = new LinkedHashMap<>(keyVals.length>>1);
+ for (int i = 0; i < keyVals.length; i+=2) {
+ propMap.put(keyVals[i].toString(), keyVals[i+1]);
+ }
+ return propMap;
+ }
+
+
+ /**
+ * Get property keys.
+ */
+ public Set<String> keySet() {
+ return propMap.keySet();
+ }
+
+ /**
+ * Get all properties as map.
+ */
+ public Map<String, Object> getProperties() {
+ return propMap;
+ }
+
+ /** Returns a shallow writable copy of the properties */
+ public Map<String,Object> shallowCopy() {
+ return new LinkedHashMap<>(propMap);
+ }
+
+ /**
+ * Create Replica from json string that is typically stored in zookeeper.
+ */
+ public static ZkNodeProps load(byte[] bytes) {
+ Map<String, Object> props = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
+ return new ZkNodeProps(props);
+ }
+
+ @Override
+ public void write(JSONWriter jsonWriter) {
+ jsonWriter.write(propMap);
+ }
+
+ /**
+ * Get a string property value.
+ */
+ public String getStr(String key) {
+ Object o = propMap.get(key);
+ return o == null ? null : o.toString();
+ }
+
+ /**
+ * Get a string property value.
+ */
+ public Integer getInt(String key, Integer def) {
+ Object o = propMap.get(key);
+ return o == null ? def : Integer.valueOf(o.toString());
+ }
+
+ /**
+ * Get a string property value.
+ */
+ public String getStr(String key,String def) {
+ Object o = propMap.get(key);
+ return o == null ? def : o.toString();
+ }
+
+ public Object get(String key) {
+ return propMap.get(key);
+ }
+
+ @Override
+ public String toString() {
+ return JSONUtil.toJSON(this);
+ /***
+ StringBuilder sb = new StringBuilder();
+ Set<Entry<String,Object>> entries = propMap.entrySet();
+ for(Entry<String,Object> entry : entries) {
+ sb.append(entry.getKey() + "=" + entry.getValue() + "\n");
+ }
+ return sb.toString();
+ ***/
+ }
+
+ /**
+ * Check if property key exists.
+ */
+ public boolean containsKey(String key) {
+ return propMap.containsKey(key);
+ }
+
+ public boolean getBool(String key, boolean b) {
+ Object o = propMap.get(key);
+ if(o==null) return b;
+ return Boolean.parseBoolean(o.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkOperation.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkOperation.java b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkOperation.java
new file mode 100644
index 0000000..b4da540
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/common/cloud/ZkOperation.java
@@ -0,0 +1,37 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.common.cloud;
+
+import java.io.IOException;
+
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * A callback object which can be used for implementing retry-able operations.
+ *
+ */
+public abstract class ZkOperation {
+
+ /**
+ * Performs the operation - which may be involved multiple times if the connection
+ * to ZooKeeper closes during this operation
+ *
+ * @return the result of the operation or null
+ */
+ public abstract Object execute() throws KeeperException, InterruptedException;
+}
[14/17] incubator-ranger git commit: Support for Solr as Audit
Destination.
Posted by bo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
new file mode 100644
index 0000000..8272813
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java
@@ -0,0 +1,1232 @@
+package org.apache.solr.client.solrj.impl;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.http.NoHttpResponseException;
+import org.apache.http.client.HttpClient;
+import org.apache.http.conn.ConnectTimeoutException;
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
+import org.apache.solr.client.solrj.request.IsUpdateRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.ImplicitDocRouter;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.ZooKeeperException;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.Hash;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+/**
+ * SolrJ client class to communicate with SolrCloud.
+ * Instances of this class communicate with Zookeeper to discover
+ * Solr endpoints for SolrCloud collections, and then use the
+ * {@link LBHttpSolrClient} to issue requests.
+ *
+ * This class assumes the id field for your documents is called
+ * 'id' - if this is not the case, you must set the right name
+ * with {@link #setIdField(String)}.
+ */
+@SuppressWarnings("serial")
+public class CloudSolrClient extends SolrClient {
+ protected static final Logger log = LoggerFactory.getLogger(CloudSolrClient.class);
+
+ private volatile ZkStateReader zkStateReader;
+ private String zkHost; // the zk server connect string
+ private int zkConnectTimeout = 10000;
+ private int zkClientTimeout = 10000;
+ private volatile String defaultCollection;
+ private final LBHttpSolrClient lbClient;
+ private final boolean shutdownLBHttpSolrServer;
+ private HttpClient myClient;
+ private final boolean clientIsInternal;
+ //no of times collection state to be reloaded if stale state error is received
+ private static final int MAX_STALE_RETRIES = 5;
+ Random rand = new Random();
+
+ private final boolean updatesToLeaders;
+ private boolean parallelUpdates = true;
+ private ExecutorService threadPool = Executors
+ .newCachedThreadPool(new SolrjNamedThreadFactory(
+ "CloudSolrServer ThreadPool"));
+ private String idField = "id";
+ public static final String STATE_VERSION = "_stateVer_";
+ private final Set<String> NON_ROUTABLE_PARAMS;
+ {
+ NON_ROUTABLE_PARAMS = new HashSet<>();
+ NON_ROUTABLE_PARAMS.add(UpdateParams.EXPUNGE_DELETES);
+ NON_ROUTABLE_PARAMS.add(UpdateParams.MAX_OPTIMIZE_SEGMENTS);
+ NON_ROUTABLE_PARAMS.add(UpdateParams.COMMIT);
+ NON_ROUTABLE_PARAMS.add(UpdateParams.WAIT_SEARCHER);
+ NON_ROUTABLE_PARAMS.add(UpdateParams.OPEN_SEARCHER);
+
+ NON_ROUTABLE_PARAMS.add(UpdateParams.SOFT_COMMIT);
+ NON_ROUTABLE_PARAMS.add(UpdateParams.PREPARE_COMMIT);
+ NON_ROUTABLE_PARAMS.add(UpdateParams.OPTIMIZE);
+
+ // Not supported via SolrCloud
+ // NON_ROUTABLE_PARAMS.add(UpdateParams.ROLLBACK);
+
+ }
+ private volatile long timeToLive = 60* 1000L;
+ private volatile List<Object> locks = objectList(3);
+
+
+ protected final Map<String, ExpiringCachedDocCollection> collectionStateCache = new ConcurrentHashMap<String, ExpiringCachedDocCollection>(){
+ @Override
+ public ExpiringCachedDocCollection get(Object key) {
+ ExpiringCachedDocCollection val = super.get(key);
+ if(val == null) return null;
+ if(val.isExpired(timeToLive)) {
+ super.remove(key);
+ return null;
+ }
+ return val;
+ }
+
+ };
+
+ class ExpiringCachedDocCollection {
+ final DocCollection cached;
+ long cachedAt;
+
+ ExpiringCachedDocCollection(DocCollection cached) {
+ this.cached = cached;
+ this.cachedAt = System.currentTimeMillis();
+ }
+
+ boolean isExpired(long timeToLive) {
+ return (System.currentTimeMillis() - cachedAt) > timeToLive;
+ }
+ }
+
+ /**
+ * Create a new client object that connects to Zookeeper and is always aware
+ * of the SolrCloud state. If there is a fully redundant Zookeeper quorum and
+ * SolrCloud has enough replicas for every shard in a collection, there is no
+ * single point of failure. Updates will be sent to shard leaders by default.
+ *
+ * @param zkHost
+ * The client endpoint of the zookeeper quorum containing the cloud
+ * state. The full specification for this string is one or more comma
+ * separated HOST:PORT values, followed by an optional chroot value
+ * that starts with a forward slash. Using a chroot allows multiple
+ * applications to coexist in one ensemble. For full details, see the
+ * Zookeeper documentation. Some examples:
+ * <p>
+ * "host1:2181"
+ * <p>
+ * "host1:2181,host2:2181,host3:2181/mysolrchroot"
+ * <p>
+ * "zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181"
+ */
+ public CloudSolrClient(String zkHost) {
+ this.zkHost = zkHost;
+ this.clientIsInternal = true;
+ this.myClient = HttpClientUtil.createClient(null);
+ this.lbClient = new LBHttpSolrClient(myClient);
+ this.lbClient.setRequestWriter(new BinaryRequestWriter());
+ this.lbClient.setParser(new BinaryResponseParser());
+ this.updatesToLeaders = true;
+ shutdownLBHttpSolrServer = true;
+ lbClient.addQueryParams(STATE_VERSION);
+ }
+
+ /**
+ * Create a new client object that connects to Zookeeper and is always aware
+ * of the SolrCloud state. If there is a fully redundant Zookeeper quorum and
+ * SolrCloud has enough replicas for every shard in a collection, there is no
+ * single point of failure. Updates will be sent to shard leaders by default.
+ *
+ * @param zkHost
+ * The client endpoint of the zookeeper quorum containing the cloud
+ * state. The full specification for this string is one or more comma
+ * separated HOST:PORT values, followed by an optional chroot value
+ * that starts with a forward slash. Using a chroot allows multiple
+ * applications to coexist in one ensemble. For full details, see the
+ * Zookeeper documentation. Some examples:
+ * <p>
+ * "host1:2181"
+ * <p>
+ * "host1:2181,host2:2181,host3:2181/mysolrchroot"
+ * <p>
+ * "zoo1.example.com:2181,zoo2.example.com:2181,zoo3.example.com:2181"
+ * @param httpClient
+ * the {@link HttpClient} instance to be used for all requests. The
+ * provided httpClient should use a multi-threaded connection manager.
+ */
+ public CloudSolrClient(String zkHost, HttpClient httpClient) {
+ this.zkHost = zkHost;
+ this.clientIsInternal = httpClient == null;
+ this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient;
+ this.lbClient = new LBHttpSolrClient(myClient);
+ this.lbClient.setRequestWriter(new BinaryRequestWriter());
+ this.lbClient.setParser(new BinaryResponseParser());
+ this.updatesToLeaders = true;
+ shutdownLBHttpSolrServer = true;
+ lbClient.addQueryParams(STATE_VERSION);
+ }
+
+ /**
+ * Create a new client object using multiple string values in a Collection
+ * instead of a standard zkHost connection string. Note that this method will
+ * not be used if there is only one String argument - that will use
+ * {@link #CloudSolrClient(String)} instead.
+ *
+ * @param zkHosts
+ * A Java Collection (List, Set, etc) of HOST:PORT strings, one for
+ * each host in the zookeeper ensemble. Note that with certain
+ * Collection types like HashSet, the order of hosts in the final
+ * connect string may not be in the same order you added them.
+ * @param chroot
+ * A chroot value for zookeeper, starting with a forward slash. If no
+ * chroot is required, use null.
+ * @throws IllegalArgumentException
+ * if the chroot value does not start with a forward slash.
+ * @see #CloudSolrClient(String)
+ */
+ public CloudSolrClient(Collection<String> zkHosts, String chroot) {
+ this(zkHosts, chroot, null);
+ }
+
+ /**
+ * Create a new client object using multiple string values in a Collection
+ * instead of a standard zkHost connection string. Note that this method will
+ * not be used if there is only one String argument - that will use
+ * {@link #CloudSolrClient(String)} instead.
+ *
+ * @param zkHosts
+ * A Java Collection (List, Set, etc) of HOST:PORT strings, one for
+ * each host in the zookeeper ensemble. Note that with certain
+ * Collection types like HashSet, the order of hosts in the final
+ * connect string may not be in the same order you added them.
+ * @param chroot
+ * A chroot value for zookeeper, starting with a forward slash. If no
+ * chroot is required, use null.
+ * @param httpClient
+ * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a
+ * multi-threaded connection manager.
+ * @throws IllegalArgumentException
+ * if the chroot value does not start with a forward slash.
+ * @see #CloudSolrClient(String)
+ */
+ public CloudSolrClient(Collection<String> zkHosts, String chroot, HttpClient httpClient) {
+ StringBuilder zkBuilder = new StringBuilder();
+ int lastIndexValue = zkHosts.size() - 1;
+ int i = 0;
+ for (String zkHost : zkHosts) {
+ zkBuilder.append(zkHost);
+ if (i < lastIndexValue) {
+ zkBuilder.append(",");
+ }
+ i++;
+ }
+ if (chroot != null) {
+ if (chroot.startsWith("/")) {
+ zkBuilder.append(chroot);
+ } else {
+ throw new IllegalArgumentException(
+ "The chroot must start with a forward slash.");
+ }
+ }
+
+ /* Log the constructed connection string and then initialize. */
+ log.info("Final constructed zkHost string: " + zkBuilder.toString());
+
+ this.zkHost = zkBuilder.toString();
+ this.clientIsInternal = httpClient == null;
+ this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient;
+ this.lbClient = new LBHttpSolrClient(myClient);
+ this.lbClient.setRequestWriter(new BinaryRequestWriter());
+ this.lbClient.setParser(new BinaryResponseParser());
+ this.updatesToLeaders = true;
+ shutdownLBHttpSolrServer = true;
+ }
+
+ /**
+ * @param zkHost
+ * A zookeeper client endpoint.
+ * @param updatesToLeaders
+ * If true, sends updates only to shard leaders.
+ * @see #CloudSolrClient(String) for full description and details on zkHost
+ */
+ public CloudSolrClient(String zkHost, boolean updatesToLeaders) {
+ this(zkHost, updatesToLeaders, null);
+ }
+
+ /**
+ * @param zkHost
+ * A zookeeper client endpoint.
+ * @param updatesToLeaders
+ * If true, sends updates only to shard leaders.
+ * @param httpClient
+ * the {@link HttpClient} instance to be used for all requests. The provided httpClient should use a
+ * multi-threaded connection manager.
+ * @see #CloudSolrClient(String) for full description and details on zkHost
+ */
+ public CloudSolrClient(String zkHost, boolean updatesToLeaders, HttpClient httpClient) {
+ this.zkHost = zkHost;
+ this.clientIsInternal = httpClient == null;
+ this.myClient = httpClient == null ? HttpClientUtil.createClient(null) : httpClient;
+ this.lbClient = new LBHttpSolrClient(myClient);
+ this.lbClient.setRequestWriter(new BinaryRequestWriter());
+ this.lbClient.setParser(new BinaryResponseParser());
+ this.updatesToLeaders = updatesToLeaders;
+ shutdownLBHttpSolrServer = true;
+ lbClient.addQueryParams(STATE_VERSION);
+ }
+
+ /**Sets the cache ttl for DocCollection Objects cached . This is only applicable for collections which are persisted outside of clusterstate.json
+ * @param seconds ttl value in seconds
+ */
+ public void setCollectionCacheTTl(int seconds){
+ assert seconds > 0;
+ timeToLive = seconds*1000L;
+ }
+
+ /**
+ * @param zkHost
+ * A zookeeper client endpoint.
+ * @param lbClient
+ * LBHttpSolrServer instance for requests.
+ * @see #CloudSolrClient(String) for full description and details on zkHost
+ */
+ public CloudSolrClient(String zkHost, LBHttpSolrClient lbClient) {
+ this(zkHost, lbClient, true);
+ }
+
+ /**
+ * @param zkHost
+ * A zookeeper client endpoint.
+ * @param lbClient
+ * LBHttpSolrServer instance for requests.
+ * @param updatesToLeaders
+ * If true, sends updates only to shard leaders.
+ * @see #CloudSolrClient(String) for full description and details on zkHost
+ */
+ public CloudSolrClient(String zkHost, LBHttpSolrClient lbClient, boolean updatesToLeaders) {
+ this.zkHost = zkHost;
+ this.lbClient = lbClient;
+ this.updatesToLeaders = updatesToLeaders;
+ shutdownLBHttpSolrServer = false;
+ this.clientIsInternal = false;
+ lbClient.addQueryParams(STATE_VERSION);
+ }
+
+ public ResponseParser getParser() {
+ return lbClient.getParser();
+ }
+
+ /**
+ * Note: This setter method is <b>not thread-safe</b>.
+ *
+ * @param processor
+ * Default Response Parser chosen to parse the response if the parser
+ * were not specified as part of the request.
+ * @see org.apache.solr.client.solrj.SolrRequest#getResponseParser()
+ */
+ public void setParser(ResponseParser processor) {
+ lbClient.setParser(processor);
+ }
+
+ public RequestWriter getRequestWriter() {
+ return lbClient.getRequestWriter();
+ }
+
+ public void setRequestWriter(RequestWriter requestWriter) {
+ lbClient.setRequestWriter(requestWriter);
+ }
+
+ /**
+ * @return the zkHost value used to connect to zookeeper.
+ */
+ public String getZkHost() {
+ return zkHost;
+ }
+
+ public ZkStateReader getZkStateReader() {
+ return zkStateReader;
+ }
+
+ /**
+ * @param idField the field to route documents on.
+ */
+ public void setIdField(String idField) {
+ this.idField = idField;
+ }
+
+ /**
+ * @return the field that updates are routed on.
+ */
+ public String getIdField() {
+ return idField;
+ }
+
+ /** Sets the default collection for request */
+ public void setDefaultCollection(String collection) {
+ this.defaultCollection = collection;
+ }
+
+ /** Gets the default collection for request */
+ public String getDefaultCollection() {
+ return defaultCollection;
+ }
+
+ /** Set the connect timeout to the zookeeper ensemble in ms */
+ public void setZkConnectTimeout(int zkConnectTimeout) {
+ this.zkConnectTimeout = zkConnectTimeout;
+ }
+
+ /** Set the timeout to the zookeeper ensemble in ms */
+ public void setZkClientTimeout(int zkClientTimeout) {
+ this.zkClientTimeout = zkClientTimeout;
+ }
+
+ /**
+ * Connect to the zookeeper ensemble.
+ * This is an optional method that may be used to force a connect before any other requests are sent.
+ *
+ */
+ public void connect() {
+ if (zkStateReader == null) {
+ synchronized (this) {
+ if (zkStateReader == null) {
+ ZkStateReader zk = null;
+ try {
+ zk = new ZkStateReader(zkHost, zkClientTimeout, zkConnectTimeout);
+ zk.createClusterStateWatchersAndUpdate();
+ zkStateReader = zk;
+ } catch (InterruptedException e) {
+ zk.close();
+ Thread.currentThread().interrupt();
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (KeeperException e) {
+ zk.close();
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ } catch (Exception e) {
+ if (zk != null) zk.close();
+ // do not wrap because clients may be relying on the underlying exception being thrown
+ throw e;
+ }
+ }
+ }
+ }
+ }
+
+ public void setParallelUpdates(boolean parallelUpdates) {
+ this.parallelUpdates = parallelUpdates;
+ }
+
+ /**
+ * Upload a set of config files to Zookeeper and give it a name
+ *
+ * NOTE: You should only allow trusted users to upload configs. If you
+ * are allowing client access to zookeeper, you should protect the
+ * /configs node against unauthorised write access.
+ *
+ * @param configPath {@link java.nio.file.Path} to the config files
+ * @param configName the name of the config
+ * @throws IOException if an IO error occurs
+ */
+ public void uploadConfig(Path configPath, String configName) throws IOException {
+ zkStateReader.getConfigManager().uploadConfigDir(configPath, configName);
+ }
+
+ /**
+ * Download a named config from Zookeeper to a location on the filesystem
+ * @param configName the name of the config
+ * @param downloadPath the path to write config files to
+ * @throws IOException if an I/O exception occurs
+ */
+ public void downloadConfig(String configName, Path downloadPath) throws IOException {
+ zkStateReader.getConfigManager().downloadConfigDir(configName, downloadPath);
+ }
+
+ private NamedList<Object> directUpdate(AbstractUpdateRequest request, ClusterState clusterState) throws SolrServerException {
+ UpdateRequest updateRequest = (UpdateRequest) request;
+ ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
+ ModifiableSolrParams routableParams = new ModifiableSolrParams();
+ ModifiableSolrParams nonRoutableParams = new ModifiableSolrParams();
+
+ if(params != null) {
+ nonRoutableParams.add(params);
+ routableParams.add(params);
+ for(String param : NON_ROUTABLE_PARAMS) {
+ routableParams.remove(param);
+ }
+ }
+
+ String collection = nonRoutableParams.get(UpdateParams.COLLECTION, defaultCollection);
+ if (collection == null) {
+ throw new SolrServerException("No collection param specified on request and no default collection has been set.");
+ }
+
+
+ //Check to see if the collection is an alias.
+ Aliases aliases = zkStateReader.getAliases();
+ if(aliases != null) {
+ Map<String, String> collectionAliases = aliases.getCollectionAliasMap();
+ if(collectionAliases != null && collectionAliases.containsKey(collection)) {
+ collection = collectionAliases.get(collection);
+ }
+ }
+
+ DocCollection col = getDocCollection(clusterState, collection,null);
+
+ DocRouter router = col.getRouter();
+
+ if (router instanceof ImplicitDocRouter) {
+ // short circuit as optimization
+ return null;
+ }
+
+ //Create the URL map, which is keyed on slice name.
+ //The value is a list of URLs for each replica in the slice.
+ //The first value in the list is the leader for the slice.
+ Map<String,List<String>> urlMap = buildUrlMap(col);
+ if (urlMap == null) {
+ // we could not find a leader yet - use unoptimized general path
+ return null;
+ }
+
+ NamedList<Throwable> exceptions = new NamedList<>();
+ NamedList<NamedList> shardResponses = new NamedList<>();
+
+ Map<String, LBHttpSolrClient.Req> routes = updateRequest.getRoutes(router, col, urlMap, routableParams, this.idField);
+ if (routes == null) {
+ return null;
+ }
+
+ long start = System.nanoTime();
+
+ if (parallelUpdates) {
+ final Map<String, Future<NamedList<?>>> responseFutures = new HashMap<>(routes.size());
+ for (final Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
+ final String url = entry.getKey();
+ final LBHttpSolrClient.Req lbRequest = entry.getValue();
+ responseFutures.put(url, threadPool.submit(new Callable<NamedList<?>>() {
+ @Override
+ public NamedList<?> call() throws Exception {
+ return lbClient.request(lbRequest).getResponse();
+ }
+ }));
+ }
+
+ for (final Map.Entry<String, Future<NamedList<?>>> entry: responseFutures.entrySet()) {
+ final String url = entry.getKey();
+ final Future<NamedList<?>> responseFuture = entry.getValue();
+ try {
+ shardResponses.add(url, responseFuture.get());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ } catch (ExecutionException e) {
+ exceptions.add(url, e.getCause());
+ }
+ }
+
+ if (exceptions.size() > 0) {
+ throw new RouteException(ErrorCode.SERVER_ERROR, exceptions, routes);
+ }
+ } else {
+ for (Map.Entry<String, LBHttpSolrClient.Req> entry : routes.entrySet()) {
+ String url = entry.getKey();
+ LBHttpSolrClient.Req lbRequest = entry.getValue();
+ try {
+ NamedList<Object> rsp = lbClient.request(lbRequest).getResponse();
+ shardResponses.add(url, rsp);
+ } catch (Exception e) {
+ throw new SolrServerException(e);
+ }
+ }
+ }
+
+ UpdateRequest nonRoutableRequest = null;
+ List<String> deleteQuery = updateRequest.getDeleteQuery();
+ if (deleteQuery != null && deleteQuery.size() > 0) {
+ UpdateRequest deleteQueryRequest = new UpdateRequest();
+ deleteQueryRequest.setDeleteQuery(deleteQuery);
+ nonRoutableRequest = deleteQueryRequest;
+ }
+
+ Set<String> paramNames = nonRoutableParams.getParameterNames();
+
+ Set<String> intersection = new HashSet<>(paramNames);
+ intersection.retainAll(NON_ROUTABLE_PARAMS);
+
+ if (nonRoutableRequest != null || intersection.size() > 0) {
+ if (nonRoutableRequest == null) {
+ nonRoutableRequest = new UpdateRequest();
+ }
+ nonRoutableRequest.setParams(nonRoutableParams);
+ List<String> urlList = new ArrayList<>();
+ urlList.addAll(routes.keySet());
+ Collections.shuffle(urlList, rand);
+ LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(nonRoutableRequest, urlList);
+ try {
+ LBHttpSolrClient.Rsp rsp = lbClient.request(req);
+ shardResponses.add(urlList.get(0), rsp.getResponse());
+ } catch (Exception e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, urlList.get(0), e);
+ }
+ }
+
+ long end = System.nanoTime();
+
+ RouteResponse rr = condenseResponse(shardResponses, (long)((end - start)/1000000));
+ rr.setRouteResponses(shardResponses);
+ rr.setRoutes(routes);
+ return rr;
+ }
+
+ private Map<String,List<String>> buildUrlMap(DocCollection col) {
+ Map<String, List<String>> urlMap = new HashMap<>();
+ Collection<Slice> slices = col.getActiveSlices();
+ Iterator<Slice> sliceIterator = slices.iterator();
+ while (sliceIterator.hasNext()) {
+ Slice slice = sliceIterator.next();
+ String name = slice.getName();
+ List<String> urls = new ArrayList<>();
+ Replica leader = slice.getLeader();
+ if (leader == null) {
+ // take unoptimized general path - we cannot find a leader yet
+ return null;
+ }
+ ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
+ String url = zkProps.getCoreUrl();
+ urls.add(url);
+ Collection<Replica> replicas = slice.getReplicas();
+ Iterator<Replica> replicaIterator = replicas.iterator();
+ while (replicaIterator.hasNext()) {
+ Replica replica = replicaIterator.next();
+ if (!replica.getNodeName().equals(leader.getNodeName()) &&
+ !replica.getName().equals(leader.getName())) {
+ ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
+ String url1 = zkProps1.getCoreUrl();
+ urls.add(url1);
+ }
+ }
+ urlMap.put(name, urls);
+ }
+ return urlMap;
+ }
+
+ public RouteResponse condenseResponse(NamedList response, long timeMillis) {
+ RouteResponse condensed = new RouteResponse();
+ int status = 0;
+ Integer rf = null;
+ Integer minRf = null;
+ for(int i=0; i<response.size(); i++) {
+ NamedList shardResponse = (NamedList)response.getVal(i);
+ NamedList header = (NamedList)shardResponse.get("responseHeader");
+ Integer shardStatus = (Integer)header.get("status");
+ int s = shardStatus.intValue();
+ if(s > 0) {
+ status = s;
+ }
+ Object rfObj = header.get(UpdateRequest.REPFACT);
+ if (rfObj != null && rfObj instanceof Integer) {
+ Integer routeRf = (Integer)rfObj;
+ if (rf == null || routeRf < rf)
+ rf = routeRf;
+ }
+ minRf = (Integer)header.get(UpdateRequest.MIN_REPFACT);
+ }
+
+ NamedList cheader = new NamedList();
+ cheader.add("status", status);
+ cheader.add("QTime", timeMillis);
+ if (rf != null)
+ cheader.add(UpdateRequest.REPFACT, rf);
+ if (minRf != null)
+ cheader.add(UpdateRequest.MIN_REPFACT, minRf);
+
+ condensed.add("responseHeader", cheader);
+ return condensed;
+ }
+
+ public static class RouteResponse extends NamedList {
+ private NamedList routeResponses;
+ private Map<String, LBHttpSolrClient.Req> routes;
+
+ public void setRouteResponses(NamedList routeResponses) {
+ this.routeResponses = routeResponses;
+ }
+
+ public NamedList getRouteResponses() {
+ return routeResponses;
+ }
+
+ public void setRoutes(Map<String, LBHttpSolrClient.Req> routes) {
+ this.routes = routes;
+ }
+
+ public Map<String, LBHttpSolrClient.Req> getRoutes() {
+ return routes;
+ }
+
+ }
+
+ public static class RouteException extends SolrException {
+
+ private NamedList<Throwable> throwables;
+ private Map<String, LBHttpSolrClient.Req> routes;
+
+ public RouteException(ErrorCode errorCode, NamedList<Throwable> throwables, Map<String, LBHttpSolrClient.Req> routes){
+ super(errorCode, throwables.getVal(0).getMessage(), throwables.getVal(0));
+ this.throwables = throwables;
+ this.routes = routes;
+ }
+
+ public NamedList<Throwable> getThrowables() {
+ return throwables;
+ }
+
+ public Map<String, LBHttpSolrClient.Req> getRoutes() {
+ return this.routes;
+ }
+ }
+
+ @Override
+ public NamedList<Object> request(SolrRequest request) throws SolrServerException, IOException {
+ SolrParams reqParams = request.getParams();
+ String collection = (reqParams != null) ? reqParams.get("collection", getDefaultCollection()) : getDefaultCollection();
+ return requestWithRetryOnStaleState(request, 0, collection);
+ }
+
+ /**
+ * As this class doesn't watch external collections on the client side,
+ * there's a chance that the request will fail due to cached stale state,
+ * which means the state must be refreshed from ZK and retried.
+ */
+ protected NamedList<Object> requestWithRetryOnStaleState(SolrRequest request, int retryCount, String collection)
+ throws SolrServerException, IOException {
+
+ connect(); // important to call this before you start working with the ZkStateReader
+
+ // build up a _stateVer_ param to pass to the server containing all of the
+ // external collection state versions involved in this request, which allows
+ // the server to notify us that our cached state for one or more of the external
+ // collections is stale and needs to be refreshed ... this code has no impact on internal collections
+ String stateVerParam = null;
+ List<DocCollection> requestedCollections = null;
+ if (collection != null && !request.getPath().startsWith("/admin")) { // don't do _stateVer_ checking for admin requests
+ Set<String> requestedCollectionNames = getCollectionNames(getZkStateReader().getClusterState(), collection);
+
+ StringBuilder stateVerParamBuilder = null;
+ for (String requestedCollection : requestedCollectionNames) {
+ // track the version of state we're using on the client side using the _stateVer_ param
+ DocCollection coll = getDocCollection(getZkStateReader().getClusterState(), requestedCollection,null);
+ int collVer = coll.getZNodeVersion();
+ if (coll.getStateFormat()>1) {
+ if(requestedCollections == null) requestedCollections = new ArrayList<>(requestedCollectionNames.size());
+ requestedCollections.add(coll);
+
+ if (stateVerParamBuilder == null) {
+ stateVerParamBuilder = new StringBuilder();
+ } else {
+ stateVerParamBuilder.append("|"); // hopefully pipe is not an allowed char in a collection name
+ }
+
+ stateVerParamBuilder.append(coll.getName()).append(":").append(collVer);
+ }
+ }
+
+ if (stateVerParamBuilder != null) {
+ stateVerParam = stateVerParamBuilder.toString();
+ }
+ }
+
+ if (request.getParams() instanceof ModifiableSolrParams) {
+ ModifiableSolrParams params = (ModifiableSolrParams) request.getParams();
+ if (stateVerParam != null) {
+ params.set(STATE_VERSION, stateVerParam);
+ } else {
+ params.remove(STATE_VERSION);
+ }
+ } // else: ??? how to set this ???
+
+ NamedList<Object> resp = null;
+ try {
+ resp = sendRequest(request);
+ //to avoid an O(n) operation we always add STATE_VERSION to the last and try to read it from there
+ Object o = resp.get(STATE_VERSION, resp.size()-1);
+ if(o != null && o instanceof Map) {
+ //remove this because no one else needs this and tests would fail if they are comparing responses
+ resp.remove(resp.size()-1);
+ Map invalidStates = (Map) o;
+ for (Object invalidEntries : invalidStates.entrySet()) {
+ Map.Entry e = (Map.Entry) invalidEntries;
+ getDocCollection(getZkStateReader().getClusterState(),(String)e.getKey(), (Integer)e.getValue());
+ }
+
+ }
+ } catch (Exception exc) {
+
+ Throwable rootCause = SolrException.getRootCause(exc);
+ // don't do retry support for admin requests or if the request doesn't have a collection specified
+ if (collection == null || request.getPath().startsWith("/admin")) {
+ if (exc instanceof SolrServerException) {
+ throw (SolrServerException)exc;
+ } else if (exc instanceof IOException) {
+ throw (IOException)exc;
+ }else if (exc instanceof RuntimeException) {
+ throw (RuntimeException) exc;
+ }
+ else {
+ throw new SolrServerException(rootCause);
+ }
+ }
+
+ int errorCode = (rootCause instanceof SolrException) ?
+ ((SolrException)rootCause).code() : SolrException.ErrorCode.UNKNOWN.code;
+
+ log.error("Request to collection {} failed due to ("+errorCode+
+ ") {}, retry? "+retryCount, collection, rootCause.toString());
+
+ boolean wasCommError =
+ (rootCause instanceof ConnectException ||
+ rootCause instanceof ConnectTimeoutException ||
+ rootCause instanceof NoHttpResponseException ||
+ rootCause instanceof SocketException);
+
+ boolean stateWasStale = false;
+ if (retryCount < MAX_STALE_RETRIES &&
+ requestedCollections != null &&
+ !requestedCollections.isEmpty() &&
+ SolrException.ErrorCode.getErrorCode(errorCode) == SolrException.ErrorCode.INVALID_STATE)
+ {
+ // cached state for one or more external collections was stale
+ // re-issue request using updated state
+ stateWasStale = true;
+
+ // just re-read state for all of them, which is a little heavy handed but hopefully a rare occurrence
+ for (DocCollection ext : requestedCollections) {
+ collectionStateCache.remove(ext.getName());
+ }
+ }
+
+ // if we experienced a communication error, it's worth checking the state
+ // with ZK just to make sure the node we're trying to hit is still part of the collection
+ if (retryCount < MAX_STALE_RETRIES &&
+ !stateWasStale &&
+ requestedCollections != null &&
+ !requestedCollections.isEmpty() &&
+ wasCommError) {
+ for (DocCollection ext : requestedCollections) {
+ DocCollection latestStateFromZk = getDocCollection(zkStateReader.getClusterState(), ext.getName(),null);
+ if (latestStateFromZk.getZNodeVersion() != ext.getZNodeVersion()) {
+ // looks like we couldn't reach the server because the state was stale == retry
+ stateWasStale = true;
+ // we just pulled state from ZK, so update the cache so that the retry uses it
+ collectionStateCache.put(ext.getName(), new ExpiringCachedDocCollection(latestStateFromZk));
+ }
+ }
+ }
+
+ if (requestedCollections != null) {
+ requestedCollections.clear(); // done with this
+ }
+
+ // if the state was stale, then we retry the request once with new state pulled from Zk
+ if (stateWasStale) {
+ log.warn("Re-trying request to collection(s) "+collection+" after stale state error from server.");
+ resp = requestWithRetryOnStaleState(request, retryCount+1, collection);
+ } else {
+ if (exc instanceof SolrServerException) {
+ throw (SolrServerException)exc;
+ } else if (exc instanceof IOException) {
+ throw (IOException)exc;
+ } else {
+ throw new SolrServerException(rootCause);
+ }
+ }
+ }
+
+ return resp;
+ }
+
+ protected NamedList<Object> sendRequest(SolrRequest request)
+ throws SolrServerException, IOException {
+ connect();
+
+ ClusterState clusterState = zkStateReader.getClusterState();
+
+ boolean sendToLeaders = false;
+ List<String> replicas = null;
+
+ if (request instanceof IsUpdateRequest) {
+ if (request instanceof UpdateRequest) {
+ NamedList<Object> response = directUpdate((AbstractUpdateRequest) request,
+ clusterState);
+ if (response != null) {
+ return response;
+ }
+ }
+ sendToLeaders = true;
+ replicas = new ArrayList<>();
+ }
+
+ SolrParams reqParams = request.getParams();
+ if (reqParams == null) {
+ reqParams = new ModifiableSolrParams();
+ }
+ List<String> theUrlList = new ArrayList<>();
+ if (request.getPath().equals("/admin/collections")
+ || request.getPath().equals("/admin/cores")) {
+ Set<String> liveNodes = clusterState.getLiveNodes();
+ for (String liveNode : liveNodes) {
+ theUrlList.add(zkStateReader.getBaseUrlForNodeName(liveNode));
+ }
+ } else {
+ String collection = reqParams.get(UpdateParams.COLLECTION, defaultCollection);
+
+ if (collection == null) {
+ throw new SolrServerException(
+ "No collection param specified on request and no default collection has been set.");
+ }
+
+ Set<String> collectionNames = getCollectionNames(clusterState, collection);
+ if (collectionNames.size() == 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Could not find collection: " + collection);
+ }
+
+ String shardKeys = reqParams.get(ShardParams._ROUTE_);
+
+ // TODO: not a big deal because of the caching, but we could avoid looking
+ // at every shard
+ // when getting leaders if we tweaked some things
+
+ // Retrieve slices from the cloud state and, for each collection
+ // specified,
+ // add it to the Map of slices.
+ Map<String,Slice> slices = new HashMap<>();
+ for (String collectionName : collectionNames) {
+ DocCollection col = getDocCollection(clusterState, collectionName, null);
+ Collection<Slice> routeSlices = col.getRouter().getSearchSlices(shardKeys, reqParams , col);
+ ClientUtils.addSlices(slices, collectionName, routeSlices, true);
+ }
+ Set<String> liveNodes = clusterState.getLiveNodes();
+
+ List<String> leaderUrlList = null;
+ List<String> urlList = null;
+ List<String> replicasList = null;
+
+ // build a map of unique nodes
+ // TODO: allow filtering by group, role, etc
+ Map<String,ZkNodeProps> nodes = new HashMap<>();
+ List<String> urlList2 = new ArrayList<>();
+ for (Slice slice : slices.values()) {
+ for (ZkNodeProps nodeProps : slice.getReplicasMap().values()) {
+ ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
+ String node = coreNodeProps.getNodeName();
+ if (!liveNodes.contains(coreNodeProps.getNodeName())
+ || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue;
+ if (nodes.put(node, nodeProps) == null) {
+ if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) {
+ String url;
+ if (reqParams.get(UpdateParams.COLLECTION) == null) {
+ url = ZkCoreNodeProps.getCoreUrl(
+ nodeProps.getStr(ZkStateReader.BASE_URL_PROP),
+ defaultCollection);
+ } else {
+ url = coreNodeProps.getCoreUrl();
+ }
+ urlList2.add(url);
+ } else if (sendToLeaders) {
+ String url;
+ if (reqParams.get(UpdateParams.COLLECTION) == null) {
+ url = ZkCoreNodeProps.getCoreUrl(
+ nodeProps.getStr(ZkStateReader.BASE_URL_PROP),
+ defaultCollection);
+ } else {
+ url = coreNodeProps.getCoreUrl();
+ }
+ replicas.add(url);
+ }
+ }
+ }
+ }
+
+ if (sendToLeaders) {
+ leaderUrlList = urlList2;
+ replicasList = replicas;
+ } else {
+ urlList = urlList2;
+ }
+
+ if (sendToLeaders) {
+ theUrlList = new ArrayList<>(leaderUrlList.size());
+ theUrlList.addAll(leaderUrlList);
+ } else {
+ theUrlList = new ArrayList<>(urlList.size());
+ theUrlList.addAll(urlList);
+ }
+ if(theUrlList.isEmpty()) {
+ for (String s : collectionNames) {
+ if(s!=null) collectionStateCache.remove(s);
+ }
+ throw new SolrException(SolrException.ErrorCode.INVALID_STATE, "Not enough nodes to handle the request");
+ }
+
+ Collections.shuffle(theUrlList, rand);
+ if (sendToLeaders) {
+ ArrayList<String> theReplicas = new ArrayList<>(
+ replicasList.size());
+ theReplicas.addAll(replicasList);
+ Collections.shuffle(theReplicas, rand);
+ theUrlList.addAll(theReplicas);
+ }
+
+ }
+
+ LBHttpSolrClient.Req req = new LBHttpSolrClient.Req(request, theUrlList);
+ LBHttpSolrClient.Rsp rsp = lbClient.request(req);
+ return rsp.getResponse();
+ }
+
+ private Set<String> getCollectionNames(ClusterState clusterState,
+ String collection) {
+ // Extract each comma separated collection name and store in a List.
+ List<String> rawCollectionsList = StrUtils.splitSmart(collection, ",", true);
+ Set<String> collectionNames = new HashSet<>();
+ // validate collections
+ for (String collectionName : rawCollectionsList) {
+ if (!clusterState.getCollections().contains(collectionName)) {
+ Aliases aliases = zkStateReader.getAliases();
+ String alias = aliases.getCollectionAlias(collectionName);
+ if (alias != null) {
+ List<String> aliasList = StrUtils.splitSmart(alias, ",", true);
+ collectionNames.addAll(aliasList);
+ continue;
+ }
+
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection not found: " + collectionName);
+ }
+
+ collectionNames.add(collectionName);
+ }
+ return collectionNames;
+ }
+
+ @Override
+ public void close() throws IOException {
+ shutdown();
+ }
+
+ @Override
+ @Deprecated
+ public void shutdown() {
+ if (zkStateReader != null) {
+ synchronized(this) {
+ if (zkStateReader!= null)
+ zkStateReader.close();
+ zkStateReader = null;
+ }
+ }
+
+ if (shutdownLBHttpSolrServer) {
+ lbClient.shutdown();
+ }
+
+ if (clientIsInternal && myClient!=null) {
+ HttpClientUtil.close(myClient);
+ }
+
+ if(this.threadPool != null && !this.threadPool.isShutdown()) {
+ this.threadPool.shutdown();
+ }
+ }
+
+ public LBHttpSolrClient getLbClient() {
+ return lbClient;
+ }
+
+ public boolean isUpdatesToLeaders() {
+ return updatesToLeaders;
+ }
+
+ /**If caches are expired they are refreshed after acquiring a lock.
+ * use this to set the number of locks
+ */
+ public void setParallelCacheRefreshes(int n){ locks = objectList(n); }
+
+ private static ArrayList<Object> objectList(int n) {
+ ArrayList<Object> l = new ArrayList<>(n);
+ for(int i=0;i<n;i++) l.add(new Object());
+ return l;
+ }
+
+
+ protected DocCollection getDocCollection(ClusterState clusterState, String collection, Integer expectedVersion) throws SolrException {
+ if (collection == null) return null;
+ DocCollection col = getFromCache(collection);
+ if (col != null) {
+ if (expectedVersion == null) return col;
+ if (expectedVersion.intValue() == col.getZNodeVersion()) return col;
+ }
+
+ ClusterState.CollectionRef ref = clusterState.getCollectionRef(collection);
+ if (ref == null) {
+ //no such collection exists
+ return null;
+ }
+ if (!ref.isLazilyLoaded()) {
+ //it is readily available just return it
+ return ref.get();
+ }
+ List locks = this.locks;
+ final Object lock = locks.get(Math.abs(Hash.murmurhash3_x86_32(collection, 0, collection.length(), 0) % locks.size()));
+ synchronized (lock) {
+ //we have waited for sometime just check once again
+ col = getFromCache(collection);
+ if (col != null) {
+ if (expectedVersion == null) return col;
+ if (expectedVersion.intValue() == col.getZNodeVersion()) {
+ return col;
+ } else {
+ collectionStateCache.remove(collection);
+ }
+ }
+ col = ref.get();//this is a call to ZK
+ }
+ if (col == null) return null;
+ if (col.getStateFormat() > 1) collectionStateCache.put(collection, new ExpiringCachedDocCollection(col));
+ return col;
+ }
+
+ private DocCollection getFromCache(String c){
+ ExpiringCachedDocCollection cachedState = collectionStateCache.get(c);
+ return cachedState != null ? cachedState.cached : null;
+ }
+
+
+ /**
+ * Useful for determining the minimum achieved replication factor across
+ * all shards involved in processing an update request, typically useful
+ * for gauging the replication factor of a batch.
+ */
+ @SuppressWarnings("rawtypes")
+ public int getMinAchievedReplicationFactor(String collection, NamedList resp) {
+ // it's probably already on the top-level header set by condense
+ NamedList header = (NamedList)resp.get("responseHeader");
+ Integer achRf = (Integer)header.get(UpdateRequest.REPFACT);
+ if (achRf != null)
+ return achRf.intValue();
+
+ // not on the top-level header, walk the shard route tree
+ Map<String,Integer> shardRf = getShardReplicationFactor(collection, resp);
+ for (Integer rf : shardRf.values()) {
+ if (achRf == null || rf < achRf) {
+ achRf = rf;
+ }
+ }
+ return (achRf != null) ? achRf.intValue() : -1;
+ }
+
+ /**
+ * Walks the NamedList response after performing an update request looking for
+ * the replication factor that was achieved in each shard involved in the request.
+ * For single doc updates, there will be only one shard in the return value.
+ */
+ @SuppressWarnings("rawtypes")
+ public Map<String,Integer> getShardReplicationFactor(String collection, NamedList resp) {
+ connect();
+
+ Map<String,Integer> results = new HashMap<String,Integer>();
+ if (resp instanceof CloudSolrClient.RouteResponse) {
+ NamedList routes = ((CloudSolrClient.RouteResponse)resp).getRouteResponses();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Map<String,String> leaders = new HashMap<String,String>();
+ for (Slice slice : clusterState.getActiveSlices(collection)) {
+ Replica leader = slice.getLeader();
+ if (leader != null) {
+ ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
+ String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
+ leaders.put(leaderUrl, slice.getName());
+ String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
+ leaders.put(altLeaderUrl, slice.getName());
+ }
+ }
+
+ Iterator<Map.Entry<String,Object>> routeIter = routes.iterator();
+ while (routeIter.hasNext()) {
+ Map.Entry<String,Object> next = routeIter.next();
+ String host = next.getKey();
+ NamedList hostResp = (NamedList)next.getValue();
+ Integer rf = (Integer)((NamedList)hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
+ if (rf != null) {
+ String shard = leaders.get(host);
+ if (shard == null) {
+ if (host.endsWith("/"))
+ shard = leaders.get(host.substring(0,host.length()-1));
+ if (shard == null) {
+ shard = host;
+ }
+ }
+ results.put(shard, rf);
+ }
+ }
+ }
+ return results;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
new file mode 100644
index 0000000..4e2a2e7
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.http.client.HttpClient;
+
+import java.util.Collection;
+
+/**
+ * @deprecated Use {@link org.apache.solr.client.solrj.impl.CloudSolrClient}
+ */
+@Deprecated
+public class CloudSolrServer extends CloudSolrClient {
+
+ public CloudSolrServer(String zkHost) {
+ super(zkHost);
+ }
+
+ public CloudSolrServer(String zkHost, HttpClient httpClient) {
+ super(zkHost, httpClient);
+ }
+
+ public CloudSolrServer(Collection<String> zkHosts, String chroot) {
+ super(zkHosts, chroot);
+ }
+
+ public CloudSolrServer(Collection<String> zkHosts, String chroot, HttpClient httpClient) {
+ super(zkHosts, chroot, httpClient);
+ }
+
+ public CloudSolrServer(String zkHost, boolean updatesToLeaders) {
+ super(zkHost, updatesToLeaders);
+ }
+
+ public CloudSolrServer(String zkHost, boolean updatesToLeaders, HttpClient httpClient) {
+ super(zkHost, updatesToLeaders, httpClient);
+ }
+
+ public CloudSolrServer(String zkHost, LBHttpSolrClient lbClient) {
+ super(zkHost, lbClient);
+ }
+
+ public CloudSolrServer(String zkHost, LBHttpSolrClient lbClient, boolean updatesToLeaders) {
+ super(zkHost, lbClient, updatesToLeaders);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
new file mode 100644
index 0000000..5e65021
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrClient.java
@@ -0,0 +1,492 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ContentProducer;
+import org.apache.http.entity.EntityTemplate;
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.util.ClientUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrjNamedThreadFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.Locale;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * ConcurrentUpdateSolrClient buffers all added documents and writes
+ * them into open HTTP connections. This class is thread safe.
+ *
+ * Params from {@link UpdateRequest} are converted to http request
+ * parameters. When params change between UpdateRequests a new HTTP
+ * request is started.
+ *
+ * Although any SolrClient request can be made with this implementation, it is
+ * only recommended to use ConcurrentUpdateSolrClient with /update
+ * requests. The class {@link HttpSolrClient} is better suited for the
+ * query interface.
+ */
+public class ConcurrentUpdateSolrClient extends SolrClient {
+ private static final long serialVersionUID = 1L;
+ static final Logger log = LoggerFactory
+ .getLogger(ConcurrentUpdateSolrClient.class);
+ private HttpSolrClient client;
+ final BlockingQueue<UpdateRequest> queue;
+ final ExecutorService scheduler;
+ final Queue<Runner> runners;
+ volatile CountDownLatch lock = null; // used to block everything
+ final int threadCount;
+ boolean shutdownExecutor = false;
+ int pollQueueTime = 250;
+ private final boolean streamDeletes;
+
+ /**
+ * Uses an internally managed HttpClient instance.
+ *
+ * @param solrServerUrl
+ * The Solr server URL
+ * @param queueSize
+ * The buffer size before the documents are sent to the server
+ * @param threadCount
+ * The number of background threads used to empty the queue
+ */
+ public ConcurrentUpdateSolrClient(String solrServerUrl, int queueSize,
+ int threadCount) {
+ this(solrServerUrl, null, queueSize, threadCount);
+ shutdownExecutor = true;
+ }
+
+ public ConcurrentUpdateSolrClient(String solrServerUrl,
+ HttpClient client, int queueSize, int threadCount) {
+ this(solrServerUrl, client, queueSize, threadCount, Executors.newCachedThreadPool(
+ new SolrjNamedThreadFactory("concurrentUpdateScheduler")));
+ shutdownExecutor = true;
+ }
+
+ /**
+ * Uses the supplied HttpClient to send documents to the Solr server.
+ */
+ public ConcurrentUpdateSolrClient(String solrServerUrl,
+ HttpClient client, int queueSize, int threadCount, ExecutorService es) {
+ this(solrServerUrl, client, queueSize, threadCount, es, false);
+ }
+
+ /**
+ * Uses the supplied HttpClient to send documents to the Solr server.
+ */
+ public ConcurrentUpdateSolrClient(String solrServerUrl,
+ HttpClient client, int queueSize, int threadCount, ExecutorService es, boolean streamDeletes) {
+ this.client = new HttpSolrClient(solrServerUrl, client);
+ this.client.setFollowRedirects(false);
+ queue = new LinkedBlockingQueue<>(queueSize);
+ this.threadCount = threadCount;
+ runners = new LinkedList<>();
+ scheduler = es;
+ this.streamDeletes = streamDeletes;
+ }
+
+ public Set<String> getQueryParams() {
+ return this.client.getQueryParams();
+ }
+
+ /**
+ * Expert Method.
+ * @param queryParams set of param keys to only send via the query string
+ */
+ public void setQueryParams(Set<String> queryParams) {
+ this.client.setQueryParams(queryParams);
+ }
+
+ /**
+ * Opens a connection and sends everything...
+ */
+ class Runner implements Runnable {
+ final Lock runnerLock = new ReentrantLock();
+
+ @Override
+ public void run() {
+ runnerLock.lock();
+
+ log.debug("starting runner: {}", this);
+ HttpPost method = null;
+ HttpResponse response = null;
+ try {
+ while (!queue.isEmpty()) {
+ try {
+ final UpdateRequest updateRequest =
+ queue.poll(pollQueueTime, TimeUnit.MILLISECONDS);
+ if (updateRequest == null)
+ break;
+
+ String contentType = client.requestWriter.getUpdateContentType();
+ final boolean isXml = ClientUtils.TEXT_XML.equals(contentType);
+
+ final ModifiableSolrParams origParams = new ModifiableSolrParams(updateRequest.getParams());
+
+ EntityTemplate template = new EntityTemplate(new ContentProducer() {
+
+ @Override
+ public void writeTo(OutputStream out) throws IOException {
+ try {
+ if (isXml) {
+ out.write("<stream>".getBytes(StandardCharsets.UTF_8)); // can be anything
+ }
+ UpdateRequest req = updateRequest;
+ while (req != null) {
+ SolrParams currentParams = new ModifiableSolrParams(req.getParams());
+ if (!origParams.toNamedList().equals(currentParams.toNamedList())) {
+ queue.add(req); // params are different, push back to queue
+ break;
+ }
+
+ client.requestWriter.write(req, out);
+ if (isXml) {
+ // check for commit or optimize
+ SolrParams params = req.getParams();
+ if (params != null) {
+ String fmt = null;
+ if (params.getBool(UpdateParams.OPTIMIZE, false)) {
+ fmt = "<optimize waitSearcher=\"%s\" />";
+ } else if (params.getBool(UpdateParams.COMMIT, false)) {
+ fmt = "<commit waitSearcher=\"%s\" />";
+ }
+ if (fmt != null) {
+ byte[] content = String.format(Locale.ROOT,
+ fmt,
+ params.getBool(UpdateParams.WAIT_SEARCHER, false)
+ + "").getBytes(StandardCharsets.UTF_8);
+ out.write(content);
+ }
+ }
+ }
+ out.flush();
+ req = queue.poll(pollQueueTime, TimeUnit.MILLISECONDS);
+ }
+
+ if (isXml) {
+ out.write("</stream>".getBytes(StandardCharsets.UTF_8));
+ }
+
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.warn("", e);
+ }
+ }
+ });
+
+ // The parser 'wt=' and 'version=' params are used instead of the
+ // original params
+ ModifiableSolrParams requestParams = new ModifiableSolrParams(origParams);
+ requestParams.set(CommonParams.WT, client.parser.getWriterType());
+ requestParams.set(CommonParams.VERSION, client.parser.getVersion());
+
+ method = new HttpPost(client.getBaseURL() + "/update"
+ + ClientUtils.toQueryString(requestParams, false));
+ method.setEntity(template);
+ method.addHeader("User-Agent", HttpSolrClient.AGENT);
+ method.addHeader("Content-Type", contentType);
+
+ response = client.getHttpClient().execute(method);
+ int statusCode = response.getStatusLine().getStatusCode();
+ if (statusCode != HttpStatus.SC_OK) {
+ StringBuilder msg = new StringBuilder();
+ msg.append(response.getStatusLine().getReasonPhrase());
+ msg.append("\n\n\n\n");
+ msg.append("request: ").append(method.getURI());
+
+ SolrException solrExc = new SolrException(ErrorCode.getErrorCode(statusCode), msg.toString());
+ // parse out the metadata from the SolrException
+ try {
+ NamedList<Object> resp =
+ client.parser.processResponse(response.getEntity().getContent(),
+ response.getEntity().getContentType().getValue());
+ NamedList<Object> error = (NamedList<Object>) resp.get("error");
+ if (error != null)
+ solrExc.setMetadata((NamedList<String>) error.get("metadata"));
+ } catch (Exception exc) {
+ // don't want to fail to report error if parsing the response fails
+ log.warn("Failed to parse error response from "+ client.getBaseURL()+" due to: "+exc);
+ }
+
+ handleError(solrExc);
+ } else {
+ onSuccess(response);
+ }
+ } finally {
+ try {
+ if (response != null) {
+ response.getEntity().getContent().close();
+ }
+ } catch (Exception ex) {
+ log.warn("", ex);
+ }
+ }
+ }
+ } catch (Throwable e) {
+ if (e instanceof OutOfMemoryError) {
+ throw (OutOfMemoryError) e;
+ }
+ handleError(e);
+ } finally {
+ synchronized (runners) {
+ if (runners.size() == 1 && !queue.isEmpty()) {
+ // keep this runner alive
+ scheduler.execute(this);
+ } else {
+ runners.remove(this);
+ if (runners.isEmpty())
+ runners.notifyAll();
+ }
+ }
+
+ log.debug("finished: {}", this);
+ runnerLock.unlock();
+ }
+ }
+ }
+
+ @Override
+ public NamedList<Object> request(final SolrRequest request)
+ throws SolrServerException, IOException {
+ if (!(request instanceof UpdateRequest)) {
+ return client.request(request);
+ }
+ UpdateRequest req = (UpdateRequest) request;
+
+ // this happens for commit...
+ if (streamDeletes) {
+ if ((req.getDocuments() == null || req.getDocuments().isEmpty())
+ && (req.getDeleteById() == null || req.getDeleteById().isEmpty())
+ && (req.getDeleteByIdMap() == null || req.getDeleteByIdMap().isEmpty())) {
+ if (req.getDeleteQuery() == null) {
+ blockUntilFinished();
+ return client.request(request);
+ }
+ }
+ } else {
+ if ((req.getDocuments() == null || req.getDocuments().isEmpty())) {
+ blockUntilFinished();
+ return client.request(request);
+ }
+ }
+
+
+ SolrParams params = req.getParams();
+ if (params != null) {
+ // check if it is waiting for the searcher
+ if (params.getBool(UpdateParams.WAIT_SEARCHER, false)) {
+ log.info("blocking for commit/optimize");
+ blockUntilFinished(); // empty the queue
+ return client.request(request);
+ }
+ }
+
+ try {
+ CountDownLatch tmpLock = lock;
+ if (tmpLock != null) {
+ tmpLock.await();
+ }
+
+ boolean success = queue.offer(req);
+
+ for (;;) {
+ synchronized (runners) {
+ // see if queue is half full and we can add more runners
+ // special case: if only using a threadCount of 1 and the queue
+ // is filling up, allow 1 add'l runner to help process the queue
+ if (runners.isEmpty() || (queue.remainingCapacity() < queue.size() && runners.size() < threadCount))
+ {
+ // We need more runners, so start a new one.
+ Runner r = new Runner();
+ runners.add(r);
+ scheduler.execute(r);
+ } else {
+ // break out of the retry loop if we added the element to the queue
+ // successfully, *and*
+ // while we are still holding the runners lock to prevent race
+ // conditions.
+ if (success)
+ break;
+ }
+ }
+
+ // Retry to add to the queue w/o the runners lock held (else we risk
+ // temporary deadlock)
+ // This retry could also fail because
+ // 1) existing runners were not able to take off any new elements in the
+ // queue
+ // 2) the queue was filled back up since our last try
+ // If we succeed, the queue may have been completely emptied, and all
+ // runners stopped.
+ // In all cases, we should loop back to the top to see if we need to
+ // start more runners.
+ //
+ if (!success) {
+ success = queue.offer(req, 100, TimeUnit.MILLISECONDS);
+ }
+ }
+ } catch (InterruptedException e) {
+ log.error("interrupted", e);
+ throw new IOException(e.getLocalizedMessage());
+ }
+
+ // RETURN A DUMMY result
+ NamedList<Object> dummy = new NamedList<>();
+ dummy.add("NOTE", "the request is processed in a background stream");
+ return dummy;
+ }
+
+ public synchronized void blockUntilFinished() {
+ lock = new CountDownLatch(1);
+ try {
+ synchronized (runners) {
+ while (!runners.isEmpty()) {
+ try {
+ runners.wait();
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ }
+
+ if (scheduler.isTerminated())
+ break;
+
+ // if we reach here, then we probably got the notifyAll, but need to check if
+ // the queue is empty before really considering this is finished (SOLR-4260)
+ int queueSize = queue.size();
+ if (queueSize > 0) {
+ log.warn("No more runners, but queue still has "+
+ queueSize+" adding more runners to process remaining requests on queue");
+ Runner r = new Runner();
+ runners.add(r);
+ scheduler.execute(r);
+ }
+ }
+ }
+ } finally {
+ lock.countDown();
+ lock = null;
+ }
+ }
+
+ public void handleError(Throwable ex) {
+ log.error("error", ex);
+ }
+
+ /**
+ * Intended to be used as an extension point for doing post processing after a request completes.
+ */
+ public void onSuccess(HttpResponse resp) {
+ // no-op by design, override to add functionality
+ }
+
+ @Override
+ public void close() {
+ shutdown();
+ }
+
+ @Override
+ @Deprecated
+ public void shutdown() {
+ client.shutdown();
+ if (shutdownExecutor) {
+ scheduler.shutdown();
+ try {
+ if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) {
+ scheduler.shutdownNow();
+ if (!scheduler.awaitTermination(60, TimeUnit.SECONDS)) log
+ .error("ExecutorService did not terminate");
+ }
+ } catch (InterruptedException ie) {
+ scheduler.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ public void setConnectionTimeout(int timeout) {
+ HttpClientUtil.setConnectionTimeout(client.getHttpClient(), timeout);
+ }
+
+ /**
+ * set soTimeout (read timeout) on the underlying HttpConnectionManager. This is desirable for queries, but probably
+ * not for indexing.
+ */
+ public void setSoTimeout(int timeout) {
+ HttpClientUtil.setSoTimeout(client.getHttpClient(), timeout);
+ }
+
+ public void shutdownNow() {
+ client.shutdown();
+ if (shutdownExecutor) {
+ scheduler.shutdownNow(); // Cancel currently executing tasks
+ try {
+ if (!scheduler.awaitTermination(30, TimeUnit.SECONDS))
+ log.error("ExecutorService did not terminate");
+ } catch (InterruptedException ie) {
+ scheduler.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ public void setParser(ResponseParser responseParser) {
+ client.setParser(responseParser);
+ }
+
+
+ /**
+ * @param pollQueueTime time for an open connection to wait for updates when
+ * the queue is empty.
+ */
+ public void setPollQueueTime(int pollQueueTime) {
+ this.pollQueueTime = pollQueueTime;
+ }
+
+ public void setRequestWriter(RequestWriter requestWriter) {
+ client.setRequestWriter(requestWriter);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
new file mode 100644
index 0000000..9ace82a
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import org.apache.http.client.HttpClient;
+
+import java.util.concurrent.ExecutorService;
+
+/**
+ * @deprecated Use {@link org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient}
+ */
+@Deprecated
+public class ConcurrentUpdateSolrServer extends ConcurrentUpdateSolrClient {
+
+ public ConcurrentUpdateSolrServer(String solrServerUrl, int queueSize, int threadCount) {
+ super(solrServerUrl, queueSize, threadCount);
+ }
+
+ public ConcurrentUpdateSolrServer(String solrServerUrl, HttpClient client, int queueSize, int threadCount) {
+ super(solrServerUrl, client, queueSize, threadCount);
+ }
+
+ public ConcurrentUpdateSolrServer(String solrServerUrl, HttpClient client, int queueSize, int threadCount, ExecutorService es) {
+ super(solrServerUrl, client, queueSize, threadCount, es);
+ }
+
+ public ConcurrentUpdateSolrServer(String solrServerUrl, HttpClient client, int queueSize, int threadCount, ExecutorService es, boolean streamDeletes) {
+ super(solrServerUrl, client, queueSize, threadCount, es, streamDeletes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java
new file mode 100644
index 0000000..1d370ff
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/impl/HttpClientConfigurer.java
@@ -0,0 +1,97 @@
+package org.apache.solr.client.solrj.impl;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.http.conn.ssl.SSLSocketFactory;
+import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.solr.common.params.SolrParams;
+
+/**
+ * The default http client configurer. If the behaviour needs to be customized a
+ * new HttpCilentConfigurer can be set by calling
+ * {@link HttpClientUtil#setConfigurer(HttpClientConfigurer)}
+ */
+public class HttpClientConfigurer {
+
+ public void configure(DefaultHttpClient httpClient, SolrParams config) {
+
+ if (config.get(HttpClientUtil.PROP_MAX_CONNECTIONS) != null) {
+ HttpClientUtil.setMaxConnections(httpClient,
+ config.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS));
+ }
+
+ if (config.get(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST) != null) {
+ HttpClientUtil.setMaxConnectionsPerHost(httpClient,
+ config.getInt(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST));
+ }
+
+ if (config.get(HttpClientUtil.PROP_CONNECTION_TIMEOUT) != null) {
+ HttpClientUtil.setConnectionTimeout(httpClient,
+ config.getInt(HttpClientUtil.PROP_CONNECTION_TIMEOUT));
+ }
+
+ if (config.get(HttpClientUtil.PROP_SO_TIMEOUT) != null) {
+ HttpClientUtil.setSoTimeout(httpClient,
+ config.getInt(HttpClientUtil.PROP_SO_TIMEOUT));
+ }
+
+ if (config.get(HttpClientUtil.PROP_FOLLOW_REDIRECTS) != null) {
+ HttpClientUtil.setFollowRedirects(httpClient,
+ config.getBool(HttpClientUtil.PROP_FOLLOW_REDIRECTS));
+ }
+
+ // always call setUseRetry, whether it is in config or not
+ HttpClientUtil.setUseRetry(httpClient,
+ config.getBool(HttpClientUtil.PROP_USE_RETRY, true));
+
+ final String basicAuthUser = config
+ .get(HttpClientUtil.PROP_BASIC_AUTH_USER);
+ final String basicAuthPass = config
+ .get(HttpClientUtil.PROP_BASIC_AUTH_PASS);
+ HttpClientUtil.setBasicAuth(httpClient, basicAuthUser, basicAuthPass);
+
+ if (config.get(HttpClientUtil.PROP_ALLOW_COMPRESSION) != null) {
+ HttpClientUtil.setAllowCompression(httpClient,
+ config.getBool(HttpClientUtil.PROP_ALLOW_COMPRESSION));
+ }
+
+ boolean sslCheckPeerName = toBooleanDefaultIfNull(
+ toBooleanObject(System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME)), true);
+ if(sslCheckPeerName == false) {
+ HttpClientUtil.setHostNameVerifier(httpClient, SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ }
+ }
+
+ public static boolean toBooleanDefaultIfNull(Boolean bool, boolean valueIfNull) {
+ if (bool == null) {
+ return valueIfNull;
+ }
+ return bool.booleanValue() ? true : false;
+ }
+
+ public static Boolean toBooleanObject(String str) {
+ if ("true".equalsIgnoreCase(str)) {
+ return Boolean.TRUE;
+ } else if ("false".equalsIgnoreCase(str)) {
+ return Boolean.FALSE;
+ }
+ // no match
+ return null;
+ }
+}