You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2014/10/15 15:33:14 UTC
svn commit: r1632017 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
Author: chetanm
Date: Wed Oct 15 13:33:14 2014
New Revision: 1632017
URL: http://svn.apache.org/r1632017
Log:
OAK-2196 - Implement sorting based on Lucene sorting
Make use of DocValues for sorting based on Amit's approach. orderedProps would be be stored in a separate field with DocVal type set.
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java Wed Oct 15 13:33:14 2014
@@ -48,4 +48,16 @@ public final class FieldNames {
public static final Set<String> PATH_SELECTOR = new HashSet<String>(
Arrays.asList(PATH));
+ /**
+ * Encodes the field name such that it can be used for storing DocValue
+ * This is done such a field if used for both sorting and querying uses
+ * a different name for docvalue field
+ *
+ * @param name name to encode
+ * @return encoded field name
+ */
+ public static String createDocValFieldName(String name){
+ return ":dv" + name;
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Wed Oct 15 13:33:14 2014
@@ -28,6 +28,7 @@ import javax.jcr.PropertyType;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.jackrabbit.oak.api.PropertyState;
@@ -42,6 +43,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FULL_TEXT_ENABLED;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES;
public class IndexDefinition {
private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class);
@@ -51,6 +53,8 @@ public class IndexDefinition {
private final Set<String> includes;
+ private final Set<String> orderedProps;
+
private final boolean fullTextEnabled;
private final boolean storageEnabled;
@@ -78,12 +82,14 @@ public class IndexDefinition {
this.excludes = toLowerCase(getMultiProperty(defn, EXCLUDE_PROPERTY_NAMES));
this.includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES);
+ this.orderedProps = getMultiProperty(defn, ORDERED_PROP_NAMES);
+
this.fullTextEnabled = getOptionalValue(defn, FULL_TEXT_ENABLED, true);
//Storage is disabled for non full text indexes
this.storageEnabled = this.fullTextEnabled && getOptionalValue(defn, EXPERIMENTAL_STORAGE, true);
Map<String, PropertyDefinition> propDefns = Maps.newHashMap();
- for(String propName : includes){
+ for(String propName : Iterables.concat(includes, orderedProps)){
if(defn.hasChildNode(propName)){
propDefns.put(propName, new PropertyDefinition(this, propName, defn.child(propName)));
}
@@ -105,6 +111,10 @@ public class IndexDefinition {
return (propertyTypes & (1 << type)) != 0;
}
+ boolean isOrdered(String name) {
+ return orderedProps.contains(name);
+ }
+
public NodeBuilder getDefinition() {
return definition;
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java Wed Oct 15 13:33:14 2014
@@ -27,8 +27,6 @@ import javax.annotation.CheckForNull;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.lucene.index.IndexReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction;
@@ -129,7 +127,7 @@ public class IndexPlanner {
//sorting can only be done for known/configured properties
// and whose types are known
//TODO Can sorting be done for array properties
- if (defn.includeProperty(o.getPropertyName())
+ if (defn.includeProperty(o.getPropertyName()) || defn.isOrdered(o.getPropertyName())
&& o.getPropertyType() != null
&& !o.getPropertyType().isArray()) {
orderEntries.add(o); //Lucene can manage any order desc/asc
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Wed Oct 15 13:33:14 2014
@@ -68,8 +68,14 @@ public interface LuceneIndexConstants {
/**
* Type of the property being indexed defined as part of property definition
- * under the given index definition. Refer to {@ling javax.jcr.PropertyType}
+ * under the given index definition. Refer to {@link javax.jcr.PropertyType}
* contants for the possible values
*/
String PROP_TYPE = "propertyType";
+
+ /**
+ * Defines properties which would be used for ordering. If range queries are to
+ * be performed with same property then it must be part of include list also
+ */
+ String ORDERED_PROP_NAMES = "orderedProps";
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Wed Oct 15 13:33:14 2014
@@ -41,12 +41,16 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.WriteOutContentHandler;
@@ -200,9 +204,16 @@ public class LuceneIndexEditor implement
boolean dirty = false;
for (PropertyState property : state.getProperties()) {
String pname = property.getName();
- if (isVisible(pname)
- && context.includeProperty(pname)) {
+ if (!isVisible(pname)) {
+ continue;
+ }
+
+ if (context.getDefinition().isOrdered(pname)) {
+ dirty = addTypedOrderedFields(fields, property);
+ }
+
+ if (context.includeProperty(pname)) {
//In case of fulltext we also check if given type is enabled for indexing
//TODO Use context.includePropertyType however that cause issue. Need
//to make filtering based on type consistent both on indexing side and
@@ -267,8 +278,6 @@ public class LuceneIndexEditor implement
for (int i = 0; i < property.count(); i++) {
Field f = null;
if (tag == Type.LONG.tag()) {
- //TODO Distinguish fields which need to be used for search and for sort
- //If a field is only used for Sort then it can be stored with less precision
f = new LongField(name, property.getValue(Type.LONG, i), Field.Store.NO);
} else if (tag == Type.DATE.tag()) {
String date = property.getValue(Type.DATE, i);
@@ -288,6 +297,38 @@ public class LuceneIndexEditor implement
return fieldAdded;
}
+ private boolean addTypedOrderedFields(List<Field> fields, PropertyState property) throws CommitFailedException {
+ int tag = property.getType().tag();
+ String name = FieldNames.createDocValFieldName(property.getName());
+ boolean fieldAdded = false;
+ for (int i = 0; i < property.count(); i++) {
+ Field f = null;
+ if (tag == Type.LONG.tag()) {
+ //TODO Distinguish fields which need to be used for search and for sort
+ //If a field is only used for Sort then it can be stored with less precision
+ f = new NumericDocValuesField(name, property.getValue(Type.LONG, i));
+ } else if (tag == Type.DATE.tag()) {
+ String date = property.getValue(Type.DATE, i);
+ f = new NumericDocValuesField(name, FieldFactory.dateToLong(date));
+ } else if (tag == Type.DOUBLE.tag()) {
+ f = new DoubleDocValuesField(name, property.getValue(Type.DOUBLE, i));
+ } else if (tag == Type.BOOLEAN.tag()) {
+ f = new SortedDocValuesField(name,
+ new BytesRef(property.getValue(Type.BOOLEAN, i).toString()));
+ } else if (tag == Type.STRING.tag()) {
+ f = new SortedDocValuesField(name,
+ new BytesRef(property.getValue(Type.STRING, i)));
+ }
+
+ if (f != null) {
+ this.context.indexUpdate();
+ fields.add(f);
+ fieldAdded = true;
+ }
+ }
+ return fieldAdded;
+ }
+
private static boolean isVisible(String name) {
return name.charAt(0) != ':';
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java Wed Oct 15 13:33:14 2014
@@ -174,4 +174,8 @@ public class LuceneIndexEditorContext {
public int getPropertyTypes() {
return definition.getPropertyTypes();
}
+
+ public IndexDefinition getDefinition() {
+ return definition;
+ }
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Wed Oct 15 13:33:14 2014
@@ -392,11 +392,16 @@ public class LucenePropertyIndex impleme
}
IndexNode indexNode = acquireIndexNode(plan);
try {
+ IndexDefinition defn = indexNode.getDefinition();
SortField[] fields = new SortField[sortOrder.size()];
for (int i = 0; i < sortOrder.size(); i++) {
OrderEntry oe = sortOrder.get(i);
boolean reverse = oe.getOrder() != OrderEntry.Order.ASCENDING;
- fields[i] = new SortField(oe.getPropertyName(), toLuceneSortType(oe, indexNode.getDefinition()), reverse);
+ String propName = oe.getPropertyName();
+ if (defn.isOrdered(propName)){
+ propName = FieldNames.createDocValFieldName(propName);
+ }
+ fields[i] = new SortField(propName, toLuceneSortType(oe, defn), reverse);
}
return new Sort(fields);
} finally {
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1632017&r1=1632016&r2=1632017&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Wed Oct 15 13:33:14 2014
@@ -49,12 +49,16 @@ import org.junit.Test;
import static com.google.common.collect.ImmutableSet.of;
import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.api.Type.STRINGS;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createCal;
import static org.apache.jackrabbit.oak.plugins.index.property.OrderedIndex.OrderDirection;
+import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.matchers.JUnitMatchers.containsString;
@@ -225,6 +229,22 @@ public class LucenePropertyIndexTest ext
propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
root.commit();
+ assertSortedtLong();
+ }
+
+ @Test
+ public void sortQueriesWithLong_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
+ root.commit();
+
+ assertSortedtLong();
+ }
+
+ void assertSortedtLong() throws CommitFailedException {
Tree test = root.getTree("/").addChild("test");
List<Long> values = createLongs(NUMBER_OF_NODES);
List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
@@ -247,6 +267,22 @@ public class LucenePropertyIndexTest ext
propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DOUBLE);
root.commit();
+ assertSortedDouble();
+ }
+
+ @Test
+ public void sortQueriesWithDouble_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DOUBLE);
+ root.commit();
+
+ assertSortedDouble();
+ }
+
+ void assertSortedDouble() throws CommitFailedException {
Tree test = root.getTree("/").addChild("test");
List<Double> values = createDoubles(NUMBER_OF_NODES);
List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
@@ -268,6 +304,21 @@ public class LucenePropertyIndexTest ext
idx.addChild("foo");
root.commit();
+ assertSortedString();
+ }
+
+ @Test
+ public void sortQueriesWithString_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ idx.addChild("foo");
+ root.commit();
+
+ assertSortedString();
+ }
+
+ void assertSortedString() throws CommitFailedException {
Tree test = root.getTree("/").addChild("test");
List<String> values = createStrings(NUMBER_OF_NODES);
List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
@@ -290,6 +341,22 @@ public class LucenePropertyIndexTest ext
propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DATE);
root.commit();
+ assertSortedDate();
+ }
+
+ @Test
+ public void sortQueriesWithDate_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DATE);
+ root.commit();
+
+ assertSortedDate();
+ }
+
+ void assertSortedDate() throws ParseException, CommitFailedException {
Tree test = root.getTree("/").addChild("test");
List<Calendar> values = createDates(NUMBER_OF_NODES);
List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());