You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2014/10/31 06:44:39 UTC
svn commit: r1635674 - in /jackrabbit/oak/branches/1.0: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
Author: chetanm
Date: Fri Oct 31 05:44:38 2014
New Revision: 1635674
URL: http://svn.apache.org/r1635674
Log:
OAK-2196 - Implement sorting based on Lucene sorting
Merging 1631969,1631986,1631990,1632017
Modified:
jackrabbit/oak/branches/1.0/ (props changed)
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java
jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Propchange: jackrabbit/oak/branches/1.0/
------------------------------------------------------------------------------
Merged /jackrabbit/oak/trunk:r1631969,1631986,1631990,1632017
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java Fri Oct 31 05:44:38 2014
@@ -48,4 +48,16 @@ public final class FieldNames {
public static final Set<String> PATH_SELECTOR = new HashSet<String>(
Arrays.asList(PATH));
+ /**
+ * Encodes the field name such that it can be used for storing DocValue
+ * This is done such a field if used for both sorting and querying uses
+ * a different name for docvalue field
+ *
+ * @param name name to encode
+ * @return encoded field name
+ */
+ public static String createDocValFieldName(String name){
+ return ":dv" + name;
+ }
+
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Fri Oct 31 05:44:38 2014
@@ -28,6 +28,7 @@ import javax.jcr.PropertyType;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.jackrabbit.oak.api.PropertyState;
@@ -42,6 +43,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FULL_TEXT_ENABLED;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES;
import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES;
public class IndexDefinition {
private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class);
@@ -51,6 +53,8 @@ public class IndexDefinition {
private final Set<String> includes;
+ private final Set<String> orderedProps;
+
private final boolean fullTextEnabled;
private final boolean storageEnabled;
@@ -78,12 +82,14 @@ public class IndexDefinition {
this.excludes = toLowerCase(getMultiProperty(defn, EXCLUDE_PROPERTY_NAMES));
this.includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES);
+ this.orderedProps = getMultiProperty(defn, ORDERED_PROP_NAMES);
+
this.fullTextEnabled = getOptionalValue(defn, FULL_TEXT_ENABLED, true);
//Storage is disabled for non full text indexes
this.storageEnabled = this.fullTextEnabled && getOptionalValue(defn, EXPERIMENTAL_STORAGE, true);
Map<String, PropertyDefinition> propDefns = Maps.newHashMap();
- for(String propName : includes){
+ for(String propName : Iterables.concat(includes, orderedProps)){
if(defn.hasChildNode(propName)){
propDefns.put(propName, new PropertyDefinition(this, propName, defn.child(propName)));
}
@@ -105,6 +111,10 @@ public class IndexDefinition {
return (propertyTypes & (1 << type)) != 0;
}
+ boolean isOrdered(String name) {
+ return orderedProps.contains(name);
+ }
+
public NodeBuilder getDefinition() {
return definition;
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java Fri Oct 31 05:44:38 2014
@@ -27,8 +27,6 @@ import javax.annotation.CheckForNull;
import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression;
import org.apache.jackrabbit.oak.spi.query.Filter;
import org.apache.lucene.index.IndexReader;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static com.google.common.collect.Lists.newArrayListWithCapacity;
import static org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction;
@@ -129,7 +127,7 @@ public class IndexPlanner {
//sorting can only be done for known/configured properties
// and whose types are known
//TODO Can sorting be done for array properties
- if (defn.includeProperty(o.getPropertyName())
+ if (defn.includeProperty(o.getPropertyName()) || defn.isOrdered(o.getPropertyName())
&& o.getPropertyType() != null
&& !o.getPropertyType().isArray()) {
orderEntries.add(o); //Lucene can manage any order desc/asc
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java Fri Oct 31 05:44:38 2014
@@ -68,8 +68,14 @@ public interface LuceneIndexConstants {
/**
* Type of the property being indexed defined as part of property definition
- * under the given index definition. Refer to {@ling javax.jcr.PropertyType}
+ * under the given index definition. Refer to {@link javax.jcr.PropertyType}
* contants for the possible values
*/
String PROP_TYPE = "propertyType";
+
+ /**
+ * Defines properties which would be used for ordering. If range queries are to
+ * be performed with same property then it must be part of include list also
+ */
+ String ORDERED_PROP_NAMES = "orderedProps";
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java Fri Oct 31 05:44:38 2014
@@ -41,12 +41,16 @@ import org.apache.jackrabbit.oak.spi.sta
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.PrefixQuery;
+import org.apache.lucene.util.BytesRef;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.sax.WriteOutContentHandler;
@@ -200,9 +204,16 @@ public class LuceneIndexEditor implement
boolean dirty = false;
for (PropertyState property : state.getProperties()) {
String pname = property.getName();
- if (isVisible(pname)
- && context.includeProperty(pname)) {
+ if (!isVisible(pname)) {
+ continue;
+ }
+
+ if (context.getDefinition().isOrdered(pname)) {
+ dirty = addTypedOrderedFields(fields, property);
+ }
+
+ if (context.includeProperty(pname)) {
//In case of fulltext we also check if given type is enabled for indexing
//TODO Use context.includePropertyType however that cause issue. Need
//to make filtering based on type consistent both on indexing side and
@@ -267,8 +278,6 @@ public class LuceneIndexEditor implement
for (int i = 0; i < property.count(); i++) {
Field f = null;
if (tag == Type.LONG.tag()) {
- //TODO Distinguish fields which need to be used for search and for sort
- //If a field is only used for Sort then it can be stored with less precision
f = new LongField(name, property.getValue(Type.LONG, i), Field.Store.NO);
} else if (tag == Type.DATE.tag()) {
String date = property.getValue(Type.DATE, i);
@@ -288,6 +297,38 @@ public class LuceneIndexEditor implement
return fieldAdded;
}
+ private boolean addTypedOrderedFields(List<Field> fields, PropertyState property) throws CommitFailedException {
+ int tag = property.getType().tag();
+ String name = FieldNames.createDocValFieldName(property.getName());
+ boolean fieldAdded = false;
+ for (int i = 0; i < property.count(); i++) {
+ Field f = null;
+ if (tag == Type.LONG.tag()) {
+ //TODO Distinguish fields which need to be used for search and for sort
+ //If a field is only used for Sort then it can be stored with less precision
+ f = new NumericDocValuesField(name, property.getValue(Type.LONG, i));
+ } else if (tag == Type.DATE.tag()) {
+ String date = property.getValue(Type.DATE, i);
+ f = new NumericDocValuesField(name, FieldFactory.dateToLong(date));
+ } else if (tag == Type.DOUBLE.tag()) {
+ f = new DoubleDocValuesField(name, property.getValue(Type.DOUBLE, i));
+ } else if (tag == Type.BOOLEAN.tag()) {
+ f = new SortedDocValuesField(name,
+ new BytesRef(property.getValue(Type.BOOLEAN, i).toString()));
+ } else if (tag == Type.STRING.tag()) {
+ f = new SortedDocValuesField(name,
+ new BytesRef(property.getValue(Type.STRING, i)));
+ }
+
+ if (f != null) {
+ this.context.indexUpdate();
+ fields.add(f);
+ fieldAdded = true;
+ }
+ }
+ return fieldAdded;
+ }
+
private static boolean isVisible(String name) {
return name.charAt(0) != ':';
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java Fri Oct 31 05:44:38 2014
@@ -174,4 +174,8 @@ public class LuceneIndexEditorContext {
public int getPropertyTypes() {
return definition.getPropertyTypes();
}
+
+ public IndexDefinition getDefinition() {
+ return definition;
+ }
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Fri Oct 31 05:44:38 2014
@@ -79,6 +79,8 @@ import org.apache.lucene.search.PhraseQu
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
@@ -264,6 +266,7 @@ public class LucenePropertyIndex impleme
@Override
public Cursor query(final IndexPlan plan, NodeState rootState) {
final Filter filter = plan.getFilter();
+ final Sort sort = getSort(plan.getSortOrder(), plan);
FullTextExpression ft = filter.getFullTextConstraint();
Set<String> relPaths = getRelativePaths(ft);
if (relPaths.size() > 1) {
@@ -329,7 +332,7 @@ public class LucenePropertyIndex impleme
private boolean loadDocs() {
ScoreDoc lastDocToRecord = null;
- IndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
+ IndexNode indexNode = acquireIndexNode(plan);
checkState(indexNode != null);
try {
IndexSearcher searcher = indexNode.getSearcher();
@@ -339,10 +342,18 @@ public class LucenePropertyIndex impleme
long time = System.currentTimeMillis();
if (lastDoc != null) {
LOG.debug("loading the next {} entries for query {}", nextBatchSize, query);
- docs = searcher.searchAfter(lastDoc, query, nextBatchSize);
+ if (sort == null) {
+ docs = searcher.searchAfter(lastDoc, query, nextBatchSize);
+ } else {
+ docs = searcher.searchAfter(lastDoc, query, LUCENE_QUERY_BATCH_SIZE, sort);
+ }
} else {
LOG.debug("loading the first {} entries for query {}", nextBatchSize, query);
- docs = searcher.search(query, nextBatchSize);
+ if (sort == null) {
+ docs = searcher.search(query, nextBatchSize);
+ } else {
+ docs = searcher.search(query, LUCENE_QUERY_BATCH_SIZE, sort);
+ }
}
time = System.currentTimeMillis() - time;
LOG.debug("... took {} ms", time);
@@ -371,6 +382,51 @@ public class LucenePropertyIndex impleme
return new LucenePathCursor(itr, settings);
}
+ private IndexNode acquireIndexNode(IndexPlan plan) {
+ return tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
+ }
+
+ private Sort getSort(List<OrderEntry> sortOrder, IndexPlan plan) {
+ if (sortOrder == null || sortOrder.isEmpty()) {
+ return null;
+ }
+ IndexNode indexNode = acquireIndexNode(plan);
+ try {
+ IndexDefinition defn = indexNode.getDefinition();
+ SortField[] fields = new SortField[sortOrder.size()];
+ for (int i = 0; i < sortOrder.size(); i++) {
+ OrderEntry oe = sortOrder.get(i);
+ boolean reverse = oe.getOrder() != OrderEntry.Order.ASCENDING;
+ String propName = oe.getPropertyName();
+ if (defn.isOrdered(propName)){
+ propName = FieldNames.createDocValFieldName(propName);
+ }
+ fields[i] = new SortField(propName, toLuceneSortType(oe, defn), reverse);
+ }
+ return new Sort(fields);
+ } finally {
+ indexNode.release();
+ }
+ }
+
+ private static SortField.Type toLuceneSortType(OrderEntry oe, IndexDefinition defn) {
+ Type<?> t = oe.getPropertyType();
+ checkState(t != null, "Type cannot be null");
+ checkState(!t.isArray(), "Array types are not supported");
+
+ int type = getPropertyType(defn, oe.getPropertyName(), t.tag());
+ switch (type) {
+ case PropertyType.LONG:
+ case PropertyType.DATE:
+ return SortField.Type.LONG;
+ case PropertyType.DOUBLE:
+ return SortField.Type.DOUBLE;
+ default:
+ //TODO Check about SortField.Type.STRING_VAL
+ return SortField.Type.STRING;
+ }
+ }
+
private static String getIndexName(IndexPlan plan){
return PathUtils.getName((String) plan.getAttribute(ATTR_INDEX_PATH));
}
@@ -627,13 +683,17 @@ public class LucenePropertyIndex impleme
}
}
+ private static int getPropertyType(IndexDefinition defn, String name, int defaultVal){
+ if (defn.hasPropertyDefinition(name)) {
+ return defn.getPropDefn(name).getPropertyType();
+ }
+ return defaultVal;
+ }
+
@CheckForNull
private static Query createQuery(PropertyRestriction pr,
IndexDefinition defn) {
- int propType = pr.propertyType;
- if (defn.hasPropertyDefinition(pr.propertyName)) {
- propType = defn.getPropDefn(pr.propertyName).getPropertyType();
- }
+ int propType = getPropertyType(defn, pr.propertyName, pr.propertyType);
switch (propType) {
case PropertyType.DATE: {
Long first = pr.first != null ? FieldFactory.dateToLong(pr.first.getValue(Type.DATE)) : null;
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java Fri Oct 31 05:44:38 2014
@@ -104,7 +104,7 @@ public class LuceneIndexEditorTest {
builder.child("test").setProperty("weight", 10.0);
builder.child("test").setProperty("bool", true);
builder.child("test").setProperty("truth", true);
- builder.child("test").setProperty("creationTime", createDate("05/06/2014"));
+ builder.child("test").setProperty("creationTime", createCal("05/06/2014"));
NodeState after = builder.getNodeState();
NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY);
@@ -179,7 +179,7 @@ public class LuceneIndexEditorTest {
return indexNode.getSearcher();
}
- static Calendar createDate(String dt) throws java.text.ParseException {
+ static Calendar createCal(String dt) throws java.text.ParseException {
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy");
Calendar cal = Calendar.getInstance();
cal.setTime(sdf.parse(dt));
@@ -187,6 +187,6 @@ public class LuceneIndexEditorTest {
}
static long dateToTime(String dt) throws java.text.ParseException {
- return FieldFactory.dateToLong(ISO8601.format(createDate(dt)));
+ return FieldFactory.dateToLong(ISO8601.format(createCal(dt)));
}
}
Modified: jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1635674&r1=1635673&r2=1635674&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/branches/1.0/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Fri Oct 31 05:44:38 2014
@@ -20,10 +20,16 @@
package org.apache.jackrabbit.oak.plugins.index.lucene;
import java.text.ParseException;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
import java.util.Set;
import javax.jcr.PropertyType;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.CommitFailedException;
@@ -43,15 +49,25 @@ import org.junit.Test;
import static com.google.common.collect.ImmutableSet.of;
import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.api.Type.STRINGS;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createDate;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createCal;
+import static org.apache.jackrabbit.oak.plugins.index.property.OrderedIndex.OrderDirection;
+import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.matchers.JUnitMatchers.containsString;
public class LucenePropertyIndexTest extends AbstractQueryTest {
+ /**
+ * Set the size to twice the batch size to test the pagination with sorting
+ */
+ static final int NUMBER_OF_NODES = LucenePropertyIndex.LUCENE_QUERY_BATCH_SIZE * 2;
@Override
protected void createTestIndexNode() throws Exception {
@@ -176,9 +192,9 @@ public class LucenePropertyIndexTest ext
root.commit();
Tree test = root.getTree("/").addChild("test");
- test.addChild("a").setProperty("propa", createDate("14/02/2014"));
- test.addChild("b").setProperty("propa", createDate("14/03/2014"));
- test.addChild("c").setProperty("propa", createDate("14/04/2014"));
+ test.addChild("a").setProperty("propa", createCal("14/02/2014"));
+ test.addChild("b").setProperty("propa", createCal("14/03/2014"));
+ test.addChild("c").setProperty("propa", createCal("14/04/2014"));
test.addChild("c").setProperty("propb", "foo");
test.addChild("d").setProperty("propb", "foo");
root.commit();
@@ -206,6 +222,187 @@ public class LucenePropertyIndexTest ext
assertQuery("select [jcr:path] from [nt:base] where propa like '%ty'", asList("/test/a", "/test/b"));
}
+ @Test
+ public void sortQueriesWithLong() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
+ root.commit();
+
+ assertSortedtLong();
+ }
+
+ @Test
+ public void sortQueriesWithLong_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
+ root.commit();
+
+ assertSortedtLong();
+ }
+
+ void assertSortedtLong() throws CommitFailedException {
+ Tree test = root.getTree("/").addChild("test");
+ List<Long> values = createLongs(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithDouble() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DOUBLE);
+ root.commit();
+
+ assertSortedDouble();
+ }
+
+ @Test
+ public void sortQueriesWithDouble_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DOUBLE);
+ root.commit();
+
+ assertSortedDouble();
+ }
+
+ void assertSortedDouble() throws CommitFailedException {
+ Tree test = root.getTree("/").addChild("test");
+ List<Double> values = createDoubles(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithString() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.addChild("foo");
+ root.commit();
+
+ assertSortedString();
+ }
+
+ @Test
+ public void sortQueriesWithString_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ idx.addChild("foo");
+ root.commit();
+
+ assertSortedString();
+ }
+
+ void assertSortedString() throws CommitFailedException {
+ Tree test = root.getTree("/").addChild("test");
+ List<String> values = createStrings(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithDate() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DATE);
+ root.commit();
+
+ assertSortedDate();
+ }
+
+ @Test
+ public void sortQueriesWithDate_OrderedProps() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("bar"), STRINGS));
+ idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("foo"), STRINGS));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DATE);
+ root.commit();
+
+ assertSortedDate();
+ }
+
+ void assertSortedDate() throws ParseException, CommitFailedException {
+ Tree test = root.getTree("/").addChild("test");
+ List<Calendar> values = createDates(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithStringAndLong() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar", "baz"));
+ Tree propIdx = idx.addChild("baz");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
+ root.commit();
+
+ Tree test = root.getTree("/").addChild("test");
+ int firstPropSize = 5;
+ List<String> values = createStrings(firstPropSize);
+ List<Long> longValues = createLongs(NUMBER_OF_NODES);
+ List<Tuple2> tuples = Lists.newArrayListWithCapacity(values.size());
+ Random r = new Random();
+ for(int i = 0; i < values.size(); i++){
+ String val = values.get(r.nextInt(firstPropSize));
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", val);
+ child.setProperty("baz", longValues.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple2(val, longValues.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] asc, [baz] desc", getSortedPaths(tuples));
+ }
+
+ private void assertOrderedQuery(String sql, List<String> paths) {
+ List<String> result = executeQuery(sql, SQL2, true);
+ assertEquals(paths, result);
+ }
+
//TODO Test for range with Date. Check for precision
private String explain(String query){
@@ -226,8 +423,119 @@ public class LucenePropertyIndexTest ext
return root.getTree("/").getChild(INDEX_DEFINITIONS_NAME).getChild(name);
}
-
private static String dt(String date) throws ParseException {
- return String.format("CAST ('%s' AS DATE)",ISO8601.format(createDate(date)));
+ return String.format("CAST ('%s' AS DATE)",ISO8601.format(createCal(date)));
+ }
+
+ private static List<String> getSortedPaths(List<Tuple> tuples, OrderDirection dir) {
+ if (OrderDirection.DESC == dir) {
+ Collections.sort(tuples, Collections.reverseOrder());
+ } else {
+ Collections.sort(tuples);
+ }
+ List<String> paths = Lists.newArrayListWithCapacity(tuples.size());
+ for (Tuple t : tuples) {
+ paths.add(t.path);
+ }
+ return paths;
+ }
+
+ private static List<String> getSortedPaths(List<Tuple2> tuples) {
+ Collections.sort(tuples);
+ List<String> paths = Lists.newArrayListWithCapacity(tuples.size());
+ for (Tuple2 t : tuples) {
+ paths.add(t.path);
+ }
+ return paths;
+ }
+
+ private static List<Long> createLongs(int n){
+ List<Long> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(i);
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static List<Double> createDoubles(int n){
+ Random rnd = new Random();
+ List<Double> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(rnd.nextDouble());
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static List<String> createStrings(int n){
+ List<String> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(String.format("value%04d",i));
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static List<Calendar> createDates(int n) throws ParseException {
+ Random rnd = new Random();
+ List<Calendar> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(createCal(String.format("%02d/%02d/2%03d", rnd.nextInt(26) + 1, rnd.nextInt(10) + 1,i)));
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static class Tuple implements Comparable<Tuple>{
+ final Comparable value;
+ final String path;
+
+ private Tuple(Comparable value, String path) {
+ this.value = value;
+ this.path = path;
+ }
+
+ @Override
+ public int compareTo(Tuple o) {
+ return value.compareTo(o.value);
+ }
+
+ @Override
+ public String toString() {
+ return "Tuple{" +
+ "value=" + value +
+ ", path='" + path + '\'' +
+ '}';
+ }
+ }
+
+ private static class Tuple2 implements Comparable<Tuple2>{
+ final Comparable value;
+ final Comparable value2;
+ final String path;
+
+ private Tuple2(Comparable value, Comparable value2, String path) {
+ this.value = value;
+ this.value2 = value2;
+ this.path = path;
+ }
+
+ @Override
+ public int compareTo(Tuple2 o) {
+ return ComparisonChain.start()
+ .compare(value, o.value)
+ .compare(value2, o.value2, Collections.reverseOrder())
+ .result();
+ }
+
+ @Override
+ public String toString() {
+ return "Tuple2{" +
+ "value=" + value +
+ ", value2=" + value2 +
+ ", path='" + path + '\'' +
+ '}';
+ }
}
}