You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2014/10/15 10:09:34 UTC
svn commit: r1631969 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/
Author: chetanm
Date: Wed Oct 15 08:09:33 2014
New Revision: 1631969
URL: http://svn.apache.org/r1631969
Log:
OAK-2196 - Implement sorting based on Lucene sorting
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1631969&r1=1631968&r2=1631969&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Wed Oct 15 08:09:33 2014
@@ -79,6 +79,8 @@ import org.apache.lucene.search.PhraseQu
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.TopDocs;
@@ -264,6 +266,7 @@ public class LucenePropertyIndex impleme
@Override
public Cursor query(final IndexPlan plan, NodeState rootState) {
final Filter filter = plan.getFilter();
+ final Sort sort = getSort(plan.getSortOrder(), plan);
FullTextExpression ft = filter.getFullTextConstraint();
Set<String> relPaths = getRelativePaths(ft);
if (relPaths.size() > 1) {
@@ -329,7 +332,7 @@ public class LucenePropertyIndex impleme
private boolean loadDocs() {
ScoreDoc lastDocToRecord = null;
- IndexNode indexNode = tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
+ IndexNode indexNode = acquireIndexNode(plan);
checkState(indexNode != null);
try {
IndexSearcher searcher = indexNode.getSearcher();
@@ -339,10 +342,18 @@ public class LucenePropertyIndex impleme
long time = System.currentTimeMillis();
if (lastDoc != null) {
LOG.debug("loading the next {} entries for query {}", nextBatchSize, query);
- docs = searcher.searchAfter(lastDoc, query, nextBatchSize);
+ if (sort == null) {
+ docs = searcher.searchAfter(lastDoc, query, nextBatchSize);
+ } else {
+ docs = searcher.searchAfter(lastDoc, query, LUCENE_QUERY_BATCH_SIZE, sort);
+ }
} else {
LOG.debug("loading the first {} entries for query {}", nextBatchSize, query);
- docs = searcher.search(query, nextBatchSize);
+ if (sort == null) {
+ docs = searcher.search(query, nextBatchSize);
+ } else {
+ docs = searcher.search(query, LUCENE_QUERY_BATCH_SIZE, sort);
+ }
}
time = System.currentTimeMillis() - time;
LOG.debug("... took {} ms", time);
@@ -371,6 +382,46 @@ public class LucenePropertyIndex impleme
return new LucenePathCursor(itr, settings);
}
+ private IndexNode acquireIndexNode(IndexPlan plan) {
+ return tracker.acquireIndexNode((String) plan.getAttribute(ATTR_INDEX_PATH));
+ }
+
+ private Sort getSort(List<OrderEntry> sortOrder, IndexPlan plan) {
+ if (sortOrder == null || sortOrder.isEmpty()) {
+ return null;
+ }
+ IndexNode indexNode = acquireIndexNode(plan);
+ try {
+ SortField[] fields = new SortField[sortOrder.size()];
+ for (int i = 0; i < sortOrder.size(); i++) {
+ OrderEntry oe = sortOrder.get(i);
+ boolean reverse = oe.getOrder() != OrderEntry.Order.ASCENDING;
+ fields[i] = new SortField(oe.getPropertyName(), toLuceneSortType(oe, indexNode.getDefinition()), reverse);
+ }
+ return new Sort(fields);
+ } finally {
+ indexNode.release();
+ }
+ }
+
+ private static SortField.Type toLuceneSortType(OrderEntry oe, IndexDefinition defn) {
+ Type<?> t = oe.getPropertyType();
+ checkState(t != null, "Type cannot be null");
+ checkState(!t.isArray(), "Array types are not supported");
+
+ int type = getPropertyType(defn, oe.getPropertyName(), t.tag());
+ switch (type) {
+ case PropertyType.LONG:
+ case PropertyType.DATE:
+ return SortField.Type.LONG;
+ case PropertyType.DOUBLE:
+ return SortField.Type.DOUBLE;
+ default:
+ //TODO Check about SortField.Type.STRING_VAL
+ return SortField.Type.STRING;
+ }
+ }
+
private static String getIndexName(IndexPlan plan){
return PathUtils.getName((String) plan.getAttribute(ATTR_INDEX_PATH));
}
@@ -627,13 +678,17 @@ public class LucenePropertyIndex impleme
}
}
+ private static int getPropertyType(IndexDefinition defn, String name, int defaultVal){
+ if (defn.hasPropertyDefinition(name)) {
+ return defn.getPropDefn(name).getPropertyType();
+ }
+ return defaultVal;
+ }
+
@CheckForNull
private static Query createQuery(PropertyRestriction pr,
IndexDefinition defn) {
- int propType = pr.propertyType;
- if (defn.hasPropertyDefinition(pr.propertyName)) {
- propType = defn.getPropDefn(pr.propertyName).getPropertyType();
- }
+ int propType = getPropertyType(defn, pr.propertyName, pr.propertyType);
switch (propType) {
case PropertyType.DATE: {
Long first = pr.first != null ? FieldFactory.dateToLong(pr.first.getValue(Type.DATE)) : null;
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java?rev=1631969&r1=1631968&r2=1631969&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java Wed Oct 15 08:09:33 2014
@@ -104,7 +104,7 @@ public class LuceneIndexEditorTest {
builder.child("test").setProperty("weight", 10.0);
builder.child("test").setProperty("bool", true);
builder.child("test").setProperty("truth", true);
- builder.child("test").setProperty("creationTime", createDate("05/06/2014"));
+ builder.child("test").setProperty("creationTime", createCal("05/06/2014"));
NodeState after = builder.getNodeState();
NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY);
@@ -179,7 +179,7 @@ public class LuceneIndexEditorTest {
return indexNode.getSearcher();
}
- static Calendar createDate(String dt) throws java.text.ParseException {
+ static Calendar createCal(String dt) throws java.text.ParseException {
SimpleDateFormat sdf = new SimpleDateFormat("dd/MM/yyyy");
Calendar cal = Calendar.getInstance();
cal.setTime(sdf.parse(dt));
@@ -187,6 +187,6 @@ public class LuceneIndexEditorTest {
}
static long dateToTime(String dt) throws java.text.ParseException {
- return FieldFactory.dateToLong(ISO8601.format(createDate(dt)));
+ return FieldFactory.dateToLong(ISO8601.format(createCal(dt)));
}
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1631969&r1=1631968&r2=1631969&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Wed Oct 15 08:09:33 2014
@@ -20,10 +20,16 @@
package org.apache.jackrabbit.oak.plugins.index.lucene;
import java.text.ParseException;
+import java.util.Calendar;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
import java.util.Set;
import javax.jcr.PropertyType;
+import com.google.common.collect.ComparisonChain;
+import com.google.common.collect.Lists;
import org.apache.jackrabbit.JcrConstants;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.CommitFailedException;
@@ -47,11 +53,17 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
-import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createDate;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorTest.createCal;
+import static org.apache.jackrabbit.oak.plugins.index.property.OrderedIndex.OrderDirection;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.matchers.JUnitMatchers.containsString;
public class LucenePropertyIndexTest extends AbstractQueryTest {
+ /**
+ * Set the size to twice the batch size to test the pagination with sorting
+ */
+ static final int NUMBER_OF_NODES = LucenePropertyIndex.LUCENE_QUERY_BATCH_SIZE * 2;
@Override
protected void createTestIndexNode() throws Exception {
@@ -176,9 +188,9 @@ public class LucenePropertyIndexTest ext
root.commit();
Tree test = root.getTree("/").addChild("test");
- test.addChild("a").setProperty("propa", createDate("14/02/2014"));
- test.addChild("b").setProperty("propa", createDate("14/03/2014"));
- test.addChild("c").setProperty("propa", createDate("14/04/2014"));
+ test.addChild("a").setProperty("propa", createCal("14/02/2014"));
+ test.addChild("b").setProperty("propa", createCal("14/03/2014"));
+ test.addChild("c").setProperty("propa", createCal("14/04/2014"));
test.addChild("c").setProperty("propb", "foo");
test.addChild("d").setProperty("propb", "foo");
root.commit();
@@ -206,6 +218,124 @@ public class LucenePropertyIndexTest ext
assertQuery("select [jcr:path] from [nt:base] where propa like '%ty'", asList("/test/a", "/test/b"));
}
+ @Test
+ public void sortQueriesWithLong() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
+ root.commit();
+
+ Tree test = root.getTree("/").addChild("test");
+ List<Long> values = createLongs(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithDouble() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DOUBLE);
+ root.commit();
+
+ Tree test = root.getTree("/").addChild("test");
+ List<Double> values = createDoubles(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithString() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ idx.addChild("foo");
+ root.commit();
+
+ Tree test = root.getTree("/").addChild("test");
+ List<String> values = createStrings(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithDate() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar"));
+ Tree propIdx = idx.addChild("foo");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_DATE);
+ root.commit();
+
+ Tree test = root.getTree("/").addChild("test");
+ List<Calendar> values = createDates(NUMBER_OF_NODES);
+ List<Tuple> tuples = Lists.newArrayListWithCapacity(values.size());
+ for(int i = 0; i < values.size(); i++){
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", values.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple(values.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo]", getSortedPaths(tuples, OrderDirection.ASC));
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] DESC", getSortedPaths(tuples, OrderDirection.DESC));
+ }
+
+ @Test
+ public void sortQueriesWithStringAndLong() throws Exception {
+ Tree idx = createIndex("test1", of("foo", "bar", "baz"));
+ Tree propIdx = idx.addChild("baz");
+ propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG);
+ root.commit();
+
+ Tree test = root.getTree("/").addChild("test");
+ int firstPropSize = 5;
+ List<String> values = createStrings(firstPropSize);
+ List<Long> longValues = createLongs(NUMBER_OF_NODES);
+ List<Tuple2> tuples = Lists.newArrayListWithCapacity(values.size());
+ Random r = new Random();
+ for(int i = 0; i < values.size(); i++){
+ String val = values.get(r.nextInt(firstPropSize));
+ Tree child = test.addChild("n"+i);
+ child.setProperty("foo", val);
+ child.setProperty("baz", longValues.get(i));
+ child.setProperty("bar", "baz");
+ tuples.add(new Tuple2(val, longValues.get(i), child.getPath()));
+ }
+ root.commit();
+
+ assertOrderedQuery("select [jcr:path] from [nt:base] where [bar] = 'baz' order by [foo] asc, [baz] desc", getSortedPaths(tuples));
+ }
+
+ private void assertOrderedQuery(String sql, List<String> paths) {
+ List<String> result = executeQuery(sql, SQL2, true);
+ assertEquals(paths, result);
+ }
+
//TODO Test for range with Date. Check for precision
private String explain(String query){
@@ -226,8 +356,119 @@ public class LucenePropertyIndexTest ext
return root.getTree("/").getChild(INDEX_DEFINITIONS_NAME).getChild(name);
}
-
private static String dt(String date) throws ParseException {
- return String.format("CAST ('%s' AS DATE)",ISO8601.format(createDate(date)));
+ return String.format("CAST ('%s' AS DATE)",ISO8601.format(createCal(date)));
+ }
+
+ private static List<String> getSortedPaths(List<Tuple> tuples, OrderDirection dir) {
+ if (OrderDirection.DESC == dir) {
+ Collections.sort(tuples, Collections.reverseOrder());
+ } else {
+ Collections.sort(tuples);
+ }
+ List<String> paths = Lists.newArrayListWithCapacity(tuples.size());
+ for (Tuple t : tuples) {
+ paths.add(t.path);
+ }
+ return paths;
+ }
+
+ private static List<String> getSortedPaths(List<Tuple2> tuples) {
+ Collections.sort(tuples);
+ List<String> paths = Lists.newArrayListWithCapacity(tuples.size());
+ for (Tuple2 t : tuples) {
+ paths.add(t.path);
+ }
+ return paths;
+ }
+
+ private static List<Long> createLongs(int n){
+ List<Long> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(i);
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static List<Double> createDoubles(int n){
+ Random rnd = new Random();
+ List<Double> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(rnd.nextDouble());
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static List<String> createStrings(int n){
+ List<String> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(String.format("value%04d",i));
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static List<Calendar> createDates(int n) throws ParseException {
+ Random rnd = new Random();
+ List<Calendar> values = Lists.newArrayListWithCapacity(n);
+ for (long i = 0; i < n; i++){
+ values.add(createCal(String.format("%02d/%02d/2014", rnd.nextInt(29) + 1, rnd.nextInt(11) + 1)));
+ }
+ Collections.shuffle(values);
+ return values;
+ }
+
+ private static class Tuple implements Comparable<Tuple>{
+ final Comparable value;
+ final String path;
+
+ private Tuple(Comparable value, String path) {
+ this.value = value;
+ this.path = path;
+ }
+
+ @Override
+ public int compareTo(Tuple o) {
+ return value.compareTo(o.value);
+ }
+
+ @Override
+ public String toString() {
+ return "Tuple{" +
+ "value=" + value +
+ ", path='" + path + '\'' +
+ '}';
+ }
+ }
+
+ private static class Tuple2 implements Comparable<Tuple2>{
+ final Comparable value;
+ final Comparable value2;
+ final String path;
+
+ private Tuple2(Comparable value, Comparable value2, String path) {
+ this.value = value;
+ this.value2 = value2;
+ this.path = path;
+ }
+
+ @Override
+ public int compareTo(Tuple2 o) {
+ return ComparisonChain.start()
+ .compare(value, o.value)
+ .compare(value2, o.value2, Collections.reverseOrder())
+ .result();
+ }
+
+ @Override
+ public String toString() {
+ return "Tuple2{" +
+ "value=" + value +
+ ", value2=" + value2 +
+ ", path='" + path + '\'' +
+ '}';
+ }
}
}