You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2015/12/15 11:03:54 UTC
svn commit: r1720108 - in /jackrabbit/oak/trunk/oak-solr-core/src:
main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/
main/resources/solr/oak/conf/ test/java/org/apache/jackrabbit/oak/jcr/query/
test/resources/solr/oak/conf/
Author: tommaso
Date: Tue Dec 15 10:03:54 2015
New Revision: 1720108
URL: http://svn.apache.org/viewvc?rev=1720108&view=rev
Log:
OAK-2510 - support for facets in solr index
Added:
jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
Modified: jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java (original)
+++ jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java Tue Dec 15 10:03:54 2015
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import java.util.Collection;
import java.util.List;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration;
import org.apache.jackrabbit.oak.query.QueryImpl;
import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd;
@@ -63,6 +64,17 @@ class FilterQueryParser {
}
}
+ // facet enable
+ List<Filter.PropertyRestriction> facetRestriction = filter.getPropertyRestrictions(QueryImpl.REP_FACET);
+ if (facetRestriction != null && facetRestriction.size() > 0) {
+ solrQuery.setFacetMinCount(1);
+ solrQuery.setFacet(true);
+ for (Filter.PropertyRestriction pr : facetRestriction) {
+ String value = pr.first.getValue(Type.STRING);
+ solrQuery.addFacetField(value.substring(QueryImpl.REP_FACET.length() + 1, value.length() - 1)+"_facet");
+ }
+ }
+
if (sortOrder != null) {
for (QueryIndex.OrderEntry orderEntry : sortOrder) {
SolrQuery.ORDER order;
Modified: jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java (original)
+++ jackrabbit/oak/trunk/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java Tue Dec 15 10:03:54 2015
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -53,6 +54,7 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
+import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SpellCheckResponse;
import org.apache.solr.common.SolrDocument;
@@ -237,6 +239,7 @@ public class SolrQueryIndex implements F
private AbstractIterator<SolrResultRow> getIterator(final Filter filter, final List<OrderEntry> sortOrder, final String parent, final int parentDepth) {
return new AbstractIterator<SolrResultRow>() {
+ public Collection<FacetField> facetFields = new LinkedList<FacetField>();
private final Set<String> seenPaths = Sets.newHashSet();
private final Deque<SolrResultRow> queue = Queues.newArrayDeque();
private int offset = 0;
@@ -270,7 +273,7 @@ public class SolrQueryIndex implements F
if (scoreObj != null) {
score = (Float) scoreObj;
}
- return new SolrResultRow(path, score, doc);
+ return new SolrResultRow(path, score, doc, facetFields);
}
@@ -343,6 +346,25 @@ public class SolrQueryIndex implements F
}
}
+ // get facets
+ List<FacetField> returnedFieldFacet = queryResponse.getFacetFields();
+ if (returnedFieldFacet != null) {
+ facetFields.addAll(returnedFieldFacet);
+ }
+
+ // filter facets on doc paths
+ if (!facetFields.isEmpty() && docs != null) {
+ for (SolrDocument doc : docs) {
+ String path = String.valueOf(doc.getFieldValue(configuration.getPathField()));
+ // if facet path doesn't exist in the node state, filter the facets
+ for (FacetField ff : facetFields) {
+ if (!filter.isAccessible(path + "/" + ff.getName())) {
+ filterFacet(doc, ff);
+ }
+ }
+ }
+ }
+
// handle spellcheck
SpellCheckResponse spellCheckResponse = queryResponse.getSpellCheckResponse();
if (spellCheckResponse != null && spellCheckResponse.getSuggestions() != null &&
@@ -374,6 +396,36 @@ public class SolrQueryIndex implements F
};
}
+ private void filterFacet(SolrDocument doc, FacetField facetField) {
+ // facet filtering by value requires that the facet values match the stored values
+ // a *_facet field must exist, storing docValues to be used for faceting and at filtering time
+ if (doc.getFieldNames().contains(facetField.getName())) {
+ // decrease facet value
+ Collection<Object> docFieldValues = doc.getFieldValues(facetField.getName());
+ if (docFieldValues != null) {
+ for (Object docFieldValue : docFieldValues) {
+ String valueString = String.valueOf(docFieldValue);
+ List<FacetField.Count> toRemove = new LinkedList<FacetField.Count>();
+ for (FacetField.Count count : facetField.getValues()) {
+ long existingCount = count.getCount();
+ if (valueString.equals(count.getName())) {
+ if (existingCount > 1) {
+ // decrease the count
+ count.setCount(existingCount - 1);
+ } else {
+ // remove the entire entry
+ toRemove.add(count);
+ }
+ }
+ }
+ for (FacetField.Count f : toRemove) {
+ assert facetField.getValues().remove(f);
+ }
+ }
+ }
+ }
+ }
+
private void putSpellChecks(SpellCheckResponse spellCheckResponse,
final Deque<SolrResultRow> queue,
Filter filter) throws SolrServerException {
@@ -450,7 +502,9 @@ public class SolrQueryIndex implements F
(!configuration.useForPropertyRestrictions() // Solr index not used for properties
|| (configuration.getUsedProperties().size() > 0 && !configuration.getUsedProperties().contains(propertyName)) // not explicitly contained in the used properties
|| propertyName.contains("/") // no child-level property restrictions
- || "rep:excerpt".equals(propertyName) // rep:excerpt is not handled at the property level
+ || QueryImpl.REP_EXCERPT.equals(propertyName) // rep:excerpt is not handled at the property level
+ || QueryImpl.OAK_SCORE_EXPLANATION.equals(propertyName) // score explain is not handled at the property level
+ || QueryImpl.REP_FACET.equals(propertyName) // rep:facet is not handled at the property level
|| QueryConstants.RESTRICTION_LOCAL_NAME.equals(propertyName)
|| configuration.getIgnoredProperties().contains(propertyName));
}
@@ -498,25 +552,31 @@ public class SolrQueryIndex implements F
final String path;
final double score;
final SolrDocument doc;
+ final Collection<FacetField> facetFields;
final String suggestion;
- private SolrResultRow(String path, double score, SolrDocument doc, String suggestion) {
+ private SolrResultRow(String path, double score, SolrDocument doc, String suggestion, Collection<FacetField> facetFields) {
this.path = path;
this.score = score;
this.doc = doc;
this.suggestion = suggestion;
+ this.facetFields = facetFields;
}
SolrResultRow(String path, double score, SolrDocument doc) {
- this (path, score, doc, null);
+ this (path, score, doc, null, null);
}
SolrResultRow(String suggestion, double score) {
- this ("/", score, null, suggestion);
+ this ("/", score, null, suggestion, null);
}
SolrResultRow(String suggestion) {
- this ("/", 1.0, null, suggestion);
+ this ("/", 1.0, null, suggestion, null);
+ }
+
+ SolrResultRow(String path, float score, SolrDocument doc, Collection<FacetField> facetFields) {
+ this(path, score, doc, null, facetFields);
}
@Override
@@ -593,6 +653,21 @@ public class SolrQueryIndex implements F
if (QueryImpl.JCR_SCORE.equals(columnName)) {
return PropertyValues.newDouble(currentRow.score);
}
+ if (columnName.startsWith(QueryImpl.REP_FACET)) {
+ String facetFieldName = columnName.substring(QueryImpl.REP_FACET.length() + 1, columnName.length() - 1);
+ FacetField facetField = null;
+ for (FacetField ff : currentRow.facetFields) {
+ if (ff.getName().equals(facetFieldName + "_facet")) {
+ facetField = ff;
+ break;
+ }
+ }
+ if (facetField != null) {
+ return PropertyValues.newString(facetFieldName + ":" + facetField.getValues().toString());
+ } else {
+ return null;
+ }
+ }
if (QueryImpl.REP_SPELLCHECK.equals(columnName) || QueryImpl.REP_SUGGEST.equals(columnName)) {
return PropertyValues.newString(currentRow.suggestion);
}
Modified: jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml (original)
+++ jackrabbit/oak/trunk/oak-solr-core/src/main/resources/solr/oak/conf/schema.xml Tue Dec 15 10:03:54 2015
@@ -135,6 +135,9 @@
<field name="path_depth" type="tint" indexed="true" stored="false"/>
<field name="_version_" type="long" indexed="true" stored="true"/>
+ <!-- facet fields -->
+ <dynamicField name="*_facet" type="string" indexed="false" stored="false" docValues="true" multiValued="true"/>
+
<!-- sorting dynamic fields -->
<dynamicField name="*_double_sort" type="tdouble" indexed="false" stored="false" multiValued="false" docValues="true"/>
<dynamicField name="*_string_sort" type="string" indexed="false" stored="false" multiValued="false" docValues="true"/>
@@ -149,6 +152,7 @@
<copyField source="path_exact" dest="path_child"/>
<copyField source="path_exact" dest=":path"/>
<copyField source="*" dest="catch_all"/>
+ <copyField source="*" dest="*_facet"/>
<copyField source="jcr:title" dest=":spellcheck"/>
<copyField source="jcr:description" dest=":spellcheck"/>
<copyField source="jcr:title" dest=":suggest"/>
Added: jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java?rev=1720108&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java (added)
+++ jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java Tue Dec 15 10:03:54 2015
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.jcr.query;
+
+import org.apache.jackrabbit.core.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.api.Type;
+
+import javax.jcr.Node;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.Row;
+import javax.jcr.query.RowIterator;
+
+/**
+ * Test for faceting capabilities via JCR API
+ */
+public class FacetTest extends AbstractQueryTest {
+
+ public void testFacetRetrieval() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("text", "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("text", "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("text", "oh hallo");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(text)] from [nt:base] " +
+ "where contains([text], 'hello OR hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "text:[hallo (1), hello (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult + ", " + facetResult, getResult(result, "rep:facet(text)"));
+ }
+
+ public void testFacetRetrievalMV() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("jcr:title", "apache jackrabbit oak");
+ n1.setProperty("tags", new String[]{"software", "repository", "apache"});
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("jcr:title", "oak furniture");
+ n2.setProperty("tags", "furniture");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("jcr:title", "oak cosmetics");
+ n3.setProperty("tags", "cosmetics");
+ Node n4 = testRootNode.addNode("node4");
+ n4.setProperty("jcr:title", "oak and aem");
+ n4.setProperty("tags", new String[]{"software", "repository", "aem"});
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(tags)] from [nt:base] " +
+ "where contains([jcr:title], 'oak') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)]";
+ assertEquals(facetResult, getResult(result, "rep:facet(tags)"));
+ }
+
+ public void testFacetRetrievalWithAnonymousUser() throws Exception {
+ Session session = superuser;
+
+ Node n1 = testRootNode.addNode("node1");
+ n1.setProperty("text", "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty("text", "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty("text", "oh hallo");
+ session.save();
+
+ session = getHelper().getReadOnlySession();
+ QueryManager qm = session.getWorkspace().getQueryManager();
+
+ String sql2 = "select [jcr:path], [rep:facet(text)] from [nt:base] " +
+ "where contains([text], 'hello OR hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = "text:[hallo (1), hello (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult + ", " + facetResult, getResult(result, "rep:facet(text)"));
+ }
+
+ public void testFacetRetrieval2() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ String pn = "jcr:title";
+ n1.setProperty(pn, "hello");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty(pn, "hallo");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty(pn, "oh hallo");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(" + pn + ")] from [nt:base] " +
+ "where contains([" + pn + "], 'hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = pn + ":[hallo (1), oh hallo (1)]";
+ assertEquals(facetResult + ", " + facetResult, getResult(result, "rep:facet(" + pn + ")"));
+ }
+
+ public void testMultipleFacetsRetrieval() throws Exception {
+ Session session = superuser;
+ QueryManager qm = session.getWorkspace().getQueryManager();
+ Node n1 = testRootNode.addNode("node1");
+ String pn = "jcr:title";
+ String pn2 = "jcr:description";
+ n1.setProperty(pn, "hello");
+ n1.setProperty(pn2, "a");
+ Node n2 = testRootNode.addNode("node2");
+ n2.setProperty(pn, "hallo");
+ n2.setProperty(pn2, "b");
+ Node n3 = testRootNode.addNode("node3");
+ n3.setProperty(pn, "oh hallo");
+ n3.setProperty(pn2, "a");
+ session.save();
+
+ String sql2 = "select [jcr:path], [rep:facet(" + pn + ")], [rep:facet(" + pn2 + ")] from [nt:base] " +
+ "where contains([" + pn + "], 'hallo') order by [jcr:path]";
+ Query q = qm.createQuery(sql2, Query.JCR_SQL2);
+ QueryResult result = q.execute();
+ String facetResult = pn + ":[hallo (1), oh hallo (1)], " + pn2 + ":[a (1), b (1)], " + pn + ":[hallo (1), oh hallo (1)], " + pn2 + ":[a (1), b (1)]";
+ assertEquals(facetResult, getResult(result, "rep:facet(" + pn + ")", "rep:facet(" + pn2 + ")"));
+ }
+
+ static String getResult(QueryResult result, String... propertyNames) throws RepositoryException {
+ StringBuilder buff = new StringBuilder();
+ RowIterator it = result.getRows();
+ while (it.hasNext()) {
+
+ Row row = it.nextRow();
+ for (String propertyName : propertyNames) {
+ Value value = row.getValue(propertyName);
+ if (value != null) {
+ if (buff.length() > 0) {
+ buff.append(", ");
+ }
+ buff.append(value.getString());
+ }
+ }
+ }
+ return buff.toString();
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml?rev=1720108&r1=1720107&r2=1720108&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml (original)
+++ jackrabbit/oak/trunk/oak-solr-core/src/test/resources/solr/oak/conf/schema.xml Tue Dec 15 10:03:54 2015
@@ -147,6 +147,9 @@
<field name="path_depth" type="tint" indexed="true" stored="false"/>
<field name="_version_" type="long" indexed="true" stored="true"/>
+ <!-- facet fields -->
+ <dynamicField name="*_facet" type="string" indexed="false" stored="false" docValues="true" multiValued="true"/>
+
<!-- sorting dynamic fields -->
<dynamicField name="*_double_sort" type="tdouble" indexed="false" stored="false" multiValued="false" docValues="true"/>
<dynamicField name="*_string_sort" type="string" indexed="false" stored="false" multiValued="false" docValues="true"/>
@@ -160,6 +163,7 @@
<copyField source="path_exact" dest="path_child"/>
<copyField source="path_exact" dest=":path"/>
<copyField source="*" dest="catch_all"/>
+ <copyField source="*" dest="*_facet"/>
<copyField source="jcr:title" dest=":spellcheck"/>
<copyField source="jcr:description" dest=":spellcheck"/>
<copyField source="jcr:title" dest=":suggest"/>