You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2018/11/22 15:55:25 UTC
svn commit: r1847180 -
/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java
Author: tommaso
Date: Thu Nov 22 15:55:25 2018
New Revision: 1847180
URL: http://svn.apache.org/viewvc?rev=1847180&view=rev
Log:
OAK-7824 - enable reranking by similarity fvs
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java?rev=1847180&r1=1847179&r2=1847180&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java Thu Nov 22 15:55:25 2018
@@ -18,10 +18,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.*;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
@@ -53,7 +50,7 @@ public class SimSearchUtils {
private static final Logger log = LoggerFactory.getLogger(SimSearchUtils.class);
public static String toDoubleString(byte[] bytes) {
- Double[] a = toDoubleArray(bytes);
+ double[] a = toDoubleArray(bytes);
StringBuilder builder = new StringBuilder();
for (Double d : a) {
if (builder.length() > 0) {
@@ -64,11 +61,6 @@ public class SimSearchUtils {
return builder.toString();
}
- private static Double[] toDoubleArray(byte[] array) {
- List<Double> doubles = toDoubles(array);
- return doubles.toArray(new Double[doubles.size()]);
- }
-
public static List<Double> toDoubles(byte[] array) {
int blockSize = Double.SIZE / Byte.SIZE;
ByteBuffer wrap = ByteBuffer.wrap(array);
@@ -81,6 +73,18 @@ public class SimSearchUtils {
return doubles;
}
+ private static double[] toDoubleArray(byte[] array) {
+ int blockSize = Double.SIZE / Byte.SIZE;
+ ByteBuffer wrap = ByteBuffer.wrap(array);
+ int capacity = array.length / blockSize;
+ double[] doubles = new double[capacity];
+ for (int i = 0; i < capacity; i++) {
+ double e = wrap.getDouble(i * blockSize);
+ doubles[i] = e;
+ }
+ return doubles;
+ }
+
private static Collection<BytesRef> getTokens(Analyzer analyzer, String field, String sampleTextString) throws IOException {
Collection<BytesRef> tokens = new LinkedList<>();
TokenStream ts = analyzer.tokenStream(field, sampleTextString);
@@ -153,6 +157,8 @@ public class SimSearchUtils {
Query simQuery = SimSearchUtils.getSimQuery(analyzer, similarityFieldName, fvString);
booleanQuery.add(new BooleanClause(simQuery, SHOULD));
log.trace("similarity query generated for {}", pd.name);
+ } else {
+ log.warn("could not create query for similarity field {}", fvString);
}
}
}
@@ -227,4 +233,49 @@ public class SimSearchUtils {
return 1;
}
-}
+ public static void bruteForceFVRerank(List<PropertyDefinition> sp, TopDocs docs, IndexSearcher indexSearcher) throws IOException {
+ double farthestDistance = 50d;
+ int k = 15;
+ ScoreDoc inputDoc = docs.scoreDocs[0]; // we assume the input doc is the first one returned
+ List<Integer> toDiscard = new LinkedList<>();
+ for (PropertyDefinition pd : sp) {
+ String fieldName = FieldNames.createBinSimilarityFieldName(pd.name);
+ BytesRef binaryValue = indexSearcher.doc(inputDoc.doc).getBinaryValue(fieldName);
+ double[] inputVector = toDoubleArray(binaryValue.bytes);
+ for (int j = 0; j < docs.scoreDocs.length; j++) {
+ double[] currentVector = toDoubleArray(indexSearcher.doc(docs.scoreDocs[j].doc)
+ .getBinaryValue(fieldName).bytes);
+ double distance = dist(inputVector, currentVector) + 1e-10; // constant term to avoid division by zero
+
+ if (distance > farthestDistance) { // a threshold distance above which current vector is discarded
+ toDiscard.add(docs.scoreDocs[j].doc);
+ }
+ if (Double.isNaN(distance) || Double.isInfinite(distance)) {
+ toDiscard.add(docs.scoreDocs[j].doc);
+ }
+ docs.scoreDocs[j].score += (float) (1d / distance); // additive similarity boosting
+ }
+ }
+ if (!toDiscard.isEmpty()) {
+ docs.scoreDocs = Arrays.stream(docs.scoreDocs).filter(e -> !toDiscard.contains(e.doc)).toArray(ScoreDoc[]::new); // remove docs that are not close enough
+ }
+ Arrays.parallelSort(docs.scoreDocs, 0, docs.scoreDocs.length, (o1, o2) -> { // rerank scoreDocs
+ return -1 * Double.compare(o1.score, o2.score);
+ });
+ if (docs.scoreDocs.length > k) {
+ docs.scoreDocs = Arrays.copyOfRange(docs.scoreDocs, 0, k); // retain only the top k nearest neighbours
+ }
+ if (docs.scoreDocs.length > 0) {
+ docs.setMaxScore(docs.scoreDocs[0].score);
+ }
+ }
+
+ private static double dist(double[] x, double[] y) { // euclidean distance
+ double d = 0;
+ for (int i = 0; i < x.length; i++) {
+ d += Math.pow(y[i] - x[i], 2);
+ }
+ return Math.sqrt(d);
+ }
+
+}
\ No newline at end of file