You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2018/11/22 15:55:25 UTC

svn commit: r1847180 - /jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java

Author: tommaso
Date: Thu Nov 22 15:55:25 2018
New Revision: 1847180

URL: http://svn.apache.org/viewvc?rev=1847180&view=rev
Log:
OAK-7824 - enable reranking by similarity fvs

Modified:
    jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java

Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java?rev=1847180&r1=1847179&r2=1847180&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/SimSearchUtils.java Thu Nov 22 15:55:25 2018
@@ -18,10 +18,7 @@ package org.apache.jackrabbit.oak.plugin
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.*;
 
 import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
 import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
@@ -53,7 +50,7 @@ public class SimSearchUtils {
     private static final Logger log = LoggerFactory.getLogger(SimSearchUtils.class);
 
     public static String toDoubleString(byte[] bytes) {
-        Double[] a = toDoubleArray(bytes);
+        double[] a = toDoubleArray(bytes);
         StringBuilder builder = new StringBuilder();
         for (Double d : a) {
             if (builder.length() > 0) {
@@ -64,11 +61,6 @@ public class SimSearchUtils {
         return builder.toString();
     }
 
-    private static Double[] toDoubleArray(byte[] array) {
-        List<Double> doubles = toDoubles(array);
-        return doubles.toArray(new Double[doubles.size()]);
-    }
-
     public static List<Double> toDoubles(byte[] array) {
         int blockSize = Double.SIZE / Byte.SIZE;
         ByteBuffer wrap = ByteBuffer.wrap(array);
@@ -81,6 +73,18 @@ public class SimSearchUtils {
         return doubles;
     }
 
+    private static double[] toDoubleArray(byte[] array) {
+        int blockSize = Double.SIZE / Byte.SIZE;
+        ByteBuffer wrap = ByteBuffer.wrap(array);
+        int capacity = array.length / blockSize;
+        double[] doubles = new double[capacity];
+        for (int i = 0; i < capacity; i++) {
+                double e = wrap.getDouble(i * blockSize);
+                doubles[i] = e;
+            }
+        return doubles;
+    }
+        
     private static Collection<BytesRef> getTokens(Analyzer analyzer, String field, String sampleTextString) throws IOException {
         Collection<BytesRef> tokens = new LinkedList<>();
         TokenStream ts = analyzer.tokenStream(field, sampleTextString);
@@ -153,6 +157,8 @@ public class SimSearchUtils {
                             Query simQuery = SimSearchUtils.getSimQuery(analyzer, similarityFieldName, fvString);
                             booleanQuery.add(new BooleanClause(simQuery, SHOULD));
                             log.trace("similarity query generated for {}", pd.name);
+                        } else {
+                            log.warn("could not create query for similarity field {}", fvString);
                         }
                     }
                 }
@@ -227,4 +233,49 @@ public class SimSearchUtils {
         return 1;
     }
 
-}
+    public static void bruteForceFVRerank(List<PropertyDefinition> sp, TopDocs docs, IndexSearcher indexSearcher) throws IOException {
+        double farthestDistance = 50d;
+        int k = 15;
+        ScoreDoc inputDoc = docs.scoreDocs[0]; // we assume the input doc is the first one returned
+        List<Integer> toDiscard = new LinkedList<>();
+        for (PropertyDefinition pd : sp) {
+            String fieldName = FieldNames.createBinSimilarityFieldName(pd.name);
+            BytesRef binaryValue = indexSearcher.doc(inputDoc.doc).getBinaryValue(fieldName);
+            double[] inputVector = toDoubleArray(binaryValue.bytes);
+            for (int j = 0; j < docs.scoreDocs.length; j++) {
+                double[] currentVector = toDoubleArray(indexSearcher.doc(docs.scoreDocs[j].doc)
+                        .getBinaryValue(fieldName).bytes);
+                double distance = dist(inputVector, currentVector) + 1e-10; // constant term to avoid division by zero
+
+                if (distance > farthestDistance) { // a threshold distance above which current vector is discarded
+                    toDiscard.add(docs.scoreDocs[j].doc);
+                }
+                if (Double.isNaN(distance) || Double.isInfinite(distance)) {
+                    toDiscard.add(docs.scoreDocs[j].doc);
+                }
+                docs.scoreDocs[j].score += (float) (1d / distance); // additive similarity boosting
+            }
+        }
+        if (!toDiscard.isEmpty()) {
+            docs.scoreDocs = Arrays.stream(docs.scoreDocs).filter(e -> !toDiscard.contains(e.doc)).toArray(ScoreDoc[]::new); // remove docs that are not close enough
+        }
+        Arrays.parallelSort(docs.scoreDocs, 0, docs.scoreDocs.length, (o1, o2) -> { // rerank scoreDocs
+            return -1 * Double.compare(o1.score, o2.score);
+        });
+        if (docs.scoreDocs.length > k) {
+            docs.scoreDocs = Arrays.copyOfRange(docs.scoreDocs, 0, k); // retain only the top k nearest neighbours
+        }
+        if (docs.scoreDocs.length > 0) {
+            docs.setMaxScore(docs.scoreDocs[0].score);
+        }
+    }
+
+    private static double dist(double[] x, double[] y) { // euclidean distance
+        double d = 0;
+        for (int i = 0; i < x.length; i++) {
+            d += Math.pow(y[i] - x[i], 2);
+        }
+        return Math.sqrt(d);
+    }
+
+}
\ No newline at end of file