You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2018/06/25 13:30:53 UTC
svn commit: r1834326 [2/3] - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util...
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/LSHAnalyzerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/LSHAnalyzerTest.java?rev=1834326&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/LSHAnalyzerTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/LSHAnalyzerTest.java Mon Jun 25 13:30:52 2018
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util.fv;
+
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+import static org.apache.jackrabbit.oak.plugins.index.lucene.util.fv.SimSearchUtils.getSimQuery;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link LSHAnalyzer}
+ */
+public class LSHAnalyzerTest {
+
+ @Test
+ public void testTextFVIndexAndSearch() throws Exception {
+ String fieldName = "text";
+ String[] texts = new String[]{
+ "0.1,0.3,0.5,0.7,0.11,0.13,0.17,0.19,0.23,0.29",
+ "0.1 0.3 0.5 0.7 0.11 0.13 0.17 0.19 0.23 0.29"
+ };
+
+ for (String text : texts) {
+ LSHAnalyzer analyzer = new LSHAnalyzer();
+ Directory directory = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_47, analyzer));
+ DirectoryReader reader = null;
+ try {
+ Document document = new Document();
+ document.add(new TextField(fieldName, text, Field.Store.YES));
+ writer.addDocument(document);
+ writer.commit();
+
+ reader = DirectoryReader.open(writer, false);
+ assertSimQuery(analyzer, fieldName, text, reader);
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ writer.close();
+ directory.close();
+ }
+ }
+ }
+
+ @Test
+ public void testBinaryFVIndexAndSearch() throws Exception {
+ LSHAnalyzer analyzer = new LSHAnalyzer();
+ Directory directory = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_47, analyzer));
+ DirectoryReader reader = null;
+ try {
+
+ List<Double> values = new LinkedList<>();
+ values.add(0.1d);
+ values.add(0.3d);
+ values.add(0.5d);
+ values.add(0.7d);
+ values.add(0.11d);
+ values.add(0.13d);
+ values.add(0.17d);
+ values.add(0.19d);
+ values.add(0.23d);
+ values.add(0.29d);
+
+ byte[] bytes = SimSearchUtils.toByteArray(values);
+ String fvString = SimSearchUtils.toDoubleString(bytes);
+
+ String fieldName = "fvs";
+ Document document = new Document();
+ document.add(new TextField(fieldName, fvString, Field.Store.YES));
+ writer.addDocument(document);
+ writer.commit();
+
+ reader = DirectoryReader.open(writer, false);
+ assertSimQuery(analyzer, fieldName, fvString, reader);
+ } finally {
+ if (reader != null) {
+ reader.close();
+ }
+ writer.close();
+ directory.close();
+ }
+ }
+
+ private void assertSimQuery(LSHAnalyzer analyzer, String fieldName, String text, DirectoryReader reader) throws IOException {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ Query booleanQuery = getSimQuery(analyzer, fieldName, text);
+ TopDocs topDocs = searcher.search(booleanQuery, 1);
+ assertEquals(1, topDocs.totalHits);
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/LSHAnalyzerTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/TruncateTokenFilterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/TruncateTokenFilterTest.java?rev=1834326&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/TruncateTokenFilterTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/TruncateTokenFilterTest.java Mon Jun 25 13:30:52 2018
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene.util.fv;
+
+import java.io.StringReader;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.Version;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link TruncateTokenFilter}
+ */
+public class TruncateTokenFilterTest {
+
+ @Test
+ public void testFiltering() throws Exception {
+ TokenStream stream = new WhitespaceTokenizer(Version.LUCENE_47, new StringReader("0.10 0.20 0.30 0.40"));
+ TruncateTokenFilter filter = new TruncateTokenFilter(stream, 3);
+ filter.reset();
+ List<String> expectedTokens = new LinkedList<>();
+ expectedTokens.add("0.1");
+ expectedTokens.add("0.2");
+ expectedTokens.add("0.3");
+ expectedTokens.add("0.4");
+ int i = 0;
+ while (filter.incrementToken()) {
+ CharTermAttribute charTermAttribute = filter.getAttribute(CharTermAttribute.class);
+ String token = new String(charTermAttribute.buffer(), 0, charTermAttribute.length());
+ assertEquals(expectedTokens.get(i), token);
+ i++;
+ }
+ filter.close();
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/fv/TruncateTokenFilterTest.java
------------------------------------------------------------------------------
svn:eol-style = native