You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/14 07:51:25 UTC
svn commit: r1594464 [3/3] - in /lucene/dev/branches/lucene5666:
lucene/analysis/common/src/java/org/apache/lucene/collation/
lucene/analysis/common/src/test/org/apache/lucene/collation/
lucene/analysis/icu/src/java/org/apache/lucene/collation/ lucene/...
Added: lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java?rev=1594464&view=auto
==============================================================================
--- lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java (added)
+++ lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestOrdValues.java Wed May 14 05:51:24 2014
@@ -0,0 +1,310 @@
+package org.apache.solr.search.function;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.queries.function.FunctionQuery;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
+import org.apache.lucene.queries.function.valuesource.IntFieldSource;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test search based on OrdFieldSource and ReverseOrdFieldSource.
+ * <p/>
+ * Tests here create an index with a few documents, each having
+ * an indexed "id" field.
+ * The ord values of this field are later used for scoring.
+ * <p/>
+ * The order tests use Hits to verify that docs are ordered as expected.
+ * <p/>
+ * The exact score tests use TopDocs top to verify the exact score.
+ */
+public class TestOrdValues extends LuceneTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ createIndex(false);
+ }
+
+ /**
+ * Test OrdFieldSource
+ */
+ @Test
+ public void testOrdFieldRank() throws Exception {
+ doTestRank(ID_FIELD, true);
+ }
+
+ /**
+ * Test ReverseOrdFieldSource
+ */
+ @Test
+ public void testReverseOrdFieldRank() throws Exception {
+ doTestRank(ID_FIELD, false);
+ }
+
+ // Test that queries based on reverse/ordFieldScore scores correctly
+ private void doTestRank(String field, boolean inOrder) throws Exception {
+ IndexReader r = DirectoryReader.open(dir);
+ IndexSearcher s = newSearcher(r);
+ ValueSource vs;
+ if (inOrder) {
+ vs = new OrdFieldSource(field);
+ } else {
+ vs = new ReverseOrdFieldSource(field);
+ }
+
+ Query q = new FunctionQuery(vs);
+ log("test: " + q);
+ QueryUtils.check(random(), q, s);
+ ScoreDoc[] h = s.search(q, null, 1000).scoreDocs;
+ assertEquals("All docs should be matched!", N_DOCS, h.length);
+ String prevID = inOrder
+ ? "IE" // greater than all ids of docs in this test ("ID0001", etc.)
+ : "IC"; // smaller than all ids of docs in this test ("ID0001", etc.)
+
+ for (int i = 0; i < h.length; i++) {
+ String resID = s.doc(h[i].doc).get(ID_FIELD);
+ log(i + ". score=" + h[i].score + " - " + resID);
+ log(s.explain(q, h[i].doc));
+ if (inOrder) {
+ assertTrue("res id " + resID + " should be < prev res id " + prevID, resID.compareTo(prevID) < 0);
+ } else {
+ assertTrue("res id " + resID + " should be > prev res id " + prevID, resID.compareTo(prevID) > 0);
+ }
+ prevID = resID;
+ }
+ r.close();
+ }
+
+ /**
+ * Test exact score for OrdFieldSource
+ */
+ @Test
+ public void testOrdFieldExactScore() throws Exception {
+ doTestExactScore(ID_FIELD, true);
+ }
+
+ /**
+ * Test exact score for ReverseOrdFieldSource
+ */
+ @Test
+ public void testReverseOrdFieldExactScore() throws Exception {
+ doTestExactScore(ID_FIELD, false);
+ }
+
+
+ // Test that queries based on reverse/ordFieldScore returns docs with expected score.
+ private void doTestExactScore(String field, boolean inOrder) throws Exception {
+ IndexReader r = DirectoryReader.open(dir);
+ IndexSearcher s = newSearcher(r);
+ ValueSource vs;
+ if (inOrder) {
+ vs = new OrdFieldSource(field);
+ } else {
+ vs = new ReverseOrdFieldSource(field);
+ }
+ Query q = new FunctionQuery(vs);
+ TopDocs td = s.search(q, null, 1000);
+ assertEquals("All docs should be matched!", N_DOCS, td.totalHits);
+ ScoreDoc sd[] = td.scoreDocs;
+ for (int i = 0; i < sd.length; i++) {
+ float score = sd[i].score;
+ String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD);
+ log("-------- " + i + ". Explain doc " + id);
+ log(s.explain(q, sd[i].doc));
+ float expectedScore = N_DOCS - i - 1;
+ assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA);
+ String expectedId = inOrder
+ ? id2String(N_DOCS - i) // in-order ==> larger values first
+ : id2String(i + 1); // reverse ==> smaller values first
+ assertTrue("id of result " + i + " shuould be " + expectedId + " != " + score, expectedId.equals(id));
+ }
+ r.close();
+ }
+
+ // LUCENE-1250
+ public void testEqualsNull() throws Exception {
+ OrdFieldSource ofs = new OrdFieldSource("f");
+ assertFalse(ofs.equals(null));
+
+ ReverseOrdFieldSource rofs = new ReverseOrdFieldSource("f");
+ assertFalse(rofs.equals(null));
+ }
+
+ /**
+ * Actual score computation order is slightly different than assumptios
+ * this allows for a small amount of variation
+ */
+ protected static float TEST_SCORE_TOLERANCE_DELTA = 0.001f;
+
+ protected static final int N_DOCS = 17; // select a primary number > 2
+
+ protected static final String ID_FIELD = "id";
+ protected static final String TEXT_FIELD = "text";
+ protected static final String INT_FIELD = "iii";
+ protected static final String FLOAT_FIELD = "fff";
+
+ protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD);
+ protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD);
+
+ private static final String DOC_TEXT_LINES[] = {
+ "Well, this is just some plain text we use for creating the ",
+ "test documents. It used to be a text from an online collection ",
+ "devoted to first aid, but if there was there an (online) lawyers ",
+ "first aid collection with legal advices, \"it\" might have quite ",
+ "probably advised one not to include \"it\"'s text or the text of ",
+ "any other online collection in one's code, unless one has money ",
+ "that one don't need and one is happy to donate for lawyers ",
+ "charity. Anyhow at some point, rechecking the usage of this text, ",
+ "it became uncertain that this text is free to use, because ",
+ "the web site in the disclaimer of he eBook containing that text ",
+ "was not responding anymore, and at the same time, in projGut, ",
+ "searching for first aid no longer found that eBook as well. ",
+ "So here we are, with a perhaps much less interesting ",
+ "text for the test, but oh much much safer. ",
+ };
+
+ protected static Directory dir;
+ protected static Analyzer anlzr;
+
+ @AfterClass
+ public static void afterClassFunctionTestSetup() throws Exception {
+ dir.close();
+ dir = null;
+ anlzr = null;
+ }
+
+ protected static void createIndex(boolean doMultiSegment) throws Exception {
+ if (VERBOSE) {
+ System.out.println("TEST: setUp");
+ }
+ // prepare a small index with just a few documents.
+ dir = newDirectory();
+ anlzr = new MockAnalyzer(random());
+ IndexWriterConfig iwc = newIndexWriterConfig( TEST_VERSION_CURRENT, anlzr).setMergePolicy(newLogMergePolicy());
+ if (doMultiSegment) {
+ iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 2, 7));
+ }
+ RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
+ // add docs not exactly in natural ID order, to verify we do check the order of docs by scores
+ int remaining = N_DOCS;
+ boolean done[] = new boolean[N_DOCS];
+ int i = 0;
+ while (remaining > 0) {
+ if (done[i]) {
+ throw new Exception("to set this test correctly N_DOCS=" + N_DOCS + " must be primary and greater than 2!");
+ }
+ addDoc(iw, i);
+ done[i] = true;
+ i = (i + 4) % N_DOCS;
+ remaining --;
+ }
+ if (!doMultiSegment) {
+ if (VERBOSE) {
+ System.out.println("TEST: setUp full merge");
+ }
+ iw.forceMerge(1);
+ }
+ iw.shutdown();
+ if (VERBOSE) {
+ System.out.println("TEST: setUp done close");
+ }
+ }
+
+ private static void addDoc(RandomIndexWriter iw, int i) throws Exception {
+ Document d = new Document();
+ Field f;
+ int scoreAndID = i + 1;
+
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.setTokenized(false);
+ customType.setOmitNorms(true);
+
+ f = newField(ID_FIELD, id2String(scoreAndID), customType); // for debug purposes
+ d.add(f);
+ d.add(new SortedDocValuesField(ID_FIELD, new BytesRef(id2String(scoreAndID))));
+
+ FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
+ customType2.setOmitNorms(true);
+ f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
+ d.add(f);
+
+ f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
+ d.add(f);
+ d.add(new NumericDocValuesField(INT_FIELD, scoreAndID));
+
+ f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
+ d.add(f);
+ d.add(new NumericDocValuesField(FLOAT_FIELD, Float.floatToRawIntBits(scoreAndID)));
+
+ iw.addDocument(d);
+ log("added: " + d);
+ }
+
+ // 17 --> ID00017
+ protected static String id2String(int scoreAndID) {
+ String s = "000000000" + scoreAndID;
+ int n = ("" + N_DOCS).length() + 3;
+ int k = s.length() - n;
+ return "ID" + s.substring(k);
+ }
+
+ // some text line for regular search
+ private static String textLine(int docNum) {
+ return DOC_TEXT_LINES[docNum % DOC_TEXT_LINES.length];
+ }
+
+ // extract expected doc score from its ID Field: "ID7" --> 7.0
+ protected static float expectedFieldScore(String docIDFieldVal) {
+ return Float.parseFloat(docIDFieldVal.substring(2));
+ }
+
+ // debug messages (change DBG to true for anything to print)
+ protected static void log(Object o) {
+ if (VERBOSE) {
+ System.out.println(o.toString());
+ }
+ }
+
+}
Modified: lucene/dev/branches/lucene5666/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java?rev=1594464&r1=1594463&r2=1594464&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java (original)
+++ lucene/dev/branches/lucene5666/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java Wed May 14 05:51:24 2014
@@ -283,8 +283,6 @@ public abstract class BaseDistributedSea
@Override
public void tearDown() throws Exception {
destroyServers();
- // nocommit: split test if needed
- // FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
super.tearDown();
}