You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2009/06/19 14:09:53 UTC
svn commit: r786470 [2/2] - in /lucene/java/trunk: ./
contrib/spatial/src/java/org/apache/lucene/spatial/
src/java/org/apache/lucene/analysis/ src/java/org/apache/lucene/document/
src/java/org/apache/lucene/search/ src/java/org/apache/lucene/util/ src/...
Added: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java?rev=786470&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java Fri Jun 19 12:09:52 2009
@@ -0,0 +1,431 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NumericUtils;
+
+public class TestNumericRangeQuery32 extends LuceneTestCase {
+ // distance of entries
+ private static final int distance = 6666;
+ // shift the starting of the values to the left, to also have negative values:
+ private static final int startOffset = - 1 << 15;
+ // number of docs to generate for testing
+ private static final int noDocs = 10000;
+
+ private static Field newField(String name, int precisionStep) {
+ NumericTokenStream stream = new NumericTokenStream(precisionStep);
+ stream.setUseNewAPI(true);
+ Field f=new Field(name, stream);
+ f.setOmitTermFreqAndPositions(true);
+ f.setOmitNorms(true);
+ return f;
+ }
+
+ private static final RAMDirectory directory;
+ private static final IndexSearcher searcher;
+ static {
+ try {
+ // set the theoretical maximum term count for 8bit (see docs for the number)
+ BooleanQuery.setMaxClauseCount(3*255*2 + 255);
+
+ directory = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(),
+ true, MaxFieldLength.UNLIMITED);
+
+ Field
+ field8 = newField("field8", 8),
+ field4 = newField("field4", 4),
+ field2 = newField("field2", 2),
+ ascfield8 = newField("ascfield8", 8),
+ ascfield4 = newField("ascfield4", 4),
+ ascfield2 = newField("ascfield2", 2);
+
+ // Add a series of noDocs docs with increasing int values
+ for (int l=0; l<noDocs; l++) {
+ Document doc=new Document();
+ // add fields, that have a distance to test general functionality
+ int val=distance*l+startOffset;
+ doc.add(new Field("value", Integer.toString(val), Field.Store.YES, Field.Index.NO));
+ ((NumericTokenStream)field8.tokenStreamValue()).setIntValue(val);
+ doc.add(field8);
+ ((NumericTokenStream)field4.tokenStreamValue()).setIntValue(val);
+ doc.add(field4);
+ ((NumericTokenStream)field2.tokenStreamValue()).setIntValue(val);
+ doc.add(field2);
+ // add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
+ val=l-(noDocs/2);
+ ((NumericTokenStream)ascfield8.tokenStreamValue()).setIntValue(val);
+ doc.add(ascfield8);
+ ((NumericTokenStream)ascfield4.tokenStreamValue()).setIntValue(val);
+ doc.add(ascfield4);
+ ((NumericTokenStream)ascfield2.tokenStreamValue()).setIntValue(val);
+ doc.add(ascfield2);
+ writer.addDocument(doc);
+ }
+
+ writer.optimize();
+ writer.close();
+ searcher=new IndexSearcher(directory);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ /** test for both constant score and boolean query, the other tests only use the constant score mode */
+ private void testRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ int lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
+ NumericRangeQuery q = NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+ NumericRangeFilter f = NumericRangeFilter.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+ int lastTerms = 0;
+ for (byte i=0; i<3; i++) {
+ TopDocs topDocs;
+ int terms;
+ String type;
+ q.clearTotalNumberOfTerms();
+ f.clearTotalNumberOfTerms();
+ switch (i) {
+ case 0:
+ type = " (constant score)";
+ q.setConstantScoreRewrite(true);
+ topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ terms = q.getTotalNumberOfTerms();
+ break;
+ case 1:
+ type = " (boolean query)";
+ q.setConstantScoreRewrite(false);
+ topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ terms = q.getTotalNumberOfTerms();
+ break;
+ case 2:
+ type = " (filter)";
+ topDocs = searcher.search(new MatchAllDocsQuery(), f, noDocs, Sort.INDEXORDER);
+ terms = f.getTotalNumberOfTerms();
+ break;
+ default:
+ return;
+ }
+ System.out.println("Found "+terms+" distinct terms in range for field '"+field+"'"+type+".");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score doc count"+type, count, sd.length );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc"+type, 2*distance+startOffset, Integer.parseInt(doc.get("value")) );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc"+type, (1+count)*distance+startOffset, Integer.parseInt(doc.get("value")) );
+ if (i>0) {
+ assertEquals("Distinct term number is equal for all query types", lastTerms, terms);
+ }
+ lastTerms = terms;
+ }
+ }
+
+ public void testRange_8bit() throws Exception {
+ testRange(8);
+ }
+
+ public void testRange_4bit() throws Exception {
+ testRange(4);
+ }
+
+ public void testRange_2bit() throws Exception {
+ testRange(2);
+ }
+
+ public void testInverseRange() throws Exception {
+ NumericRangeFilter f = NumericRangeFilter.newIntRange("field8", 8, new Integer(1000), new Integer(-1000), true, true);
+ assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(searcher.getIndexReader()));
+ }
+
+ private void testLeftOpenRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ int upper=(count-1)*distance + (distance/3) + startOffset;
+ NumericRangeQuery q=NumericRangeQuery.newIntRange(field, precisionStep, null, new Integer(upper), true, true);
+ TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in left open range for field '"+field+"'.");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score doc count", count, sd.length );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc", startOffset, Integer.parseInt(doc.get("value")) );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc", (count-1)*distance+startOffset, Integer.parseInt(doc.get("value")) );
+ }
+
+ public void testLeftOpenRange_8bit() throws Exception {
+ testLeftOpenRange(8);
+ }
+
+ public void testLeftOpenRange_4bit() throws Exception {
+ testLeftOpenRange(4);
+ }
+
+ public void testLeftOpenRange_2bit() throws Exception {
+ testLeftOpenRange(2);
+ }
+
+ private void testRightOpenRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ int lower=(count-1)*distance + (distance/3) +startOffset;
+ NumericRangeQuery q=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), null, true, true);
+ TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in right open range for field '"+field+"'.");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score doc count", noDocs-count, sd.length );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc", count*distance+startOffset, Integer.parseInt(doc.get("value")) );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc", (noDocs-1)*distance+startOffset, Integer.parseInt(doc.get("value")) );
+ }
+
+ public void testRightOpenRange_8bit() throws Exception {
+ testRightOpenRange(8);
+ }
+
+ public void testRightOpenRange_4bit() throws Exception {
+ testRightOpenRange(4);
+ }
+
+ public void testRightOpenRange_2bit() throws Exception {
+ testRightOpenRange(2);
+ }
+
+ private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="field"+precisionStep;
+ int termCountT=0,termCountC=0;
+ for (int i=0; i<50; i++) {
+ int lower=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ int upper=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ if (lower>upper) {
+ int a=lower; lower=upper; upper=a;
+ }
+ // test inclusive range
+ NumericRangeQuery tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+ RangeQuery cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, true);
+ cq.setConstantScoreRewrite(true);
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ TopDocs cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ // test exclusive range
+ tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, false);
+ cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, false);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ // test left exclusive range
+ tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, true);
+ cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, true);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ // test right exclusive range
+ tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, false);
+ cq=new RangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, false);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ }
+ System.out.println("Average number of terms during random search on '" + field + "':");
+ System.out.println(" Trie query: " + (((double)termCountT)/(50*4)));
+ System.out.println(" Classical query: " + (((double)termCountC)/(50*4)));
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(8);
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(4);
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(2);
+ }
+
+ private void testRangeSplit(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="ascfield"+precisionStep;
+ // 50 random tests
+ for (int i=0; i<50; i++) {
+ int lower=(int)(rnd.nextDouble()*noDocs - noDocs/2);
+ int upper=(int)(rnd.nextDouble()*noDocs - noDocs/2);
+ if (lower>upper) {
+ int a=lower; lower=upper; upper=a;
+ }
+ // test inclusive range
+ Query tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+ // test exclusive range
+ tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, false);
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to exclusive range length", Math.max(upper-lower-1, 0), tTopDocs.totalHits );
+ // test left exclusive range
+ tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, true);
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+ // test right exclusive range
+ tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, false);
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+ }
+ }
+
+ public void testRangeSplit_8bit() throws Exception {
+ testRangeSplit(8);
+ }
+
+ public void testRangeSplit_4bit() throws Exception {
+ testRangeSplit(4);
+ }
+
+ public void testRangeSplit_2bit() throws Exception {
+ testRangeSplit(2);
+ }
+
+ /** we fake a float test using int2float conversion of NumericUtils */
+ private void testFloatRange(int precisionStep) throws Exception {
+ final String field="ascfield"+precisionStep;
+ final int lower=-1000, upper=+2000;
+
+ Query tq=NumericRangeQuery.newFloatRange(field, precisionStep,
+ new Float(NumericUtils.sortableIntToFloat(lower)), new Float(NumericUtils.sortableIntToFloat(upper)), true, true);
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+
+ Filter tf=NumericRangeFilter.newFloatRange(field, precisionStep,
+ new Float(NumericUtils.sortableIntToFloat(lower)), new Float(NumericUtils.sortableIntToFloat(upper)), true, true);
+ tTopDocs = searcher.search(new MatchAllDocsQuery(), tf, 1);
+ assertEquals("Returned count of range filter must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+ }
+
+ public void testFloatRange_8bit() throws Exception {
+ testFloatRange(8);
+ }
+
+ public void testFloatRange_4bit() throws Exception {
+ testFloatRange(4);
+ }
+
+ public void testFloatRange_2bit() throws Exception {
+ testFloatRange(2);
+ }
+
+ private void testSorting(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="field"+precisionStep;
+ // 10 random tests, the index order is ascending,
+ // so using a reverse sort field should retun descending documents
+ for (int i=0; i<10; i++) {
+ int lower=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ int upper=(int)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ if (lower>upper) {
+ int a=lower; lower=upper; upper=a;
+ }
+ Query tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true);
+ TopDocs topDocs = searcher.search(tq, null, noDocs, new Sort(NumericUtils.getIntSortField(field, true)));
+ if (topDocs.totalHits==0) continue;
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ int last=Integer.parseInt(searcher.doc(sd[0].doc).get("value"));
+ for (int j=1; j<sd.length; j++) {
+ int act=Integer.parseInt(searcher.doc(sd[j].doc).get("value"));
+ assertTrue("Docs should be sorted backwards", last>act );
+ last=act;
+ }
+ }
+ }
+
+ public void testSorting_8bit() throws Exception {
+ testSorting(8);
+ }
+
+ public void testSorting_4bit() throws Exception {
+ testSorting(4);
+ }
+
+ public void testSorting_2bit() throws Exception {
+ testSorting(2);
+ }
+
+ public void testEqualsAndHash() throws Exception {
+ QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test1", 4, new Integer(10), new Integer(20), true, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test2", 4, new Integer(10), new Integer(20), false, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test3", 4, new Integer(10), new Integer(20), true, false));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test4", 4, new Integer(10), new Integer(20), false, false));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test5", 4, new Integer(10), null, true, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test6", 4, null, new Integer(20), true, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newIntRange("test7", 4, null, null, true, true));
+ QueryUtils.checkEqual(
+ NumericRangeQuery.newIntRange("test8", 4, new Integer(10), new Integer(20), true, true),
+ NumericRangeQuery.newIntRange("test8", 4, new Integer(10), new Integer(20), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newIntRange("test9", 4, new Integer(10), new Integer(20), true, true),
+ NumericRangeQuery.newIntRange("test9", 8, new Integer(10), new Integer(20), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newIntRange("test10a", 4, new Integer(10), new Integer(20), true, true),
+ NumericRangeQuery.newIntRange("test10b", 4, new Integer(10), new Integer(20), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newIntRange("test11", 4, new Integer(10), new Integer(20), true, true),
+ NumericRangeQuery.newIntRange("test11", 4, new Integer(20), new Integer(10), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newIntRange("test12", 4, new Integer(10), new Integer(20), true, true),
+ NumericRangeQuery.newIntRange("test12", 4, new Integer(10), new Integer(20), false, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newIntRange("test13", 4, new Integer(10), new Integer(20), true, true),
+ NumericRangeQuery.newFloatRange("test13", 4, new Float(10f), new Float(20f), true, true)
+ );
+ // the following produces a hash collision, because Long and Integer have the same hashcode, so only test equality:
+ Query q1 = NumericRangeQuery.newIntRange("test14", 4, new Integer(10), new Integer(20), true, true);
+ Query q2 = NumericRangeQuery.newLongRange("test14", 4, new Long(10L), new Long(20L), true, true);
+ assertFalse(q1.equals(q2));
+ assertFalse(q2.equals(q1));
+ }
+
+}
Propchange: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery32.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java?rev=786470&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java Fri Jun 19 12:09:52 2009
@@ -0,0 +1,427 @@
+package org.apache.lucene.search;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+
+import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.NumericUtils;
+
+public class TestNumericRangeQuery64 extends LuceneTestCase {
+ // distance of entries
+ private static final long distance = 66666L;
+ // shift the starting of the values to the left, to also have negative values:
+ private static final long startOffset = - 1L << 31;
+ // number of docs to generate for testing
+ private static final int noDocs = 10000;
+
+ private static Field newField(String name, int precisionStep) {
+ NumericTokenStream stream = new NumericTokenStream(precisionStep);
+ stream.setUseNewAPI(true);
+ Field f=new Field(name, stream);
+ f.setOmitTermFreqAndPositions(true);
+ f.setOmitNorms(true);
+ return f;
+ }
+
+ private static final RAMDirectory directory;
+ private static final IndexSearcher searcher;
+ static {
+ try {
+ // set the theoretical maximum term count for 8bit (see docs for the number)
+ BooleanQuery.setMaxClauseCount(7*255*2 + 255);
+
+ directory = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(),
+ true, MaxFieldLength.UNLIMITED);
+
+ Field
+ field8 = newField("field8", 8),
+ field4 = newField("field4", 4),
+ field2 = newField("field2", 2),
+ ascfield8 = newField("ascfield8", 8),
+ ascfield4 = newField("ascfield4", 4),
+ ascfield2 = newField("ascfield2", 2);
+
+ // Add a series of noDocs docs with increasing long values
+ for (int l=0; l<noDocs; l++) {
+ Document doc=new Document();
+ // add fields, that have a distance to test general functionality
+ long val=distance*l+startOffset;
+ doc.add(new Field("value", Long.toString(val), Field.Store.YES, Field.Index.NO));
+ ((NumericTokenStream)field8.tokenStreamValue()).setLongValue(val);
+ doc.add(field8);
+ ((NumericTokenStream)field4.tokenStreamValue()).setLongValue(val);
+ doc.add(field4);
+ ((NumericTokenStream)field2.tokenStreamValue()).setLongValue(val);
+ doc.add(field2);
+ // add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
+ val=l-(noDocs/2);
+ ((NumericTokenStream)ascfield8.tokenStreamValue()).setLongValue(val);
+ doc.add(ascfield8);
+ ((NumericTokenStream)ascfield4.tokenStreamValue()).setLongValue(val);
+ doc.add(ascfield4);
+ ((NumericTokenStream)ascfield2.tokenStreamValue()).setLongValue(val);
+ doc.add(ascfield2);
+ writer.addDocument(doc);
+ }
+
+ writer.optimize();
+ writer.close();
+ searcher=new IndexSearcher(directory);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ /** test for constant score + boolean query + filter, the other tests only use the constant score mode */
+ private void testRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ long lower=(distance*3/2)+startOffset, upper=lower + count*distance + (distance/3);
+ NumericRangeQuery q = NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+ NumericRangeFilter f = NumericRangeFilter.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+ int lastTerms = 0;
+ for (byte i=0; i<3; i++) {
+ TopDocs topDocs;
+ int terms;
+ String type;
+ q.clearTotalNumberOfTerms();
+ f.clearTotalNumberOfTerms();
+ switch (i) {
+ case 0:
+ type = " (constant score)";
+ q.setConstantScoreRewrite(true);
+ topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ terms = q.getTotalNumberOfTerms();
+ break;
+ case 1:
+ type = " (boolean query)";
+ q.setConstantScoreRewrite(false);
+ topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ terms = q.getTotalNumberOfTerms();
+ break;
+ case 2:
+ type = " (filter)";
+ topDocs = searcher.search(new MatchAllDocsQuery(), f, noDocs, Sort.INDEXORDER);
+ terms = f.getTotalNumberOfTerms();
+ break;
+ default:
+ return;
+ }
+ System.out.println("Found "+terms+" distinct terms in range for field '"+field+"'"+type+".");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score doc count"+type, count, sd.length );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc"+type, 2*distance+startOffset, Long.parseLong(doc.get("value")) );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc"+type, (1+count)*distance+startOffset, Long.parseLong(doc.get("value")) );
+ if (i>0) {
+ assertEquals("Distinct term number is equal for all query types", lastTerms, terms);
+ }
+ lastTerms = terms;
+ }
+ }
+
+ public void testRange_8bit() throws Exception {
+ testRange(8);
+ }
+
+ public void testRange_4bit() throws Exception {
+ testRange(4);
+ }
+
+ public void testRange_2bit() throws Exception {
+ testRange(2);
+ }
+
+ public void testInverseRange() throws Exception {
+ NumericRangeFilter f = NumericRangeFilter.newLongRange("field8", 8, new Long(1000L), new Long(-1000L), true, true);
+ assertSame("A inverse range should return the EMPTY_DOCIDSET instance", DocIdSet.EMPTY_DOCIDSET, f.getDocIdSet(searcher.getIndexReader()));
+ }
+
+ private void testLeftOpenRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ long upper=(count-1)*distance + (distance/3) + startOffset;
+ NumericRangeQuery q=NumericRangeQuery.newLongRange(field, precisionStep, null, new Long(upper), true, true);
+ TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in left open range for field '"+field+"'.");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score doc count", count, sd.length );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc", startOffset, Long.parseLong(doc.get("value")) );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc", (count-1)*distance+startOffset, Long.parseLong(doc.get("value")) );
+ }
+
+ public void testLeftOpenRange_8bit() throws Exception {
+ testLeftOpenRange(8);
+ }
+
+ public void testLeftOpenRange_4bit() throws Exception {
+ testLeftOpenRange(4);
+ }
+
+ public void testLeftOpenRange_2bit() throws Exception {
+ testLeftOpenRange(2);
+ }
+
+ private void testRightOpenRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ long lower=(count-1)*distance + (distance/3) +startOffset;
+ NumericRangeQuery q=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), null, true, true);
+ TopDocs topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER);
+ System.out.println("Found "+q.getTotalNumberOfTerms()+" distinct terms in right open range for field '"+field+"'.");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score doc count", noDocs-count, sd.length );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc", count*distance+startOffset, Long.parseLong(doc.get("value")) );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc", (noDocs-1)*distance+startOffset, Long.parseLong(doc.get("value")) );
+ }
+
+ public void testRightOpenRange_8bit() throws Exception {
+ testRightOpenRange(8);
+ }
+
+ public void testRightOpenRange_4bit() throws Exception {
+ testRightOpenRange(4);
+ }
+
+ public void testRightOpenRange_2bit() throws Exception {
+ testRightOpenRange(2);
+ }
+
+ private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="field"+precisionStep;
+ int termCountT=0,termCountC=0;
+ for (int i=0; i<50; i++) {
+ long lower=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ long upper=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ if (lower>upper) {
+ long a=lower; lower=upper; upper=a;
+ }
+ // test inclusive range
+ NumericRangeQuery tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+ RangeQuery cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, true);
+ cq.setConstantScoreRewrite(true);
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ TopDocs cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ // test exclusive range
+ tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, false);
+ cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, false);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ // test left exclusive range
+ tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, true);
+ cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, true);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ // test right exclusive range
+ tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, false);
+ cq=new RangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, false);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for NumericRangeQuery and RangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits );
+ termCountT += tq.getTotalNumberOfTerms();
+ termCountC += cq.getTotalNumberOfTerms();
+ }
+ System.out.println("Average number of terms during random search on '" + field + "':");
+ System.out.println(" Trie query: " + (((double)termCountT)/(50*4)));
+ System.out.println(" Classical query: " + (((double)termCountC)/(50*4)));
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(8);
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(4);
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(2);
+ }
+
+ private void testRangeSplit(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="ascfield"+precisionStep;
+ // 50 random tests
+ for (int i=0; i<50; i++) {
+ long lower=(long)(rnd.nextDouble()*noDocs - noDocs/2);
+ long upper=(long)(rnd.nextDouble()*noDocs - noDocs/2);
+ if (lower>upper) {
+ long a=lower; lower=upper; upper=a;
+ }
+ // test inclusive range
+ Query tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+ // test exclusive range
+ tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, false);
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to exclusive range length", Math.max(upper-lower-1, 0), tTopDocs.totalHits );
+ // test left exclusive range
+ tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, true);
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+ // test right exclusive range
+ tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, false);
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to half exclusive range length", upper-lower, tTopDocs.totalHits );
+ }
+ }
+
+ public void testRangeSplit_8bit() throws Exception {
+ testRangeSplit(8);
+ }
+
+ public void testRangeSplit_4bit() throws Exception {
+ testRangeSplit(4);
+ }
+
+ public void testRangeSplit_2bit() throws Exception {
+ testRangeSplit(2);
+ }
+
+ /** we fake a double test using long2double conversion of NumericUtils */
+ private void testDoubleRange(int precisionStep) throws Exception {
+ final String field="ascfield"+precisionStep;
+ final long lower=-1000L, upper=+2000L;
+
+ Query tq=NumericRangeQuery.newDoubleRange(field, precisionStep,
+ new Double(NumericUtils.sortableLongToDouble(lower)), new Double(NumericUtils.sortableLongToDouble(upper)), true, true);
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+
+ Filter tf=NumericRangeFilter.newDoubleRange(field, precisionStep,
+ new Double(NumericUtils.sortableLongToDouble(lower)), new Double(NumericUtils.sortableLongToDouble(upper)), true, true);
+ tTopDocs = searcher.search(new MatchAllDocsQuery(), tf, 1);
+ assertEquals("Returned count of range filter must be equal to inclusive range length", upper-lower+1, tTopDocs.totalHits );
+ }
+
+ public void testDoubleRange_8bit() throws Exception {
+ testDoubleRange(8);
+ }
+
+ public void testDoubleRange_4bit() throws Exception {
+ testDoubleRange(4);
+ }
+
+ public void testDoubleRange_2bit() throws Exception {
+ testDoubleRange(2);
+ }
+
+ private void testSorting(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="field"+precisionStep;
+ // 10 random tests, the index order is ascending,
+ // so using a reverse sort field should retun descending documents
+ for (int i=0; i<10; i++) {
+ long lower=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ long upper=(long)(rnd.nextDouble()*noDocs*distance)+startOffset;
+ if (lower>upper) {
+ long a=lower; lower=upper; upper=a;
+ }
+ Query tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true);
+ TopDocs topDocs = searcher.search(tq, null, noDocs, new Sort(NumericUtils.getLongSortField(field, true)));
+ if (topDocs.totalHits==0) continue;
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ long last=Long.parseLong(searcher.doc(sd[0].doc).get("value"));
+ for (int j=1; j<sd.length; j++) {
+ long act=Long.parseLong(searcher.doc(sd[j].doc).get("value"));
+ assertTrue("Docs should be sorted backwards", last>act );
+ last=act;
+ }
+ }
+ }
+
+ public void testSorting_8bit() throws Exception {
+ testSorting(8);
+ }
+
+ public void testSorting_4bit() throws Exception {
+ testSorting(4);
+ }
+
+ public void testSorting_2bit() throws Exception {
+ testSorting(2);
+ }
+
+ public void testEqualsAndHash() throws Exception {
+ QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test1", 4, new Long(10L), new Long(20L), true, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test2", 4, new Long(10L), new Long(20L), false, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test3", 4, new Long(10L), new Long(20L), true, false));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test4", 4, new Long(10L), new Long(20L), false, false));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test5", 4, new Long(10L), null, true, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test6", 4, null, new Long(20L), true, true));
+ QueryUtils.checkHashEquals(NumericRangeQuery.newLongRange("test7", 4, null, null, true, true));
+ QueryUtils.checkEqual(
+ NumericRangeQuery.newLongRange("test8", 4, new Long(10L), new Long(20L), true, true),
+ NumericRangeQuery.newLongRange("test8", 4, new Long(10L), new Long(20L), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newLongRange("test9", 4, new Long(10L), new Long(20L), true, true),
+ NumericRangeQuery.newLongRange("test9", 8, new Long(10L), new Long(20L), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newLongRange("test10a", 4, new Long(10L), new Long(20L), true, true),
+ NumericRangeQuery.newLongRange("test10b", 4, new Long(10L), new Long(20L), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newLongRange("test11", 4, new Long(10L), new Long(20L), true, true),
+ NumericRangeQuery.newLongRange("test11", 4, new Long(20L), new Long(10L), true, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newLongRange("test12", 4, new Long(10L), new Long(20L), true, true),
+ NumericRangeQuery.newLongRange("test12", 4, new Long(10L), new Long(20L), false, true)
+ );
+ QueryUtils.checkUnequal(
+ NumericRangeQuery.newLongRange("test13", 4, new Long(10L), new Long(20L), true, true),
+ NumericRangeQuery.newFloatRange("test13", 4, new Float(10f), new Float(20f), true, true)
+ );
+ // difference to int range is tested in TestNumericRangeQuery32
+ }
+
+}
Propchange: lucene/java/trunk/src/test/org/apache/lucene/search/TestNumericRangeQuery64.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java?rev=786470&view=auto
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java (added)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java Fri Jun 19 12:09:52 2009
@@ -0,0 +1,339 @@
+package org.apache.lucene.util;
+
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.OpenBitSet;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+
+public class TestNumericUtils extends LuceneTestCase {
+
+ public void testLongConversionAndOrdering() throws Exception {
+ // generate a series of encoded longs, each numerical one bigger than the one before
+ String last=null;
+ for (long l=-100000L; l<100000L; l++) {
+ String act=NumericUtils.longToPrefixCoded(l);
+ if (last!=null) {
+ // test if smaller
+ assertTrue("actual bigger than last", last.compareTo(act) < 0 );
+ }
+ // test is back and forward conversion works
+ assertEquals("forward and back conversion should generate same long", l, NumericUtils.prefixCodedToLong(act));
+ // next step
+ last=act;
+ }
+ }
+
+ public void testIntConversionAndOrdering() throws Exception {
+ // generate a series of encoded ints, each numerical one bigger than the one before
+ String last=null;
+ for (int i=-100000; i<100000; i++) {
+ String act=NumericUtils.intToPrefixCoded(i);
+ if (last!=null) {
+ // test if smaller
+ assertTrue("actual bigger than last", last.compareTo(act) < 0 );
+ }
+ // test is back and forward conversion works
+ assertEquals("forward and back conversion should generate same int", i, NumericUtils.prefixCodedToInt(act));
+ // next step
+ last=act;
+ }
+ }
+
+ public void testLongSpecialValues() throws Exception {
+ long[] vals=new long[]{
+ Long.MIN_VALUE, Long.MIN_VALUE+1, Long.MIN_VALUE+2, -5003400000000L,
+ -4000L, -3000L, -2000L, -1000L, -1L, 0L, 1L, 10L, 300L, 50006789999999999L, Long.MAX_VALUE-2, Long.MAX_VALUE-1, Long.MAX_VALUE
+ };
+ String[] prefixVals=new String[vals.length];
+
+ for (int i=0; i<vals.length; i++) {
+ prefixVals[i]=NumericUtils.longToPrefixCoded(vals[i]);
+
+ // check forward and back conversion
+ assertEquals( "forward and back conversion should generate same long", vals[i], NumericUtils.prefixCodedToLong(prefixVals[i]) );
+
+ // test if decoding values as int fails correctly
+ try {
+ NumericUtils.prefixCodedToInt(prefixVals[i]);
+ fail("decoding a prefix coded long value as int should fail");
+ } catch (NumberFormatException e) {
+ // worked
+ }
+ }
+
+ // check sort order (prefixVals should be ascending)
+ for (int i=1; i<prefixVals.length; i++) {
+ assertTrue( "check sort order", prefixVals[i-1].compareTo( prefixVals[i] ) < 0 );
+ }
+
+ // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
+ for (int i=0; i<vals.length; i++) {
+ for (int j=0; j<64; j++) {
+ long prefixVal=NumericUtils.prefixCodedToLong(NumericUtils.longToPrefixCoded(vals[i], j));
+ long mask=(1L << j) - 1L;
+ assertEquals( "difference between prefix val and original value for "+vals[i]+" with shift="+j, vals[i] & mask, vals[i]-prefixVal );
+ }
+ }
+ }
+
+ public void testIntSpecialValues() throws Exception {
+ int[] vals=new int[]{
+ Integer.MIN_VALUE, Integer.MIN_VALUE+1, Integer.MIN_VALUE+2, -64765767,
+ -4000, -3000, -2000, -1000, -1, 0, 1, 10, 300, 765878989, Integer.MAX_VALUE-2, Integer.MAX_VALUE-1, Integer.MAX_VALUE
+ };
+ String[] prefixVals=new String[vals.length];
+
+ for (int i=0; i<vals.length; i++) {
+ prefixVals[i]=NumericUtils.intToPrefixCoded(vals[i]);
+
+ // check forward and back conversion
+ assertEquals( "forward and back conversion should generate same int", vals[i], NumericUtils.prefixCodedToInt(prefixVals[i]) );
+
+ // test if decoding values as long fails correctly
+ try {
+ NumericUtils.prefixCodedToLong(prefixVals[i]);
+ fail("decoding a prefix coded int value as long should fail");
+ } catch (NumberFormatException e) {
+ // worked
+ }
+ }
+
+ // check sort order (prefixVals should be ascending)
+ for (int i=1; i<prefixVals.length; i++) {
+ assertTrue( "check sort order", prefixVals[i-1].compareTo( prefixVals[i] ) < 0 );
+ }
+
+ // check the prefix encoding, lower precision should have the difference to original value equal to the lower removed bits
+ for (int i=0; i<vals.length; i++) {
+ for (int j=0; j<32; j++) {
+ int prefixVal=NumericUtils.prefixCodedToInt(NumericUtils.intToPrefixCoded(vals[i], j));
+ int mask=(1 << j) - 1;
+ assertEquals( "difference between prefix val and original value for "+vals[i]+" with shift="+j, vals[i] & mask, vals[i]-prefixVal );
+ }
+ }
+ }
+
+ public void testDoubles() throws Exception {
+ double[] vals=new double[]{
+ Double.NEGATIVE_INFINITY, -2.3E25, -1.0E15, -1.0, -1.0E-1, -1.0E-2, -0.0,
+ +0.0, 1.0E-2, 1.0E-1, 1.0, 1.0E15, 2.3E25, Double.POSITIVE_INFINITY
+ };
+ long[] longVals=new long[vals.length];
+
+ // check forward and back conversion
+ for (int i=0; i<vals.length; i++) {
+ longVals[i]=NumericUtils.doubleToSortableLong(vals[i]);
+ assertTrue( "forward and back conversion should generate same double", Double.compare(vals[i], NumericUtils.sortableLongToDouble(longVals[i]))==0 );
+ }
+
+ // check sort order (prefixVals should be ascending)
+ for (int i=1; i<longVals.length; i++) {
+ assertTrue( "check sort order", longVals[i-1] < longVals[i] );
+ }
+ }
+
+ public void testFloats() throws Exception {
+ float[] vals=new float[]{
+ Float.NEGATIVE_INFINITY, -2.3E25f, -1.0E15f, -1.0f, -1.0E-1f, -1.0E-2f, -0.0f,
+ +0.0f, 1.0E-2f, 1.0E-1f, 1.0f, 1.0E15f, 2.3E25f, Float.POSITIVE_INFINITY
+ };
+ int[] intVals=new int[vals.length];
+
+ // check forward and back conversion
+ for (int i=0; i<vals.length; i++) {
+ intVals[i]=NumericUtils.floatToSortableInt(vals[i]);
+ assertTrue( "forward and back conversion should generate same double", Float.compare(vals[i], NumericUtils.sortableIntToFloat(intVals[i]))==0 );
+ }
+
+ // check sort order (prefixVals should be ascending)
+ for (int i=1; i<intVals.length; i++) {
+ assertTrue( "check sort order", intVals[i-1] < intVals[i] );
+ }
+ }
+
+ // INFO: Tests for trieCodeLong()/trieCodeInt() not needed because implicitely tested by range filter tests
+
+ /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
+ protected void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
+ final boolean useBitSet, final Iterator neededBounds
+ ) throws Exception {
+ final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
+
+ NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() {
+ //@Override
+ public void addRange(long min, long max, int shift) {
+ assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
+ if (useBitSet) for (long l=min; l<=max; l++) {
+ assertFalse("ranges should not overlap", bits.getAndSet(l-lower) );
+ }
+ // make unsigned longs for easier display and understanding
+ min ^= 0x8000000000000000L;
+ max ^= 0x8000000000000000L;
+ //System.out.println("new Long(0x"+Long.toHexString(min>>>shift)+"L),new Long(0x"+Long.toHexString(max>>>shift)+"L),");
+ assertEquals( "inner min bound", ((Long)neededBounds.next()).longValue(), min>>>shift);
+ assertEquals( "inner max bound", ((Long)neededBounds.next()).longValue(), max>>>shift);
+ }
+ }, precisionStep, lower, upper);
+
+ if (useBitSet) {
+ // after flipping all bits in the range, the cardinality should be zero
+ bits.flip(0,upper-lower+1);
+ assertTrue("The sub-range concenated should match the whole range", bits.isEmpty());
+ }
+ }
+
+ public void testSplitLongRange() throws Exception {
+ // a hard-coded "standard" range
+ assertLongRangeSplit(-5000L, 9500L, 4, true, Arrays.asList(new Long[]{
+ new Long(0x7fffffffffffec78L),new Long(0x7fffffffffffec7fL),
+ new Long(0x8000000000002510L),new Long(0x800000000000251cL),
+ new Long(0x7fffffffffffec8L), new Long(0x7fffffffffffecfL),
+ new Long(0x800000000000250L), new Long(0x800000000000250L),
+ new Long(0x7fffffffffffedL), new Long(0x7fffffffffffefL),
+ new Long(0x80000000000020L), new Long(0x80000000000024L),
+ new Long(0x7ffffffffffffL), new Long(0x8000000000001L)
+ }).iterator());
+
+ // the same with no range splitting
+ assertLongRangeSplit(-5000L, 9500L, 64, true, Arrays.asList(new Long[]{
+ new Long(0x7fffffffffffec78L),new Long(0x800000000000251cL)
+ }).iterator());
+
+ // this tests optimized range splitting, if one of the inner bounds
+ // is also the bound of the next lower precision, it should be used completely
+ assertLongRangeSplit(0L, 1024L+63L, 4, true, Arrays.asList(new Long[]{
+ new Long(0x800000000000040L), new Long(0x800000000000043L),
+ new Long(0x80000000000000L), new Long(0x80000000000003L)
+ }).iterator());
+
+ // the full long range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 8, false, Arrays.asList(new Long[]{
+ new Long(0x00L),new Long(0xffL)
+ }).iterator());
+
+ // the same with precisionStep=4
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 4, false, Arrays.asList(new Long[]{
+ new Long(0x0L),new Long(0xfL)
+ }).iterator());
+
+ // the same with precisionStep=2
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 2, false, Arrays.asList(new Long[]{
+ new Long(0x0L),new Long(0x3L)
+ }).iterator());
+
+ // the same with precisionStep=1
+ assertLongRangeSplit(Long.MIN_VALUE, Long.MAX_VALUE, 1, false, Arrays.asList(new Long[]{
+ new Long(0x0L),new Long(0x1L)
+ }).iterator());
+
+ // a inverse range should produce no sub-ranges
+ assertLongRangeSplit(9500L, -5000L, 4, false, Collections.EMPTY_LIST.iterator());
+
+ // a 0-length range should reproduce the range itsself
+ assertLongRangeSplit(9500L, 9500L, 4, false, Arrays.asList(new Long[]{
+ new Long(0x800000000000251cL),new Long(0x800000000000251cL)
+ }).iterator());
+ }
+
+ /** Note: The neededBounds iterator must be unsigned (easier understanding what's happening) */
+ protected void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
+ final boolean useBitSet, final Iterator neededBounds
+ ) throws Exception {
+ final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null;
+
+ NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() {
+ //@Override
+ public void addRange(int min, int max, int shift) {
+ assertTrue("min, max should be inside bounds", min>=lower && min<=upper && max>=lower && max<=upper);
+ if (useBitSet) for (int i=min; i<=max; i++) {
+ assertFalse("ranges should not overlap", bits.getAndSet(i-lower) );
+ }
+ // make unsigned ints for easier display and understanding
+ min ^= 0x80000000;
+ max ^= 0x80000000;
+ //System.out.println("new Integer(0x"+Integer.toHexString(min>>>shift)+"),new Integer(0x"+Integer.toHexString(max>>>shift)+"),");
+ assertEquals( "inner min bound", ((Integer)neededBounds.next()).intValue(), min>>>shift);
+ assertEquals( "inner max bound", ((Integer)neededBounds.next()).intValue(), max>>>shift);
+ }
+ }, precisionStep, lower, upper);
+
+ if (useBitSet) {
+ // after flipping all bits in the range, the cardinality should be zero
+ bits.flip(0,upper-lower+1);
+ assertTrue("The sub-range concenated should match the whole range", bits.isEmpty());
+ }
+ }
+
+ public void testSplitIntRange() throws Exception {
+ // a hard-coded "standard" range
+ assertIntRangeSplit(-5000, 9500, 4, true, Arrays.asList(new Integer[]{
+ new Integer(0x7fffec78),new Integer(0x7fffec7f),
+ new Integer(0x80002510),new Integer(0x8000251c),
+ new Integer(0x7fffec8), new Integer(0x7fffecf),
+ new Integer(0x8000250), new Integer(0x8000250),
+ new Integer(0x7fffed), new Integer(0x7fffef),
+ new Integer(0x800020), new Integer(0x800024),
+ new Integer(0x7ffff), new Integer(0x80001)
+ }).iterator());
+
+ // the same with no range splitting
+ assertIntRangeSplit(-5000, 9500, 32, true, Arrays.asList(new Integer[]{
+ new Integer(0x7fffec78),new Integer(0x8000251c)
+ }).iterator());
+
+ // this tests optimized range splitting, if one of the inner bounds
+ // is also the bound of the next lower precision, it should be used completely
+ assertIntRangeSplit(0, 1024+63, 4, true, Arrays.asList(new Integer[]{
+ new Integer(0x8000040), new Integer(0x8000043),
+ new Integer(0x800000), new Integer(0x800003)
+ }).iterator());
+
+ // the full int range should only consist of a lowest precision range; no bitset testing here, as too much memory needed :-)
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 8, false, Arrays.asList(new Integer[]{
+ new Integer(0x00),new Integer(0xff)
+ }).iterator());
+
+ // the same with precisionStep=4
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 4, false, Arrays.asList(new Integer[]{
+ new Integer(0x0),new Integer(0xf)
+ }).iterator());
+
+ // the same with precisionStep=2
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 2, false, Arrays.asList(new Integer[]{
+ new Integer(0x0),new Integer(0x3)
+ }).iterator());
+
+ // the same with precisionStep=1
+ assertIntRangeSplit(Integer.MIN_VALUE, Integer.MAX_VALUE, 1, false, Arrays.asList(new Integer[]{
+ new Integer(0x0),new Integer(0x1)
+ }).iterator());
+
+ // a inverse range should produce no sub-ranges
+ assertIntRangeSplit(9500, -5000, 4, false, Collections.EMPTY_LIST.iterator());
+
+ // a 0-length range should reproduce the range itsself
+ assertIntRangeSplit(9500, 9500, 4, false, Arrays.asList(new Integer[]{
+ new Integer(0x8000251c),new Integer(0x8000251c)
+ }).iterator());
+ }
+
+}
Propchange: lucene/java/trunk/src/test/org/apache/lucene/util/TestNumericUtils.java
------------------------------------------------------------------------------
svn:eol-style = native