You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/08 04:26:21 UTC
svn commit: r1443834 [11/16] - in /lucene/dev/branches/branch_4x: ./
dev-tools/ lucene/ lucene/analysis/
lucene/analysis/icu/src/java/org/apache/lucene/collation/
lucene/analysis/icu/src/test/org/apache/lucene/collation/ lucene/backwards/
lucene/benchm...
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/TestSort.java Fri Feb 8 03:26:14 2013
@@ -31,20 +31,18 @@ import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.document.DerefBytesDocValuesField;
+import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.PackedLongDocValuesField;
-import org.apache.lucene.document.SortedBytesDocValuesField;
-import org.apache.lucene.document.StraightBytesDocValuesField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -65,7 +63,6 @@ import org.apache.lucene.util.LuceneTest
import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
import org.junit.BeforeClass;
-
/**
* Unit tests for sorting code.
*
@@ -127,24 +124,23 @@ public class TestSort extends LuceneTest
};
// create an index of all the documents, or just the x, or just the y documents
- private IndexSearcher getIndex (boolean even, boolean odd)
+ private IndexSearcher getIndex(boolean even, boolean odd)
throws IOException {
Directory indexStore = newDirectory();
dirs.add(indexStore);
RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- final DocValues.Type stringDVType;
+ final DocValuesType stringDVType;
if (dvStringSorted) {
// Index sorted
- stringDVType = random().nextBoolean() ? DocValues.Type.BYTES_VAR_SORTED : DocValues.Type.BYTES_FIXED_SORTED;
+ stringDVType = DocValuesType.SORTED;
} else {
- // Index non-sorted
if (random().nextBoolean()) {
- // Fixed
- stringDVType = random().nextBoolean() ? DocValues.Type.BYTES_FIXED_STRAIGHT : DocValues.Type.BYTES_FIXED_DEREF;
+ // Index non-sorted
+ stringDVType = DocValuesType.BINARY;
} else {
- // Var
- stringDVType = random().nextBoolean() ? DocValues.Type.BYTES_VAR_STRAIGHT : DocValues.Type.BYTES_VAR_DEREF;
+ // sorted anyway
+ stringDVType = DocValuesType.SORTED;
}
}
@@ -152,62 +148,63 @@ public class TestSort extends LuceneTest
ft1.setStored(true);
FieldType ft2 = new FieldType();
ft2.setIndexed(true);
- for (int i=0; i<data.length; ++i) {
+ for(int i=0; i<data.length; ++i) {
if (((i%2)==0 && even) || ((i%2)==1 && odd)) {
Document doc = new Document();
- doc.add (new Field ("tracer", data[i][0], ft1));
- doc.add (new TextField ("contents", data[i][1], Field.Store.NO));
+ doc.add(new Field("tracer", data[i][0], ft1));
+ doc.add(new TextField("contents", data[i][1], Field.Store.NO));
if (data[i][2] != null) {
doc.add(new StringField ("int", data[i][2], Field.Store.NO));
if (supportsDocValues) {
- doc.add(new PackedLongDocValuesField("int", Integer.parseInt(data[i][2])));
+ doc.add(new NumericDocValuesField("int_dv", Integer.parseInt(data[i][2])));
}
}
if (data[i][3] != null) {
doc.add(new StringField ("float", data[i][3], Field.Store.NO));
if (supportsDocValues) {
- doc.add(new FloatDocValuesField("float", Float.parseFloat(data[i][3])));
+ doc.add(new FloatDocValuesField("float_dv", Float.parseFloat(data[i][3])));
}
}
if (data[i][4] != null) {
- doc.add(new StringField ("string", data[i][4], Field.Store.NO));
+ doc.add(new StringField("string", data[i][4], Field.Store.NO));
if (supportsDocValues) {
switch(stringDVType) {
- case BYTES_FIXED_SORTED:
- doc.add(new SortedBytesDocValuesField("string", new BytesRef(data[i][4]), true));
- break;
- case BYTES_VAR_SORTED:
- doc.add(new SortedBytesDocValuesField("string", new BytesRef(data[i][4]), false));
- break;
- case BYTES_FIXED_STRAIGHT:
- doc.add(new StraightBytesDocValuesField("string", new BytesRef(data[i][4]), true));
- break;
- case BYTES_VAR_STRAIGHT:
- doc.add(new StraightBytesDocValuesField("string", new BytesRef(data[i][4]), false));
- break;
- case BYTES_FIXED_DEREF:
- doc.add(new DerefBytesDocValuesField("string", new BytesRef(data[i][4]), true));
+ case SORTED:
+ doc.add(new SortedDocValuesField("string_dv", new BytesRef(data[i][4])));
break;
- case BYTES_VAR_DEREF:
- doc.add(new DerefBytesDocValuesField("string", new BytesRef(data[i][4]), false));
+ case BINARY:
+ doc.add(new BinaryDocValuesField("string_dv", new BytesRef(data[i][4])));
break;
default:
throw new IllegalStateException("unknown type " + stringDVType);
+ }
+ } else {
+ if (supportsDocValues) {
+ switch(stringDVType) {
+ case SORTED:
+ doc.add(new SortedDocValuesField("string_dv", new BytesRef()));
+ break;
+ case BINARY:
+ doc.add(new BinaryDocValuesField("string_dv", new BytesRef()));
+ break;
+ default:
+ throw new IllegalStateException("unknown type " + stringDVType);
+ }
}
}
}
- if (data[i][5] != null) doc.add (new StringField ("custom", data[i][5], Field.Store.NO));
- if (data[i][6] != null) doc.add (new StringField ("i18n", data[i][6], Field.Store.NO));
- if (data[i][7] != null) doc.add (new StringField ("long", data[i][7], Field.Store.NO));
+ if (data[i][5] != null) doc.add(new StringField("custom", data[i][5], Field.Store.NO));
+ if (data[i][6] != null) doc.add(new StringField("i18n", data[i][6], Field.Store.NO));
+ if (data[i][7] != null) doc.add(new StringField("long", data[i][7], Field.Store.NO));
if (data[i][8] != null) {
doc.add(new StringField ("double", data[i][8], Field.Store.NO));
if (supportsDocValues) {
- doc.add(new DoubleDocValuesField("double", Double.parseDouble(data[i][8])));
+ doc.add(new NumericDocValuesField("double_dv", Double.doubleToRawLongBits(Double.parseDouble(data[i][8]))));
}
}
- if (data[i][9] != null) doc.add (new StringField ("short", data[i][9], Field.Store.NO));
- if (data[i][10] != null) doc.add (new StringField ("byte", data[i][10], Field.Store.NO));
- if (data[i][11] != null) doc.add (new StringField ("parser", data[i][11], Field.Store.NO));
+ if (data[i][9] != null) doc.add(new StringField("short", data[i][9], Field.Store.NO));
+ if (data[i][10] != null) doc.add(new StringField("byte", data[i][10], Field.Store.NO));
+ if (data[i][11] != null) doc.add(new StringField("parser", data[i][11], Field.Store.NO));
for(IndexableField f : doc.getFields()) {
if (f.fieldType().indexed() && !f.fieldType().omitNorms()) {
@@ -215,9 +212,10 @@ public class TestSort extends LuceneTest
}
}
- writer.addDocument (doc);
+ writer.addDocument(doc);
}
}
+
IndexReader reader = writer.getReader();
writer.close ();
IndexSearcher s = newSearcher(reader);
@@ -226,34 +224,41 @@ public class TestSort extends LuceneTest
private IndexSearcher getFullIndex()
throws IOException {
- return getIndex (true, true);
+ return getIndex(true, true);
}
private IndexSearcher getFullStrings() throws IOException {
Directory indexStore = newDirectory();
dirs.add(indexStore);
IndexWriter writer = new IndexWriter(
- indexStore,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
- setMergePolicy(newLogMergePolicy(97))
- );
+ indexStore,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
+ setMergePolicy(newLogMergePolicy(97)));
FieldType onlyStored = new FieldType();
onlyStored.setStored(true);
final int fixedLen = getRandomNumber(2, 8);
final int fixedLen2 = getRandomNumber(1, 4);
- for (int i=0; i<NUM_STRINGS; i++) {
+ for(int i=0; i<NUM_STRINGS; i++) {
Document doc = new Document();
String num = getRandomCharString(getRandomNumber(2, 8), 48, 52);
- doc.add (new Field ("tracer", num, onlyStored));
- //doc.add (new Field ("contents", Integer.toString(i), Field.Store.NO, Field.Index.ANALYZED));
+ doc.add(new Field("tracer", num, onlyStored));
+ //doc.add(new Field("contents", Integer.toString(i), Field.Store.NO, Field.Index.ANALYZED));
doc.add(new StringField("string", num, Field.Store.NO));
if (supportsDocValues) {
- doc.add(new SortedBytesDocValuesField("string", new BytesRef(num)));
+ if (dvStringSorted) {
+ doc.add(new SortedDocValuesField("string_dv", new BytesRef(num)));
+ } else {
+ doc.add(new BinaryDocValuesField("string_dv", new BytesRef(num)));
+ }
}
String num2 = getRandomCharString(getRandomNumber(1, 4), 48, 50);
doc.add(new StringField ("string2", num2, Field.Store.NO));
if (supportsDocValues) {
- doc.add(new SortedBytesDocValuesField("string2", new BytesRef(num2)));
+ if (dvStringSorted) {
+ doc.add(new SortedDocValuesField("string2_dv", new BytesRef(num2)));
+ } else {
+ doc.add(new BinaryDocValuesField("string2_dv", new BytesRef(num2)));
+ }
}
doc.add (new Field ("tracer2", num2, onlyStored));
for(IndexableField f2 : doc.getFields()) {
@@ -263,16 +268,24 @@ public class TestSort extends LuceneTest
}
String numFixed = getRandomCharString(fixedLen, 48, 52);
- doc.add (new Field ("fixed_tracer", numFixed, onlyStored));
- //doc.add (new Field ("contents", Integer.toString(i), Field.Store.NO, Field.Index.ANALYZED));
+ doc.add(new Field("tracer_fixed", numFixed, onlyStored));
+ //doc.add(new Field("contents", Integer.toString(i), Field.Store.NO, Field.Index.ANALYZED));
doc.add(new StringField("string_fixed", numFixed, Field.Store.NO));
if (supportsDocValues) {
- doc.add(new SortedBytesDocValuesField("string_fixed", new BytesRef(numFixed), true));
+ if (dvStringSorted) {
+ doc.add(new SortedDocValuesField("string_fixed_dv", new BytesRef(numFixed)));
+ } else {
+ doc.add(new BinaryDocValuesField("string_fixed_dv", new BytesRef(numFixed)));
+ }
}
String num2Fixed = getRandomCharString(fixedLen2, 48, 52);
doc.add(new StringField ("string2_fixed", num2Fixed, Field.Store.NO));
if (supportsDocValues) {
- doc.add(new SortedBytesDocValuesField("string2_fixed", new BytesRef(num2Fixed), true));
+ if (dvStringSorted) {
+ doc.add(new SortedDocValuesField("string2_fixed_dv", new BytesRef(num2Fixed)));
+ } else {
+ doc.add(new BinaryDocValuesField("string2_fixed_dv", new BytesRef(num2Fixed)));
+ }
}
doc.add (new Field ("tracer2_fixed", num2Fixed, onlyStored));
@@ -282,18 +295,47 @@ public class TestSort extends LuceneTest
}
}
- writer.addDocument (doc);
+ writer.addDocument(doc);
}
- //writer.forceMerge(1);
//System.out.println(writer.getSegmentCount());
writer.close();
IndexReader reader = DirectoryReader.open(indexStore);
- return newSearcher(reader);
+ IndexSearcher searcher = newSearcher(reader);
+
+ /*
+ for(int docID=0;docID<reader.maxDoc();docID++) {
+ StoredDocument doc = reader.document(docID);
+ String s = doc.get("tracer");
+ TopDocs hits = searcher.search(new TermQuery(new Term("string", s)), NUM_STRINGS);
+ System.out.println("string=" + s + " has " + hits.totalHits + " docs");
+ boolean found = false;
+ for(int hit=0;!found && hit<hits.totalHits;hit++) {
+ if (hits.scoreDocs[hit].doc == docID) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue(found);
+ s = doc.get("tracer2");
+ hits = searcher.search(new TermQuery(new Term("string2", s)), NUM_STRINGS);
+ System.out.println("string2=" + s + " has " + hits.totalHits + " docs");
+ found = false;
+ for(int hit=0;!found && hit<hits.totalHits;hit++) {
+ if (hits.scoreDocs[hit].doc == docID) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue(found);
+ }
+ */
+
+ return searcher;
}
public String getRandomNumberString(int num, int low, int high) {
StringBuilder sb = new StringBuilder();
- for (int i = 0; i < num; i++) {
+ for(int i = 0; i < num; i++) {
sb.append(getRandomNumber(low, high));
}
return sb.toString();
@@ -305,7 +347,7 @@ public class TestSort extends LuceneTest
public String getRandomCharString(int num, int start, int end) {
StringBuilder sb = new StringBuilder();
- for (int i = 0; i < num; i++) {
+ for(int i = 0; i < num; i++) {
sb.append(new Character((char) getRandomNumber(start, end)));
}
return sb.toString();
@@ -320,17 +362,17 @@ public class TestSort extends LuceneTest
private IndexSearcher getXIndex()
throws IOException {
- return getIndex (true, false);
+ return getIndex(true, false);
}
private IndexSearcher getYIndex()
throws IOException {
- return getIndex (false, true);
+ return getIndex(false, true);
}
private IndexSearcher getEmptyIndex()
throws IOException {
- return getIndex (false, false);
+ return getIndex(false, false);
}
// Set to true if the DV "string" field is indexed as a
@@ -345,13 +387,13 @@ public class TestSort extends LuceneTest
full = getFullIndex();
searchX = getXIndex();
searchY = getYIndex();
- queryX = new TermQuery (new Term ("contents", "x"));
- queryY = new TermQuery (new Term ("contents", "y"));
- queryA = new TermQuery (new Term ("contents", "a"));
- queryE = new TermQuery (new Term ("contents", "e"));
- queryF = new TermQuery (new Term ("contents", "f"));
- queryG = new TermQuery (new Term ("contents", "g"));
- queryM = new TermQuery (new Term ("contents", "m"));
+ queryX = new TermQuery(new Term("contents", "x"));
+ queryY = new TermQuery(new Term("contents", "y"));
+ queryA = new TermQuery(new Term("contents", "a"));
+ queryE = new TermQuery(new Term("contents", "e"));
+ queryF = new TermQuery(new Term("contents", "f"));
+ queryG = new TermQuery(new Term("contents", "g"));
+ queryM = new TermQuery(new Term("contents", "m"));
sort = new Sort();
}
@@ -363,77 +405,78 @@ public class TestSort extends LuceneTest
full.reader.close();
searchX.reader.close();
searchY.reader.close();
- for (Directory dir : dirs)
+ for(Directory dir : dirs) {
dir.close();
+ }
super.tearDown();
}
// test the sorts by score and document number
public void testBuiltInSorts() throws Exception {
sort = new Sort();
- assertMatches (full, queryX, sort, "ACEGI");
- assertMatches (full, queryY, sort, "BDFHJ");
+ assertMatches(full, queryX, sort, "ACEGI");
+ assertMatches(full, queryY, sort, "BDFHJ");
sort.setSort(SortField.FIELD_DOC);
- assertMatches (full, queryX, sort, "ACEGI");
- assertMatches (full, queryY, sort, "BDFHJ");
+ assertMatches(full, queryX, sort, "ACEGI");
+ assertMatches(full, queryY, sort, "BDFHJ");
}
- private static SortField useDocValues(SortField field) {
- field.setUseIndexValues(true);
- return field;
- }
// test sorts where the type of field is specified
public void testTypedSort() throws Exception {
- sort.setSort (new SortField ("int", SortField.Type.INT), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "IGAEC");
- assertMatches (full, queryY, sort, "DHFJB");
-
- sort.setSort (new SortField ("float", SortField.Type.FLOAT), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "GCIEA");
- assertMatches (full, queryY, sort, "DHJFB");
-
- sort.setSort (new SortField ("long", SortField.Type.LONG), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "EACGI");
- assertMatches (full, queryY, sort, "FBJHD");
-
- sort.setSort (new SortField ("double", SortField.Type.DOUBLE), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "AGICE");
- assertMatches (full, queryY, sort, "DJHBF");
-
- sort.setSort (new SortField ("byte", SortField.Type.BYTE), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "CIGAE");
- assertMatches (full, queryY, sort, "DHFBJ");
-
- sort.setSort (new SortField ("short", SortField.Type.SHORT), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "IAGCE");
- assertMatches (full, queryY, sort, "DFHBJ");
-
- sort.setSort (new SortField ("string", SortField.Type.STRING), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "AIGEC");
- assertMatches (full, queryY, sort, "DJHFB");
+ sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "IGAEC");
+ assertMatches(full, queryY, sort, "DHFJB");
+
+ sort.setSort(new SortField("float", SortField.Type.FLOAT), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "GCIEA");
+ assertMatches(full, queryY, sort, "DHJFB");
+
+ sort.setSort(new SortField("long", SortField.Type.LONG), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "EACGI");
+ assertMatches(full, queryY, sort, "FBJHD");
+
+ sort.setSort(new SortField("double", SortField.Type.DOUBLE), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "AGICE");
+ assertMatches(full, queryY, sort, "DJHBF");
+
+ sort.setSort(new SortField("byte", SortField.Type.BYTE), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "CIGAE");
+ assertMatches(full, queryY, sort, "DHFBJ");
+
+ sort.setSort(new SortField("short", SortField.Type.SHORT), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "IAGCE");
+ assertMatches(full, queryY, sort, "DFHBJ");
+
+ sort.setSort(new SortField("string", SortField.Type.STRING), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "AIGEC");
+ assertMatches(full, queryY, sort, "DJHFB");
if (supportsDocValues) {
- sort.setSort (useDocValues(new SortField ("int", SortField.Type.INT)), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "IGAEC");
- assertMatches (full, queryY, sort, "DHFJB");
-
- sort.setSort (useDocValues(new SortField ("float", SortField.Type.FLOAT)), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "GCIEA");
- assertMatches (full, queryY, sort, "DHJFB");
+ sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "IGAEC");
+ assertMatches(full, queryY, sort, "DHFJB");
+
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "GCIEA");
+ assertMatches(full, queryY, sort, "DHJFB");
- sort.setSort (useDocValues(new SortField ("double", SortField.Type.DOUBLE)), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "AGICE");
- assertMatches (full, queryY, sort, "DJHBF");
-
- sort.setSort (useDocValues(new SortField ("string", getDVStringSortType())), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "AIGEC");
- assertMatches (full, queryY, sort, "DJHFB");
+ sort.setSort(new SortField("double_dv", SortField.Type.DOUBLE), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "AGICE");
+ assertMatches(full, queryY, sort, "DJHBF");
+
+ sort.setSort(new SortField("string_dv", getDVStringSortType()), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "AIGEC");
+ assertMatches(full, queryY, sort, "DJHFB");
}
}
private SortField.Type getDVStringSortType() {
- if (dvStringSorted) {
+ return getDVStringSortType(true);
+ }
+
+ private SortField.Type getDVStringSortType(boolean allowSorted) {
+ if (dvStringSorted && allowSorted) {
// If you index as sorted source you can still sort by
// value instead:
return random().nextBoolean() ? SortField.Type.STRING : SortField.Type.STRING_VAL;
@@ -524,20 +567,23 @@ public class TestSort extends LuceneTest
assumeFalse("cannot work with preflex codec",
"Lucene3x".equals(Codec.getDefault().getName()));
sort.setSort(
- useDocValues(new SortField("string", getDVStringSortType())),
- useDocValues(new SortField("string2", getDVStringSortType(), true)),
+ new SortField("string_dv", getDVStringSortType()),
+ new SortField("string2_dv", getDVStringSortType(), true),
SortField.FIELD_DOC);
verifyStringSort(sort);
// Doc values field, fixed length
sort.setSort(
- useDocValues(new SortField("string_fixed", getDVStringSortType())),
- useDocValues(new SortField("string2_fixed", getDVStringSortType(), true)),
+ new SortField("string_fixed_dv", getDVStringSortType()),
+ new SortField("string2_fixed_dv", getDVStringSortType(), true),
SortField.FIELD_DOC);
verifyStringSort(sort);
}
private void verifyStringSort(Sort sort) throws Exception {
+ if (VERBOSE) {
+ System.out.println("verifySort sort=" + sort);
+ }
final IndexSearcher searcher = getFullStrings();
final ScoreDoc[] result = searcher.search(new MatchAllDocsQuery(), null, _TestUtil.nextInt(random(), 500, searcher.getIndexReader().maxDoc()), sort).scoreDocs;
StringBuilder buff = new StringBuilder();
@@ -546,13 +592,13 @@ public class TestSort extends LuceneTest
String lastSub = null;
int lastDocId = 0;
boolean fail = false;
- final String fieldSuffix = sort.getSort()[0].getField().endsWith("_fixed") ? "_fixed" : "";
- for (int x = 0; x < n; ++x) {
+ final String fieldSuffix = (sort.getSort()[0].getField().indexOf("_fixed") != -1) ? "_fixed" : "";
+ for(int x = 0; x < n; ++x) {
Document doc2 = searcher.doc(result[x].doc);
IndexableField[] v = doc2.getFields("tracer" + fieldSuffix);
IndexableField[] v2 = doc2.getFields("tracer2" + fieldSuffix);
- for (int j = 0; j < v.length; ++j) {
- buff.append(v[j] + "(" + v2[j] + ")(" + result[x].doc+")\n");
+ for(int j = 0; j < v.length; ++j) {
+ buff.append(v[j].stringValue() + "(" + v2[j].stringValue() + ")(" + result[x].doc+")\n");
if (last != null) {
int cmp = v[j].stringValue().compareTo(last);
if (!(cmp >= 0)) { // ensure first field is in order
@@ -599,7 +645,7 @@ public class TestSort extends LuceneTest
FieldCache fc = FieldCache.DEFAULT;
- sort.setSort (new SortField ("parser", new FieldCache.IntParser(){
+ sort.setSort(new SortField("parser", new FieldCache.IntParser(){
@Override
public final int parseInt(final BytesRef term) {
return (term.bytes[term.offset]-'A') * 123456;
@@ -614,10 +660,10 @@ public class TestSort extends LuceneTest
assertSaneFieldCaches(getTestName() + " IntParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.FloatParser(){
+ sort.setSort(new SortField("parser", new FieldCache.FloatParser(){
@Override
public final float parseFloat(final BytesRef term) {
- return (float) Math.sqrt( term.bytes[term.offset] );
+ return (float) Math.sqrt( term.bytes[term.offset]);
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
@@ -628,7 +674,7 @@ public class TestSort extends LuceneTest
assertSaneFieldCaches(getTestName() + " FloatParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.LongParser(){
+ sort.setSort(new SortField("parser", new FieldCache.LongParser(){
@Override
public final long parseLong(final BytesRef term) {
return (term.bytes[term.offset]-'A') * 1234567890L;
@@ -643,10 +689,10 @@ public class TestSort extends LuceneTest
assertSaneFieldCaches(getTestName() + " LongParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.DoubleParser(){
+ sort.setSort(new SortField("parser", new FieldCache.DoubleParser(){
@Override
public final double parseDouble(final BytesRef term) {
- return Math.pow( term.bytes[term.offset], (term.bytes[term.offset]-'A') );
+ return Math.pow( term.bytes[term.offset], (term.bytes[term.offset]-'A'));
}
@Override
public TermsEnum termsEnum(Terms terms) throws IOException {
@@ -657,7 +703,7 @@ public class TestSort extends LuceneTest
assertSaneFieldCaches(getTestName() + " DoubleParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.ByteParser(){
+ sort.setSort(new SortField("parser", new FieldCache.ByteParser(){
@Override
public final byte parseByte(final BytesRef term) {
return (byte) (term.bytes[term.offset]-'A');
@@ -672,7 +718,7 @@ public class TestSort extends LuceneTest
assertSaneFieldCaches(getTestName() + " ByteParser");
fc.purgeAllCaches();
- sort.setSort (new SortField ("parser", new FieldCache.ShortParser(){
+ sort.setSort(new SortField("parser", new FieldCache.ShortParser(){
@Override
public final short parseShort(final BytesRef term) {
return (short) (term.bytes[term.offset]-'A');
@@ -692,39 +738,39 @@ public class TestSort extends LuceneTest
IndexSearcher empty = getEmptyIndex();
sort = new Sort();
- assertMatches (empty, queryX, sort, "");
+ assertMatches(empty, queryX, sort, "");
sort.setSort(SortField.FIELD_DOC);
- assertMatches (empty, queryX, sort, "");
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (new SortField ("int", SortField.Type.INT), SortField.FIELD_DOC );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC);
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (useDocValues(new SortField ("int", SortField.Type.INT)), SortField.FIELD_DOC );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC);
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (new SortField ("string", SortField.Type.STRING, true), SortField.FIELD_DOC );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("string", SortField.Type.STRING, true), SortField.FIELD_DOC);
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (new SortField ("float", SortField.Type.FLOAT), new SortField ("string", SortField.Type.STRING) );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("float", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING));
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (useDocValues(new SortField ("float", SortField.Type.FLOAT)), new SortField ("string", SortField.Type.STRING) );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING));
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (useDocValues(new SortField ("string", getDVStringSortType(), true)), SortField.FIELD_DOC );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("string_dv", getDVStringSortType(false), true), SortField.FIELD_DOC);
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (useDocValues(new SortField ("float", SortField.Type.FLOAT)),
- useDocValues(new SortField ("string", getDVStringSortType())) );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT),
+ new SortField("string_dv", getDVStringSortType(false)));
+ assertMatches(empty, queryX, sort, "");
- sort.setSort (useDocValues(new SortField ("float", SortField.Type.FLOAT)), useDocValues(new SortField ("string", getDVStringSortType())) );
- assertMatches (empty, queryX, sort, "");
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string_dv", getDVStringSortType(false)));
+ assertMatches(empty, queryX, sort, "");
}
static class MyFieldComparator extends FieldComparator<Integer> {
- int[] docValues;
+ FieldCache.Ints docValues;
int[] slotValues;
int bottomValue;
@@ -734,7 +780,7 @@ public class TestSort extends LuceneTest
@Override
public void copy(int slot, int doc) {
- slotValues[slot] = docValues[doc];
+ slotValues[slot] = docValues.get(doc);
}
@Override
@@ -745,7 +791,7 @@ public class TestSort extends LuceneTest
@Override
public int compareBottom(int doc) {
- return bottomValue - docValues[doc];
+ return bottomValue - docValues.get(doc);
}
@Override
@@ -779,7 +825,7 @@ public class TestSort extends LuceneTest
@Override
public int compareDocToValue(int doc, Integer valueObj) {
final int value = valueObj.intValue();
- final int docValue = docValues[doc];
+ final int docValue = docValues.get(doc);
// values are small enough that overflow won't happen
return docValue - value;
@@ -795,44 +841,44 @@ public class TestSort extends LuceneTest
// Test sorting w/ custom FieldComparator
public void testNewCustomFieldParserSort() throws Exception {
- sort.setSort (new SortField ("parser", new MyFieldComparatorSource()));
- assertMatches (full, queryA, sort, "JIHGFEDCBA");
+ sort.setSort(new SortField("parser", new MyFieldComparatorSource()));
+ assertMatches(full, queryA, sort, "JIHGFEDCBA");
}
// test sorts in reverse
public void testReverseSort() throws Exception {
- sort.setSort (new SortField (null, SortField.Type.SCORE, true), SortField.FIELD_DOC );
- assertMatches (full, queryX, sort, "IEGCA");
- assertMatches (full, queryY, sort, "JFHDB");
-
- sort.setSort (new SortField (null, SortField.Type.DOC, true));
- assertMatches (full, queryX, sort, "IGECA");
- assertMatches (full, queryY, sort, "JHFDB");
-
- sort.setSort (new SortField ("int", SortField.Type.INT, true) );
- assertMatches (full, queryX, sort, "CAEGI");
- assertMatches (full, queryY, sort, "BJFHD");
-
- sort.setSort (new SortField ("float", SortField.Type.FLOAT, true) );
- assertMatches (full, queryX, sort, "AECIG");
- assertMatches (full, queryY, sort, "BFJHD");
-
- sort.setSort (new SortField ("string", SortField.Type.STRING, true) );
- assertMatches (full, queryX, sort, "CEGIA");
- assertMatches (full, queryY, sort, "BFHJD");
+ sort.setSort(new SortField(null, SortField.Type.SCORE, true), SortField.FIELD_DOC);
+ assertMatches(full, queryX, sort, "IEGCA");
+ assertMatches(full, queryY, sort, "JFHDB");
+
+ sort.setSort(new SortField(null, SortField.Type.DOC, true));
+ assertMatches(full, queryX, sort, "IGECA");
+ assertMatches(full, queryY, sort, "JHFDB");
+
+ sort.setSort(new SortField("int", SortField.Type.INT, true));
+ assertMatches(full, queryX, sort, "CAEGI");
+ assertMatches(full, queryY, sort, "BJFHD");
+
+ sort.setSort(new SortField("float", SortField.Type.FLOAT, true));
+ assertMatches(full, queryX, sort, "AECIG");
+ assertMatches(full, queryY, sort, "BFJHD");
+
+ sort.setSort(new SortField("string", SortField.Type.STRING, true));
+ assertMatches(full, queryX, sort, "CEGIA");
+ assertMatches(full, queryY, sort, "BFHJD");
if (supportsDocValues) {
- sort.setSort (useDocValues(new SortField ("int", SortField.Type.INT, true)) );
- assertMatches (full, queryX, sort, "CAEGI");
- assertMatches (full, queryY, sort, "BJFHD");
-
- sort.setSort (useDocValues(new SortField ("float", SortField.Type.FLOAT, true)) );
- assertMatches (full, queryX, sort, "AECIG");
- assertMatches (full, queryY, sort, "BFJHD");
-
- sort.setSort (useDocValues(new SortField ("string", getDVStringSortType(), true)) );
- assertMatches (full, queryX, sort, "CEGIA");
- assertMatches (full, queryY, sort, "BFHJD");
+ sort.setSort(new SortField("int_dv", SortField.Type.INT, true));
+ assertMatches(full, queryX, sort, "CAEGI");
+ assertMatches(full, queryY, sort, "BJFHD");
+
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT, true));
+ assertMatches(full, queryX, sort, "AECIG");
+ assertMatches(full, queryY, sort, "BFJHD");
+
+ sort.setSort(new SortField("string_dv", getDVStringSortType(), true));
+ assertMatches(full, queryX, sort, "CEGIA");
+ assertMatches(full, queryY, sort, "BFHJD");
}
}
@@ -841,81 +887,81 @@ public class TestSort extends LuceneTest
// NOTE: do not test DocValues fields here, since you
// can't sort when some documents don't have the field
- sort.setSort (new SortField ("string", SortField.Type.STRING) );
- assertMatches (full, queryF, sort, "ZJI");
+ sort.setSort(new SortField("string", SortField.Type.STRING));
+ assertMatches(full, queryF, sort, "ZJI");
- sort.setSort (new SortField ("string", SortField.Type.STRING, true) );
- assertMatches (full, queryF, sort, "IJZ");
+ sort.setSort(new SortField("string", SortField.Type.STRING, true));
+ assertMatches(full, queryF, sort, "IJZ");
- sort.setSort (new SortField ("int", SortField.Type.INT) );
- assertMatches (full, queryF, sort, "IZJ");
+ sort.setSort(new SortField("int", SortField.Type.INT));
+ assertMatches(full, queryF, sort, "IZJ");
- sort.setSort (new SortField ("int", SortField.Type.INT, true) );
- assertMatches (full, queryF, sort, "JZI");
+ sort.setSort(new SortField("int", SortField.Type.INT, true));
+ assertMatches(full, queryF, sort, "JZI");
- sort.setSort (new SortField ("float", SortField.Type.FLOAT) );
- assertMatches (full, queryF, sort, "ZJI");
+ sort.setSort(new SortField("float", SortField.Type.FLOAT));
+ assertMatches(full, queryF, sort, "ZJI");
// using a nonexisting field as first sort key shouldn't make a difference:
- sort.setSort (new SortField ("nosuchfield", SortField.Type.STRING),
- new SortField ("float", SortField.Type.FLOAT) );
- assertMatches (full, queryF, sort, "ZJI");
+ sort.setSort(new SortField("nosuchfield", SortField.Type.STRING),
+ new SortField("float", SortField.Type.FLOAT));
+ assertMatches(full, queryF, sort, "ZJI");
- sort.setSort (new SortField ("float", SortField.Type.FLOAT, true) );
- assertMatches (full, queryF, sort, "IJZ");
+ sort.setSort(new SortField("float", SortField.Type.FLOAT, true));
+ assertMatches(full, queryF, sort, "IJZ");
// When a field is null for both documents, the next SortField should be used.
- sort.setSort (new SortField ("int", SortField.Type.INT),
- new SortField ("string", SortField.Type.STRING),
- new SortField ("float", SortField.Type.FLOAT) );
- assertMatches (full, queryG, sort, "ZWXY");
+ sort.setSort(new SortField("int", SortField.Type.INT),
+ new SortField("string", SortField.Type.STRING),
+ new SortField("float", SortField.Type.FLOAT));
+ assertMatches(full, queryG, sort, "ZWXY");
// Reverse the last criterium to make sure the test didn't pass by chance
- sort.setSort (new SortField ("int", SortField.Type.INT),
- new SortField ("string", SortField.Type.STRING),
- new SortField ("float", SortField.Type.FLOAT, true) );
- assertMatches (full, queryG, sort, "ZYXW");
+ sort.setSort(new SortField("int", SortField.Type.INT),
+ new SortField("string", SortField.Type.STRING),
+ new SortField("float", SortField.Type.FLOAT, true));
+ assertMatches(full, queryG, sort, "ZYXW");
// Do the same for a ParallelMultiSearcher
ExecutorService exec = Executors.newFixedThreadPool(_TestUtil.nextInt(random(), 2, 8), new NamedThreadFactory("testEmptyFieldSort"));
- IndexSearcher parallelSearcher=new IndexSearcher (full.getIndexReader(), exec);
+ IndexSearcher parallelSearcher = new IndexSearcher(full.getIndexReader(), exec);
- sort.setSort (new SortField ("int", SortField.Type.INT),
- new SortField ("string", SortField.Type.STRING),
- new SortField ("float", SortField.Type.FLOAT) );
- assertMatches (parallelSearcher, queryG, sort, "ZWXY");
-
- sort.setSort (new SortField ("int", SortField.Type.INT),
- new SortField ("string", SortField.Type.STRING),
- new SortField ("float", SortField.Type.FLOAT, true) );
- assertMatches (parallelSearcher, queryG, sort, "ZYXW");
+ sort.setSort(new SortField("int", SortField.Type.INT),
+ new SortField("string", SortField.Type.STRING),
+ new SortField("float", SortField.Type.FLOAT));
+ assertMatches(parallelSearcher, queryG, sort, "ZWXY");
+
+ sort.setSort(new SortField("int", SortField.Type.INT),
+ new SortField("string", SortField.Type.STRING),
+ new SortField("float", SortField.Type.FLOAT, true));
+ assertMatches(parallelSearcher, queryG, sort, "ZYXW");
exec.shutdown();
exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
}
// test sorts using a series of fields
public void testSortCombos() throws Exception {
- sort.setSort (new SortField ("int", SortField.Type.INT), new SortField ("float", SortField.Type.FLOAT) );
- assertMatches (full, queryX, sort, "IGEAC");
+ sort.setSort(new SortField("int", SortField.Type.INT), new SortField("float", SortField.Type.FLOAT));
+ assertMatches(full, queryX, sort, "IGEAC");
- sort.setSort (new SortField ("int", SortField.Type.INT, true), new SortField (null, SortField.Type.DOC, true) );
- assertMatches (full, queryX, sort, "CEAGI");
+ sort.setSort(new SortField("int", SortField.Type.INT, true), new SortField(null, SortField.Type.DOC, true));
+ assertMatches(full, queryX, sort, "CEAGI");
- sort.setSort (new SortField ("float", SortField.Type.FLOAT), new SortField ("string", SortField.Type.STRING) );
- assertMatches (full, queryX, sort, "GICEA");
+ sort.setSort(new SortField("float", SortField.Type.FLOAT), new SortField("string", SortField.Type.STRING));
+ assertMatches(full, queryX, sort, "GICEA");
if (supportsDocValues) {
- sort.setSort (useDocValues(new SortField ("int", SortField.Type.INT)),
- useDocValues(new SortField ("float", SortField.Type.FLOAT)));
- assertMatches (full, queryX, sort, "IGEAC");
-
- sort.setSort (useDocValues(new SortField ("int", SortField.Type.INT, true)),
- useDocValues(new SortField (null, SortField.Type.DOC, true)));
- assertMatches (full, queryX, sort, "CEAGI");
-
- sort.setSort (useDocValues(new SortField ("float", SortField.Type.FLOAT)),
- useDocValues(new SortField ("string", getDVStringSortType())));
- assertMatches (full, queryX, sort, "GICEA");
+ sort.setSort(new SortField("int_dv", SortField.Type.INT),
+ new SortField("float_dv", SortField.Type.FLOAT));
+ assertMatches(full, queryX, sort, "IGEAC");
+
+ sort.setSort(new SortField("int_dv", SortField.Type.INT, true),
+ new SortField(null, SortField.Type.DOC, true));
+ assertMatches(full, queryX, sort, "CEAGI");
+
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT),
+ new SortField("string_dv", getDVStringSortType()));
+ assertMatches(full, queryX, sort, "GICEA");
}
}
@@ -925,9 +971,12 @@ public class TestSort extends LuceneTest
IndexSearcher searcher = new IndexSearcher(
new MultiReader(searchX.getIndexReader(),
searchY.getIndexReader()), exec);
- runMultiSorts(searcher, false);
- exec.shutdown();
- exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
+ try {
+ runMultiSorts(searcher, false);
+ } finally {
+ exec.shutdown();
+ exec.awaitTermination(1000, TimeUnit.MILLISECONDS);
+ }
}
public void testTopDocsScores() throws Exception {
@@ -944,7 +993,7 @@ public class TestSort extends LuceneTest
// a filter that only allows through the first hit
Filter filt = new Filter() {
@Override
- public DocIdSet getDocIdSet (AtomicReaderContext context, Bits acceptDocs) {
+ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) {
assertNull("acceptDocs should be null, as we have no deletions", acceptDocs);
BitSet bs = new BitSet(context.reader().maxDoc());
bs.set(0, context.reader().maxDoc());
@@ -966,7 +1015,7 @@ public class TestSort extends LuceneTest
// does not use Searcher's default search methods (with Sort) since all set
// fillFields to true.
Sort[] sort = new Sort[] { new Sort(SortField.FIELD_DOC), new Sort() };
- for (int i = 0; i < sort.length; i++) {
+ for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, false,
false, false, true);
@@ -974,7 +1023,7 @@ public class TestSort extends LuceneTest
full.search(q, tdc);
ScoreDoc[] sd = tdc.topDocs().scoreDocs;
- for (int j = 1; j < sd.length; j++) {
+ for(int j = 1; j < sd.length; j++) {
assertTrue(sd[j].doc != sd[j - 1].doc);
}
@@ -985,7 +1034,7 @@ public class TestSort extends LuceneTest
// Two Sort criteria to instantiate the multi/single comparators.
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for (int i = 0; i < sort.length; i++) {
+ for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, false,
false, true);
@@ -994,7 +1043,7 @@ public class TestSort extends LuceneTest
TopDocs td = tdc.topDocs();
ScoreDoc[] sd = td.scoreDocs;
- for (int j = 0; j < sd.length; j++) {
+ for(int j = 0; j < sd.length; j++) {
assertTrue(Float.isNaN(sd[j].score));
}
assertTrue(Float.isNaN(td.getMaxScore()));
@@ -1005,7 +1054,7 @@ public class TestSort extends LuceneTest
// Two Sort criteria to instantiate the multi/single comparators.
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for (int i = 0; i < sort.length; i++) {
+ for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
false, true);
@@ -1014,7 +1063,7 @@ public class TestSort extends LuceneTest
TopDocs td = tdc.topDocs();
ScoreDoc[] sd = td.scoreDocs;
- for (int j = 0; j < sd.length; j++) {
+ for(int j = 0; j < sd.length; j++) {
assertTrue(!Float.isNaN(sd[j].score));
}
assertTrue(Float.isNaN(td.getMaxScore()));
@@ -1026,7 +1075,7 @@ public class TestSort extends LuceneTest
// Two Sort criteria to instantiate the multi/single comparators.
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC, SortField.FIELD_SCORE) };
- for (int i = 0; i < sort.length; i++) {
+ for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
false, true);
@@ -1035,7 +1084,7 @@ public class TestSort extends LuceneTest
TopDocs td = tdc.topDocs();
ScoreDoc[] sd = td.scoreDocs;
- for (int j = 0; j < sd.length; j++) {
+ for(int j = 0; j < sd.length; j++) {
assertTrue(!Float.isNaN(sd[j].score));
}
assertTrue(Float.isNaN(td.getMaxScore()));
@@ -1046,7 +1095,7 @@ public class TestSort extends LuceneTest
// Two Sort criteria to instantiate the multi/single comparators.
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for (int i = 0; i < sort.length; i++) {
+ for(int i = 0; i < sort.length; i++) {
Query q = new MatchAllDocsQuery();
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true,
true, true);
@@ -1055,7 +1104,7 @@ public class TestSort extends LuceneTest
TopDocs td = tdc.topDocs();
ScoreDoc[] sd = td.scoreDocs;
- for (int j = 0; j < sd.length; j++) {
+ for(int j = 0; j < sd.length; j++) {
assertTrue(!Float.isNaN(sd[j].score));
}
assertTrue(!Float.isNaN(td.getMaxScore()));
@@ -1094,8 +1143,8 @@ public class TestSort extends LuceneTest
// Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
// the clause instead of BQ.
bq.setMinimumNumberShouldMatch(1);
- for (int i = 0; i < sort.length; i++) {
- for (int j = 0; j < tfcOptions.length; j++) {
+ for(int i = 0; i < sort.length; i++) {
+ for(int j = 0; j < tfcOptions.length; j++) {
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10,
tfcOptions[j][0], tfcOptions[j][1], tfcOptions[j][2], false);
@@ -1143,8 +1192,8 @@ public class TestSort extends LuceneTest
// Set minNrShouldMatch to 1 so that BQ will not optimize rewrite to return
// the clause instead of BQ.
bq.setMinimumNumberShouldMatch(1);
- for (int i = 0; i < sort.length; i++) {
- for (int j = 0; j < tfcOptions.length; j++) {
+ for(int i = 0; i < sort.length; i++) {
+ for(int j = 0; j < tfcOptions.length; j++) {
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10,
tfcOptions[j][0], tfcOptions[j][1], tfcOptions[j][2], false);
@@ -1163,7 +1212,7 @@ public class TestSort extends LuceneTest
// Two Sort criteria to instantiate the multi/single comparators.
Sort[] sort = new Sort[] {new Sort(SortField.FIELD_DOC), new Sort() };
- for (int i = 0; i < sort.length; i++) {
+ for(int i = 0; i < sort.length; i++) {
TopDocsCollector<Entry> tdc = TopFieldCollector.create(sort[i], 10, true, true, true, true);
TopDocs td = tdc.topDocs();
assertEquals(0, td.totalHits);
@@ -1177,11 +1226,11 @@ public class TestSort extends LuceneTest
String expected = isFull ? "ABCDEFGHIJ" : "ACEGIBDFHJ";
assertMatches(multi, queryA, sort, expected);
- sort.setSort(new SortField ("int", SortField.Type.INT));
+ sort.setSort(new SortField("int", SortField.Type.INT));
expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC";
assertMatches(multi, queryA, sort, expected);
- sort.setSort(new SortField ("int", SortField.Type.INT), SortField.FIELD_DOC);
+ sort.setSort(new SortField("int", SortField.Type.INT), SortField.FIELD_DOC);
expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC";
assertMatches(multi, queryA, sort, expected);
@@ -1189,7 +1238,7 @@ public class TestSort extends LuceneTest
expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC";
assertMatches(multi, queryA, sort, expected);
- sort.setSort(new SortField ("float", SortField.Type.FLOAT), SortField.FIELD_DOC);
+ sort.setSort(new SortField("float", SortField.Type.FLOAT), SortField.FIELD_DOC);
assertMatches(multi, queryA, sort, "GDHJCIEFAB");
sort.setSort(new SortField("float", SortField.Type.FLOAT));
@@ -1214,66 +1263,66 @@ public class TestSort extends LuceneTest
sort.setSort(new SortField("float", SortField.Type.FLOAT),new SortField("string", SortField.Type.STRING));
assertMatches(multi, queryA, sort, "GDHJICEFAB");
- sort.setSort(new SortField ("int", SortField.Type.INT));
+ sort.setSort(new SortField("int", SortField.Type.INT));
assertMatches(multi, queryF, sort, "IZJ");
- sort.setSort(new SortField ("int", SortField.Type.INT, true));
+ sort.setSort(new SortField("int", SortField.Type.INT, true));
assertMatches(multi, queryF, sort, "JZI");
- sort.setSort(new SortField ("float", SortField.Type.FLOAT));
+ sort.setSort(new SortField("float", SortField.Type.FLOAT));
assertMatches(multi, queryF, sort, "ZJI");
- sort.setSort(new SortField ("string", SortField.Type.STRING));
+ sort.setSort(new SortField("string", SortField.Type.STRING));
assertMatches(multi, queryF, sort, "ZJI");
- sort.setSort(new SortField ("string", SortField.Type.STRING, true));
+ sort.setSort(new SortField("string", SortField.Type.STRING, true));
assertMatches(multi, queryF, sort, "IJZ");
if (supportsDocValues) {
- sort.setSort(useDocValues(new SortField ("int", SortField.Type.INT)));
+ sort.setSort(new SortField("int_dv", SortField.Type.INT));
expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC";
assertMatches(multi, queryA, sort, expected);
- sort.setSort(useDocValues(new SortField ("int", SortField.Type.INT)), SortField.FIELD_DOC);
+ sort.setSort(new SortField("int_dv", SortField.Type.INT), SortField.FIELD_DOC);
expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC";
assertMatches(multi, queryA, sort, expected);
- sort.setSort(useDocValues(new SortField("int", SortField.Type.INT)));
+ sort.setSort(new SortField("int_dv", SortField.Type.INT));
expected = isFull ? "IDHFGJABEC" : "IDHFGJAEBC";
assertMatches(multi, queryA, sort, expected);
- sort.setSort(useDocValues(new SortField ("float", SortField.Type.FLOAT)), SortField.FIELD_DOC);
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), SortField.FIELD_DOC);
assertMatches(multi, queryA, sort, "GDHJCIEFAB");
- sort.setSort(useDocValues(new SortField("float", SortField.Type.FLOAT)));
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT));
assertMatches(multi, queryA, sort, "GDHJCIEFAB");
-
- sort.setSort(useDocValues(new SortField("int", SortField.Type.INT, true)));
+
+ sort.setSort(new SortField("int_dv", SortField.Type.INT, true));
expected = isFull ? "CABEJGFHDI" : "CAEBJGFHDI";
assertMatches(multi, queryA, sort, expected);
-
- sort.setSort(useDocValues(new SortField("int", SortField.Type.INT)), useDocValues(new SortField("float", SortField.Type.FLOAT)));
+
+ sort.setSort(new SortField("int_dv", SortField.Type.INT), new SortField("float_dv", SortField.Type.FLOAT));
assertMatches(multi, queryA, sort, "IDHFGJEABC");
-
- sort.setSort(useDocValues(new SortField ("int", SortField.Type.INT)));
+
+ sort.setSort(new SortField("int_dv", SortField.Type.INT));
assertMatches(multi, queryF, sort, "IZJ");
- sort.setSort(useDocValues(new SortField ("int", SortField.Type.INT, true)));
+ sort.setSort(new SortField("int_dv", SortField.Type.INT, true));
assertMatches(multi, queryF, sort, "JZI");
- sort.setSort(useDocValues(new SortField("string", getDVStringSortType())));
+ sort.setSort(new SortField("string_dv", getDVStringSortType()));
assertMatches(multi, queryA, sort, "DJAIHGFEBC");
-
- sort.setSort(useDocValues(new SortField("string", getDVStringSortType(), true)));
+
+ sort.setSort(new SortField("string_dv", getDVStringSortType(), true));
assertMatches(multi, queryA, sort, "CBEFGHIAJD");
-
- sort.setSort(useDocValues(new SortField("float", SortField.Type.FLOAT)),useDocValues(new SortField("string", getDVStringSortType())));
+
+ sort.setSort(new SortField("float_dv", SortField.Type.FLOAT), new SortField("string_dv", getDVStringSortType()));
assertMatches(multi, queryA, sort, "GDHJICEFAB");
- sort.setSort(useDocValues(new SortField ("string", getDVStringSortType())));
+ sort.setSort(new SortField("string_dv", getDVStringSortType()));
assertMatches(multi, queryF, sort, "ZJI");
- sort.setSort(useDocValues(new SortField ("string", getDVStringSortType(), true)));
+ sort.setSort(new SortField("string_dv", getDVStringSortType(), true));
assertMatches(multi, queryF, sort, "IJZ");
}
@@ -1285,13 +1334,16 @@ public class TestSort extends LuceneTest
}
private void assertMatches(IndexSearcher searcher, Query query, Sort sort, String expectedResult) throws IOException {
- assertMatches( null, searcher, query, sort, expectedResult );
+ assertMatches( null, searcher, query, sort, expectedResult);
}
// make sure the documents returned by the search match the expected list
private void assertMatches(String msg, IndexSearcher searcher, Query query, Sort sort,
String expectedResult) throws IOException {
+ if (VERBOSE) {
+ System.out.println("assertMatches searcher=" + searcher + " sort=" + sort);
+ }
//ScoreDoc[] result = searcher.search (query, null, 1000, sort).scoreDocs;
TopDocs hits = searcher.search(query, null, Math.max(1, expectedResult.length()), sort, true, true);
@@ -1299,10 +1351,10 @@ public class TestSort extends LuceneTest
assertEquals(expectedResult.length(),hits.totalHits);
StringBuilder buff = new StringBuilder(10);
int n = result.length;
- for (int i=0; i<n; ++i) {
+ for(int i=0; i<n; ++i) {
Document doc = searcher.doc(result[i].doc);
IndexableField[] v = doc.getFields("tracer");
- for (int j=0; j<v.length; ++j) {
+ for(int j=0; j<v.length; ++j) {
buff.append (v[j].stringValue());
}
}
@@ -1338,17 +1390,17 @@ public class TestSort extends LuceneTest
Directory indexStore = newDirectory();
IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- for (int i=0; i<5; i++) {
+ for(int i=0; i<5; i++) {
Document doc = new Document();
- doc.add (new StringField ("string", "a"+i, Field.Store.NO));
- doc.add (new StringField ("string", "b"+i, Field.Store.NO));
- writer.addDocument (doc);
+ doc.add(new StringField("string", "a"+i, Field.Store.NO));
+ doc.add(new StringField("string", "b"+i, Field.Store.NO));
+ writer.addDocument(doc);
}
writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases
writer.close();
sort.setSort(
new SortField("string", SortField.Type.STRING),
- SortField.FIELD_DOC );
+ SortField.FIELD_DOC);
// this should not throw AIOOBE or RuntimeEx
IndexReader reader = DirectoryReader.open(indexStore);
IndexSearcher searcher = new IndexSearcher(reader);
@@ -1360,11 +1412,11 @@ public class TestSort extends LuceneTest
public void testCountingCollector() throws Exception {
Directory indexStore = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);
- for (int i=0; i<5; i++) {
+ for(int i=0; i<5; i++) {
Document doc = new Document();
- doc.add (new StringField ("string", "a"+i, Field.Store.NO));
- doc.add (new StringField ("string", "b"+i, Field.Store.NO));
- writer.addDocument (doc);
+ doc.add(new StringField("string", "a"+i, Field.Store.NO));
+ doc.add(new StringField("string", "b"+i, Field.Store.NO));
+ writer.addDocument(doc);
}
IndexReader reader = writer.getReader();
writer.close();
@@ -1393,14 +1445,14 @@ public class TestSort extends LuceneTest
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
final int maxDoc = context.reader().maxDoc();
- final DocValues.Source idSource = context.reader().docValues("id").getSource();
+ final FieldCache.Ints idSource = FieldCache.DEFAULT.getInts(context.reader(), "id", false);
assertNotNull(idSource);
final FixedBitSet bits = new FixedBitSet(maxDoc);
for(int docID=0;docID<maxDoc;docID++) {
if (random.nextFloat() <= density && (acceptDocs == null || acceptDocs.get(docID))) {
bits.set(docID);
//System.out.println(" acc id=" + idSource.getInt(docID) + " docID=" + docID);
- matchValues.add(docValues.get((int) idSource.getInt(docID)));
+ matchValues.add(docValues.get(idSource.get(docID)));
}
}
@@ -1447,9 +1499,9 @@ public class TestSort extends LuceneTest
}
final Document doc = new Document();
- doc.add(new SortedBytesDocValuesField("stringdv", br));
+ doc.add(new SortedDocValuesField("stringdv", br));
doc.add(newStringField("string", s, Field.Store.NO));
- doc.add(new PackedLongDocValuesField("id", numDocs));
+ doc.add(new NumericDocValuesField("id", numDocs));
docValues.add(br);
writer.addDocument(doc);
numDocs++;
@@ -1474,7 +1526,6 @@ public class TestSort extends LuceneTest
final SortField sf;
if (random.nextBoolean()) {
sf = new SortField("stringdv", SortField.Type.STRING, reverse);
- sf.setUseIndexValues(true);
} else {
sf = new SortField("string", SortField.Type.STRING, reverse);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Fri Feb 8 03:26:14 2013
@@ -24,7 +24,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.CollectionStatistics;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java Fri Feb 8 03:26:14 2013
@@ -36,7 +36,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Norm;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java Fri Feb 8 03:26:14 2013
@@ -96,18 +96,18 @@ public class TestBasics extends LuceneTe
}
}
- static Analyzer simplePayloadAnalyzer = new Analyzer() {
+ static Analyzer simplePayloadAnalyzer;
- @Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
- return new TokenStreamComponents(tokenizer, new SimplePayloadFilter(tokenizer));
- }
-
- };
-
@BeforeClass
public static void beforeClass() throws Exception {
+ simplePayloadAnalyzer = new Analyzer() {
+ @Override
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(tokenizer, new SimplePayloadFilter(tokenizer));
+ }
+ };
+
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, simplePayloadAnalyzer)
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/Test2BPagedBytes.java Fri Feb 8 03:26:14 2013
@@ -17,52 +17,59 @@ package org.apache.lucene.util;
* limitations under the License.
*/
-import java.util.Arrays;
import java.util.Random;
-import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
-import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput;
+import org.apache.lucene.store.BaseDirectoryWrapper;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.Ignore;
@Ignore("You must increase heap to > 2 G to run this")
public class Test2BPagedBytes extends LuceneTestCase {
public void test() throws Exception {
+ BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("test2BPagedBytes"));
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
+ }
PagedBytes pb = new PagedBytes(15);
- PagedBytesDataOutput dataOutput = pb.getDataOutput();
+ IndexOutput dataOutput = dir.createOutput("foo", IOContext.DEFAULT);
long netBytes = 0;
long seed = random().nextLong();
long lastFP = 0;
Random r2 = new Random(seed);
while(netBytes < 1.1*Integer.MAX_VALUE) {
- int numBytes = _TestUtil.nextInt(r2, 1, 100000);
+ int numBytes = _TestUtil.nextInt(r2, 1, 32768);
byte[] bytes = new byte[numBytes];
r2.nextBytes(bytes);
dataOutput.writeBytes(bytes, bytes.length);
- long fp = dataOutput.getPosition();
+ long fp = dataOutput.getFilePointer();
assert fp == lastFP + numBytes;
lastFP = fp;
netBytes += numBytes;
}
- pb.freeze(true);
+ dataOutput.close();
+ IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
+ pb.copy(input, input.length());
+ input.close();
+ PagedBytes.Reader reader = pb.freeze(true);
- PagedBytesDataInput dataInput = pb.getDataInput();
- lastFP = 0;
r2 = new Random(seed);
netBytes = 0;
while(netBytes < 1.1*Integer.MAX_VALUE) {
- int numBytes = _TestUtil.nextInt(r2, 1, 100000);
+ int numBytes = _TestUtil.nextInt(r2, 1, 32768);
byte[] bytes = new byte[numBytes];
r2.nextBytes(bytes);
+ BytesRef expected = new BytesRef(bytes);
- byte[] bytesIn = new byte[numBytes];
- dataInput.readBytes(bytesIn, 0, numBytes);
- assertTrue(Arrays.equals(bytes, bytesIn));
+ BytesRef actual = new BytesRef();
+ reader.fillSlice(actual, netBytes, numBytes);
+ assertEquals(expected, actual);
- long fp = dataInput.getPosition();
- assert fp == lastFP + numBytes;
- lastFP = fp;
netBytes += numBytes;
}
+ dir.close();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestByteBlockPool.java Fri Feb 8 03:26:14 2013
@@ -1,14 +1,9 @@
package org.apache.lucene.util;
-import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.RAMDirectory;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with this
@@ -27,45 +22,32 @@ import org.apache.lucene.store.RAMDirect
*/
public class TestByteBlockPool extends LuceneTestCase {
- public void testCopyRefAndWrite() throws IOException {
+ public void testReadAndWrite() throws IOException {
Counter bytesUsed = Counter.newCounter();
ByteBlockPool pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed));
pool.nextBuffer();
boolean reuseFirst = random().nextBoolean();
for (int j = 0; j < 2; j++) {
- List<String> list = new ArrayList<String>();
+ List<BytesRef> list = new ArrayList<BytesRef>();
int maxLength = atLeast(500);
final int numValues = atLeast(100);
BytesRef ref = new BytesRef();
for (int i = 0; i < numValues; i++) {
final String value = _TestUtil.randomRealisticUnicodeString(random(),
maxLength);
- list.add(value);
+ list.add(new BytesRef(value));
ref.copyChars(value);
- pool.copy(ref);
- }
- RAMDirectory dir = new RAMDirectory();
- IndexOutput stream = dir.createOutput("foo.txt", newIOContext(random()));
- pool.writePool(stream);
- stream.flush();
- stream.close();
- IndexInput input = dir.openInput("foo.txt", newIOContext(random()));
- assertEquals(pool.byteOffset + pool.byteUpto, stream.length());
- BytesRef expected = new BytesRef();
- BytesRef actual = new BytesRef();
- for (String string : list) {
- expected.copyChars(string);
- actual.grow(expected.length);
- actual.length = expected.length;
- input.readBytes(actual.bytes, 0, actual.length);
- assertEquals(expected, actual);
+ pool.append(ref);
}
- try {
- input.readByte();
- fail("must be EOF");
- } catch (EOFException e) {
- // expected - read past EOF
+ // verify
+ long position = 0;
+ for (BytesRef expected : list) {
+ ref.grow(expected.length);
+ ref.length = expected.length;
+ pool.readBytes(position, ref.bytes, ref.offset, ref.length);
+ assertEquals(expected, ref);
+ position += ref.length;
}
pool.reset(random().nextBoolean(), reuseFirst);
if (reuseFirst) {
@@ -74,7 +56,6 @@ public class TestByteBlockPool extends L
assertEquals(0, bytesUsed.get());
pool.nextBuffer(); // prepare for next iter
}
- dir.close();
}
- }
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/TestPagedBytes.java Fri Feb 8 03:26:14 2013
@@ -20,10 +20,12 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.*;
+import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.PagedBytes.PagedBytesDataInput;
-import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.Ignore;
public class TestPagedBytes extends LuceneTestCase {
@@ -31,10 +33,14 @@ public class TestPagedBytes extends Luce
public void testDataInputOutput() throws Exception {
Random random = random();
for(int iter=0;iter<5*RANDOM_MULTIPLIER;iter++) {
+ BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("testOverflow"));
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
+ }
final int blockBits = _TestUtil.nextInt(random, 1, 20);
final int blockSize = 1 << blockBits;
final PagedBytes p = new PagedBytes(blockBits);
- final DataOutput out = p.getDataOutput();
+ final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
final int numBytes = _TestUtil.nextInt(random(), 2, 10000000);
final byte[] answer = new byte[numBytes];
@@ -49,11 +55,14 @@ public class TestPagedBytes extends Luce
written += chunk;
}
}
-
+
+ out.close();
+ final IndexInput input = dir.openInput("foo", IOContext.DEFAULT);
+ final DataInput in = input.clone();
+
+ p.copy(input, input.length());
final PagedBytes.Reader reader = p.freeze(random.nextBoolean());
- final DataInput in = p.getDataInput();
-
final byte[] verify = new byte[numBytes];
int read = 0;
while(read < numBytes) {
@@ -76,68 +85,17 @@ public class TestPagedBytes extends Luce
assertEquals(answer[pos + byteUpto], slice.bytes[slice.offset + byteUpto]);
}
}
- }
- }
-
- public void testLengthPrefix() throws Exception {
- Random random = random();
- for(int iter=0;iter<5*RANDOM_MULTIPLIER;iter++) {
- final int blockBits = _TestUtil.nextInt(random, 2, 20);
- final int blockSize = 1 << blockBits;
- final PagedBytes p = new PagedBytes(blockBits);
- final List<Integer> addresses = new ArrayList<Integer>();
- final List<BytesRef> answers = new ArrayList<BytesRef>();
- int totBytes = 0;
- while(totBytes < 10000000 && answers.size() < 100000) {
- final int len = random.nextInt(Math.min(blockSize-2, 32768));
- final BytesRef b = new BytesRef();
- b.bytes = new byte[len];
- b.length = len;
- b.offset = 0;
- random.nextBytes(b.bytes);
- answers.add(b);
- addresses.add((int) p.copyUsingLengthPrefix(b));
-
- totBytes += len;
- }
-
- final PagedBytes.Reader reader = p.freeze(random.nextBoolean());
-
- final BytesRef slice = new BytesRef();
-
- for(int idx=0;idx<answers.size();idx++) {
- reader.fillSliceWithPrefix(slice, addresses.get(idx));
- assertEquals(answers.get(idx), slice);
- }
- }
- }
-
- // LUCENE-3841: even though
- // copyUsingLengthPrefix will never span two blocks, make
- // sure if caller writes their own prefix followed by the
- // bytes, it still works:
- public void testLengthPrefixAcrossTwoBlocks() throws Exception {
- Random random = random();
- final PagedBytes p = new PagedBytes(10);
- final DataOutput out = p.getDataOutput();
- final byte[] bytes1 = new byte[1000];
- random.nextBytes(bytes1);
- out.writeBytes(bytes1, 0, bytes1.length);
- out.writeByte((byte) 40);
- final byte[] bytes2 = new byte[40];
- random.nextBytes(bytes2);
- out.writeBytes(bytes2, 0, bytes2.length);
-
- final PagedBytes.Reader reader = p.freeze(random.nextBoolean());
- BytesRef answer = reader.fillSliceWithPrefix(new BytesRef(), 1000);
- assertEquals(40, answer.length);
- for(int i=0;i<40;i++) {
- assertEquals(bytes2[i], answer.bytes[answer.offset + i]);
+ input.close();
+ dir.close();
}
}
@Ignore // memory hole
public void testOverflow() throws IOException {
+ BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("testOverflow"));
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
+ }
final int blockBits = _TestUtil.nextInt(random(), 14, 28);
final int blockSize = 1 << blockBits;
byte[] arr = new byte[_TestUtil.nextInt(random(), blockSize / 2, blockSize * 2)];
@@ -146,23 +104,26 @@ public class TestPagedBytes extends Luce
}
final long numBytes = (1L << 31) + _TestUtil.nextInt(random(), 1, blockSize * 3);
final PagedBytes p = new PagedBytes(blockBits);
- final PagedBytesDataOutput out = p.getDataOutput();
+ final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
for (long i = 0; i < numBytes; ) {
- assertEquals(i, out.getPosition());
+ assertEquals(i, out.getFilePointer());
final int len = (int) Math.min(arr.length, numBytes - i);
out.writeBytes(arr, len);
i += len;
}
- assertEquals(numBytes, out.getPosition());
- p.freeze(random().nextBoolean());
- final PagedBytesDataInput in = p.getDataInput();
+ assertEquals(numBytes, out.getFilePointer());
+ out.close();
+ final IndexInput in = dir.openInput("foo", IOContext.DEFAULT);
+ p.copy(in, numBytes);
+ final PagedBytes.Reader reader = p.freeze(random().nextBoolean());
for (long offset : new long[] {0L, Integer.MAX_VALUE, numBytes - 1,
_TestUtil.nextLong(random(), 1, numBytes - 2)}) {
- in.setPosition(offset);
- assertEquals(offset, in.getPosition());
- assertEquals(arr[(int) (offset % arr.length)], in.readByte());
- assertEquals(offset+1, in.getPosition());
+ BytesRef b = new BytesRef();
+ reader.fillSlice(b, offset, 1);
+ assertEquals(arr[(int) (offset % arr.length)], b.bytes[b.offset]);
}
+ in.close();
+ dir.close();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfDirectoryNotClosed.java Fri Feb 8 03:26:14 2013
@@ -30,7 +30,7 @@ public class TestFailIfDirectoryNotClose
}
public static class Nested1 extends WithNestedTests.AbstractNestedTest {
- public void testDummy() {
+ public void testDummy() throws Exception {
Directory dir = newDirectory();
System.out.println(dir.toString());
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1443834&r1=1443833&r2=1443834&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Fri Feb 8 03:26:14 2013
@@ -37,9 +37,9 @@ import org.apache.lucene.store.RAMDirect
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.packed.PackedInts.Reader;
-
import org.junit.Ignore;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
@@ -103,7 +103,11 @@ public class TestPackedInts extends Luce
final int actualValueCount = random().nextBoolean() ? valueCount : _TestUtil.nextInt(random(), 0, valueCount);
final long[] values = new long[valueCount];
for(int i=0;i<actualValueCount;i++) {
- values[i] = _TestUtil.nextLong(random(), 0, maxValue);
+ if (nbits == 64) {
+ values[i] = random().nextLong();
+ } else {
+ values[i] = _TestUtil.nextLong(random(), 0, maxValue);
+ }
w.add(values[i]);
}
w.finish();
@@ -135,6 +139,11 @@ public class TestPackedInts extends Luce
+ r.getClass().getSimpleName(), values[i], r.get(i));
}
in.close();
+
+ final long expectedBytesUsed = RamUsageEstimator.sizeOf(r);
+ final long computedBytesUsed = r.ramBytesUsed();
+ assertEquals(r.getClass() + "expected " + expectedBytesUsed + ", got: " + computedBytesUsed,
+ expectedBytesUsed, computedBytesUsed);
}
{ // test reader iterator next
@@ -795,6 +804,46 @@ public class TestPackedInts extends Luce
return true;
}
+ public void testAppendingLongBuffer() {
+ final long[] arr = new long[RandomInts.randomIntBetween(random(), 1, 2000000)];
+ for (int bpv : new int[] {0, 1, 63, 64, RandomInts.randomIntBetween(random(), 2, 61)}) {
+ if (bpv == 0) {
+ Arrays.fill(arr, random().nextLong());
+ } else if (bpv == 64) {
+ for (int i = 0; i < arr.length; ++i) {
+ arr[i] = random().nextLong();
+ }
+ } else {
+ final long minValue = _TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE - PackedInts.maxValue(bpv));
+ for (int i = 0; i < arr.length; ++i) {
+ arr[i] = minValue + random().nextLong() & PackedInts.maxValue(bpv); // _TestUtil.nextLong is too slow
+ }
+ }
+ AppendingLongBuffer buf = new AppendingLongBuffer();
+ for (int i = 0; i < arr.length; ++i) {
+ buf.add(arr[i]);
+ }
+ assertEquals(arr.length, buf.size());
+ final AppendingLongBuffer.Iterator it = buf.iterator();
+ for (int i = 0; i < arr.length; ++i) {
+ if (random().nextBoolean()) {
+ assertTrue(it.hasNext());
+ }
+ assertEquals(arr[i], it.next());
+ }
+ assertFalse(it.hasNext());
+
+ for (int i = 0; i < arr.length; ++i) {
+ assertEquals(arr[i], buf.get(i));
+ }
+
+ final long expectedBytesUsed = RamUsageEstimator.sizeOf(buf);
+ final long computedBytesUsed = buf.ramBytesUsed();
+ assertEquals("got " + computedBytesUsed + ", expected: " + expectedBytesUsed,
+ expectedBytesUsed, computedBytesUsed);
+ }
+ }
+
public void testPackedInputOutput() throws IOException {
final long[] longs = new long[random().nextInt(8192)];
final int[] bitsPerValues = new int[longs.length];
@@ -841,7 +890,7 @@ public class TestPackedInts extends Luce
public void testBlockPackedReaderWriter() throws IOException {
final int iters = atLeast(2);
for (int iter = 0; iter < iters; ++iter) {
- final int blockSize = 64 * _TestUtil.nextInt(random(), 1, 1 << 12);
+ final int blockSize = 1 << _TestUtil.nextInt(random(), 6, 18);
final int valueCount = random().nextInt(1 << 18);
final long[] values = new long[valueCount];
long minValue = 0;
@@ -873,30 +922,29 @@ public class TestPackedInts extends Luce
final long fp = out.getFilePointer();
out.close();
- DataInput in = dir.openInput("out.bin", IOContext.DEFAULT);
- if (random().nextBoolean()) {
- byte[] buf = new byte[(int) fp];
- in.readBytes(buf, 0, (int) fp);
- ((IndexInput) in).close();
- in = new ByteArrayDataInput(buf);
- }
- final BlockPackedReader reader = new BlockPackedReader(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
+ IndexInput in1 = dir.openInput("out.bin", IOContext.DEFAULT);
+ byte[] buf = new byte[(int) fp];
+ in1.readBytes(buf, 0, (int) fp);
+ in1.seek(0L);
+ ByteArrayDataInput in2 = new ByteArrayDataInput(buf);
+ final DataInput in = random().nextBoolean() ? in1 : in2;
+ final BlockPackedReaderIterator it = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
for (int i = 0; i < valueCount; ) {
if (random().nextBoolean()) {
- assertEquals("" + i, values[i], reader.next());
+ assertEquals("" + i, values[i], it.next());
++i;
} else {
- final LongsRef nextValues = reader.next(_TestUtil.nextInt(random(), 1, 1024));
+ final LongsRef nextValues = it.next(_TestUtil.nextInt(random(), 1, 1024));
for (int j = 0; j < nextValues.length; ++j) {
assertEquals("" + (i + j), values[i + j], nextValues.longs[nextValues.offset + j]);
}
i += nextValues.length;
}
- assertEquals(i, reader.ord());
+ assertEquals(i, it.ord());
}
assertEquals(fp, in instanceof ByteArrayDataInput ? ((ByteArrayDataInput) in).getPosition() : ((IndexInput) in).getFilePointer());
try {
- reader.next();
+ it.next();
assertTrue(false);
} catch (IOException e) {
// OK
@@ -907,33 +955,121 @@ public class TestPackedInts extends Luce
} else {
((IndexInput) in).seek(0L);
}
- final BlockPackedReader reader2 = new BlockPackedReader(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
+ final BlockPackedReaderIterator it2 = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
int i = 0;
while (true) {
final int skip = _TestUtil.nextInt(random(), 0, valueCount - i);
- reader2.skip(skip);
+ it2.skip(skip);
i += skip;
- assertEquals(i, reader2.ord());
+ assertEquals(i, it2.ord());
if (i == valueCount) {
break;
} else {
- assertEquals(values[i], reader2.next());
+ assertEquals(values[i], it2.next());
++i;
}
}
assertEquals(fp, in instanceof ByteArrayDataInput ? ((ByteArrayDataInput) in).getPosition() : ((IndexInput) in).getFilePointer());
try {
- reader2.skip(1);
+ it2.skip(1);
assertTrue(false);
} catch (IOException e) {
// OK
}
- if (in instanceof IndexInput) {
- ((IndexInput) in).close();
+ in1.seek(0L);
+ final BlockPackedReader reader = new BlockPackedReader(in1, PackedInts.VERSION_CURRENT, blockSize, valueCount, random().nextBoolean());
+ assertEquals(in1.getFilePointer(), in1.length());
+ for (i = 0; i < valueCount; ++i) {
+ assertEquals("i=" + i, values[i], reader.get(i));
}
+ in1.close();
dir.close();
}
}
+ public void testMonotonicBlockPackedReaderWriter() throws IOException {
+ final int iters = atLeast(2);
+ for (int iter = 0; iter < iters; ++iter) {
+ final int blockSize = 1 << _TestUtil.nextInt(random(), 6, 18);
+ final int valueCount = random().nextInt(1 << 18);
+ final long[] values = new long[valueCount];
+ if (valueCount > 0) {
+ values[0] = random().nextBoolean() ? random().nextInt(10) : random().nextInt(Integer.MAX_VALUE);
+ int maxDelta = random().nextInt(64);
+ for (int i = 1; i < valueCount; ++i) {
+ if (random().nextDouble() < 0.1d) {
+ maxDelta = random().nextInt(64);
+ }
+ values[i] = Math.max(0, values[i-1] + _TestUtil.nextInt(random(), -16, maxDelta));
+ }
+ }
+
+ final Directory dir = newDirectory();
+ final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
+ final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(out, blockSize);
+ for (int i = 0; i < valueCount; ++i) {
+ assertEquals(i, writer.ord());
+ writer.add(values[i]);
+ }
+ assertEquals(valueCount, writer.ord());
+ writer.finish();
+ assertEquals(valueCount, writer.ord());
+ final long fp = out.getFilePointer();
+ out.close();
+
+ final IndexInput in = dir.openInput("out.bin", IOContext.DEFAULT);
+ final MonotonicBlockPackedReader reader = new MonotonicBlockPackedReader(in, PackedInts.VERSION_CURRENT, blockSize, valueCount, random().nextBoolean());
+ assertEquals(fp, in.getFilePointer());
+ for (int i = 0; i < valueCount; ++i) {
+ assertEquals("i=" +i, values[i], reader.get(i));
+ }
+ in.close();
+ dir.close();
+ }
+ }
+
+ @Nightly
+ public void testBlockReaderOverflow() throws IOException {
+ final long valueCount = _TestUtil.nextLong(random(), 1L + Integer.MAX_VALUE, (long) Integer.MAX_VALUE * 2);
+ final int blockSize = 1 << _TestUtil.nextInt(random(), 20, 22);
+ final Directory dir = newDirectory();
+ final IndexOutput out = dir.createOutput("out.bin", IOContext.DEFAULT);
+ final BlockPackedWriter writer = new BlockPackedWriter(out, blockSize);
+ long value = random().nextInt() & 0xFFFFFFFFL;
+ long valueOffset = _TestUtil.nextLong(random(), 0, valueCount - 1);
+ for (long i = 0; i < valueCount; ) {
+ assertEquals(i, writer.ord());
+ if ((i & (blockSize - 1)) == 0 && (i + blockSize < valueOffset || i > valueOffset && i + blockSize < valueCount)) {
+ writer.addBlockOfZeros();
+ i += blockSize;
+ } else if (i == valueOffset) {
+ writer.add(value);
+ ++i;
+ } else {
+ writer.add(0);
+ ++i;
+ }
+ }
+ writer.finish();
+ out.close();
+ final IndexInput in = dir.openInput("out.bin", IOContext.DEFAULT);
+ final BlockPackedReaderIterator it = new BlockPackedReaderIterator(in, PackedInts.VERSION_CURRENT, blockSize, valueCount);
+ it.skip(valueOffset);
+ assertEquals(value, it.next());
+ in.seek(0L);
+ final BlockPackedReader reader = new BlockPackedReader(in, PackedInts.VERSION_CURRENT, blockSize, valueCount, random().nextBoolean());
+ assertEquals(value, reader.get(valueOffset));
+ for (int i = 0; i < 5; ++i) {
+ final long offset = _TestUtil.nextLong(random(), 0, valueCount - 1);
+ if (offset == valueOffset) {
+ assertEquals(value, reader.get(offset));
+ } else {
+ assertEquals(0, reader.get(offset));
+ }
+ }
+ in.close();
+ dir.close();
+ }
+
}