You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/16 18:40:00 UTC
svn commit: r1595259 [6/8] - in /lucene/dev/trunk: ./ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/collation/
lucene/analysis/common/src/test/org/apache/lucene/collation/
lucene/analysis/icu/src/jav...
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FieldCacheSource.java Fri May 16 16:39:56 2014
@@ -18,26 +18,20 @@
package org.apache.lucene.queries.function.valuesource;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.FieldCache;
/**
* A base class for ValueSource implementations that retrieve values for
- * a single field from the {@link org.apache.lucene.search.FieldCache}.
+ * a single field from DocValues.
*
*
*/
public abstract class FieldCacheSource extends ValueSource {
protected final String field;
- protected final FieldCache cache = FieldCache.DEFAULT;
public FieldCacheSource(String field) {
this.field=field;
}
- public FieldCache getFieldCache() {
- return cache;
- }
-
public String getField() {
return field;
}
@@ -51,13 +45,12 @@ public abstract class FieldCacheSource e
public boolean equals(Object o) {
if (!(o instanceof FieldCacheSource)) return false;
FieldCacheSource other = (FieldCacheSource)o;
- return this.field.equals(other.field)
- && this.cache == other.cache;
+ return this.field.equals(other.field);
}
@Override
public int hashCode() {
- return cache.hashCode() + field.hashCode();
+ return field.hashCode();
}
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java Fri May 16 16:39:56 2014
@@ -20,29 +20,24 @@ package org.apache.lucene.queries.functi
import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.FloatDocValues;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueFloat;
/**
- * Obtains float field values from {@link FieldCache#getFloats} and makes those
+ * Obtains float field values from {@link AtomicReader#getNumericDocValues} and makes those
* values available as other numeric types, casting as needed.
*/
public class FloatFieldSource extends FieldCacheSource {
- protected final FieldCache.FloatParser parser;
-
public FloatFieldSource(String field) {
- this(field, null);
- }
-
- public FloatFieldSource(String field, FieldCache.FloatParser parser) {
super(field);
- this.parser = parser;
}
@Override
@@ -52,18 +47,13 @@ public class FloatFieldSource extends Fi
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- final FieldCache.Floats arr = cache.getFloats(readerContext.reader(), field, parser, true);
- final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
+ final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
+ final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
return new FloatDocValues(this) {
@Override
public float floatVal(int doc) {
- return arr.get(doc);
- }
-
- @Override
- public Object objectVal(int doc) {
- return valid.get(doc) ? arr.get(doc) : null;
+ return Float.intBitsToFloat((int)arr.get(doc));
}
@Override
@@ -83,7 +73,7 @@ public class FloatFieldSource extends Fi
@Override
public void fillValue(int doc) {
- mval.value = arr.get(doc);
+ mval.value = floatVal(doc);
mval.exists = mval.value != 0 || valid.get(doc);
}
};
@@ -96,14 +86,12 @@ public class FloatFieldSource extends Fi
public boolean equals(Object o) {
if (o.getClass() != FloatFieldSource.class) return false;
FloatFieldSource other = (FloatFieldSource)o;
- return super.equals(other)
- && (this.parser==null ? other.parser==null :
- this.parser.getClass() == other.parser.getClass());
+ return super.equals(other);
}
@Override
public int hashCode() {
- int h = parser==null ? Float.class.hashCode() : parser.getClass().hashCode();
+ int h = Float.class.hashCode();
h += super.hashCode();
return h;
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java Fri May 16 16:39:56 2014
@@ -20,30 +20,26 @@ package org.apache.lucene.queries.functi
import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
/**
- * Obtains int field values from {@link FieldCache#getInts} and makes those
+ * Obtains int field values from {@link AtomicReader#getNumericDocValues} and makes those
* values available as other numeric types, casting as needed.
*/
public class IntFieldSource extends FieldCacheSource {
- final FieldCache.IntParser parser;
public IntFieldSource(String field) {
- this(field, null);
- }
-
- public IntFieldSource(String field, FieldCache.IntParser parser) {
super(field);
- this.parser = parser;
}
@Override
@@ -54,40 +50,20 @@ public class IntFieldSource extends Fiel
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- final FieldCache.Ints arr = cache.getInts(readerContext.reader(), field, parser, true);
- final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
+ final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
+ final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
return new IntDocValues(this) {
final MutableValueInt val = new MutableValueInt();
-
- @Override
- public float floatVal(int doc) {
- return (float)arr.get(doc);
- }
@Override
public int intVal(int doc) {
- return arr.get(doc);
- }
-
- @Override
- public long longVal(int doc) {
- return (long)arr.get(doc);
- }
-
- @Override
- public double doubleVal(int doc) {
- return (double)arr.get(doc);
+ return (int) arr.get(doc);
}
@Override
public String strVal(int doc) {
- return Integer.toString(arr.get(doc));
- }
-
- @Override
- public Object objectVal(int doc) {
- return valid.get(doc) ? arr.get(doc) : null;
+ return Integer.toString(intVal(doc));
}
@Override
@@ -96,11 +72,6 @@ public class IntFieldSource extends Fiel
}
@Override
- public String toString(int doc) {
- return description() + '=' + intVal(doc);
- }
-
- @Override
public ValueFiller getValueFiller() {
return new ValueFiller() {
private final MutableValueInt mval = new MutableValueInt();
@@ -112,13 +83,11 @@ public class IntFieldSource extends Fiel
@Override
public void fillValue(int doc) {
- mval.value = arr.get(doc);
+ mval.value = intVal(doc);
mval.exists = mval.value != 0 || valid.get(doc);
}
};
}
-
-
};
}
@@ -126,14 +95,12 @@ public class IntFieldSource extends Fiel
public boolean equals(Object o) {
if (o.getClass() != IntFieldSource.class) return false;
IntFieldSource other = (IntFieldSource)o;
- return super.equals(other)
- && (this.parser==null ? other.parser==null :
- this.parser.getClass() == other.parser.getClass());
+ return super.equals(other);
}
@Override
public int hashCode() {
- int h = parser==null ? Integer.class.hashCode() : parser.getClass().hashCode();
+ int h = Integer.class.hashCode();
h += super.hashCode();
return h;
}
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java Fri May 16 16:39:56 2014
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.ReaderUtil;
@@ -30,7 +31,6 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.packed.PackedInts;
/**
* Use a field value and find the Document Frequency within another field.
@@ -56,7 +56,7 @@ public class JoinDocFreqValueSource exte
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException
{
- final BinaryDocValues terms = cache.getTerms(readerContext.reader(), field, false, PackedInts.FAST);
+ final BinaryDocValues terms = DocValues.getBinary(readerContext.reader(), field);
final IndexReader top = ReaderUtil.getTopLevelContext(readerContext).reader();
Terms t = MultiFields.getTerms(top, qfield);
final TermsEnum termsEnum = t == null ? TermsEnum.EMPTY : t.iterator(null);
Modified: lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java (original)
+++ lucene/dev/trunk/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/LongFieldSource.java Fri May 16 16:39:56 2014
@@ -20,31 +20,24 @@ package org.apache.lucene.queries.functi
import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
-import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.queries.function.docvalues.LongDocValues;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueLong;
/**
- * Obtains long field values from {@link FieldCache#getLongs} and makes those
+ * Obtains long field values from {@link AtomicReader#getNumericDocValues} and makes those
* values available as other numeric types, casting as needed.
*/
public class LongFieldSource extends FieldCacheSource {
- protected final FieldCache.LongParser parser;
-
public LongFieldSource(String field) {
- this(field, null);
- }
-
- public LongFieldSource(String field, FieldCache.LongParser parser) {
super(field);
- this.parser = parser;
}
@Override
@@ -66,8 +59,8 @@ public class LongFieldSource extends Fie
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- final FieldCache.Longs arr = cache.getLongs(readerContext.reader(), field, parser, true);
- final Bits valid = cache.getDocsWithField(readerContext.reader(), field);
+ final NumericDocValues arr = DocValues.getNumeric(readerContext.reader(), field);
+ final Bits valid = DocValues.getDocsWithField(readerContext.reader(), field);
return new LongDocValues(this) {
@Override
@@ -124,14 +117,12 @@ public class LongFieldSource extends Fie
public boolean equals(Object o) {
if (o.getClass() != this.getClass()) return false;
LongFieldSource other = (LongFieldSource) o;
- return super.equals(other)
- && (this.parser == null ? other.parser == null :
- this.parser.getClass() == other.parser.getClass());
+ return super.equals(other);
}
@Override
public int hashCode() {
- int h = parser == null ? this.getClass().hashCode() : parser.getClass().hashCode();
+ int h = getClass().hashCode();
h += super.hashCode();
return h;
}
Modified: lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java (original)
+++ lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java Fri May 16 16:39:56 2014
@@ -24,7 +24,6 @@ import org.apache.lucene.search.BooleanC
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryUtils;
@@ -39,7 +38,9 @@ import java.util.Map;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.Term;
/**
@@ -66,11 +67,6 @@ public class TestCustomScoreQuery extend
*/
@Test
public void testCustomScoreFloat() throws Exception {
- // INT field can be parsed as float
- doTestCustomScore(INT_AS_FLOAT_VALUESOURCE, 1.0);
- doTestCustomScore(INT_AS_FLOAT_VALUESOURCE, 5.0);
-
- // same values, but in float format
doTestCustomScore(FLOAT_VALUESOURCE, 1.0);
doTestCustomScore(FLOAT_VALUESOURCE, 6.0);
}
@@ -164,7 +160,7 @@ public class TestCustomScoreQuery extend
@Override
protected CustomScoreProvider getCustomScoreProvider(AtomicReaderContext context) throws IOException {
- final FieldCache.Ints values = FieldCache.DEFAULT.getInts(context.reader(), INT_FIELD, false);
+ final NumericDocValues values = DocValues.getNumeric(context.reader(), INT_FIELD);
return new CustomScoreProvider(context) {
@Override
public float customScore(int doc, float subScore, float valSrcScore) {
Modified: lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java (original)
+++ lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java Fri May 16 16:39:56 2014
@@ -1,7 +1,5 @@
package org.apache.lucene.queries.function;
-import java.io.IOException;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -9,15 +7,14 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -60,21 +57,7 @@ public abstract class FunctionTestSetup
protected static final String INT_FIELD = "iii";
protected static final String FLOAT_FIELD = "fff";
- private static final FieldCache.FloatParser CUSTOM_FLOAT_PARSER = new FieldCache.FloatParser() {
-
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return FieldCache.NUMERIC_UTILS_INT_PARSER.termsEnum(terms);
- }
-
- @Override
- public float parseFloat(BytesRef term) {
- return (float) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term);
- }
- };
-
protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD);
- protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD, CUSTOM_FLOAT_PARSER);
protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD);
private static final String DOC_TEXT_LINES[] = {
@@ -152,6 +135,7 @@ public abstract class FunctionTestSetup
f = newField(ID_FIELD, id2String(scoreAndID), customType); // for debug purposes
d.add(f);
+ d.add(new SortedDocValuesField(ID_FIELD, new BytesRef(id2String(scoreAndID))));
FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
customType2.setOmitNorms(true);
@@ -160,9 +144,11 @@ public abstract class FunctionTestSetup
f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
+ d.add(new NumericDocValuesField(INT_FIELD, scoreAndID));
f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
+ d.add(new NumericDocValuesField(FLOAT_FIELD, Float.floatToRawIntBits(scoreAndID)));
iw.addDocument(d);
log("added: " + d);
Modified: lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java (original)
+++ lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java Fri May 16 16:39:56 2014
@@ -53,8 +53,6 @@ public class TestFieldScoreQuery extends
/** Test that FieldScoreQuery of Type.FLOAT returns docs in expected order. */
@Test
public void testRankFloat () throws Exception {
- // INT field can be parsed as float
- doTestRank(INT_AS_FLOAT_VALUESOURCE);
// same values, but in flot format
doTestRank(FLOAT_VALUESOURCE);
}
@@ -88,8 +86,6 @@ public class TestFieldScoreQuery extends
/** Test that FieldScoreQuery of Type.FLOAT returns the expected scores. */
@Test
public void testExactScoreFloat () throws Exception {
- // INT field can be parsed as float
- doTestExactScore(INT_AS_FLOAT_VALUESOURCE);
// same values, but in flot format
doTestExactScore(FLOAT_VALUESOURCE);
}
Modified: lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (original)
+++ lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java Fri May 16 16:39:56 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
@@ -48,12 +49,15 @@ public class TestFunctionQuerySort exten
Document doc = new Document();
Field field = new IntField("value", 0, Field.Store.YES);
+ Field dvField = new NumericDocValuesField("value", 0);
doc.add(field);
+ doc.add(dvField);
// Save docs unsorted (decreasing value n, n-1, ...)
final int NUM_VALS = 5;
for (int val = NUM_VALS; val > 0; val--) {
field.setIntValue(val);
+ dvField.setLongValue(val);
writer.addDocument(doc);
}
Modified: lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (original)
+++ lucene/dev/trunk/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java Fri May 16 16:39:56 2014
@@ -27,6 +27,8 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
@@ -101,26 +103,44 @@ public class TestValueSources extends Lu
Document document = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
document.add(idField);
+ Field idDVField = new SortedDocValuesField("id", new BytesRef());
+ document.add(idDVField);
Field doubleField = new DoubleField("double", 0d, Field.Store.NO);
document.add(doubleField);
+ Field doubleDVField = new NumericDocValuesField("double", 0);
+ document.add(doubleDVField);
Field floatField = new FloatField("float", 0f, Field.Store.NO);
document.add(floatField);
+ Field floatDVField = new NumericDocValuesField("float", 0);
+ document.add(floatDVField);
Field intField = new IntField("int", 0, Field.Store.NO);
document.add(intField);
+ Field intDVField = new NumericDocValuesField("int", 0);
+ document.add(intDVField);
Field longField = new LongField("long", 0L, Field.Store.NO);
document.add(longField);
+ Field longDVField = new NumericDocValuesField("long", 0);
+ document.add(longDVField);
Field stringField = new StringField("string", "", Field.Store.NO);
document.add(stringField);
+ Field stringDVField = new SortedDocValuesField("string", new BytesRef());
+ document.add(stringDVField);
Field textField = new TextField("text", "", Field.Store.NO);
document.add(textField);
for (String [] doc : documents) {
idField.setStringValue(doc[0]);
+ idDVField.setBytesValue(new BytesRef(doc[0]));
doubleField.setDoubleValue(Double.valueOf(doc[1]));
+ doubleDVField.setLongValue(Double.doubleToRawLongBits(Double.valueOf(doc[1])));
floatField.setFloatValue(Float.valueOf(doc[2]));
+ floatDVField.setLongValue(Float.floatToRawIntBits(Float.valueOf(doc[2])));
intField.setIntValue(Integer.valueOf(doc[3]));
+ intDVField.setLongValue(Integer.valueOf(doc[3]));
longField.setLongValue(Long.valueOf(doc[4]));
+ longDVField.setLongValue(Long.valueOf(doc[4]));
stringField.setStringValue(doc[5]);
+ stringDVField.setBytesValue(new BytesRef(doc[5]));
textField.setStringValue(doc[6]);
iw.addDocument(document);
}
Modified: lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java Fri May 16 16:39:56 2014
@@ -22,7 +22,7 @@ import java.text.Collator;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -95,8 +95,8 @@ public final class SlowCollatedStringCom
@Override
public FieldComparator<String> setNextReader(AtomicReaderContext context) throws IOException {
- currentDocTerms = FieldCache.DEFAULT.getTerms(context.reader(), field, true);
- docsWithField = FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
+ currentDocTerms = DocValues.getBinary(context.reader(), field);
+ docsWithField = DocValues.getDocsWithField(context.reader(), field);
return this;
}
Modified: lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedTermRangeFilter.java Fri May 16 16:39:56 2014
@@ -21,7 +21,7 @@ import java.text.Collator;
import org.apache.lucene.search.MultiTermQueryWrapperFilter;
import org.apache.lucene.search.NumericRangeFilter; // javadoc
-import org.apache.lucene.search.FieldCacheRangeFilter; // javadoc
+import org.apache.lucene.search.DocValuesRangeFilter; // javadoc
/**
* A Filter that restricts search results to a range of term
@@ -33,7 +33,7 @@ import org.apache.lucene.search.FieldCac
* for numerical ranges; use {@link NumericRangeFilter} instead.
*
* <p>If you construct a large number of range filters with different ranges but on the
- * same field, {@link FieldCacheRangeFilter} may have significantly better performance.
+ * same field, {@link DocValuesRangeFilter} may have significantly better performance.
* @deprecated Index collation keys with CollationKeyAnalyzer or ICUCollationKeyAnalyzer instead.
* This class will be removed in Lucene 5.0
*/
Modified: lucene/dev/trunk/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java (original)
+++ lucene/dev/trunk/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowCollationMethods.java Fri May 16 16:39:56 2014
@@ -5,11 +5,13 @@ import java.util.Locale;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.AfterClass;
@@ -58,6 +60,8 @@ public class TestSlowCollationMethods ex
String value = TestUtil.randomUnicodeString(random());
Field field = newStringField("field", value, Field.Store.YES);
doc.add(field);
+ Field dvField = new SortedDocValuesField("field", new BytesRef(value));
+ doc.add(dvField);
iw.addDocument(doc);
}
splitDoc = TestUtil.randomUnicodeString(random());
Modified: lucene/dev/trunk/lucene/spatial/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/build.xml?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/build.xml (original)
+++ lucene/dev/trunk/lucene/spatial/build.xml Fri May 16 16:39:56 2014
@@ -32,6 +32,7 @@
<path refid="base.classpath"/>
<path refid="spatialjar"/>
<pathelement path="${queries.jar}" />
+ <pathelement path="${misc.jar}" />
</path>
<path id="test.classpath">
@@ -40,12 +41,13 @@
<pathelement path="src/test-files" />
</path>
- <target name="compile-core" depends="jar-queries,common.compile-core" />
+ <target name="compile-core" depends="jar-queries,jar-misc,common.compile-core" />
- <target name="javadocs" depends="javadocs-queries,compile-core">
+ <target name="javadocs" depends="javadocs-queries,javadocs-misc,compile-core">
<invoke-module-javadoc>
<links>
<link href="../queries"/>
+ <link href="../misc"/>
</links>
</invoke-module-javadoc>
</target>
Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/DisjointSpatialFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/DisjointSpatialFilter.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/DisjointSpatialFilter.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/DisjointSpatialFilter.java Fri May 16 16:39:56 2014
@@ -17,11 +17,12 @@ package org.apache.lucene.spatial;
* limitations under the License.
*/
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.queries.ChainedFilter;
import org.apache.lucene.search.BitsFilteredDocIdSet;
import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.Filter;
import org.apache.lucene.spatial.query.SpatialArgs;
import org.apache.lucene.spatial.query.SpatialOperation;
@@ -48,7 +49,7 @@ public class DisjointSpatialFilter exten
* @param strategy Needed to compute intersects
* @param args Used in spatial intersection
* @param field This field is used to determine which docs have spatial data via
- * {@link org.apache.lucene.search.FieldCache#getDocsWithField(org.apache.lucene.index.AtomicReader, String)}.
+ * {@link AtomicReader#getDocsWithField(String)}.
* Passing null will assume all docs have spatial data.
*/
public DisjointSpatialFilter(SpatialStrategy strategy, SpatialArgs args, String field) {
@@ -92,7 +93,7 @@ public class DisjointSpatialFilter exten
// which is nice but loading it in this way might be slower than say using an
// intersects filter against the world bounds. So do we add a method to the
// strategy, perhaps? But the strategy can't cache it.
- docsWithField = FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
+ docsWithField = DocValues.getDocsWithField(context.reader(), field);
final int maxDoc = context.reader().maxDoc();
if (docsWithField.length() != maxDoc )
Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java Fri May 16 16:39:56 2014
@@ -41,8 +41,7 @@ import org.apache.lucene.spatial.query.S
* <li>What types of query shapes can be used?</li>
* <li>What types of query operations are supported?
* This might vary per shape.</li>
- * <li>Does it use the {@link org.apache.lucene.search.FieldCache},
- * or some other type of cache? When?
+ * <li>Does it use some type of cache? When?
* </ul>
* If a strategy only supports certain shapes at index or query time, then in
* general it will throw an exception if given an incompatible one. It will not
Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java Fri May 16 16:39:56 2014
@@ -20,10 +20,11 @@ package org.apache.lucene.spatial.bbox;
import com.spatial4j.core.shape.Rectangle;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import java.io.IOException;
@@ -64,13 +65,13 @@ public class BBoxSimilarityValueSource e
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
AtomicReader reader = readerContext.reader();
- final FieldCache.Doubles minX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minX, true);
- final FieldCache.Doubles minY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minY, true);
- final FieldCache.Doubles maxX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxX, true);
- final FieldCache.Doubles maxY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxY, true);
+ final NumericDocValues minX = DocValues.getNumeric(reader, strategy.field_minX);
+ final NumericDocValues minY = DocValues.getNumeric(reader, strategy.field_minY);
+ final NumericDocValues maxX = DocValues.getNumeric(reader, strategy.field_maxX);
+ final NumericDocValues maxY = DocValues.getNumeric(reader, strategy.field_maxY);
- final Bits validMinX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_minX);
- final Bits validMaxX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_maxX);
+ final Bits validMinX = DocValues.getDocsWithField(reader, strategy.field_minX);
+ final Bits validMaxX = DocValues.getDocsWithField(reader, strategy.field_maxX);
return new FunctionValues() {
//reused
@@ -78,13 +79,13 @@ public class BBoxSimilarityValueSource e
@Override
public float floatVal(int doc) {
- double minXVal = minX.get(doc);
- double maxXVal = maxX.get(doc);
+ double minXVal = Double.longBitsToDouble(minX.get(doc));
+ double maxXVal = Double.longBitsToDouble(maxX.get(doc));
// make sure it has minX and area
if ((minXVal != 0 || validMinX.get(doc)) && (maxXVal != 0 || validMaxX.get(doc))) {
rect.reset(
minXVal, maxXVal,
- minY.get(doc), maxY.get(doc));
+ Double.longBitsToDouble(minY.get(doc)), Double.longBitsToDouble(maxY.get(doc)));
return (float) similarity.score(rect, null);
} else {
return (float) similarity.score(null, null);
@@ -96,8 +97,8 @@ public class BBoxSimilarityValueSource e
// make sure it has minX and area
if (validMinX.get(doc) && validMaxX.get(doc)) {
rect.reset(
- minX.get(doc), maxX.get(doc),
- minY.get(doc), maxY.get(doc));
+ Double.longBitsToDouble(minX.get(doc)), Double.longBitsToDouble(maxX.get(doc)),
+ Double.longBitsToDouble(minY.get(doc)), Double.longBitsToDouble(maxY.get(doc)));
Explanation exp = new Explanation();
similarity.score(rect, exp);
return exp;
Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java Fri May 16 16:39:56 2014
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Double
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
@@ -64,7 +65,7 @@ import org.apache.lucene.spatial.query.U
* The {@link #makeBBoxAreaSimilarityValueSource(com.spatial4j.core.shape.Rectangle)}
* works by calculating the query bbox overlap percentage against the indexed
* shape overlap percentage. The indexed shape's coordinates are retrieved from
- * the {@link org.apache.lucene.search.FieldCache}.
+ * {@link AtomicReader#getNumericDocValues}
*
* @lucene.experimental
*/
Modified: lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/java/org/apache/lucene/spatial/vector/DistanceValueSource.java Fri May 16 16:39:56 2014
@@ -21,9 +21,10 @@ import com.spatial4j.core.distance.Dista
import com.spatial4j.core.shape.Point;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.util.Bits;
import java.io.IOException;
@@ -65,10 +66,10 @@ public class DistanceValueSource extends
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
AtomicReader reader = readerContext.reader();
- final FieldCache.Doubles ptX = FieldCache.DEFAULT.getDoubles(reader, strategy.getFieldNameX(), true);
- final FieldCache.Doubles ptY = FieldCache.DEFAULT.getDoubles(reader, strategy.getFieldNameY(), true);
- final Bits validX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.getFieldNameX());
- final Bits validY = FieldCache.DEFAULT.getDocsWithField(reader, strategy.getFieldNameY());
+ final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
+ final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());
+ final Bits validX = DocValues.getDocsWithField(reader, strategy.getFieldNameX());
+ final Bits validY = DocValues.getDocsWithField(reader, strategy.getFieldNameY());
return new FunctionValues() {
@@ -87,7 +88,7 @@ public class DistanceValueSource extends
// make sure it has minX and area
if (validX.get(doc)) {
assert validY.get(doc);
- return calculator.distance(from, ptX.get(doc), ptY.get(doc)) * multiplier;
+ return calculator.distance(from, Double.longBitsToDouble(ptX.get(doc)), Double.longBitsToDouble(ptY.get(doc))) * multiplier;
}
return nullValue;
}
Modified: lucene/dev/trunk/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java (original)
+++ lucene/dev/trunk/lucene/spatial/src/test/org/apache/lucene/spatial/SpatialTestCase.java Fri May 16 16:39:56 2014
@@ -32,6 +32,8 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.uninverting.UninvertingReader;
+import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -42,7 +44,9 @@ import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Random;
import static com.carrotsearch.randomizedtesting.RandomizedTest.randomGaussian;
@@ -59,15 +63,26 @@ public abstract class SpatialTestCase ex
protected SpatialContext ctx;//subclass must initialize
+ Map<String,Type> uninvertMap = new HashMap<>();
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
-
+ // TODO: change this module to index docvalues instead of uninverting
+ uninvertMap.clear();
+ uninvertMap.put("bbox__minX", Type.DOUBLE);
+ uninvertMap.put("bbox__maxX", Type.DOUBLE);
+ uninvertMap.put("bbox__minY", Type.DOUBLE);
+ uninvertMap.put("bbox__maxY", Type.DOUBLE);
+ uninvertMap.put("pointvector__x", Type.DOUBLE);
+ uninvertMap.put("pointvector__y", Type.DOUBLE);
+ uninvertMap.put("SpatialOpRecursivePrefixTreeTest", Type.SORTED);
+
directory = newDirectory();
final Random random = random();
indexWriter = new RandomIndexWriter(random,directory, newIndexWriterConfig(random));
- indexReader = indexWriter.getReader();
+ indexReader = UninvertingReader.wrap(indexWriter.getReader(), uninvertMap);
indexSearcher = newSearcher(indexReader);
}
@@ -110,8 +125,11 @@ public abstract class SpatialTestCase ex
protected void commit() throws IOException {
indexWriter.commit();
- IOUtils.close(indexReader);
- indexReader = indexWriter.getReader();
+ DirectoryReader newReader = DirectoryReader.openIfChanged(indexReader);
+ if (newReader != null) {
+ IOUtils.close(indexReader);
+ indexReader = newReader;
+ }
indexSearcher = newSearcher(indexReader);
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/CollationTestBase.java Fri May 16 16:39:56 2014
@@ -154,83 +154,6 @@ public abstract class CollationTestBase
farsiIndex.close();
}
- // Test using various international locales with accented characters (which
- // sort differently depending on locale)
- //
- // Copied (and slightly modified) from
- // org.apache.lucene.search.TestSort.testInternationalSort()
- //
- // TODO: this test is really fragile. there are already 3 different cases,
- // depending upon unicode version.
- public void testCollationKeySort(Analyzer usAnalyzer,
- Analyzer franceAnalyzer,
- Analyzer swedenAnalyzer,
- Analyzer denmarkAnalyzer,
- String usResult,
- String frResult,
- String svResult,
- String dkResult) throws Exception {
- Directory indexStore = newDirectory();
- IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
-
- // document data:
- // the tracer field is used to determine which document was hit
- String[][] sortData = new String[][] {
- // tracer contents US France Sweden (sv_SE) Denmark (da_DK)
- { "A", "x", "p\u00EAche", "p\u00EAche", "p\u00EAche", "p\u00EAche" },
- { "B", "y", "HAT", "HAT", "HAT", "HAT" },
- { "C", "x", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9" },
- { "D", "y", "HUT", "HUT", "HUT", "HUT" },
- { "E", "x", "peach", "peach", "peach", "peach" },
- { "F", "y", "H\u00C5T", "H\u00C5T", "H\u00C5T", "H\u00C5T" },
- { "G", "x", "sin", "sin", "sin", "sin" },
- { "H", "y", "H\u00D8T", "H\u00D8T", "H\u00D8T", "H\u00D8T" },
- { "I", "x", "s\u00EDn", "s\u00EDn", "s\u00EDn", "s\u00EDn" },
- { "J", "y", "HOT", "HOT", "HOT", "HOT" },
- };
-
- FieldType customType = new FieldType();
- customType.setStored(true);
-
- for (int i = 0 ; i < sortData.length ; ++i) {
- Document doc = new Document();
- doc.add(new Field("tracer", sortData[i][0], customType));
- doc.add(new TextField("contents", sortData[i][1], Field.Store.NO));
- if (sortData[i][2] != null)
- doc.add(new TextField("US", usAnalyzer.tokenStream("US", sortData[i][2])));
- if (sortData[i][3] != null)
- doc.add(new TextField("France", franceAnalyzer.tokenStream("France", sortData[i][3])));
- if (sortData[i][4] != null)
- doc.add(new TextField("Sweden", swedenAnalyzer.tokenStream("Sweden", sortData[i][4])));
- if (sortData[i][5] != null)
- doc.add(new TextField("Denmark", denmarkAnalyzer.tokenStream("Denmark", sortData[i][5])));
- writer.addDocument(doc);
- }
- writer.forceMerge(1);
- writer.shutdown();
- IndexReader reader = DirectoryReader.open(indexStore);
- IndexSearcher searcher = new IndexSearcher(reader);
-
- Sort sort = new Sort();
- Query queryX = new TermQuery(new Term ("contents", "x"));
- Query queryY = new TermQuery(new Term ("contents", "y"));
-
- sort.setSort(new SortField("US", SortField.Type.STRING));
- assertMatches(searcher, queryY, sort, usResult);
-
- sort.setSort(new SortField("France", SortField.Type.STRING));
- assertMatches(searcher, queryX, sort, frResult);
-
- sort.setSort(new SortField("Sweden", SortField.Type.STRING));
- assertMatches(searcher, queryY, sort, svResult);
-
- sort.setSort(new SortField("Denmark", SortField.Type.STRING));
- assertMatches(searcher, queryY, sort, dkResult);
- reader.close();
- indexStore.close();
- }
-
// Make sure the documents returned by the search match the expected list
// Copied from TestSort.java
private void assertMatches(IndexSearcher searcher, Query query, Sort sort,
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Fri May 16 16:39:56 2014
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -33,7 +32,6 @@ import java.util.concurrent.CountDownLat
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene42.Lucene42DocValuesFormat;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -47,7 +45,6 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
@@ -1278,73 +1275,6 @@ public abstract class BaseDocValuesForma
dir.close();
}
- private void doTestMissingVsFieldCache(final long minValue, final long maxValue) throws Exception {
- doTestMissingVsFieldCache(new LongProducer() {
- @Override
- long next() {
- return TestUtil.nextLong(random(), minValue, maxValue);
- }
- });
- }
-
- private void doTestMissingVsFieldCache(LongProducer longs) throws Exception {
- assumeTrue("Codec does not support getDocsWithField", defaultCodecSupportsDocsWithField());
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
- Field idField = new StringField("id", "", Field.Store.NO);
- Field indexedField = newStringField("indexed", "", Field.Store.NO);
- Field dvField = new NumericDocValuesField("dv", 0);
-
-
- // index some docs
- int numDocs = atLeast(300);
- // numDocs should be always > 256 so that in case of a codec that optimizes
- // for numbers of values <= 256, all storage layouts are tested
- assert numDocs > 256;
- for (int i = 0; i < numDocs; i++) {
- idField.setStringValue(Integer.toString(i));
- long value = longs.next();
- indexedField.setStringValue(Long.toString(value));
- dvField.setLongValue(value);
- Document doc = new Document();
- doc.add(idField);
- // 1/4 of the time we neglect to add the fields
- if (random().nextInt(4) > 0) {
- doc.add(indexedField);
- doc.add(dvField);
- }
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
-
- // delete some docs
- int numDeletions = random().nextInt(numDocs/10);
- for (int i = 0; i < numDeletions; i++) {
- int id = random().nextInt(numDocs);
- writer.deleteDocuments(new Term("id", Integer.toString(id)));
- }
-
- // merge some segments and ensure that at least one of them has more than
- // 256 values
- writer.forceMerge(numDocs / 256);
-
- writer.shutdown();
-
- // compare
- DirectoryReader ir = DirectoryReader.open(dir);
- for (AtomicReaderContext context : ir.leaves()) {
- AtomicReader r = context.reader();
- Bits expected = FieldCache.DEFAULT.getDocsWithField(r, "indexed");
- Bits actual = FieldCache.DEFAULT.getDocsWithField(r, "dv");
- assertEquals(expected, actual);
- }
- ir.close();
- dir.close();
- }
-
public void testBooleanNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1359,13 +1289,6 @@ public abstract class BaseDocValuesForma
}
}
- public void testByteMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Byte.MIN_VALUE, Byte.MAX_VALUE);
- }
- }
-
public void testShortNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1373,13 +1296,6 @@ public abstract class BaseDocValuesForma
}
}
- public void testShortMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Short.MIN_VALUE, Short.MAX_VALUE);
- }
- }
-
public void testIntNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1387,13 +1303,6 @@ public abstract class BaseDocValuesForma
}
}
- public void testIntMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Integer.MIN_VALUE, Integer.MAX_VALUE);
- }
- }
-
public void testLongNumericsVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1401,13 +1310,6 @@ public abstract class BaseDocValuesForma
}
}
- public void testLongMissingVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestMissingVsFieldCache(Long.MIN_VALUE, Long.MAX_VALUE);
- }
- }
-
private void doTestBinaryVsStoredFields(int minLength, int maxLength) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
@@ -1535,57 +1437,6 @@ public abstract class BaseDocValuesForma
dir.close();
}
- private void doTestSortedVsFieldCache(int minLength, int maxLength) throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
- Document doc = new Document();
- Field idField = new StringField("id", "", Field.Store.NO);
- Field indexedField = new StringField("indexed", "", Field.Store.NO);
- Field dvField = new SortedDocValuesField("dv", new BytesRef());
- doc.add(idField);
- doc.add(indexedField);
- doc.add(dvField);
-
- // index some docs
- int numDocs = atLeast(300);
- for (int i = 0; i < numDocs; i++) {
- idField.setStringValue(Integer.toString(i));
- final int length;
- if (minLength == maxLength) {
- length = minLength; // fixed length
- } else {
- length = TestUtil.nextInt(random(), minLength, maxLength);
- }
- String value = TestUtil.randomSimpleString(random(), length);
- indexedField.setStringValue(value);
- dvField.setBytesValue(new BytesRef(value));
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
-
- // delete some docs
- int numDeletions = random().nextInt(numDocs/10);
- for (int i = 0; i < numDeletions; i++) {
- int id = random().nextInt(numDocs);
- writer.deleteDocuments(new Term("id", Integer.toString(id)));
- }
- writer.shutdown();
-
- // compare
- DirectoryReader ir = DirectoryReader.open(dir);
- for (AtomicReaderContext context : ir.leaves()) {
- AtomicReader r = context.reader();
- SortedDocValues expected = FieldCache.DEFAULT.getTermsIndex(r, "indexed");
- SortedDocValues actual = r.getSortedDocValues("dv");
- assertEquals(r.maxDoc(), expected, actual);
- }
- ir.close();
- dir.close();
- }
-
public void testSortedFixedLengthVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -1594,21 +1445,6 @@ public abstract class BaseDocValuesForma
}
}
- public void testSortedFixedLengthVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- int fixedLength = TestUtil.nextInt(random(), 1, 10);
- doTestSortedVsFieldCache(fixedLength, fixedLength);
- }
- }
-
- public void testSortedVariableLengthVsFieldCache() throws Exception {
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSortedVsFieldCache(1, 10);
- }
- }
-
public void testSortedVariableLengthVsStoredFields() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -2173,206 +2009,6 @@ public abstract class BaseDocValuesForma
}
}
- private void assertEquals(Bits expected, Bits actual) throws Exception {
- assertEquals(expected.length(), actual.length());
- for (int i = 0; i < expected.length(); i++) {
- assertEquals(expected.get(i), actual.get(i));
- }
- }
-
- private void assertEquals(int maxDoc, SortedDocValues expected, SortedDocValues actual) throws Exception {
- assertEquals(maxDoc, new SingletonSortedSetDocValues(expected), new SingletonSortedSetDocValues(actual));
- }
-
- private void assertEquals(int maxDoc, SortedSetDocValues expected, SortedSetDocValues actual) throws Exception {
- // can be null for the segment if no docs actually had any SortedDocValues
- // in this case FC.getDocTermsOrds returns EMPTY
- if (actual == null) {
- assertEquals(DocValues.EMPTY_SORTED_SET, expected);
- return;
- }
- assertEquals(expected.getValueCount(), actual.getValueCount());
- // compare ord lists
- for (int i = 0; i < maxDoc; i++) {
- expected.setDocument(i);
- actual.setDocument(i);
- long expectedOrd;
- while ((expectedOrd = expected.nextOrd()) != NO_MORE_ORDS) {
- assertEquals(expectedOrd, actual.nextOrd());
- }
- assertEquals(NO_MORE_ORDS, actual.nextOrd());
- }
-
- // compare ord dictionary
- BytesRef expectedBytes = new BytesRef();
- BytesRef actualBytes = new BytesRef();
- for (long i = 0; i < expected.getValueCount(); i++) {
- expected.lookupTerm(expectedBytes);
- actual.lookupTerm(actualBytes);
- assertEquals(expectedBytes, actualBytes);
- }
-
- // compare termsenum
- assertEquals(expected.getValueCount(), expected.termsEnum(), actual.termsEnum());
- }
-
- private void assertEquals(long numOrds, TermsEnum expected, TermsEnum actual) throws Exception {
- BytesRef ref;
-
- // sequential next() through all terms
- while ((ref = expected.next()) != null) {
- assertEquals(ref, actual.next());
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
- assertNull(actual.next());
-
- // sequential seekExact(ord) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- actual.seekExact(i);
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // sequential seekExact(BytesRef) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- assertTrue(actual.seekExact(expected.term()));
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // sequential seekCeil(BytesRef) through all terms
- for (long i = 0; i < numOrds; i++) {
- expected.seekExact(i);
- assertEquals(SeekStatus.FOUND, actual.seekCeil(expected.term()));
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekExact(ord)
- for (long i = 0; i < numOrds; i++) {
- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
- expected.seekExact(randomOrd);
- actual.seekExact(randomOrd);
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekExact(BytesRef)
- for (long i = 0; i < numOrds; i++) {
- long randomOrd = TestUtil.nextLong(random(), 0, numOrds - 1);
- expected.seekExact(randomOrd);
- actual.seekExact(expected.term());
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
-
- // random seekCeil(BytesRef)
- for (long i = 0; i < numOrds; i++) {
- BytesRef target = new BytesRef(TestUtil.randomUnicodeString(random()));
- SeekStatus expectedStatus = expected.seekCeil(target);
- assertEquals(expectedStatus, actual.seekCeil(target));
- if (expectedStatus != SeekStatus.END) {
- assertEquals(expected.ord(), actual.ord());
- assertEquals(expected.term(), actual.term());
- }
- }
- }
-
- private void doTestSortedSetVsUninvertedField(int minLength, int maxLength) throws Exception {
- Directory dir = newDirectory();
- IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
- RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
-
- // index some docs
- int numDocs = atLeast(300);
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- Field idField = new StringField("id", Integer.toString(i), Field.Store.NO);
- doc.add(idField);
- final int length;
- if (minLength == maxLength) {
- length = minLength; // fixed length
- } else {
- length = TestUtil.nextInt(random(), minLength, maxLength);
- }
- int numValues = random().nextInt(17);
- // create a random list of strings
- List<String> values = new ArrayList<>();
- for (int v = 0; v < numValues; v++) {
- values.add(TestUtil.randomSimpleString(random(), length));
- }
-
- // add in any order to the indexed field
- ArrayList<String> unordered = new ArrayList<>(values);
- Collections.shuffle(unordered, random());
- for (String v : values) {
- doc.add(newStringField("indexed", v, Field.Store.NO));
- }
-
- // add in any order to the dv field
- ArrayList<String> unordered2 = new ArrayList<>(values);
- Collections.shuffle(unordered2, random());
- for (String v : unordered2) {
- doc.add(new SortedSetDocValuesField("dv", new BytesRef(v)));
- }
-
- writer.addDocument(doc);
- if (random().nextInt(31) == 0) {
- writer.commit();
- }
- }
-
- // delete some docs
- int numDeletions = random().nextInt(numDocs/10);
- for (int i = 0; i < numDeletions; i++) {
- int id = random().nextInt(numDocs);
- writer.deleteDocuments(new Term("id", Integer.toString(id)));
- }
-
- // compare per-segment
- DirectoryReader ir = writer.getReader();
- for (AtomicReaderContext context : ir.leaves()) {
- AtomicReader r = context.reader();
- SortedSetDocValues expected = FieldCache.DEFAULT.getDocTermOrds(r, "indexed");
- SortedSetDocValues actual = r.getSortedSetDocValues("dv");
- assertEquals(r.maxDoc(), expected, actual);
- }
- ir.close();
-
- writer.forceMerge(1);
-
- // now compare again after the merge
- ir = writer.getReader();
- AtomicReader ar = getOnlySegmentReader(ir);
- SortedSetDocValues expected = FieldCache.DEFAULT.getDocTermOrds(ar, "indexed");
- SortedSetDocValues actual = ar.getSortedSetDocValues("dv");
- assertEquals(ir.maxDoc(), expected, actual);
- ir.close();
-
- writer.shutdown();
- dir.close();
- }
-
- public void testSortedSetFixedLengthVsUninvertedField() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- int fixedLength = TestUtil.nextInt(random(), 1, 10);
- doTestSortedSetVsUninvertedField(fixedLength, fixedLength);
- }
- }
-
- public void testSortedSetVariableLengthVsUninvertedField() throws Exception {
- assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
- int numIterations = atLeast(1);
- for (int i = 0; i < numIterations; i++) {
- doTestSortedSetVsUninvertedField(1, 10);
- }
- }
-
public void testGCDCompression() throws Exception {
int numIterations = atLeast(1);
for (int i = 0; i < numIterations; i++) {
@@ -2606,172 +2242,6 @@ public abstract class BaseDocValuesForma
ir.close();
directory.close();
}
-
- // LUCENE-4853
- public void testHugeBinaryValues() throws Exception {
- Analyzer analyzer = new MockAnalyzer(random());
- // FSDirectory because SimpleText will consume gobbs of
- // space when storing big binary values:
- Directory d = newFSDirectory(createTempDir("hugeBinaryValues"));
- boolean doFixed = random().nextBoolean();
- int numDocs;
- int fixedLength = 0;
- if (doFixed) {
- // Sometimes make all values fixed length since some
- // codecs have different code paths for this:
- numDocs = TestUtil.nextInt(random(), 10, 20);
- fixedLength = TestUtil.nextInt(random(), 65537, 256 * 1024);
- } else {
- numDocs = TestUtil.nextInt(random(), 100, 200);
- }
- IndexWriter w = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
- List<byte[]> docBytes = new ArrayList<>();
- long totalBytes = 0;
- for(int docID=0;docID<numDocs;docID++) {
- // we don't use RandomIndexWriter because it might add
- // more docvalues than we expect !!!!
-
- // Must be > 64KB in size to ensure more than 2 pages in
- // PagedBytes would be needed:
- int numBytes;
- if (doFixed) {
- numBytes = fixedLength;
- } else if (docID == 0 || random().nextInt(5) == 3) {
- numBytes = TestUtil.nextInt(random(), 65537, 3 * 1024 * 1024);
- } else {
- numBytes = TestUtil.nextInt(random(), 1, 1024 * 1024);
- }
- totalBytes += numBytes;
- if (totalBytes > 5 * 1024*1024) {
- break;
- }
- byte[] bytes = new byte[numBytes];
- random().nextBytes(bytes);
- docBytes.add(bytes);
- Document doc = new Document();
- BytesRef b = new BytesRef(bytes);
- b.length = bytes.length;
- doc.add(new BinaryDocValuesField("field", b));
- doc.add(new StringField("id", ""+docID, Field.Store.YES));
- try {
- w.addDocument(doc);
- } catch (IllegalArgumentException iae) {
- if (iae.getMessage().indexOf("is too large") == -1) {
- throw iae;
- } else {
- // OK: some codecs can't handle binary DV > 32K
- assertFalse(codecAcceptsHugeBinaryValues("field"));
- w.rollback();
- d.close();
- return;
- }
- }
- }
-
- DirectoryReader r;
- try {
- r = w.getReader();
- } catch (IllegalArgumentException iae) {
- if (iae.getMessage().indexOf("is too large") == -1) {
- throw iae;
- } else {
- assertFalse(codecAcceptsHugeBinaryValues("field"));
-
- // OK: some codecs can't handle binary DV > 32K
- w.rollback();
- d.close();
- return;
- }
- }
- w.shutdown();
-
- AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
-
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
- for(int docID=0;docID<docBytes.size();docID++) {
- StoredDocument doc = ar.document(docID);
- BytesRef bytes = new BytesRef();
- s.get(docID, bytes);
- byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
- assertEquals(expected.length, bytes.length);
- assertEquals(new BytesRef(expected), bytes);
- }
-
- assertTrue(codecAcceptsHugeBinaryValues("field"));
-
- ar.close();
- d.close();
- }
-
- // TODO: get this out of here and into the deprecated codecs (4.0, 4.2)
- public void testHugeBinaryValueLimit() throws Exception {
- // We only test DVFormats that have a limit
- assumeFalse("test requires codec with limits on max binary field length", codecAcceptsHugeBinaryValues("field"));
- Analyzer analyzer = new MockAnalyzer(random());
- // FSDirectory because SimpleText will consume gobbs of
- // space when storing big binary values:
- Directory d = newFSDirectory(createTempDir("hugeBinaryValues"));
- boolean doFixed = random().nextBoolean();
- int numDocs;
- int fixedLength = 0;
- if (doFixed) {
- // Sometimes make all values fixed length since some
- // codecs have different code paths for this:
- numDocs = TestUtil.nextInt(random(), 10, 20);
- fixedLength = Lucene42DocValuesFormat.MAX_BINARY_FIELD_LENGTH;
- } else {
- numDocs = TestUtil.nextInt(random(), 100, 200);
- }
- IndexWriter w = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
- List<byte[]> docBytes = new ArrayList<>();
- long totalBytes = 0;
- for(int docID=0;docID<numDocs;docID++) {
- // we don't use RandomIndexWriter because it might add
- // more docvalues than we expect !!!!
-
- // Must be > 64KB in size to ensure more than 2 pages in
- // PagedBytes would be needed:
- int numBytes;
- if (doFixed) {
- numBytes = fixedLength;
- } else if (docID == 0 || random().nextInt(5) == 3) {
- numBytes = Lucene42DocValuesFormat.MAX_BINARY_FIELD_LENGTH;
- } else {
- numBytes = TestUtil.nextInt(random(), 1, Lucene42DocValuesFormat.MAX_BINARY_FIELD_LENGTH);
- }
- totalBytes += numBytes;
- if (totalBytes > 5 * 1024*1024) {
- break;
- }
- byte[] bytes = new byte[numBytes];
- random().nextBytes(bytes);
- docBytes.add(bytes);
- Document doc = new Document();
- BytesRef b = new BytesRef(bytes);
- b.length = bytes.length;
- doc.add(new BinaryDocValuesField("field", b));
- doc.add(new StringField("id", ""+docID, Field.Store.YES));
- w.addDocument(doc);
- }
-
- DirectoryReader r = w.getReader();
- w.shutdown();
-
- AtomicReader ar = SlowCompositeReaderWrapper.wrap(r);
-
- BinaryDocValues s = FieldCache.DEFAULT.getTerms(ar, "field", false);
- for(int docID=0;docID<docBytes.size();docID++) {
- StoredDocument doc = ar.document(docID);
- BytesRef bytes = new BytesRef();
- s.get(docID, bytes);
- byte[] expected = docBytes.get(Integer.parseInt(doc.get("id")));
- assertEquals(expected.length, bytes.length);
- assertEquals(new BytesRef(expected), bytes);
- }
-
- ar.close();
- d.close();
- }
/** Tests dv against stored fields with threads (binary/numeric/sorted, no missing) */
public void testThreads() throws Exception {
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseStoredFieldsFormatTestCase.java Fri May 16 16:39:56 2014
@@ -43,10 +43,10 @@ import org.apache.lucene.document.FieldT
import org.apache.lucene.document.FloatField;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.Query;
@@ -289,6 +289,7 @@ public abstract class BaseStoredFieldsFo
FieldType ft = new FieldType(IntField.TYPE_STORED);
ft.setNumericPrecisionStep(Integer.MAX_VALUE);
doc.add(new IntField("id", id, ft));
+ doc.add(new NumericDocValuesField("id", id));
w.addDocument(doc);
}
final DirectoryReader r = w.getReader();
@@ -298,12 +299,12 @@ public abstract class BaseStoredFieldsFo
for(AtomicReaderContext ctx : r.leaves()) {
final AtomicReader sub = ctx.reader();
- final FieldCache.Ints ids = FieldCache.DEFAULT.getInts(sub, "id", false);
+ final NumericDocValues ids = DocValues.getNumeric(sub, "id");
for(int docID=0;docID<sub.numDocs();docID++) {
final StoredDocument doc = sub.document(docID);
final Field f = (Field) doc.getField("nf");
assertTrue("got f=" + f, f instanceof StoredField);
- assertEquals(answers[ids.get(docID)], f.numericValue());
+ assertEquals(answers[(int) ids.get(docID)], f.numericValue());
}
}
r.close();
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/search/QueryUtils.java Fri May 16 16:39:56 2014
@@ -130,11 +130,6 @@ public class QueryUtils {
}
}
- public static void purgeFieldCache(IndexReader r) throws IOException {
- // this is just a hack, to get an atomic reader that contains all subreaders for insanity checks
- FieldCache.DEFAULT.purgeByCacheKey(SlowCompositeReaderWrapper.wrap(r).getCoreCacheKey());
- }
-
/** This is a MultiReader that can be used for randomly wrapping other readers
* without creating FieldCache insanity.
* The trick is to use an opaque/fake cache key. */
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Fri May 16 16:39:56 2014
@@ -105,8 +105,6 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.AssertingIndexSearcher;
import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.FieldCache.CacheEntry;
-import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils.FCInvisibleMultiReader;
import org.apache.lucene.store.BaseDirectoryWrapper;
@@ -121,7 +119,6 @@ import org.apache.lucene.store.MockDirec
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NRTCachingDirectory;
import org.apache.lucene.store.RateLimitedDirectoryWrapper;
-import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
@@ -630,7 +627,6 @@ public abstract class LuceneTestCase ext
.around(threadAndTestNameRule)
.around(new SystemPropertiesInvariantRule(IGNORED_INVARIANT_PROPERTIES))
.around(new TestRuleSetupAndRestoreInstanceEnv())
- .around(new TestRuleFieldCacheSanity())
.around(parentChainCallRule);
private static final Map<String,FieldType> fieldToType = new HashMap<String,FieldType>();
@@ -742,48 +738,6 @@ public abstract class LuceneTestCase ext
}
/**
- * Asserts that FieldCacheSanityChecker does not detect any
- * problems with FieldCache.DEFAULT.
- * <p>
- * If any problems are found, they are logged to System.err
- * (allong with the msg) when the Assertion is thrown.
- * </p>
- * <p>
- * This method is called by tearDown after every test method,
- * however IndexReaders scoped inside test methods may be garbage
- * collected prior to this method being called, causing errors to
- * be overlooked. Tests are encouraged to keep their IndexReaders
- * scoped at the class level, or to explicitly call this method
- * directly in the same scope as the IndexReader.
- * </p>
- *
- * @see org.apache.lucene.util.FieldCacheSanityChecker
- */
- protected static void assertSaneFieldCaches(final String msg) {
- final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
- Insanity[] insanity = null;
- try {
- try {
- insanity = FieldCacheSanityChecker.checkSanity(entries);
- } catch (RuntimeException e) {
- dumpArray(msg + ": FieldCache", entries, System.err);
- throw e;
- }
-
- assertEquals(msg + ": Insane FieldCache usage(s) found",
- 0, insanity.length);
- insanity = null;
- } finally {
-
- // report this in the event of any exception/failure
- // if no failure, then insanity will be null anyway
- if (null != insanity) {
- dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err);
- }
- }
- }
-
- /**
* Returns a number of at least <code>i</code>
* <p>
* The actual number returned will be influenced by whether {@link #TEST_NIGHTLY}
Modified: lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java (original)
+++ lucene/dev/trunk/solr/contrib/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java Fri May 16 16:39:56 2014
@@ -34,10 +34,11 @@ import org.apache.lucene.document.Sorted
import org.apache.lucene.index.StorableField;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocTermOrdsRangeFilter;
-import org.apache.lucene.search.FieldCacheRangeFilter;
+import org.apache.lucene.search.DocValuesRangeFilter;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.uninverting.UninvertingReader.Type;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
import org.apache.lucene.analysis.util.ResourceLoader;
@@ -224,6 +225,15 @@ public class ICUCollationField extends F
public SortField getSortField(SchemaField field, boolean top) {
return getStringSort(field, top);
}
+
+ @Override
+ public Type getUninversionType(SchemaField sf) {
+ if (sf.multiValued()) {
+ return Type.SORTED_SET_BINARY;
+ } else {
+ return Type.SORTED;
+ }
+ }
@Override
public Analyzer getIndexAnalyzer() {
@@ -270,7 +280,7 @@ public class ICUCollationField extends F
return new ConstantScoreQuery(DocTermOrdsRangeFilter.newBytesRefRange(
field.getName(), low, high, minInclusive, maxInclusive));
} else {
- return new ConstantScoreQuery(FieldCacheRangeFilter.newBytesRefRange(
+ return new ConstantScoreQuery(DocValuesRangeFilter.newBytesRefRange(
field.getName(), low, high, minInclusive, maxInclusive));
}
} else {
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/statistics/StatsCollectorSupplierFactory.java Fri May 16 16:39:56 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
-import org.apache.lucene.search.FieldCache;
import org.apache.solr.analytics.expression.ExpressionFactory;
import org.apache.solr.analytics.request.ExpressionRequest;
import org.apache.solr.analytics.util.AnalyticsParams;
@@ -357,7 +356,7 @@ public class StatsCollectorSupplierFacto
if (sourceType!=NUMBER_TYPE&&sourceType!=FIELD_TYPE) {
return null;
}
- return new IntFieldSource(expressionString, FieldCache.NUMERIC_UTILS_INT_PARSER) {
+ return new IntFieldSource(expressionString) {
public String description() {
return field;
}
@@ -366,7 +365,7 @@ public class StatsCollectorSupplierFacto
if (sourceType!=NUMBER_TYPE&&sourceType!=FIELD_TYPE) {
return null;
}
- return new LongFieldSource(expressionString, FieldCache.NUMERIC_UTILS_LONG_PARSER) {
+ return new LongFieldSource(expressionString) {
public String description() {
return field;
}
@@ -375,7 +374,7 @@ public class StatsCollectorSupplierFacto
if (sourceType!=NUMBER_TYPE&&sourceType!=FIELD_TYPE) {
return null;
}
- return new FloatFieldSource(expressionString, FieldCache.NUMERIC_UTILS_FLOAT_PARSER) {
+ return new FloatFieldSource(expressionString) {
public String description() {
return field;
}
@@ -384,7 +383,7 @@ public class StatsCollectorSupplierFacto
if (sourceType!=NUMBER_TYPE&&sourceType!=FIELD_TYPE) {
return null;
}
- return new DoubleFieldSource(expressionString, FieldCache.NUMERIC_UTILS_DOUBLE_PARSER) {
+ return new DoubleFieldSource(expressionString) {
public String description() {
return field;
}
@@ -393,7 +392,7 @@ public class StatsCollectorSupplierFacto
if (sourceType!=DATE_TYPE&&sourceType!=FIELD_TYPE) {
return null;
}
- return new DateFieldSource(expressionString, AnalyticsParsers.DEFAULT_DATE_PARSER) {
+ return new DateFieldSource(expressionString) {
public String description() {
return field;
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java?rev=1595259&r1=1595258&r2=1595259&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/analytics/util/AnalyticsParsers.java Fri May 16 16:39:56 2014
@@ -18,14 +18,9 @@
package org.apache.solr.analytics.util;
import java.io.IOException;
-import java.text.ParseException;
import java.util.Arrays;
import java.util.Date;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.search.FieldCache.LongParser;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.solr.schema.FieldType;
@@ -61,31 +56,7 @@ public class AnalyticsParsers {
return AnalyticsParsers.STRING_PARSER;
}
}
-
- /** Long Parser that takes in String representations of dates and
- * converts them into longs
- */
- public final static LongParser DEFAULT_DATE_PARSER = new LongParser() {
- @SuppressWarnings("deprecation")
- @Override
- public long parseLong(BytesRef term) {
- try {
- return TrieDateField.parseDate(term.utf8ToString()).getTime();
- } catch (ParseException e) {
- System.err.println("Cannot parse date "+term.utf8ToString());
- return 0;
- }
- }
- @Override
- public String toString() {
- return FieldCache.class.getName()+".DEFAULT_DATE_PARSER";
- }
- @Override
- public TermsEnum termsEnum(Terms terms) throws IOException {
- return terms.iterator(null);
- }
- };
-
+
/**
* For use in classes that grab values by docValue.
* Converts a BytesRef object into the correct readable text.