You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/01/22 22:24:49 UTC
svn commit: r1437183 - in /lucene/dev/branches/lucene4547/lucene:
core/src/java/org/apache/lucene/search/
queries/src/java/org/apache/lucene/queries/function/docvalues/
queries/src/java/org/apache/lucene/queries/function/valuesource/
queries/src/test/o...
Author: jpountz
Date: Tue Jan 22 21:24:49 2013
New Revision: 1437183
URL: http://svn.apache.org/viewvc?rev=1437183&view=rev
Log:
Re-enable test.
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
lucene/dev/branches/lucene4547/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1437183&r1=1437182&r2=1437183&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Tue Jan 22 21:24:49 2013
@@ -20,7 +20,6 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
-import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -29,14 +28,12 @@ import java.util.WeakHashMap;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocTermOrds;
-import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.NumericDocValues;
-import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.SegmentReader;
import org.apache.lucene.index.SortedDocValues;
-import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.ArrayUtil;
@@ -628,10 +625,21 @@ class FieldCacheImpl implements FieldCac
@Override
protected Object createValue(AtomicReader reader, CacheKey key, boolean setDocsWithField /* ignored */)
throws IOException {
- final String field = key.field;
+ final String field = key.field;
+ final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+ final int maxDoc = reader.maxDoc();
+
+ if (fieldInfo == null) {
+ // field does not exist or has no value
+ return new Bits.MatchNoBits(maxDoc);
+ } else if (fieldInfo.hasDocValues()) {
+ // doc values are dense
+ return new Bits.MatchAllBits(maxDoc);
+ }
+
+ // Visit all docs that have terms for this field
FixedBitSet res = null;
Terms terms = reader.terms(field);
- final int maxDoc = reader.maxDoc();
if (terms != null) {
final int termsDocCount = terms.getDocCount();
assert termsDocCount <= maxDoc;
Modified: lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java?rev=1437183&r1=1437182&r2=1437183&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java Tue Jan 22 21:24:49 2013
@@ -26,6 +26,7 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.UnicodeUtil;
@@ -33,11 +34,12 @@ import org.apache.lucene.util.mutable.Mu
import org.apache.lucene.util.mutable.MutableValueStr;
/**
- * Internal class, subject to change.
* Serves as base class for FunctionValues based on DocTermsIndex.
+ * @lucene.internal
*/
public abstract class DocTermsIndexDocValues extends FunctionValues {
protected final SortedDocValues termsIndex;
+ protected final Bits valid;
protected final ValueSource vs;
protected final MutableValueStr val = new MutableValueStr();
protected final BytesRef spare = new BytesRef();
@@ -46,40 +48,42 @@ public abstract class DocTermsIndexDocVa
public DocTermsIndexDocValues(ValueSource vs, AtomicReaderContext context, String field) throws IOException {
try {
termsIndex = FieldCache.DEFAULT.getTermsIndex(context.reader(), field);
+ valid = FieldCache.DEFAULT.getDocsWithField(context.reader(), field);
} catch (RuntimeException e) {
throw new DocTermsIndexException(field, e);
}
this.vs = vs;
}
- public SortedDocValues getSortedDocValues() {
- return termsIndex;
- }
-
protected abstract String toTerm(String readableValue);
@Override
public boolean exists(int doc) {
- return termsIndex.getOrd(doc) != -1;
+ return valid.get(doc);
+ }
+
+ @Override
+ public int ordVal(int doc) {
+ return termsIndex.getOrd(doc);
}
+ @Override
+ public int numOrd() {
+ return termsIndex.getValueCount();
+ }
@Override
public boolean bytesVal(int doc, BytesRef target) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) {
- target.length = 0;
- return false;
- }
- termsIndex.lookupOrd(ord, target);
- return true;
+ termsIndex.get(doc, target);
+ return target.length > 0;
}
@Override
public String strVal(int doc) {
- int ord=termsIndex.getOrd(doc);
- if (ord==-1) return null;
- termsIndex.lookupOrd(ord, spare);
+ termsIndex.get(doc, spare);
+ if (spare.length == 0) {
+ return null;
+ }
UnicodeUtil.UTF8toUTF16(spare, spareChars);
return spareChars.toString();
}
@@ -149,13 +153,7 @@ public abstract class DocTermsIndexDocVa
@Override
public void fillValue(int doc) {
- int ord = termsIndex.getOrd(doc);
- mval.exists = ord != -1;
- if (!mval.exists) {
- mval.value.length = 0;
- } else {
- termsIndex.lookupOrd(ord, mval.value);
- }
+ termsIndex.get(doc, mval.value);
}
};
}
Modified: lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java?rev=1437183&r1=1437182&r2=1437183&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java (original)
+++ lucene/dev/branches/lucene4547/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/BytesRefFieldSource.java Tue Jan 22 21:24:49 2013
@@ -17,12 +17,18 @@ package org.apache.lucene.queries.functi
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Map;
+
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.docvalues.DocTermsIndexDocValues;
-
-import java.io.IOException;
-import java.util.Map;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
/**
* An implementation for retrieving {@link FunctionValues} instances for string based fields.
@@ -35,23 +41,59 @@ public class BytesRefFieldSource extends
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
- return new DocTermsIndexDocValues(this, readerContext, field) {
-
- @Override
- protected String toTerm(String readableValue) {
- return readableValue;
- }
-
- @Override
- public Object objectVal(int doc) {
- return strVal(doc);
- }
-
- @Override
- public String toString(int doc) {
- return description() + '=' + strVal(doc);
- }
-
- };
+ final FieldInfo fieldInfo = readerContext.reader().getFieldInfos().fieldInfo(field);
+ // To be sorted or not to be sorted, that is the question
+ // nocommit: do it cleaner?
+ if (fieldInfo != null && fieldInfo.hasDocValues() && fieldInfo.getDocValuesType() == DocValuesType.BINARY) {
+ final BinaryDocValues binaryValues = FieldCache.DEFAULT.getTerms(readerContext.reader(), field);
+ return new FunctionValues() {
+
+ @Override
+ public boolean exists(int doc) {
+ return true; // doc values are dense
+ }
+
+ @Override
+ public boolean bytesVal(int doc, BytesRef target) {
+ binaryValues.get(doc, target);
+ return target.length > 0;
+ }
+
+ public String strVal(int doc) {
+ final BytesRef bytes = new BytesRef();
+ return bytesVal(doc, bytes)
+ ? bytes.utf8ToString()
+ : null;
+ }
+
+ @Override
+ public Object objectVal(int doc) {
+ return strVal(doc);
+ }
+
+ @Override
+ public String toString(int doc) {
+ return description() + '=' + strVal(doc);
+ }
+ };
+ } else {
+ return new DocTermsIndexDocValues(this, readerContext, field) {
+
+ @Override
+ protected String toTerm(String readableValue) {
+ return readableValue;
+ }
+
+ @Override
+ public Object objectVal(int doc) {
+ return strVal(doc);
+ }
+
+ @Override
+ public String toString(int doc) {
+ return description() + '=' + strVal(doc);
+ }
+ };
+ }
}
}
Modified: lucene/dev/branches/lucene4547/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java?rev=1437183&r1=1437182&r2=1437183&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java (original)
+++ lucene/dev/branches/lucene4547/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java Tue Jan 22 21:24:49 2013
@@ -37,7 +37,6 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.packed.PackedInts;
-import org.junit.Ignore;
import com.carrotsearch.randomizedtesting.generators.RandomInts;
@@ -71,11 +70,13 @@ public class TestDocValuesFieldSources e
RandomIndexWriter iw = new RandomIndexWriter(random(), d, iwConfig);
for (int i = 0; i < nDocs; ++i) {
- id.setIntValue(i);
+ id.setLongValue(i);
switch (type) {
case SORTED:
case BINARY:
- vals[i] = _TestUtil.randomSimpleString(random(), 20);
+ do {
+ vals[i] = _TestUtil.randomSimpleString(random(), 20);
+ } while (((String) vals[i]).isEmpty());
f.setBytesValue(new BytesRef((String) vals[i]));
break;
case NUMERIC:
@@ -115,7 +116,6 @@ public class TestDocValuesFieldSources e
} else if (vs instanceof LongFieldSource) {
assertTrue(values.objectVal(i) instanceof Long);
assertTrue(values.bytesVal(i, bytes));
- assertEquals(8, bytes.length);
} else {
throw new AssertionError();
}
@@ -143,8 +143,6 @@ public class TestDocValuesFieldSources e
d.close();
}
- // nocommit
- @Ignore("fix this test")
public void test() throws IOException {
for (DocValuesType type : DocValuesType.values()) {
test(type);