You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/01/21 21:39:25 UTC
svn commit: r1436600 - in /lucene/dev/branches/lucene4547/lucene/facet/src:
java/org/apache/lucene/facet/index/ java/org/apache/lucene/facet/search/
test/org/apache/lucene/util/
Author: mikemccand
Date: Mon Jan 21 20:39:25 2013
New Revision: 1436600
URL: http://svn.apache.org/viewvc?rev=1436600&view=rev
Log:
fix facet tests
Modified:
lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadMigrationReader.java
lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java
lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java
Modified: lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadMigrationReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadMigrationReader.java?rev=1436600&r1=1436599&r2=1436600&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadMigrationReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/FacetsPayloadMigrationReader.java Mon Jan 21 20:39:25 2013
@@ -30,6 +30,7 @@ import org.apache.lucene.index.AtomicRea
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
@@ -74,39 +75,37 @@ import org.apache.lucene.util.BytesRef;
*/
public class FacetsPayloadMigrationReader extends FilterAtomicReader {
- private class PayloadMigratingDocValues extends DocValues {
+ private class PayloadMigratingBinaryDocValues extends BinaryDocValues {
- private final DocsAndPositionsEnum dpe;
-
- public PayloadMigratingDocValues(DocsAndPositionsEnum dpe) {
- this.dpe = dpe;
- }
-
- @Override
- protected Source loadDirectSource() throws IOException {
- return new PayloadMigratingSource(getType(), dpe);
- }
-
- @Override
- protected Source loadSource() throws IOException {
- throw new UnsupportedOperationException("in-memory Source is not supported by this reader");
- }
+ private Fields fields;
+ private Term term;
+ private DocsAndPositionsEnum dpe;
+ private int curDocID = -1;
+ private int lastRequestedDocID;
- @Override
- public Type getType() {
- return Type.BYTES_VAR_STRAIGHT;
+ private DocsAndPositionsEnum getDPE() {
+ try {
+ DocsAndPositionsEnum dpe = null;
+ if (fields != null) {
+ Terms terms = fields.terms(term.field());
+ if (terms != null) {
+ TermsEnum te = terms.iterator(null); // no use for reusing
+ if (te.seekExact(term.bytes(), true)) {
+ // we're not expected to be called for deleted documents
+ dpe = te.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_PAYLOADS);
+ }
+ }
+ }
+ return dpe;
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
}
- }
-
- private class PayloadMigratingSource extends Source {
-
- private final DocsAndPositionsEnum dpe;
- private int curDocID;
-
- protected PayloadMigratingSource(Type type, DocsAndPositionsEnum dpe) {
- super(type);
- this.dpe = dpe;
+ protected PayloadMigratingBinaryDocValues(Fields fields, Term term) {
+ this.fields = fields;
+ this.term = term;
+ this.dpe = getDPE();
if (dpe == null) {
curDocID = DocIdSetIterator.NO_MORE_DOCS;
} else {
@@ -119,31 +118,41 @@ public class FacetsPayloadMigrationReade
}
@Override
- public BytesRef getBytes(int docID, BytesRef ref) {
- if (curDocID > docID) {
- // document does not exist
- ref.length = 0;
- return ref;
- }
-
+ public void get(int docID, BytesRef result) {
try {
+ // If caller is moving backwards (eg, during merge,
+ // the consuming DV format is free to iterate over
+ // our values as many times as it wants), we must
+ // re-init the dpe:
+ if (docID <= lastRequestedDocID) {
+ dpe = getDPE();
+ if (dpe == null) {
+ curDocID = DocIdSetIterator.NO_MORE_DOCS;
+ } else{
+ curDocID = dpe.nextDoc();
+ }
+ }
+ lastRequestedDocID = docID;
+ if (curDocID > docID) {
+ // document does not exist
+ result.length = 0;
+ return;
+ }
+
if (curDocID < docID) {
curDocID = dpe.advance(docID);
if (curDocID != docID) { // requested document does not have a payload
- ref.length = 0;
- return ref;
+ result.length = 0;
+ return;
}
}
- // we're on the document
dpe.nextPosition();
- ref.copyBytes(dpe.getPayload());
- return ref;
+ result.copyBytes(dpe.getPayload());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
-
}
/** The {@link Term} text of the ordinals payload. */
@@ -202,26 +211,14 @@ public class FacetsPayloadMigrationReade
}
@Override
- public DocValues docValues(String field) throws IOException {
+ public BinaryDocValues getBinaryDocValues(String field) throws IOException {
Term term = fieldTerms.get(field);
if (term == null) {
- return super.docValues(field);
+ return super.getBinaryDocValues(field);
} else {
- DocsAndPositionsEnum dpe = null;
- Fields fields = fields();
- if (fields != null) {
- Terms terms = fields.terms(term.field());
- if (terms != null) {
- TermsEnum te = terms.iterator(null); // no use for reusing
- if (te.seekExact(term.bytes(), true)) {
- // we're not expected to be called for deleted documents
- dpe = te.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_PAYLOADS);
- }
- }
- }
// we shouldn't return null, even if the term does not exist or has no
// payloads, since we already marked the field as having DocValues.
- return new PayloadMigratingDocValues(dpe);
+ return new PayloadMigratingBinaryDocValues(fields(), term);
}
}
@@ -238,7 +235,7 @@ public class FacetsPayloadMigrationReade
// mark this field as having a DocValues
infos.add(new FieldInfo(info.name, true, info.number,
info.hasVectors(), info.omitsNorms(), info.hasPayloads(),
- info.getIndexOptions(), Type.BYTES_VAR_STRAIGHT,
+ info.getIndexOptions(), DocValuesType.BINARY,
info.getNormType(), info.attributes()));
leftoverFields.remove(info.name);
} else {
@@ -248,9 +245,8 @@ public class FacetsPayloadMigrationReade
}
for (String field : leftoverFields) {
infos.add(new FieldInfo(field, false, ++number, false, false, false,
- null, Type.BYTES_VAR_STRAIGHT, null, null));
+ null, DocValuesType.BINARY, null, null));
}
return new FieldInfos(infos.toArray(new FieldInfo[infos.size()]));
}
-
}
Modified: lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java?rev=1436600&r1=1436599&r2=1436600&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/index/OrdinalMappingAtomicReader.java Mon Jan 21 20:39:25 2013
@@ -25,9 +25,7 @@ import org.apache.lucene.facet.index.par
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter.OrdinalMap;
import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValues.Source;
-import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.FilterAtomicReader;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
@@ -92,8 +90,8 @@ public class OrdinalMappingAtomicReader
}
@Override
- public DocValues docValues(String field) throws IOException {
- DocValues inner = super.docValues(field);
+ public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+ BinaryDocValues inner = super.getBinaryDocValues(field);
if (inner == null) {
return inner;
}
@@ -102,46 +100,19 @@ public class OrdinalMappingAtomicReader
if (clp == null) {
return inner;
} else {
- return new OrdinalMappingDocValues(inner, clp);
+ return new OrdinalMappingBinaryDocValues(clp, inner);
}
}
- private class OrdinalMappingDocValues extends DocValues {
-
- private final CategoryListParams clp;
- private final DocValues delegate;
-
- public OrdinalMappingDocValues(DocValues delegate, CategoryListParams clp) {
- this.delegate = delegate;
- this.clp = clp;
- }
-
- @Override
- protected Source loadSource() throws IOException {
- return new OrdinalMappingSource(getType(), clp, delegate.getSource());
- }
-
- @Override
- protected Source loadDirectSource() throws IOException {
- return new OrdinalMappingSource(getType(), clp, delegate.getDirectSource());
- }
-
- @Override
- public Type getType() {
- return Type.BYTES_VAR_STRAIGHT;
- }
-
- }
-
- private class OrdinalMappingSource extends Source {
+ private class OrdinalMappingBinaryDocValues extends BinaryDocValues {
private final IntEncoder encoder;
private final IntDecoder decoder;
private final IntsRef ordinals = new IntsRef(32);
- private final Source delegate;
+ private final BinaryDocValues delegate;
+ private final BytesRef scratch = new BytesRef();
- protected OrdinalMappingSource(Type type, CategoryListParams clp, Source delegate) {
- super(type);
+ protected OrdinalMappingBinaryDocValues(CategoryListParams clp, BinaryDocValues delegate) {
this.delegate = delegate;
encoder = clp.createEncoder();
decoder = encoder.createMatchingDecoder();
@@ -149,23 +120,26 @@ public class OrdinalMappingAtomicReader
@SuppressWarnings("synthetic-access")
@Override
- public BytesRef getBytes(int docID, BytesRef ref) {
- ref = delegate.getBytes(docID, ref);
- if (ref == null || ref.length == 0) {
- return ref;
- } else {
- decoder.decode(ref, ordinals);
+ public void get(int docID, BytesRef result) {
+ // NOTE: this isn't quite koscher, because in general
+ // multiple threads can call BinaryDV.get which would
+ // then conflict on the single scratch instance, but
+ // because this impl is only used for merging, we know
+ // only 1 thread calls us:
+ delegate.get(docID, scratch);
+ if (scratch.length > 0) {
+ // We must use scratch (and not re-use result) here,
+ // else encoder may overwrite the DV provider's
+ // private byte[]:
+ decoder.decode(scratch, ordinals);
// map the ordinals
for (int i = 0; i < ordinals.length; i++) {
ordinals.ints[i] = ordinalMap[ordinals.ints[i]];
}
- encoder.encode(ordinals, ref);
- return ref;
+ encoder.encode(ordinals, result);
}
}
-
}
-
}
Modified: lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java?rev=1436600&r1=1436599&r2=1436600&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java Mon Jan 21 20:39:25 2013
@@ -11,17 +11,16 @@ import org.apache.lucene.facet.index.cat
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
-import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest.SortBy;
import org.apache.lucene.facet.search.params.FacetRequest.SortOrder;
+import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.ParallelTaxonomyArrays;
import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.index.DocValues;
-import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.util.BytesRef;
@@ -89,9 +88,9 @@ public class CountingFacetsCollector ext
private final int[] counts;
private final String facetsField;
private final boolean useDirectSource;
- private final HashMap<Source,FixedBitSet> matchingDocs = new HashMap<Source,FixedBitSet>();
+ private final HashMap<BinaryDocValues,FixedBitSet> matchingDocs = new HashMap<BinaryDocValues,FixedBitSet>();
- private DocValues facetsValues;
+ private BinaryDocValues facetsValues;
private FixedBitSet bits;
public CountingFacetsCollector(FacetSearchParams fsp, TaxonomyReader taxoReader) {
@@ -158,11 +157,10 @@ public class CountingFacetsCollector ext
@Override
public void setNextReader(AtomicReaderContext context) throws IOException {
- facetsValues = context.reader().docValues(facetsField);
+ facetsValues = context.reader().getBinaryDocValues(facetsField);
if (facetsValues != null) {
- Source facetSource = useDirectSource ? facetsValues.getDirectSource() : facetsValues.getSource();
bits = new FixedBitSet(context.reader().maxDoc());
- matchingDocs.put(facetSource, bits);
+ matchingDocs.put(facetsValues, bits);
}
}
@@ -176,13 +174,13 @@ public class CountingFacetsCollector ext
}
private void countFacets() {
- for (Entry<Source,FixedBitSet> entry : matchingDocs.entrySet()) {
- Source facetsSource = entry.getKey();
+ for (Entry<BinaryDocValues,FixedBitSet> entry : matchingDocs.entrySet()) {
+ BinaryDocValues facetsSource = entry.getKey();
FixedBitSet bits = entry.getValue();
int doc = 0;
int length = bits.length();
while (doc < length && (doc = bits.nextSetBit(doc)) != -1) {
- facetsSource .getBytes(doc, buf);
+ facetsSource.get(doc, buf);
if (buf.length > 0) {
// this document has facets
int upto = buf.offset + buf.length;
Modified: lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java?rev=1436600&r1=1436599&r2=1436600&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java (original)
+++ lucene/dev/branches/lucene4547/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java Mon Jan 21 20:39:25 2013
@@ -33,7 +33,7 @@ public class SlowRAMDirectory extends RA
private static final int IO_SLEEP_THRESHOLD = 50;
- private Random random;
+ Random random;
private int sleepMillis;
public void setSleepMillis(int sleepMillis) {
@@ -62,7 +62,7 @@ public class SlowRAMDirectory extends RA
return super.openInput(name, context);
}
- void doSleep(int length) {
+ void doSleep(Random random, int length) {
int sTime = length<10 ? sleepMillis : (int) (sleepMillis * Math.log(length));
if (random!=null) {
sTime = random.nextInt(sTime);
@@ -74,6 +74,14 @@ public class SlowRAMDirectory extends RA
}
}
+ /** Make a private random. */
+ Random forkRandom() {
+ if (random == null) {
+ return null;
+ }
+ return new Random(random.nextLong());
+ }
+
/**
* Delegate class to wrap an IndexInput and delay reading bytes by some
* specified time.
@@ -81,16 +89,18 @@ public class SlowRAMDirectory extends RA
private class SlowIndexInput extends IndexInput {
private IndexInput ii;
private int numRead = 0;
+ private Random random;
public SlowIndexInput(IndexInput ii) {
super("SlowIndexInput(" + ii + ")");
+ this.random = forkRandom();
this.ii = ii;
}
@Override
public byte readByte() throws IOException {
if (numRead >= IO_SLEEP_THRESHOLD) {
- doSleep(0);
+ doSleep(random, 0);
numRead = 0;
}
++numRead;
@@ -100,7 +110,7 @@ public class SlowRAMDirectory extends RA
@Override
public void readBytes(byte[] b, int offset, int len) throws IOException {
if (numRead >= IO_SLEEP_THRESHOLD) {
- doSleep(len);
+ doSleep(random, len);
numRead = 0;
}
numRead += len;
@@ -125,15 +135,17 @@ public class SlowRAMDirectory extends RA
private IndexOutput io;
private int numWrote;
+ private final Random random;
public SlowIndexOutput(IndexOutput io) {
this.io = io;
+ this.random = forkRandom();
}
@Override
public void writeByte(byte b) throws IOException {
if (numWrote >= IO_SLEEP_THRESHOLD) {
- doSleep(0);
+ doSleep(random, 0);
numWrote = 0;
}
++numWrote;
@@ -143,7 +155,7 @@ public class SlowRAMDirectory extends RA
@Override
public void writeBytes(byte[] b, int offset, int length) throws IOException {
if (numWrote >= IO_SLEEP_THRESHOLD) {
- doSleep(length);
+ doSleep(random, length);
numWrote = 0;
}
numWrote += length;