You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/10 08:55:39 UTC
svn commit: r1454801 - in /lucene/dev/trunk/lucene: ./
misc/src/java/org/apache/lucene/index/sorter/
misc/src/test/org/apache/lucene/index/sorter/
Author: shaie
Date: Sun Mar 10 07:55:38 2013
New Revision: 1454801
URL: http://svn.apache.org/r1454801
Log:
LUCENE-3918: port IndexSorter to trunk API
Added:
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java (with props)
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (with props)
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (with props)
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/package.html (with props)
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java (with props)
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (with props)
lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1454801&r1=1454800&r2=1454801&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Sun Mar 10 07:55:38 2013
@@ -31,6 +31,11 @@ New Features
* LUCENE-4815: DrillSideways now allows more than one FacetRequest per
dimension (Mike McCandless)
+
+* LUCENE-3918: IndexSorter has been ported to 4.3 API and now supports
+ sorting documents by a numeric DocValues field, or reverse the order of
+ the documents in the index. Additionally, apps can implement their own
+ sort criteria. (Anat Hashavit, Shai Erera)
* LUCENE-4817: Added KeywordRepeatFilter that allows to emit a token twice
once as a keyword and once as an ordinary token allow stemmers to emit
Added: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java?rev=1454801&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java (added)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/NumericDocValuesSorter.java Sun Mar 10 07:55:38 2013
@@ -0,0 +1,65 @@
+package org.apache.lucene.index.sorter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.AbstractList;
+import java.util.List;
+
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.NumericDocValues;
+
+/**
+ * A {@link Sorter} which sorts documents according to their
+ * {@link NumericDocValues}.
+ *
+ * @lucene.experimental
+ */
+public class NumericDocValuesSorter extends Sorter {
+
+ private final String fieldName;
+
+ public NumericDocValuesSorter(final String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public int[] oldToNew(final AtomicReader reader) throws IOException {
+ final NumericDocValues ndv = reader.getNumericDocValues(fieldName);
+ final int maxDoc = reader.maxDoc();
+ final int[] docs = new int[maxDoc];
+ final List<Long> values = new AbstractList<Long>() {
+
+ @Override
+ public Long get(int doc) {
+ return ndv.get(doc);
+ }
+
+ @Override
+ public int size() {
+ return reader.maxDoc();
+ }
+
+ };
+ for (int i = 0; i < maxDoc; i++) {
+ docs[i] = i;
+ }
+ return compute(docs, values);
+ }
+
+}
Added: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java?rev=1454801&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (added)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java Sun Mar 10 07:55:38 2013
@@ -0,0 +1,106 @@
+package org.apache.lucene.index.sorter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.util.SorterTemplate;
+
+/**
+ * Sorts documents in a given index by returning a permutation on the docs.
+ * Implementations can call {@link #compute(int[], List)} to compute the
+ * old-to-new permutation over the given documents and values.
+ *
+ * @lucene.experimental
+ */
+public abstract class Sorter {
+
+ /** Sorts documents in reverse order. */
+ public static final Sorter REVERSE_DOCS = new Sorter() {
+ @Override
+ public int[] oldToNew(final AtomicReader reader) throws IOException {
+ final int maxDoc = reader.maxDoc();
+ int[] reverseDocs = new int[maxDoc];
+ for (int i = 0; i < maxDoc; i++) {
+ reverseDocs[i] = maxDoc - (i + 1);
+ }
+ return reverseDocs;
+ }
+ };
+
+ private static final class DocValueSorterTemplate<T extends Comparable<? super T>> extends SorterTemplate {
+
+ private final int[] docs;
+ private final List<T> values;
+
+ private T pivot;
+
+ public DocValueSorterTemplate(int[] docs, List<T> values) {
+ this.docs = docs;
+ this.values = values;
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ return values.get(docs[i]).compareTo(values.get(docs[j]));
+ }
+
+ @Override
+ protected int comparePivot(int j) {
+ return pivot.compareTo(values.get(docs[j]));
+ }
+
+ @Override
+ protected void setPivot(int i) {
+ pivot = values.get(docs[i]);
+ }
+
+ @Override
+ protected void swap(int i, int j) {
+ int tmpDoc = docs[i];
+ docs[i] = docs[j];
+ docs[j] = tmpDoc;
+ }
+ }
+
+ /** Computes the old-to-new permutation over the given documents and values. */
+ protected static <T extends Comparable<? super T>> int[] compute(int[] docs, List<T> values) {
+ SorterTemplate sorter = new DocValueSorterTemplate<T>(docs, values);
+ sorter.quickSort(0, docs.length - 1);
+
+ final int[] oldToNew = new int[docs.length];
+ for (int i = 0; i < docs.length; i++) {
+ oldToNew[docs[i]] = i;
+ }
+ return oldToNew;
+ }
+
+ /**
+ * Returns a mapping from the old document ID to its new location in the
+ * sorted index. Implementations can use the auxiliary
+ * {@link #compute(int[], List)} to compute the old-to-new permutation
+ * given an array of documents and their corresponding values.
+ * <p>
+ * <b>NOTE:</b> deleted documents are expected to appear in the mapping as
+ * well, they will however be dropped when the index is actually sorted.
+ */
+ public abstract int[] oldToNew(AtomicReader reader) throws IOException;
+
+}
Added: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java?rev=1454801&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java (added)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java Sun Mar 10 07:55:38 2013
@@ -0,0 +1,652 @@
+package org.apache.lucene.index.sorter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FilterAtomicReader;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMFile;
+import org.apache.lucene.store.RAMInputStream;
+import org.apache.lucene.store.RAMOutputStream;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.SorterTemplate;
+
+/**
+ * An {@link AtomicReader} which supports sorting documents by a given
+ * {@link Sorter}. You can use this class to sort an index as follows:
+ *
+ * <pre class="prettyprint">
+ * IndexWriter writer; // writer to which the sorted index will be added
+ * DirectoryReader reader; // reader on the input index
+ * Sorter sorter; // determines how the documents are sorted
+ * AtomicReader sortingReader = new SortingAtomicReader(reader, sorter);
+ * writer.addIndexes(reader);
+ * writer.close();
+ * reader.close();
+ * </pre>
+ *
+ * @lucene.experimental
+ */
+public class SortingAtomicReader extends FilterAtomicReader {
+
+ private static class SortingFields extends FilterFields {
+
+ private final int[] old2new;
+ private final Bits inLiveDocs;
+ private final FieldInfos infos;
+
+ public SortingFields(final Fields in, final Bits inLiveDocs, FieldInfos infos, final int[] old2new) {
+ super(in);
+ this.old2new = old2new;
+ this.inLiveDocs = inLiveDocs;
+ this.infos = infos;
+ }
+
+ @Override
+ public Terms terms(final String field) throws IOException {
+ Terms terms = in.terms(field);
+ if (terms == null) {
+ return null;
+ } else {
+ return new SortingTerms(terms, inLiveDocs, infos.fieldInfo(field).getIndexOptions(), old2new);
+ }
+ }
+
+ }
+
+ private static class SortingTerms extends FilterTerms {
+
+ private final int[] old2new;
+ private final Bits inLiveDocs;
+ private final IndexOptions indexOptions;
+
+ public SortingTerms(final Terms in, final Bits inLiveDocs, IndexOptions indexOptions, final int[] old2new) {
+ super(in);
+ this.old2new = old2new;
+ this.inLiveDocs = inLiveDocs;
+ this.indexOptions = indexOptions;
+ }
+
+ @Override
+ public TermsEnum iterator(final TermsEnum reuse) throws IOException {
+ return new SortingTermsEnum(in.iterator(reuse), inLiveDocs, old2new, indexOptions);
+ }
+
+ }
+
+ private static class SortingTermsEnum extends FilterTermsEnum {
+
+ private final int[] old2new;
+ private final Bits inLiveDocs;
+ private final IndexOptions indexOptions;
+
+ public SortingTermsEnum(final TermsEnum in, final Bits inLiveDocs, final int[] old2new, IndexOptions indexOptions) {
+ super(in);
+ this.old2new = old2new;
+ this.inLiveDocs = inLiveDocs;
+ this.indexOptions = indexOptions;
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, final int flags) throws IOException {
+ if (liveDocs != null) {
+ liveDocs = inLiveDocs;
+ }
+
+ // if we're asked to reuse the given DocsEnum and it is Sorting, return
+ // the wrapped one, since some Codecs expect it.
+ if (reuse != null && reuse instanceof SortingDocsEnum) {
+ reuse = ((SortingDocsEnum) reuse).getWrapped();
+ }
+ boolean withFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >=0 && (flags & DocsEnum.FLAG_FREQS) != 0;
+ return new SortingDocsEnum(in.docs(liveDocs, reuse, flags), withFreqs, old2new);
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, final int flags) throws IOException {
+ if (liveDocs != null) {
+ liveDocs = inLiveDocs;
+ }
+
+ // if we're asked to reuse the given DocsAndPositionsEnum and it is
+ // Sorting, return the wrapped one, since some Codecs expect it.
+ if (reuse != null && reuse instanceof SortingDocsAndPositionsEnum) {
+ reuse = ((SortingDocsAndPositionsEnum) reuse).getWrapped();
+ }
+
+ final DocsAndPositionsEnum positions = in.docsAndPositions(liveDocs, reuse, flags);
+ if (positions == null) {
+ return null;
+ } else {
+ // we ignore the fact that offsets may be stored but not asked for,
+ // since this code is expected to be used during addIndexes which will
+ // ask for everything. if that assumption changes in the future, we can
+ // factor in whether 'flags' says offsets are not required.
+ boolean storeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ return new SortingDocsAndPositionsEnum(positions, old2new, storeOffsets);
+ }
+ }
+
+ }
+
+ private static class SortingBinaryDocValues extends BinaryDocValues {
+
+ private final BinaryDocValues in;
+ private final int[] new2old;
+
+ SortingBinaryDocValues(BinaryDocValues in, int[] new2old) {
+ this.in = in;
+ this.new2old = new2old;
+ }
+
+ @Override
+ public void get(int docID, BytesRef result) {
+ in.get(new2old[docID], result);
+ }
+ }
+
+ private static class SortingNumericDocValues extends NumericDocValues {
+
+ private final NumericDocValues in;
+ private final int[] new2old;
+
+ public SortingNumericDocValues(final NumericDocValues in, final int[] new2old) {
+ this.in = in;
+ this.new2old = new2old;
+ }
+
+ @Override
+ public long get(int docID) {
+ return in.get(new2old[docID]);
+ }
+ }
+
+ private static class SortingSortedDocValues extends SortedDocValues {
+
+ private final SortedDocValues in;
+ private final int[] new2old;
+
+ SortingSortedDocValues(SortedDocValues in, int[] new2old) {
+ this.in = in;
+ this.new2old = new2old;
+ }
+
+ @Override
+ public int getOrd(int docID) {
+ return in.getOrd(new2old[docID]);
+ }
+
+ @Override
+ public void lookupOrd(int ord, BytesRef result) {
+ in.lookupOrd(ord, result);
+ }
+
+ @Override
+ public int getValueCount() {
+ return in.getValueCount();
+ }
+
+ @Override
+ public void get(int docID, BytesRef result) {
+ in.get(new2old[docID], result);
+ }
+
+ @Override
+ public int lookupTerm(BytesRef key) {
+ return in.lookupTerm(key);
+ }
+ }
+
+ private static class SortingSortedSetDocValues extends SortedSetDocValues {
+
+ private final SortedSetDocValues in;
+ private final int[] new2old;
+
+ SortingSortedSetDocValues(SortedSetDocValues in, int[] new2old) {
+ this.in = in;
+ this.new2old = new2old;
+ }
+
+ @Override
+ public long nextOrd() {
+ return in.nextOrd();
+ }
+
+ @Override
+ public void setDocument(int docID) {
+ in.setDocument(new2old[docID]);
+ }
+
+ @Override
+ public void lookupOrd(long ord, BytesRef result) {
+ in.lookupOrd(ord, result);
+ }
+
+ @Override
+ public long getValueCount() {
+ return in.getValueCount();
+ }
+
+ @Override
+ public long lookupTerm(BytesRef key) {
+ return in.lookupTerm(key);
+ }
+ }
+
+ private static class SortingDocsEnum extends FilterDocsEnum {
+
+ private static final class DocFreqSorterTemplate extends SorterTemplate {
+
+ private final int[] docs;
+ private final int[] freqs;
+
+ private int pivot;
+
+ public DocFreqSorterTemplate(int[] docs, int[] freqs) {
+ this.docs = docs;
+ this.freqs = freqs;
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ return docs[i] - docs[j];
+ }
+
+ @Override
+ protected int comparePivot(int j) {
+ return pivot - docs[j];
+ }
+
+ @Override
+ protected void setPivot(int i) {
+ pivot = docs[i];
+ }
+
+ @Override
+ protected void swap(int i, int j) {
+ int tmpDoc = docs[i];
+ docs[i] = docs[j];
+ docs[j] = tmpDoc;
+
+ int tmpFreq = freqs[i];
+ freqs[i] = freqs[j];
+ freqs[j] = tmpFreq;
+ }
+ }
+
+ private int[] docs = new int[64];
+ private int[] freqs;
+ private int docIt = -1;
+ private final int upto;
+ private final boolean withFreqs;
+
+ public SortingDocsEnum(final DocsEnum in, boolean withFreqs, final int[] old2new) throws IOException {
+ super(in);
+ this.withFreqs = withFreqs;
+ int i = 0;
+ int doc;
+ if (withFreqs) {
+ freqs = new int[docs.length];
+ while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){
+ if (i >= docs.length) {
+ docs = ArrayUtil.grow(docs, docs.length + 1);
+ freqs = ArrayUtil.grow(freqs, freqs.length + 1);
+ }
+ docs[i] = old2new[doc];
+ freqs[i] = in.freq();
+ ++i;
+ }
+ SorterTemplate sorter = new DocFreqSorterTemplate(docs, freqs);
+ sorter.quickSort(0, i - 1);
+ } else {
+ freqs = null;
+ while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS){
+ if (i >= docs.length) {
+ docs = ArrayUtil.grow(docs, docs.length + 1);
+ }
+ docs[i++] = old2new[doc];
+ }
+ Arrays.sort(docs, 0, i);
+ }
+ upto = i;
+ }
+
+ @Override
+ public int advance(final int target) throws IOException {
+ // need to support it for checkIndex, but in practice it won't be called, so
+ // don't bother to implement efficiently for now.
+ while (nextDoc() < target) {}
+ return docID();
+ }
+
+ @Override
+ public int docID() {
+ return docIt >= upto ? NO_MORE_DOCS : docs[docIt];
+ }
+
+ @Override
+ public int freq() throws IOException {
+ return withFreqs && docIt < upto ? freqs[docIt] : 1;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (++docIt >= upto) return NO_MORE_DOCS;
+ return docs[docIt];
+ }
+
+ /** Returns the wrapped {@link DocsEnum}. */
+ DocsEnum getWrapped() {
+ return in;
+ }
+ }
+
+ private static class SortingDocsAndPositionsEnum extends FilterDocsAndPositionsEnum {
+
+ /**
+ * A {@link SorterTemplate} which sorts two parallel arrays of doc IDs and
+ * offsets in one go. Everytime a doc ID is 'swapped', its correponding offset
+ * is swapped too.
+ */
+ private static final class DocOffsetSorterTemplate extends SorterTemplate {
+
+ private final int[] docs;
+ private final long[] offsets;
+
+ private int pivot;
+
+ public DocOffsetSorterTemplate(int[] docs, long[] offsets) {
+ this.docs = docs;
+ this.offsets = offsets;
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ return docs[i] - docs[j];
+ }
+
+ @Override
+ protected int comparePivot(int j) {
+ return pivot - docs[j];
+ }
+
+ @Override
+ protected void setPivot(int i) {
+ pivot = docs[i];
+ }
+
+ @Override
+ protected void swap(int i, int j) {
+ int tmpDoc = docs[i];
+ docs[i] = docs[j];
+ docs[j] = tmpDoc;
+
+ long tmpOffset = offsets[i];
+ offsets[i] = offsets[j];
+ offsets[j] = tmpOffset;
+ }
+ }
+
+ private int[] docs;
+ private long[] offsets;
+ private final int upto;
+
+ private final IndexInput postingInput;
+ private final boolean storeOffsets;
+
+ private int docIt = -1;
+ private int pos;
+ private int startOffset = -1;
+ private int endOffset = -1;
+ private final BytesRef payload = new BytesRef(32);
+ private int currFreq;
+
+ public SortingDocsAndPositionsEnum(final DocsAndPositionsEnum in, final int[] old2new, boolean storeOffsets) throws IOException {
+ super(in);
+ this.storeOffsets = storeOffsets;
+ final RAMFile file = new RAMFile();
+ final IndexOutput out = new RAMOutputStream(file);
+ docs = new int[32];
+ offsets = new long[32];
+ int doc;
+ int i = 0;
+ while ((doc = in.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ if (i == docs.length) {
+ docs = ArrayUtil.grow(docs, i + 1);
+ offsets = ArrayUtil.grow(offsets, i + 1);
+ }
+ docs[i] = old2new[doc];
+ offsets[i] = out.getFilePointer();
+ addPositions(in, out);
+ i++;
+ }
+ upto = i;
+ SorterTemplate sorter = new DocOffsetSorterTemplate(docs, offsets);
+ sorter.quickSort(0, upto - 1);
+ out.close();
+ this.postingInput = new RAMInputStream("", file);
+ }
+
+ private void addPositions(final DocsAndPositionsEnum in, final IndexOutput out) throws IOException {
+ int freq = in.freq();
+ out.writeVInt(freq);
+ for (int i = 0; i < freq; i++) {
+ final int pos = in.nextPosition();
+ out.writeVInt(pos);
+ if (storeOffsets) { // don't encode offsets if they are not stored
+ out.writeVInt(in.startOffset());
+ out.writeVInt(in.endOffset());
+ }
+ BytesRef payload = in.getPayload();
+ if (payload != null) {
+ out.writeVInt(payload.length);
+ out.writeBytes(payload.bytes, payload.offset, payload.length);
+ } else {
+ out.writeVInt(0);
+ }
+ }
+ }
+
+ @Override
+ public int advance(final int target) throws IOException {
+ // need to support it for checkIndex, but in practice it won't be called, so
+ // don't bother to implement efficiently for now.
+ while (nextDoc() < target) {}
+ return docID();
+ }
+
+ @Override
+ public int docID() {
+ return docIt >= upto ? NO_MORE_DOCS : docs[docIt];
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ return endOffset;
+ }
+
+ @Override
+ public int freq() throws IOException {
+ return currFreq;
+ }
+
+ @Override
+ public BytesRef getPayload() throws IOException {
+ return payload.length == 0 ? null : payload;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (++docIt >= upto) return DocIdSetIterator.NO_MORE_DOCS;
+ postingInput.seek(offsets[docIt]);
+ currFreq = postingInput.readVInt();
+ return docs[docIt];
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ pos = postingInput.readVInt();
+ if (storeOffsets) {
+ startOffset = postingInput.readVInt();
+ endOffset = postingInput.readVInt();
+ }
+ int length = postingInput.readVInt();
+ if (length > 0) {
+ if (length >= payload.bytes.length) {
+ payload.grow(length + 1);
+ }
+ postingInput.readBytes(payload.bytes, 0, length);
+ }
+ payload.length = length;
+ return pos;
+ }
+
+ @Override
+ public int startOffset() throws IOException {
+ return startOffset;
+ }
+
+ /** Returns the wrapped {@link DocsAndPositionsEnum}. */
+ DocsAndPositionsEnum getWrapped() {
+ return in;
+ }
+ }
+
+ private final int[] old2new, new2old;
+ private final FixedBitSet mappedLiveDocs;
+
+ public SortingAtomicReader(final AtomicReader in, final Sorter sorter) throws IOException {
+ super(in);
+ old2new = sorter.oldToNew(in);
+ if (old2new.length != in.maxDoc()) {
+ throw new IllegalArgumentException("sorter should provide mapping for every document in the index, including deleted ones");
+ }
+ new2old = new int[old2new.length];
+ for (int i = 0; i < new2old.length; i++) {
+ new2old[old2new[i]] = i;
+ }
+
+ if (!in.hasDeletions()) {
+ mappedLiveDocs = null;
+ } else {
+ mappedLiveDocs = new FixedBitSet(in.maxDoc());
+ mappedLiveDocs.set(0, in.maxDoc());
+ Bits liveDocs = in.getLiveDocs();
+ int len = liveDocs.length();
+ for (int i = 0; i < len; i++) {
+ if (!liveDocs.get(i)) {
+ mappedLiveDocs.clear(old2new[i]);
+ }
+ }
+ }
+ }
+
+ @Override
+ public void document(final int docID, final StoredFieldVisitor visitor) throws IOException {
+ in.document(new2old[docID], visitor);
+ }
+
+ @Override
+ public Fields fields() throws IOException {
+ Fields fields = in.fields();
+ if (fields == null) {
+ return null;
+ } else {
+ return new SortingFields(fields, in.getLiveDocs(), in.getFieldInfos(), old2new);
+ }
+ }
+
+ @Override
+ public BinaryDocValues getBinaryDocValues(String field) throws IOException {
+ BinaryDocValues oldDocValues = in.getBinaryDocValues(field);
+ if (oldDocValues == null) {
+ return null;
+ } else {
+ return new SortingBinaryDocValues(oldDocValues, new2old);
+ }
+ }
+
+ @Override
+ public Bits getLiveDocs() {
+ ensureOpen();
+ return mappedLiveDocs;
+ }
+
+ @Override
+ public NumericDocValues getNormValues(String field) throws IOException {
+ final NumericDocValues norm = in.getNormValues(field);
+ if (norm == null) {
+ return null;
+ } else {
+ return new SortingNumericDocValues(norm, new2old);
+ }
+ }
+
+ @Override
+ public NumericDocValues getNumericDocValues(String field) throws IOException {
+ final NumericDocValues oldDocValues = in.getNumericDocValues(field);
+ if (oldDocValues == null) return null;
+ return new SortingNumericDocValues(oldDocValues, new2old);
+ }
+
+ @Override
+ public SortedDocValues getSortedDocValues(String field) throws IOException {
+ SortedDocValues sortedDV = in.getSortedDocValues(field);
+ if (sortedDV == null) {
+ return null;
+ } else {
+ return new SortingSortedDocValues(sortedDV, new2old);
+ }
+ }
+
+ @Override
+ public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
+ SortedSetDocValues sortedSetDV = in.getSortedSetDocValues(field);
+ if (sortedSetDV == null) {
+ return null;
+ } else {
+ return new SortingSortedSetDocValues(sortedSetDV, new2old);
+ }
+ }
+
+ @Override
+ public Fields getTermVectors(final int docID) throws IOException {
+ return in.getTermVectors(new2old[docID]);
+ }
+
+}
Added: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/package.html?rev=1454801&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/package.html (added)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/package.html Sun Mar 10 07:55:38 2013
@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<body>
+Provides index sorting capablities. The application can use one of the
+pre-existing Sorter implementations, e.g. to sort by a numeric
+DocValues or reverse the order of the documents. Additionally, the
+application can implement a Sorter which returns a permutation on
+a source Directory's document IDs, to sort the input documents by additional
+values.
+</body>
+</html>
Added: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java?rev=1454801&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java (added)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/IndexSortingTest.java Sun Mar 10 07:55:38 2013
@@ -0,0 +1,79 @@
+package org.apache.lucene.index.sorter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util._TestUtil;
+import org.junit.BeforeClass;
+
+public class IndexSortingTest extends SorterTestBase {
+
+ private static final Sorter[] SORTERS = new Sorter[] {
+ new NumericDocValuesSorter(NUMERIC_DV_FIELD),
+ Sorter.REVERSE_DOCS,
+ };
+
+ @BeforeClass
+ public static void beforeClassSorterUtilTest() throws Exception {
+ // only read the values of the undeleted documents, since after addIndexes,
+ // the deleted ones will be dropped from the index.
+ Bits liveDocs = reader.getLiveDocs();
+ List<Integer> values = new ArrayList<Integer>();
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ if (liveDocs == null || liveDocs.get(i)) {
+ values.add(Integer.valueOf(reader.document(i).get(ID_FIELD)));
+ }
+ }
+ Sorter sorter = SORTERS[random().nextInt(SORTERS.length)];
+ if (sorter == Sorter.REVERSE_DOCS) {
+ Collections.reverse(values);
+ } else {
+ Collections.sort(values);
+ }
+ sortedValues = values.toArray(new Integer[values.size()]);
+ if (VERBOSE) {
+ System.out.println("sortedValues: " + sortedValues);
+ System.out.println("Sorter: " + sorter);
+ }
+
+ Directory target = newDirectory();
+ IndexWriter writer = new IndexWriter(target, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+ reader = new SortingAtomicReader(reader, sorter);
+ writer.addIndexes(reader);
+ writer.close();
+ reader.close();
+ dir.close();
+
+ // CheckIndex the target directory
+ dir = target;
+ _TestUtil.checkIndex(dir);
+
+ // set reader for tests
+ reader = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(dir));
+ assertFalse("index should not have deletions", reader.hasDeletions());
+ }
+
+}
Added: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java?rev=1454801&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (added)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java Sun Mar 10 07:55:38 2013
@@ -0,0 +1,377 @@
+package org.apache.lucene.index.sorter;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SlowCompositeReaderWrapper;
+import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.search.CollectionStatistics;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.TermStatistics;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public abstract class SorterTestBase extends LuceneTestCase {
+
+ static final class NormsSimilarity extends Similarity {
+
+ private final Similarity in;
+
+ public NormsSimilarity(Similarity in) {
+ this.in = in;
+ }
+
+ @Override
+ public long computeNorm(FieldInvertState state) {
+ if (state.getName().equals(NORMS_FIELD)) {
+ return Float.floatToIntBits(state.getBoost());
+ } else {
+ return in.computeNorm(state);
+ }
+ }
+
+ @Override
+ public SimWeight computeWeight(float queryBoost, CollectionStatistics collectionStats, TermStatistics... termStats) {
+ return in.computeWeight(queryBoost, collectionStats, termStats);
+ }
+
+ @Override
+ public ExactSimScorer exactSimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+ return in.exactSimScorer(weight, context);
+ }
+
+ @Override
+ public SloppySimScorer sloppySimScorer(SimWeight weight, AtomicReaderContext context) throws IOException {
+ return in.sloppySimScorer(weight, context);
+ }
+
+ }
+
+ static final class PositionsTokenStream extends TokenStream {
+
+ private final CharTermAttribute term;
+ private final PayloadAttribute payload;
+ private final OffsetAttribute offset;
+
+ private int pos, off;
+
+ public PositionsTokenStream() {
+ term = addAttribute(CharTermAttribute.class);
+ term.append(DOC_POSITIONS_TERM);
+ payload = addAttribute(PayloadAttribute.class);
+ offset = addAttribute(OffsetAttribute.class);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (pos == 0) {
+ return false;
+ }
+
+ payload.setPayload(new BytesRef(Integer.toString(pos)));
+ offset.setOffset(off, off);
+ --pos;
+ ++off;
+ return true;
+ }
+
+ void setId(int id) {
+ pos = id / 10 + 1;
+ off = 0;
+ }
+ }
+
+ protected static final String ID_FIELD = "id";
+ protected static final String DOCS_ENUM_FIELD = "docs";
+ protected static final String DOCS_ENUM_TERM = "$all$";
+ protected static final String DOC_POSITIONS_FIELD = "positions";
+ protected static final String DOC_POSITIONS_TERM = "$all$";
+ protected static final String NUMERIC_DV_FIELD = "numeric";
+ protected static final String NORMS_FIELD = "norm";
+ protected static final String BINARY_DV_FIELD = "binary";
+ protected static final String SORTED_DV_FIELD = "sorted";
+ protected static final String SORTED_SET_DV_FIELD = "sorted_set";
+ protected static final String TERM_VECTORS_FIELD = "term_vectors";
+
+ private static final FieldType TERM_VECTORS_TYPE = new FieldType(TextField.TYPE_NOT_STORED);
+ static {
+ TERM_VECTORS_TYPE.setStoreTermVectors(true);
+ TERM_VECTORS_TYPE.freeze();
+ }
+
+ private static final FieldType POSITIONS_TYPE = new FieldType(TextField.TYPE_NOT_STORED);
+ static {
+ POSITIONS_TYPE.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ POSITIONS_TYPE.freeze();
+ }
+
+ protected static Directory dir;
+ protected static AtomicReader reader;
+ protected static Integer[] sortedValues;
+
+ private static Document doc(final int id, PositionsTokenStream positions) {
+ final Document doc = new Document();
+ doc.add(new StringField(ID_FIELD, Integer.toString(id), Store.YES));
+ doc.add(new StringField(DOCS_ENUM_FIELD, DOCS_ENUM_TERM, Store.NO));
+ positions.setId(id);
+ if (doesntSupportOffsets.contains(_TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) {
+ // codec doesnt support offsets: just index positions for the field
+ doc.add(new Field(DOC_POSITIONS_FIELD, positions, TextField.TYPE_NOT_STORED));
+ } else {
+ doc.add(new Field(DOC_POSITIONS_FIELD, positions, POSITIONS_TYPE));
+ }
+ doc.add(new NumericDocValuesField(NUMERIC_DV_FIELD, id));
+ TextField norms = new TextField(NORMS_FIELD, Integer.toString(id), Store.NO);
+ norms.setBoost(Float.intBitsToFloat(id));
+ doc.add(norms);
+ doc.add(new BinaryDocValuesField(BINARY_DV_FIELD, new BytesRef(Integer.toString(id))));
+ doc.add(new SortedDocValuesField(SORTED_DV_FIELD, new BytesRef(Integer.toString(id))));
+ if (defaultCodecSupportsSortedSet()) {
+ doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id))));
+ doc.add(new SortedSetDocValuesField(SORTED_SET_DV_FIELD, new BytesRef(Integer.toString(id + 1))));
+ }
+ doc.add(new Field(TERM_VECTORS_FIELD, Integer.toString(id), TERM_VECTORS_TYPE));
+ return doc;
+ }
+
+ /** Creates an index for sorting. */
+ public static void createIndex(Directory dir, int numDocs, Random random) throws IOException {
+ List<Integer> ids = new ArrayList<Integer>();
+ for (int i = 0; i < numDocs; i++) {
+ ids.add(Integer.valueOf(i * 10));
+ }
+ // shuffle them for indexing
+ Collections.shuffle(ids, random);
+ if (VERBOSE) {
+ System.out.println("Shuffled IDs for indexing: " + Arrays.toString(ids.toArray()));
+ }
+
+ PositionsTokenStream positions = new PositionsTokenStream();
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ conf.setMaxBufferedDocs(4); // create some segments
+ conf.setSimilarity(new NormsSimilarity(conf.getSimilarity())); // for testing norms field
+ RandomIndexWriter writer = new RandomIndexWriter(random, dir, conf);
+ writer.setDoRandomForceMerge(false);
+ for (int id : ids) {
+ writer.addDocument(doc(id, positions));
+ }
+ // delete some documents
+ writer.commit();
+ for (Integer id : ids) {
+ if (random.nextDouble() < 0.2) {
+ if (VERBOSE) {
+ System.out.println("delete doc_id " + id);
+ }
+ writer.deleteDocuments(new Term(ID_FIELD, id.toString()));
+ }
+ }
+ writer.close();
+ }
+
+ @BeforeClass
+ public static void beforeClassSorterTestBase() throws Exception {
+ dir = newDirectory();
+ int numDocs = atLeast(20);
+ createIndex(dir, numDocs, random());
+
+ reader = new SlowCompositeReaderWrapper(DirectoryReader.open(dir));
+ }
+
+ @AfterClass
+ public static void afterClassSorterTestBase() throws Exception {
+ reader.close();
+ dir.close();
+ }
+
+ @Test
+ public void testBinaryDocValuesField() throws Exception {
+ BinaryDocValues dv = reader.getBinaryDocValues(BINARY_DV_FIELD);
+ BytesRef bytes = new BytesRef();
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ dv.get(i, bytes);
+ assertEquals("incorrect binary DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
+ }
+ }
+
+ @Test
+ public void testDocsAndPositionsEnum() throws Exception {
+ Term term = new Term(DOC_POSITIONS_FIELD, DOC_POSITIONS_TERM);
+ DocsAndPositionsEnum sortedPositions = reader.termPositionsEnum(term);
+ int doc;
+
+ // test nextDoc()
+ while ((doc = sortedPositions.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ int freq = sortedPositions.freq();
+ assertEquals("incorrect freq for doc=" + doc, sortedValues[doc].intValue() / 10 + 1, freq);
+ for (int i = 0; i < freq; i++) {
+ assertEquals("incorrect position for doc=" + doc, i, sortedPositions.nextPosition());
+ if (!doesntSupportOffsets.contains(_TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) {
+ assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset());
+ assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset());
+ }
+ assertEquals("incorrect payload for doc=" + doc, freq - i, Integer.parseInt(sortedPositions.getPayload().utf8ToString()));
+ }
+ }
+
+ // test advance()
+ sortedPositions = reader.termPositionsEnum(term);
+ doc = 0;
+ while ((doc = sortedPositions.advance(doc)) != DocIdSetIterator.NO_MORE_DOCS) {
+ int freq = sortedPositions.freq();
+ assertEquals("incorrect freq for doc=" + doc, sortedValues[doc].intValue() / 10 + 1, freq);
+ for (int i = 0; i < freq; i++) {
+ assertEquals("incorrect position for doc=" + doc, i, sortedPositions.nextPosition());
+ if (!doesntSupportOffsets.contains(_TestUtil.getPostingsFormat(DOC_POSITIONS_FIELD))) {
+ assertEquals("incorrect startOffset for doc=" + doc, i, sortedPositions.startOffset());
+ assertEquals("incorrect endOffset for doc=" + doc, i, sortedPositions.endOffset());
+ }
+ assertEquals("incorrect payload for doc=" + doc, freq - i, Integer.parseInt(sortedPositions.getPayload().utf8ToString()));
+ }
+ }
+ }
+
+ @Test
+ public void testDocsEnum() throws Exception {
+ Term term = new Term(DOCS_ENUM_FIELD, DOCS_ENUM_TERM);
+ DocsEnum docs = reader.termDocsEnum(term);
+ Bits mappedLiveDocs = reader.getLiveDocs();
+ int doc;
+ int prev = -1;
+ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
+ if (mappedLiveDocs != null) {
+ assertTrue("document " + doc + " marked as deleted", mappedLiveDocs.get(doc));
+ }
+ assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD)));
+ while (++prev < doc) {
+ assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs.get(prev));
+ }
+ }
+
+ docs = reader.termDocsEnum(term);
+ doc = 0;
+ prev = -1;
+ while ((doc = docs.advance(doc)) != DocIdSetIterator.NO_MORE_DOCS) {
+ if (mappedLiveDocs != null) {
+ assertTrue("document " + doc + " marked as deleted", mappedLiveDocs.get(doc));
+ }
+ assertEquals("incorrect value; doc " + doc, sortedValues[doc].intValue(), Integer.parseInt(reader.document(doc).get(ID_FIELD)));
+ while (++prev < doc) {
+ assertFalse("document " + prev + " not marked as deleted", mappedLiveDocs.get(prev));
+ }
+ }
+ }
+
+ @Test
+ public void testNormValues() throws Exception {
+ NumericDocValues dv = reader.getNormValues(NORMS_FIELD);
+ int maxDoc = reader.maxDoc();
+ for (int i = 0; i < maxDoc; i++) {
+ assertEquals("incorrect norm value for doc " + i, sortedValues[i].intValue(), dv.get(i));
+ }
+ }
+
+ @Test
+ public void testNumericDocValuesField() throws Exception {
+ NumericDocValues dv = reader.getNumericDocValues(NUMERIC_DV_FIELD);
+ int maxDoc = reader.maxDoc();
+ for (int i = 0; i < maxDoc; i++) {
+ assertEquals("incorrect numeric DocValues for doc " + i, sortedValues[i].intValue(), dv.get(i));
+ }
+ }
+
+ @Test
+ public void testSortedDocValuesField() throws Exception {
+ SortedDocValues dv = reader.getSortedDocValues(SORTED_DV_FIELD);
+ int maxDoc = reader.maxDoc();
+ BytesRef bytes = new BytesRef();
+ for (int i = 0; i < maxDoc; i++) {
+ dv.get(i, bytes);
+ assertEquals("incorrect sorted DocValues for doc " + i, sortedValues[i].toString(), bytes.utf8ToString());
+ }
+ }
+
+ @Test
+ public void testSortedSetDocValuesField() throws Exception {
+ assumeTrue("default codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
+ SortedSetDocValues dv = reader.getSortedSetDocValues(SORTED_SET_DV_FIELD);
+ int maxDoc = reader.maxDoc();
+ BytesRef bytes = new BytesRef();
+ for (int i = 0; i < maxDoc; i++) {
+ dv.setDocument(i);
+ dv.lookupOrd(dv.nextOrd(), bytes);
+ int value = sortedValues[i].intValue();
+ assertEquals("incorrect sorted-set DocValues for doc " + i, Integer.valueOf(value).toString(), bytes.utf8ToString());
+ dv.lookupOrd(dv.nextOrd(), bytes);
+ assertEquals("incorrect sorted-set DocValues for doc " + i, Integer.valueOf(value + 1).toString(), bytes.utf8ToString());
+ assertEquals(SortedSetDocValues.NO_MORE_ORDS, dv.nextOrd());
+ }
+ }
+
+ @Test
+ public void testTermVectors() throws Exception {
+ int maxDoc = reader.maxDoc();
+ for (int i = 0; i < maxDoc; i++) {
+ Terms terms = reader.getTermVector(i, TERM_VECTORS_FIELD);
+ assertNotNull("term vectors not found for doc " + i + " field [" + TERM_VECTORS_FIELD + "]", terms);
+ assertEquals("incorrect term vector for doc " + i, sortedValues[i].toString(), terms.iterator(null).next().utf8ToString());
+ }
+ }
+
+}
Added: lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java?rev=1454801&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java (added)
+++ lucene/dev/trunk/lucene/misc/src/test/org/apache/lucene/index/sorter/SortingAtomicReaderTest.java Sun Mar 10 07:55:38 2013
@@ -0,0 +1,75 @@
+package org.apache.lucene.index.sorter;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+
+import org.apache.lucene.index.AtomicReader;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util._TestUtil;
+import org.junit.BeforeClass;
+
+public class SortingAtomicReaderTest extends SorterTestBase {
+
+ @BeforeClass
+ public static void beforeClassSortingAtomicReaderTest() throws Exception {
+ // build the mapping from the reader, since we deleted documents, some of
+ // them might have disappeared from the index (e.g. if an entire segment is
+ // dropped b/c all its docs are deleted)
+ Integer[] values = new Integer[reader.maxDoc()];
+ int[] docs = new int[reader.maxDoc()];
+ for (int i = 0; i < reader.maxDoc(); i++) {
+ docs[i] = i;
+ values[i] = Integer.valueOf(reader.document(i).get(ID_FIELD));
+ }
+
+ final int[] oldToNew = Sorter.compute(docs, Collections.unmodifiableList(Arrays.asList(values)));
+ // Sorter.compute also sorts the values
+ sortedValues = new Integer[reader.maxDoc()];
+ for (int i = 0; i < reader.maxDoc(); ++i) {
+ sortedValues[oldToNew[i]] = values[i];
+ }
+ if (VERBOSE) {
+ System.out.println("oldToNew: " + Arrays.toString(oldToNew));
+ System.out.println("sortedValues: " + Arrays.toString(sortedValues));
+ }
+
+ reader = new SortingAtomicReader(reader, new Sorter() {
+ @Override
+ public int[] oldToNew(AtomicReader reader) throws IOException {
+ return oldToNew;
+ }
+ });
+
+ if (VERBOSE) {
+ System.out.print("mapped-deleted-docs: ");
+ Bits mappedLiveDocs = reader.getLiveDocs();
+ for (int i = 0; i < mappedLiveDocs.length(); i++) {
+ if (!mappedLiveDocs.get(i)) {
+ System.out.print(i + " ");
+ }
+ }
+ System.out.println();
+ }
+
+ _TestUtil.checkReader(reader);
+ }
+
+}