You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/02/11 03:54:54 UTC
svn commit: r1444647 [2/2] - in /lucene/dev/branches/lucene4765:
lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/
lucene/core/src/java/org/apache/lucene/codecs/ lucene/core/src/java/org...
Added: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java?rev=1444647&view=auto
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java (added)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java Mon Feb 11 02:54:53 2013
@@ -0,0 +1,141 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Comparator;
+
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+/** Implements a {@link TermsEnum} wrapping a provided
+ * {@link SortedSetDocValues}. */
+
+// nocommit: if we are ok with ords being 'long' for SortedDocValues,
+// then we don't need this...
+public class SortedSetDocValuesTermsEnum extends TermsEnum {
+ private final SortedSetDocValues values;
+ private long currentOrd = -1;
+ private final BytesRef term = new BytesRef();
+
+ /** Creates a new TermsEnum over the provided values */
+ public SortedSetDocValuesTermsEnum(SortedSetDocValues values) {
+ this.values = values;
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef text, boolean useCache /* ignored */) throws IOException {
+ long ord = values.lookupTerm(text);
+ if (ord >= 0) {
+ currentOrd = ord;
+ term.offset = 0;
+ // TODO: is there a cleaner way?
+ // term.bytes may be pointing to codec-private byte[]
+ // storage, so we must force new byte[] allocation:
+ term.bytes = new byte[text.length];
+ term.copyBytes(text);
+ return SeekStatus.FOUND;
+ } else {
+ currentOrd = -ord-1;
+ if (currentOrd == values.getValueCount()) {
+ return SeekStatus.END;
+ } else {
+ // TODO: hmm can we avoid this "extra" lookup?:
+ values.lookupOrd(currentOrd, term);
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+ }
+
+ @Override
+ public boolean seekExact(BytesRef text, boolean useCache) throws IOException {
+ long ord = values.lookupTerm(text);
+ if (ord >= 0) {
+ currentOrd = ord;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void seekExact(long ord) throws IOException {
+ assert ord >= 0 && ord < values.getValueCount();
+ currentOrd = (int) ord;
+ values.lookupOrd(currentOrd, term);
+ }
+
+ @Override
+ public BytesRef next() throws IOException {
+ currentOrd++;
+ if (currentOrd >= values.getValueCount()) {
+ return null;
+ }
+ values.lookupOrd(currentOrd, term);
+ return term;
+ }
+
+ @Override
+ public BytesRef term() throws IOException {
+ return term;
+ }
+
+ @Override
+ public long ord() throws IOException {
+ return currentOrd;
+ }
+
+ @Override
+ public int docFreq() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long totalTermFreq() {
+ return -1;
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
+
+ @Override
+ public void seekExact(BytesRef term, TermState state) throws IOException {
+ assert state != null && state instanceof OrdTermState;
+ this.seekExact(((OrdTermState)state).ord);
+ }
+
+ @Override
+ public TermState termState() throws IOException {
+ OrdTermState state = new OrdTermState();
+ state.ord = currentOrd;
+ return state;
+ }
+}
+
Added: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java?rev=1444647&view=auto
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java (added)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java Mon Feb 11 02:54:53 2013
@@ -0,0 +1,285 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
+import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.packed.AppendingLongBuffer;
+
+/** Buffers up pending byte[]s per doc, deref and sorting via
+ * int ord, then flushes when segment flushes. */
+class SortedSetDocValuesWriter extends DocValuesWriter {
+ final BytesRefHash hash;
+ private AppendingLongBuffer pending; // stream of all ords
+ private AppendingLongBuffer pendingCounts; // ords per doc
+ private final Counter iwBytesUsed;
+ private long bytesUsed; // this only tracks differences in 'pending' and 'pendingCounts'
+ private final FieldInfo fieldInfo;
+ private int currentDoc;
+ private int currentValues[] = new int[8];
+ private int currentUpto = 0;
+
+ public SortedSetDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
+ this.fieldInfo = fieldInfo;
+ this.iwBytesUsed = iwBytesUsed;
+ hash = new BytesRefHash(
+ new ByteBlockPool(
+ new ByteBlockPool.DirectTrackingAllocator(iwBytesUsed)),
+ BytesRefHash.DEFAULT_CAPACITY,
+ new DirectBytesStartArray(BytesRefHash.DEFAULT_CAPACITY, iwBytesUsed));
+ pending = new AppendingLongBuffer();
+ pendingCounts = new AppendingLongBuffer();
+ bytesUsed = pending.ramBytesUsed() + pendingCounts.ramBytesUsed();
+ iwBytesUsed.addAndGet(bytesUsed);
+ }
+
+ public void addValue(int docID, BytesRef value) {
+ if (value == null) {
+ throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": null value not allowed");
+ }
+ if (value.length > (BYTE_BLOCK_SIZE - 2)) {
+ throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" is too large, must be <= " + (BYTE_BLOCK_SIZE - 2));
+ }
+
+ if (docID != currentDoc) {
+ finishCurrentDoc();
+ }
+
+ // Fill in any holes:
+ while(currentDoc < docID) {
+ pendingCounts.add(0); // no values
+ currentDoc++;
+ }
+
+ addOneValue(value);
+ updateBytesUsed();
+ }
+
+ // finalize currentDoc
+ private void finishCurrentDoc() {
+ Arrays.sort(currentValues, 0, currentUpto);
+ int lastValue = -1;
+ int count = 0;
+ for (int i = 0; i < currentUpto; i++) {
+ int v = currentValues[i];
+ // if its not a duplicate
+ if (v != lastValue) {
+ pending.add(v); // record the ord
+ count++;
+ }
+ lastValue = v;
+ }
+ // record the number of unique ords for this doc
+ pendingCounts.add(count);
+ currentUpto = 0;
+ currentDoc++;
+ }
+
+ @Override
+ public void finish(int maxDoc) {
+ finishCurrentDoc();
+
+ // fill in any holes
+ for (int i = currentDoc; i < maxDoc; i++) {
+ pendingCounts.add(0); // no values
+ }
+ }
+
+ private void addOneValue(BytesRef value) {
+ int ord = hash.add(value);
+ if (ord < 0) {
+ ord = -ord-1;
+ } else {
+ // reserve additional space for each unique value:
+ // 1. when indexing, when hash is 50% full, rehash() suddenly needs 2*size ints.
+ // TODO: can this same OOM happen in THPF?
+ // 2. when flushing, we need 1 int per value (slot in the ordMap).
+ iwBytesUsed.addAndGet(2 * RamUsageEstimator.NUM_BYTES_INT);
+ }
+
+ if (currentUpto == currentValues.length) {
+ currentValues = ArrayUtil.grow(currentValues, currentValues.length+1);
+ iwBytesUsed.addAndGet((currentValues.length - currentUpto) * RamUsageEstimator.NUM_BYTES_INT);
+ }
+
+ currentValues[currentUpto] = ord;
+ currentUpto++;
+ }
+
+ private void updateBytesUsed() {
+ final long newBytesUsed = pending.ramBytesUsed() + pendingCounts.ramBytesUsed();
+ iwBytesUsed.addAndGet(newBytesUsed - bytesUsed);
+ bytesUsed = newBytesUsed;
+ }
+
+ @Override
+ public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException {
+ final int maxDoc = state.segmentInfo.getDocCount();
+
+ assert pendingCounts.size() == maxDoc;
+ final int valueCount = hash.size();
+
+ final int[] sortedValues = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
+ final int[] ordMap = new int[valueCount];
+
+ for(int ord=0;ord<valueCount;ord++) {
+ ordMap[sortedValues[ord]] = ord;
+ }
+
+ dvConsumer.addSortedSetField(fieldInfo,
+
+ // ord -> value
+ new Iterable<BytesRef>() {
+ @Override
+ public Iterator<BytesRef> iterator() {
+ return new ValuesIterator(sortedValues, valueCount);
+ }
+ },
+
+ // doc -> ordCount
+ new Iterable<Number>() {
+ @Override
+ public Iterator<Number> iterator() {
+ return new OrdCountIterator(maxDoc);
+ }
+ },
+
+ // ords
+ new Iterable<Number>() {
+ @Override
+ public Iterator<Number> iterator() {
+ return new OrdsIterator(ordMap);
+ }
+ });
+ }
+
+ @Override
+ public void abort() {
+ }
+
+ // iterates over the unique values we have in ram
+ private class ValuesIterator implements Iterator<BytesRef> {
+ final int sortedValues[];
+ final BytesRef scratch = new BytesRef();
+ final int valueCount;
+ int ordUpto;
+
+ ValuesIterator(int sortedValues[], int valueCount) {
+ this.sortedValues = sortedValues;
+ this.valueCount = valueCount;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return ordUpto < valueCount;
+ }
+
+ @Override
+ public BytesRef next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ hash.get(sortedValues[ordUpto], scratch);
+ ordUpto++;
+ return scratch;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ // iterates over the ords for each doc we have in ram
+ private class OrdsIterator implements Iterator<Number> {
+ final AppendingLongBuffer.Iterator iter = pending.iterator();
+ final int ordMap[];
+ final long numOrds;
+ long ordUpto;
+
+ OrdsIterator(int ordMap[]) {
+ this.ordMap = ordMap;
+ this.numOrds = pending.size();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return ordUpto < numOrds;
+ }
+
+ @Override
+ public Number next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ int ord = (int) iter.next();
+ ordUpto++;
+ // TODO: make reusable Number
+ return ordMap[ord];
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private class OrdCountIterator implements Iterator<Number> {
+ final AppendingLongBuffer.Iterator iter = pendingCounts.iterator();
+ final int maxDoc;
+ int docUpto;
+
+ OrdCountIterator(int maxDoc) {
+ this.maxDoc = maxDoc;
+ assert pendingCounts.size() == maxDoc;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return docUpto < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ docUpto++;
+ // TODO: make reusable Number
+ return iter.next();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
Modified: lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java?rev=1444647&r1=1444646&r2=1444647&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java (original)
+++ lucene/dev/branches/lucene4765/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java Mon Feb 11 02:54:53 2013
@@ -61,11 +61,11 @@ public class AppendingLongBuffer {
* <p>
* <b>NOTE</b>: This class is not really designed for random access!
* You will likely get better performance by using packed ints in another way! */
- public long get(int index) {
- assert index < size(); // TODO: do a better check, and throw IndexOutOfBoundsException?
+ public long get(long index) {
+ assert index < size() : "index=" + index + ",size=" + size(); // TODO: do a better check, and throw IndexOutOfBoundsException?
// This class is currently only used by the indexer.
- int block = index >> BLOCK_BITS;
- int element = index & BLOCK_MASK;
+ int block = (int) (index >> BLOCK_BITS);
+ int element = (int) (index & BLOCK_MASK);
if (block == valuesOff) {
return pending[element];
} else if (values[block] == null) {
@@ -115,8 +115,8 @@ public class AppendingLongBuffer {
}
/** Get the number of values that have been added to the buffer. */
- public int size() {
- return valuesOff * MAX_PENDING_COUNT + pendingOff;
+ public long size() {
+ return valuesOff * (long)MAX_PENDING_COUNT + pendingOff;
}
/** Return an iterator over the values of this buffer. */
Added: lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java?rev=1444647&view=auto
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java (added)
+++ lucene/dev/branches/lucene4765/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java Mon Feb 11 02:54:53 2013
@@ -0,0 +1,216 @@
+package org.apache.lucene;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.index.SortedSetDocValues.OrdIterator;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+
+/**
+ * A very simple demo used in the API documentation (src/java/overview.html).
+ *
+ * Please try to keep src/java/overview.html up-to-date when making changes
+ * to this class.
+ */
+public class TestDemoDocValue extends LuceneTestCase {
+
+ // nocommit: only Lucene42/Asserting implemented right now
+ private Codec saved;
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ saved = Codec.getDefault();
+ Codec.setDefault(_TestUtil.alwaysDocValuesFormat(DocValuesFormat.forName("Asserting")));
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ Codec.setDefault(saved);
+ super.tearDown();
+ }
+
+ public void testOneValue() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ // Store the index in memory:
+ Directory directory = newDirectory();
+ // To store an index on disk, use this instead:
+ // Directory directory = FSDirectory.open(new File("/tmp/testindex"));
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, analyzer);
+ Document doc = new Document();
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ DirectoryReader ireader = DirectoryReader.open(directory); // read-only=true
+ SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");
+ OrdIterator oi = dv.getOrds(0, null);
+ assertEquals(0, oi.nextOrd());
+ assertEquals(OrdIterator.NO_MORE_ORDS, oi.nextOrd());
+
+ BytesRef bytes = new BytesRef();
+ dv.lookupOrd(0, bytes);
+ assertEquals(new BytesRef("hello"), bytes);
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testTwoDocumentsMerged() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ // Store the index in memory:
+ Directory directory = newDirectory();
+ // To store an index on disk, use this instead:
+ // Directory directory = FSDirectory.open(new File("/tmp/testindex"));
+ IndexWriterConfig iwconfig = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+ Document doc = new Document();
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ doc = new Document();
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("world")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ DirectoryReader ireader = DirectoryReader.open(directory); // read-only=true
+ SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");
+ OrdIterator oi = dv.getOrds(0, null);
+ assertEquals(0, oi.nextOrd());
+ assertEquals(OrdIterator.NO_MORE_ORDS, oi.nextOrd());
+
+ BytesRef bytes = new BytesRef();
+ dv.lookupOrd(0, bytes);
+ assertEquals(new BytesRef("hello"), bytes);
+
+ oi = dv.getOrds(1, oi);
+ assertEquals(1, oi.nextOrd());
+ assertEquals(OrdIterator.NO_MORE_ORDS, oi.nextOrd());
+
+ dv.lookupOrd(1, bytes);
+ assertEquals(new BytesRef("world"), bytes);
+
+ assertEquals(2, dv.getValueCount());
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testTwoValues() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ // Store the index in memory:
+ Directory directory = newDirectory();
+ // To store an index on disk, use this instead:
+ // Directory directory = FSDirectory.open(new File("/tmp/testindex"));
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, analyzer);
+ Document doc = new Document();
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("world")));
+ iwriter.addDocument(doc);
+ iwriter.close();
+
+ // Now search the index:
+ DirectoryReader ireader = DirectoryReader.open(directory); // read-only=true
+ SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");
+ OrdIterator oi = dv.getOrds(0, null);
+ assertEquals(0, oi.nextOrd());
+ assertEquals(1, oi.nextOrd());
+ assertEquals(OrdIterator.NO_MORE_ORDS, oi.nextOrd());
+
+ BytesRef bytes = new BytesRef();
+ dv.lookupOrd(0, bytes);
+ assertEquals(new BytesRef("hello"), bytes);
+
+ dv.lookupOrd(1, bytes);
+ assertEquals(new BytesRef("world"), bytes);
+
+ ireader.close();
+ directory.close();
+ }
+
+ public void testThreeValuesTwoDocs() throws IOException {
+ Analyzer analyzer = new MockAnalyzer(random());
+
+ // Store the index in memory:
+ Directory directory = newDirectory();
+ // To store an index on disk, use this instead:
+ // Directory directory = FSDirectory.open(new File("/tmp/testindex"));
+ IndexWriterConfig iwconfig = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+ Document doc = new Document();
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("world")));
+ iwriter.addDocument(doc);
+ iwriter.commit();
+ doc = new Document();
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("beer")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+ iwriter.close();
+
+ // Now search the index:
+ DirectoryReader ireader = DirectoryReader.open(directory); // read-only=true
+ SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");
+ assertEquals(3, dv.getValueCount());
+
+ OrdIterator oi = dv.getOrds(0, null);
+ assertEquals(1, oi.nextOrd());
+ assertEquals(2, oi.nextOrd());
+ assertEquals(OrdIterator.NO_MORE_ORDS, oi.nextOrd());
+
+ oi = dv.getOrds(1, null);
+ assertEquals(0, oi.nextOrd());
+ assertEquals(1, oi.nextOrd());
+ assertEquals(OrdIterator.NO_MORE_ORDS, oi.nextOrd());
+
+ BytesRef bytes = new BytesRef();
+ dv.lookupOrd(0, bytes);
+ assertEquals(new BytesRef("beer"), bytes);
+
+ dv.lookupOrd(1, bytes);
+ assertEquals(new BytesRef("hello"), bytes);
+
+ dv.lookupOrd(2, bytes);
+ assertEquals(new BytesRef("world"), bytes);
+
+ ireader.close();
+ directory.close();
+ }
+}
Modified: lucene/dev/branches/lucene4765/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1444647&r1=1444646&r2=1444647&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene4765/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Mon Feb 11 02:54:53 2013
@@ -46,6 +46,7 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.OrdTermState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
@@ -752,6 +753,11 @@ public class MemoryIndex {
public SortedDocValues getSortedDocValues(String field) {
return null;
}
+
+ @Override
+ public SortedSetDocValues getSortedSetDocValues(String field) {
+ return null;
+ }
private class MemoryFields extends Fields {
@Override
Modified: lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1444647&r1=1444646&r2=1444647&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Mon Feb 11 02:54:53 2013
@@ -32,6 +32,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
@@ -127,6 +128,12 @@ public class AssertingDocValuesFormat ex
in.addSortedField(field, values, docToOrd);
}
+ @Override
+ public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
+ // nocommit: add checks
+ in.addSortedSetField(field, values, docToOrdCount, ords);
+ }
+
private <T> void checkIterator(Iterator<T> iterator, int expectedSize) {
for (int i = 0; i < expectedSize; i++) {
boolean hasNext = iterator.hasNext();
@@ -190,6 +197,14 @@ public class AssertingDocValuesFormat ex
}
@Override
+ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+ assert field.getDocValuesType() == FieldInfo.DocValuesType.SORTED_SET;
+ SortedSetDocValues values = in.getSortedSet(field);
+ assert values != null;
+ return new AssertingAtomicReader.AssertingSortedSetDocValues(values, maxDoc);
+ }
+
+ @Override
public void close() throws IOException {
in.close();
}
Modified: lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java?rev=1444647&r1=1444646&r2=1444647&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java Mon Feb 11 02:54:53 2013
@@ -31,6 +31,7 @@ import org.apache.lucene.index.IndexFile
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -232,6 +233,11 @@ class CheapBastardDocValuesProducer exte
}
@Override
+ public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
+ throw new UnsupportedOperationException(); // nocommit
+ }
+
+ @Override
public void close() throws IOException {
data.close();
}
Modified: lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java?rev=1444647&r1=1444646&r2=1444647&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java (original)
+++ lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40DocValuesWriter.java Mon Feb 11 02:54:53 2013
@@ -494,6 +494,11 @@ class Lucene40DocValuesWriter extends Do
}
@Override
+ public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
+ throw new UnsupportedOperationException("Lucene 4.0 does not support SortedSet docvalues");
+ }
+
+ @Override
public void close() throws IOException {
dir.close();
}
Modified: lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java?rev=1444647&r1=1444646&r2=1444647&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java (original)
+++ lucene/dev/branches/lucene4765/lucene/test-framework/src/java/org/apache/lucene/index/AssertingAtomicReader.java Mon Feb 11 02:54:53 2013
@@ -3,6 +3,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Iterator;
+import org.apache.lucene.index.SortedSetDocValues.OrdIterator;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -453,6 +454,78 @@ public class AssertingAtomicReader exten
return result;
}
}
+
+ /** Wraps a SortedSetDocValues but with additional asserts */
+ public static class AssertingSortedSetDocValues extends SortedSetDocValues {
+ private final SortedSetDocValues in;
+ private final int maxDoc;
+ private final long valueCount;
+
+ public AssertingSortedSetDocValues(SortedSetDocValues in, int maxDoc) {
+ this.in = in;
+ this.maxDoc = maxDoc;
+ this.valueCount = in.getValueCount();
+ assert valueCount >= 0;
+ }
+
+ @Override
+ public OrdIterator getOrds(int docID, OrdIterator reuse) {
+ assert docID >= 0 && docID < maxDoc : "docid=" + docID + ",maxDoc=" + maxDoc;
+ if (reuse instanceof AssertingOrdIterator) {
+ reuse = ((AssertingOrdIterator) reuse).in;
+ }
+ OrdIterator iterator = in.getOrds(docID, reuse);
+ assert iterator != null;
+ return new AssertingOrdIterator(iterator, valueCount);
+ }
+
+ @Override
+ public void lookupOrd(long ord, BytesRef result) {
+ assert ord >= 0 && ord < valueCount;
+ assert result.isValid();
+ in.lookupOrd(ord, result);
+ assert result.isValid();
+ }
+
+ @Override
+ public long getValueCount() {
+ long valueCount = in.getValueCount();
+ assert valueCount == this.valueCount; // should not change
+ return valueCount;
+ }
+
+ @Override
+ public long lookupTerm(BytesRef key) {
+ assert key.isValid();
+ long result = in.lookupTerm(key);
+ assert result < valueCount;
+ assert key.isValid();
+ return result;
+ }
+ }
+
+ /** Wraps a OrdIterator but with additional asserts */
+ public static class AssertingOrdIterator extends OrdIterator {
+ final OrdIterator in;
+ final long valueCount;
+ long lastOrd = Long.MIN_VALUE;
+
+ AssertingOrdIterator(OrdIterator in, long valueCount) {
+ this.in = in;
+ this.valueCount = valueCount;
+ assert lastOrd != NO_MORE_ORDS;
+ }
+
+ @Override
+ public long nextOrd() {
+ assert lastOrd != NO_MORE_ORDS;
+ long ord = in.nextOrd();
+ assert ord == NO_MORE_ORDS || ord < valueCount;
+ assert ord > lastOrd;
+ lastOrd = ord;
+ return ord;
+ }
+ }
@Override
public NumericDocValues getNumericDocValues(String field) throws IOException {
@@ -497,6 +570,20 @@ public class AssertingAtomicReader exten
}
@Override
+ public SortedSetDocValues getSortedSetDocValues(String field) throws IOException {
+ SortedSetDocValues dv = super.getSortedSetDocValues(field);
+ FieldInfo fi = getFieldInfos().fieldInfo(field);
+ if (dv != null) {
+ assert fi != null;
+ assert fi.getDocValuesType() == FieldInfo.DocValuesType.SORTED_SET;
+ return new AssertingSortedSetDocValues(dv, maxDoc());
+ } else {
+ assert fi == null || fi.getDocValuesType() != FieldInfo.DocValuesType.SORTED_SET;
+ return null;
+ }
+ }
+
+ @Override
public NumericDocValues getNormValues(String field) throws IOException {
NumericDocValues dv = super.getNormValues(field);
FieldInfo fi = getFieldInfos().fieldInfo(field);
Modified: lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/search/TestDocSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/search/TestDocSet.java?rev=1444647&r1=1444646&r2=1444647&view=diff
==============================================================================
--- lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/search/TestDocSet.java (original)
+++ lucene/dev/branches/lucene4765/solr/core/src/test/org/apache/solr/search/TestDocSet.java Mon Feb 11 02:54:53 2013
@@ -33,6 +33,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
@@ -401,6 +402,11 @@ public class TestDocSet extends LuceneTe
public SortedDocValues getSortedDocValues(String field) {
return null;
}
+
+ @Override
+ public SortedSetDocValues getSortedSetDocValues(String field) {
+ return null;
+ }
@Override
public NumericDocValues getNormValues(String field) {