You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2013/01/30 02:20:46 UTC
svn commit: r1440234 - in /lucene/dev/branches/lucene4547/lucene:
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/index/
test-framework/src/java/org/apache/lucene/codecs/asserting/
Author: rmuir
Date: Wed Jan 30 01:20:46 2013
New Revision: 1440234
URL: http://svn.apache.org/viewvc?rev=1440234&view=rev
Log:
clean up iterators / iterators bugs / style
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java?rev=1440234&r1=1440233&r2=1440234&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/DocValuesConsumer.java Wed Jan 30 01:20:46 2013
@@ -406,6 +406,9 @@ public abstract class DocValuesConsumer
@Override
public BytesRef next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
return merger.mergedTerms.get(ordUpto++);
}
};
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java?rev=1440234&r1=1440233&r2=1440234&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java Wed Jan 30 01:20:46 2013
@@ -21,6 +21,7 @@ import static org.apache.lucene.util.Byt
import java.io.IOException;
import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.store.RAMFile;
@@ -99,60 +100,66 @@ class BinaryDocValuesWriter extends DocV
@Override
public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException {
final int maxDoc = state.segmentInfo.getDocCount();
- final int size = addedValues;
-
dvConsumer.addBinaryField(fieldInfo,
new Iterable<BytesRef>() {
-
@Override
public Iterator<BytesRef> iterator() {
- return new Iterator<BytesRef>() {
- RAMInputStream bytesReader;
- AppendingLongBuffer.Iterator iter = lengths.iterator();
- BytesRef value = new BytesRef();
- int upto;
-
- {
- try {
- bytesReader = new RAMInputStream("bogus", bytes);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public boolean hasNext() {
- return upto < maxDoc;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public BytesRef next() {
- if (upto < size) {
- int length = (int) iter.next();
- value.grow(length);
- try {
- bytesReader.readBytes(value.bytes, 0, length);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- value.length = length;
- } else {
- value.length = 0;
- }
- upto++;
- return value;
- }
- };
- }
- });
+ return new BytesIterator(maxDoc);
+ }
+ });
}
@Override
public void abort() {
}
-}
\ No newline at end of file
+
+ // iterates over the values we have in ram
+ private class BytesIterator implements Iterator<BytesRef> {
+ final BytesRef value = new BytesRef();
+ final AppendingLongBuffer.Iterator lengthsIterator = lengths.iterator();
+ final int size = lengths.size();
+ final int maxDoc;
+ final RAMInputStream bytesReader;
+ int upto;
+
+ BytesIterator(int maxDoc) {
+ this.maxDoc = maxDoc;
+ try {
+ bytesReader = new RAMInputStream("BinaryDocValuesWriter", bytes);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return upto < maxDoc;
+ }
+
+ @Override
+ public BytesRef next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ if (upto < size) {
+ int length = (int) lengthsIterator.next();
+ value.grow(length);
+ try {
+ bytesReader.readBytes(value.bytes, 0, length);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ value.length = length;
+ } else {
+ value.length = 0;
+ }
+ upto++;
+ return value;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java?rev=1440234&r1=1440233&r2=1440234&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java Wed Jan 30 01:20:46 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.util.Counter;
@@ -75,36 +76,9 @@ class NumericDocValuesWriter extends Doc
dvConsumer.addNumericField(fieldInfo,
new Iterable<Number>() {
-
@Override
public Iterator<Number> iterator() {
- return new Iterator<Number>() {
- int upto;
- AppendingLongBuffer.Iterator iter = pending.iterator();
-
- @Override
- public boolean hasNext() {
- return upto < maxDoc;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Number next() {
- long value;
- if (upto < pending.size()) {
- value = iter.next();
- } else {
- value = 0;
- }
- upto++;
- // TODO: make reusable Number
- return value;
- }
- };
+ return new NumericIterator(maxDoc);
}
});
}
@@ -112,4 +86,42 @@ class NumericDocValuesWriter extends Doc
@Override
public void abort() {
}
-}
\ No newline at end of file
+
+ // iterates over the values we have in ram
+ private class NumericIterator implements Iterator<Number> {
+ final AppendingLongBuffer.Iterator iter = pending.iterator();
+ final int size = pending.size();
+ final int maxDoc;
+ int upto;
+
+ NumericIterator(int maxDoc) {
+ this.maxDoc = maxDoc;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return upto < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ long value;
+ if (upto < size) {
+ value = iter.next();
+ } else {
+ value = 0;
+ }
+ upto++;
+ // TODO: make reusable Number
+ return value;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java?rev=1440234&r1=1440233&r2=1440234&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java Wed Jan 30 01:20:46 2013
@@ -21,6 +21,7 @@ import static org.apache.lucene.util.Byt
import java.io.IOException;
import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.util.ArrayUtil;
@@ -110,11 +111,12 @@ class SortedDocValuesWriter extends DocV
emptyOrd = ord;
}
} else {
- emptyOrd = -1;
+ emptyOrd = -1; // nocommit: HUH? how can this possibly work?
}
final int valueCount = hash.size();
+ // nocommit: account for both sortedValues and ordMap as-we-go...
final int[] sortedValues = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator());
final int sortedValueRamUsage = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_INT*valueCount;
final int[] ordMap = new int[valueCount];
@@ -131,27 +133,7 @@ class SortedDocValuesWriter extends DocV
new Iterable<BytesRef>() {
@Override
public Iterator<BytesRef> iterator() {
- return new Iterator<BytesRef>() {
- int ordUpto;
- BytesRef scratch = new BytesRef();
-
- @Override
- public boolean hasNext() {
- return ordUpto < valueCount;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public BytesRef next() {
- hash.get(sortedValues[ordUpto], scratch);
- ordUpto++;
- return scratch;
- }
- };
+ return new ValuesIterator(sortedValues, valueCount);
}
},
@@ -159,32 +141,7 @@ class SortedDocValuesWriter extends DocV
new Iterable<Number>() {
@Override
public Iterator<Number> iterator() {
- return new Iterator<Number>() {
- int docUpto;
-
- @Override
- public boolean hasNext() {
- return docUpto < maxDoc;
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Number next() {
- int ord;
- if (docUpto < bufferedDocCount) {
- ord = pending[docUpto];
- } else {
- ord = emptyOrd;
- }
- docUpto++;
- // TODO: make reusable Number
- return ordMap[ord];
- }
- };
+ return new OrdsIterator(ordMap, bufferedDocCount, maxDoc, emptyOrd);
}
});
}
@@ -192,4 +149,79 @@ class SortedDocValuesWriter extends DocV
@Override
public void abort() {
}
+
+ // iterates over the unique values we have in ram
+ private class ValuesIterator implements Iterator<BytesRef> {
+ final int sortedValues[];
+ final BytesRef scratch = new BytesRef();
+ final int valueCount;
+ int ordUpto;
+
+ ValuesIterator(int sortedValues[], int valueCount) {
+ this.sortedValues = sortedValues;
+ this.valueCount = valueCount;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return ordUpto < valueCount;
+ }
+
+ @Override
+ public BytesRef next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ hash.get(sortedValues[ordUpto], scratch);
+ ordUpto++;
+ return scratch;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ // iterates over the ords for each doc we have in ram
+ private class OrdsIterator implements Iterator<Number> {
+ final int ordMap[];
+ final int size;
+ final int maxDoc;
+ final int emptyOrd; // nocommit
+ int docUpto;
+
+ OrdsIterator(int ordMap[], int size, int maxDoc, int emptyOrd) {
+ this.ordMap = ordMap;
+ this.size = size;
+ this.maxDoc = maxDoc;
+ this.emptyOrd = emptyOrd;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return docUpto < maxDoc;
+ }
+
+ @Override
+ public Number next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ int ord;
+ if (docUpto < size) {
+ ord = pending[docUpto];
+ } else {
+ ord = emptyOrd;
+ }
+ docUpto++;
+ // TODO: make reusable Number
+ return ordMap[ord];
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
}
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java?rev=1440234&r1=1440233&r2=1440234&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingDocValuesFormat.java Wed Jan 30 01:20:46 2013
@@ -18,6 +18,8 @@ package org.apache.lucene.codecs.asserti
*/
import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesFormat;
@@ -75,6 +77,7 @@ public class AssertingDocValuesFormat ex
count++;
}
assert count == maxDoc;
+ checkIterator(values.iterator(), maxDoc);
in.addNumericField(field, values);
}
@@ -87,6 +90,7 @@ public class AssertingDocValuesFormat ex
count++;
}
assert count == maxDoc;
+ checkIterator(values.iterator(), maxDoc);
in.addBinaryField(field, values);
}
@@ -118,9 +122,33 @@ public class AssertingDocValuesFormat ex
assert count == maxDoc;
assert seenOrds.cardinality() == valueCount;
+ checkIterator(values.iterator(), valueCount);
+ checkIterator(docToOrd.iterator(), maxDoc);
in.addSortedField(field, values, docToOrd);
}
+ private <T> void checkIterator(Iterator<T> iterator, int expectedSize) {
+ for (int i = 0; i < expectedSize; i++) {
+ boolean hasNext = iterator.hasNext();
+ assert hasNext;
+ T v = iterator.next();
+ assert v != null;
+ try {
+ iterator.remove();
+ throw new AssertionError("broken iterator (supports remove): " + iterator);
+ } catch (UnsupportedOperationException expected) {
+ // ok
+ }
+ }
+ assert !iterator.hasNext();
+ try {
+ iterator.next();
+ throw new AssertionError("broken iterator (allows next() when hasNext==false) " + iterator);
+ } catch (NoSuchElementException expected) {
+ // ok
+ }
+ }
+
@Override
public void close() throws IOException {
in.close();