You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/01/07 23:29:34 UTC
svn commit: r1430064 - in /lucene/dev/branches/lucene4547/lucene:
codecs/src/java/org/apache/lucene/codecs/memory/
codecs/src/java/org/apache/lucene/codecs/simpletext/
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/codecs/lucen...
Author: mikemccand
Date: Mon Jan 7 22:29:33 2013
New Revision: 1430064
URL: http://svn.apache.org/viewvc?rev=1430064&view=rev
Log:
cutover to Iterable<BytesRef> for BinaryDV
Removed:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/BinaryDocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValues.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41BinaryDocValuesConsumer.java
Modified:
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesFormat.java Mon Jan 7 22:29:33 2013
@@ -94,8 +94,6 @@ public class MemoryDocValuesFormat exten
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
BinaryDocValues valuesIn = producer.getBinary(field);
final int maxDoc = valuesIn.size();
- final int maxLength = valuesIn.maxLength();
- final boolean fixedLength = valuesIn.isFixedLength();
// nocommit more ram efficient
final byte[][] values = new byte[maxDoc][];
BytesRef scratch = new BytesRef();
@@ -118,16 +116,6 @@ public class MemoryDocValuesFormat exten
public int size() {
return maxDoc;
}
-
- @Override
- public boolean isFixedLength() {
- return fixedLength;
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
};
}
Modified: lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextSimpleDocValuesFormat.java Mon Jan 7 22:29:33 2013
@@ -29,7 +29,6 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
import org.apache.lucene.codecs.SimpleDocValuesFormat;
@@ -63,10 +62,10 @@ public class SimpleTextSimpleDocValuesFo
final static BytesRef MINVALUE = new BytesRef(" minvalue ");
final static BytesRef PATTERN = new BytesRef(" pattern ");
// used for bytes
- final static BytesRef FIXEDLENGTH = new BytesRef(" fixedlength ");
- final static BytesRef MAXLENGTH = new BytesRef(" maxlength ");
final static BytesRef LENGTH = new BytesRef("length ");
+ final static BytesRef MAXLENGTH = new BytesRef(" maxlength ");
// used for sorted bytes
+ final static BytesRef FIXEDLENGTH = new BytesRef(" fixedlength ");
final static BytesRef NUMVALUES = new BytesRef(" numvalues ");
final static BytesRef ORDPATTERN = new BytesRef(" ordpattern ");
@@ -101,8 +100,7 @@ public class SimpleTextSimpleDocValuesFo
* for bytes this is also a "fixed-width" file, for example:
* <pre>
* field myField
- * fixedlength false
- * maxlength 8
+ * maxlength 6
* pattern 0
* length 6
* foobar[space][space]
@@ -221,15 +219,16 @@ public class SimpleTextSimpleDocValuesFo
}
@Override
- public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, final int maxLength) throws IOException {
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
assert fieldSeen(field.name);
assert DocValues.isBytes(field.getDocValuesType());
assert !isNorms;
+ int maxLength = 0;
+ for(BytesRef value : values) {
+ maxLength = Math.max(maxLength, value.length);
+ }
writeFieldEntry(field);
- // write fixedlength
- SimpleTextUtil.write(data, FIXEDLENGTH);
- SimpleTextUtil.write(data, Boolean.toString(fixedLength), scratch);
- SimpleTextUtil.writeNewline(data);
+
// write maxLength
SimpleTextUtil.write(data, MAXLENGTH);
SimpleTextUtil.write(data, Integer.toString(maxLength), scratch);
@@ -245,34 +244,27 @@ public class SimpleTextSimpleDocValuesFo
SimpleTextUtil.write(data, sb.toString(), scratch);
SimpleTextUtil.writeNewline(data);
final DecimalFormat encoder = new DecimalFormat(sb.toString(), new DecimalFormatSymbols(Locale.ROOT));
-
- return new BinaryDocValuesConsumer() {
- int numDocsWritten = 0;
-
- @Override
- public void add(BytesRef value) throws IOException {
- // write length
- SimpleTextUtil.write(data, LENGTH);
- SimpleTextUtil.write(data, encoder.format(value.length), scratch);
- SimpleTextUtil.writeNewline(data);
-
- // write bytes -- don't use SimpleText.write
- // because it escapes:
- data.writeBytes(value.bytes, value.offset, value.length);
- // pad to fit
- for (int i = value.length; i < maxLength; i++) {
- data.writeByte((byte)' ');
- }
- SimpleTextUtil.writeNewline(data);
- numDocsWritten++;
+ int numDocsWritten = 0;
+ for(BytesRef value : values) {
+ // write length
+ SimpleTextUtil.write(data, LENGTH);
+ SimpleTextUtil.write(data, encoder.format(value.length), scratch);
+ SimpleTextUtil.writeNewline(data);
+
+ // write bytes -- don't use SimpleText.write
+ // because it escapes:
+ data.writeBytes(value.bytes, value.offset, value.length);
+
+ // pad to fit
+ for (int i = value.length; i < maxLength; i++) {
+ data.writeByte((byte)' ');
}
+ SimpleTextUtil.writeNewline(data);
+ numDocsWritten++;
+ }
- @Override
- public void finish() throws IOException {
- assert numDocs == numDocsWritten;
- }
- };
+ assert numDocs == numDocsWritten;
}
@Override
@@ -446,9 +438,6 @@ public class SimpleTextSimpleDocValuesFo
data.seek(data.getFilePointer() + (1+field.pattern.length()) * maxDoc);
} else if (DocValues.isBytes(dvType)) {
readLine();
- assert startsWith(FIXEDLENGTH);
- field.fixedLength = Boolean.parseBoolean(stripPrefix(FIXEDLENGTH));
- readLine();
assert startsWith(MAXLENGTH);
field.maxLength = Integer.parseInt(stripPrefix(MAXLENGTH));
readLine();
@@ -574,16 +563,6 @@ public class SimpleTextSimpleDocValuesFo
public int size() {
return maxDoc;
}
-
- @Override
- public boolean isFixedLength() {
- return field.fixedLength;
- }
-
- @Override
- public int maxLength() {
- return field.maxLength;
- }
};
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/SimpleDVConsumer.java Mon Jan 7 22:29:33 2013
@@ -40,7 +40,8 @@ public abstract class SimpleDVConsumer i
// nocommit should we pass SegmentWriteState...?
public abstract void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException;
- public abstract BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, int maxLength) throws IOException;
+ public abstract void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException;
+
// nocommit: figure out whats fair here.
public abstract SortedDocValuesConsumer addSortedField(FieldInfo field, int valueCount, boolean fixedLength, int maxLength) throws IOException;
@@ -114,34 +115,72 @@ public abstract class SimpleDVConsumer i
}
// dead simple impl: codec can optimize
- public void mergeBinaryField(FieldInfo fieldInfo, MergeState mergeState, List<BinaryDocValues> toMerge) throws IOException {
- // first compute fixedLength and maxLength of live ones to be merged.
- // nocommit: messy, and can be simplified by using docValues.maxLength/fixedLength in many cases.
- boolean fixedLength = true;
- int maxLength = -1;
- BytesRef bytes = new BytesRef();
- for (int readerIDX=0;readerIDX<toMerge.size();readerIDX++) {
- AtomicReader reader = mergeState.readers.get(readerIDX);
- int maxDoc = reader.maxDoc();
- Bits liveDocs = reader.getLiveDocs();
- BinaryDocValues values = toMerge.get(readerIDX);
- for (int i = 0; i < maxDoc; i++) {
- if (liveDocs == null || liveDocs.get(i)) {
- values.get(i, bytes);
- if (maxLength == -1) {
- maxLength = bytes.length;
- } else {
- fixedLength &= bytes.length == maxLength;
- maxLength = Math.max(bytes.length, maxLength);
- }
- }
- mergeState.checkAbort.work(300);
- }
- }
- // now we can merge
- assert maxLength >= 0; // could this happen (nothing to do?)
- BinaryDocValuesConsumer field = addBinaryField(fieldInfo, fixedLength, maxLength);
- field.merge(mergeState, toMerge);
+ public void mergeBinaryField(FieldInfo fieldInfo, final MergeState mergeState, final List<BinaryDocValues> toMerge) throws IOException {
+
+ addBinaryField(fieldInfo,
+ new Iterable<BytesRef>() {
+ @Override
+ public Iterator<BytesRef> iterator() {
+ return new Iterator<BytesRef>() {
+ int readerUpto = -1;
+ int docIDUpto;
+ BytesRef nextValue = new BytesRef();
+ AtomicReader currentReader;
+ BinaryDocValues currentValues;
+ Bits currentLiveDocs;
+ boolean nextIsSet;
+
+ @Override
+ public boolean hasNext() {
+ return nextIsSet || setNext();
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BytesRef next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ assert nextIsSet;
+ nextIsSet = false;
+ // nocommit make a mutable number
+ return nextValue;
+ }
+
+ private boolean setNext() {
+ while (true) {
+ if (readerUpto == toMerge.size()) {
+ return false;
+ }
+
+ if (currentReader == null || docIDUpto == currentReader.maxDoc()) {
+ readerUpto++;
+ if (readerUpto < toMerge.size()) {
+ currentReader = mergeState.readers.get(readerUpto);
+ currentValues = toMerge.get(readerUpto);
+ currentLiveDocs = currentReader.getLiveDocs();
+ }
+ docIDUpto = 0;
+ continue;
+ }
+
+ if (currentLiveDocs == null || currentLiveDocs.get(docIDUpto)) {
+ nextIsSet = true;
+ currentValues.get(docIDUpto, nextValue);
+ docIDUpto++;
+ return true;
+ }
+
+ docIDUpto++;
+ }
+ }
+ };
+ }
+ });
}
public void mergeSortedField(FieldInfo fieldInfo, MergeState mergeState, List<SortedDocValues> toMerge) throws IOException {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SimpleDocValuesFormat.java Mon Jan 7 22:29:33 2013
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
@@ -108,23 +107,23 @@ public class Lucene41SimpleDocValuesForm
}
@Override
- public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, int maxLength) throws IOException {
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
meta.writeVInt(field.number);
- meta.writeByte(fixedLength ? (byte)1 : 0);
- meta.writeVInt(maxLength);
+ // nocommit handle var length too!!
+ int length = -1;
+ for(BytesRef v : values) {
+ if (length == -1) {
+ length = v.length;
+ } else if (length != v.length) {
+ throw new UnsupportedOperationException();
+ }
+ }
+ // nocommit don't hardwire fixedLength to 1:
+ meta.writeByte((byte) 1);
+ meta.writeVInt(length);
meta.writeLong(data.getFilePointer());
- if (fixedLength) {
- return new BinaryDocValuesConsumer() {
- @Override
- public void add(BytesRef value) throws IOException {
- data.writeBytes(value.bytes, value.offset, value.length);
- }
-
- @Override
- public void finish() throws IOException {}
- };
- } else {
- throw new UnsupportedOperationException();
+ for(BytesRef value : values) {
+ data.writeBytes(value.bytes, value.offset, value.length);
}
}
@@ -267,16 +266,6 @@ public class Lucene41SimpleDocValuesForm
}
@Override
- public boolean isFixedLength() {
- return entry.fixedLength;
- }
-
- @Override
- public int maxLength() {
- return entry.maxLength;
- }
-
- @Override
public int size() {
return maxDoc;
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesConsumer.java Mon Jan 7 22:29:33 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.codecs.lucene4
*/
import java.io.IOException;
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SortedDocValuesConsumer;
import org.apache.lucene.index.FieldInfo;
@@ -29,6 +28,7 @@ import org.apache.lucene.store.CompoundF
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
public class Lucene41DocValuesConsumer extends SimpleDVConsumer {
@@ -86,25 +86,8 @@ public class Lucene41DocValuesConsumer e
}
@Override
- public BinaryDocValuesConsumer addBinaryField(FieldInfo field,
- boolean fixedLength, int maxLength) throws IOException {
- String nameData = getDocValuesFileName(info, field, DATA_EXTENSION);
- String idxOut = getDocValuesFileName(info, field, INDEX_EXTENSION);
- boolean success = false;
- IndexOutput dataOut = null;
- IndexOutput indexOut = null;
- try {
- dataOut = getDirectory().createOutput(nameData, context);
- indexOut = getDirectory().createOutput(idxOut, context);
- Lucene41BinaryDocValuesConsumer consumer = new Lucene41BinaryDocValuesConsumer(
- dataOut, indexOut, fixedLength, maxLength);
- success = true;
- return consumer;
- } finally {
- if (!success) {
- IOUtils.close(dataOut, indexOut);
- }
- }
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) {
+ // ncommit
}
// nocommit: bogus to put segmentName in here. think about copySegmentAsIs!!!!!!
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41DocValuesProducer.java Mon Jan 7 22:29:33 2013
@@ -72,11 +72,8 @@ public class Lucene41DocValuesProducer e
@Override
public BinaryDocValues getBinary(FieldInfo field) throws IOException {
- if (DocValues.isBytes(field.getDocValuesType()) || DocValues.isSortedBytes(field.getDocValuesType())) {
- return new Lucene41BinaryDocValues.Factory(this.cfs, this.info, field, context).getDirect();
- } else {
- return null;
- }
+ // nocommit
+ return null;
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/lucene41/values/Lucene41SortedDocValues.java Mon Jan 7 22:29:33 2013
@@ -146,7 +146,8 @@ public class Lucene41SortedDocValues ext
this.valueCount = datIn.readInt();
this.baseOffset = datIn.getFilePointer();
- if (size == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
+ //if (size == Lucene41BinaryDocValuesConsumer.VALUE_SIZE_VAR) {
+ if (size == -1) {
offsetIn = dir.openInput(Lucene41DocValuesConsumer
.getDocValuesFileName(segmentInfo, field,
Lucene41DocValuesConsumer.OFFSET_EXTENSION), context);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldDocValuesFormat.java Mon Jan 7 22:29:33 2013
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.ServiceLoader; // javadocs
import java.util.TreeMap;
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.codecs.SimpleDVProducer;
@@ -37,6 +36,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SortedDocValues;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
/**
@@ -106,8 +106,8 @@ public abstract class PerFieldDocValuesF
}
@Override
- public BinaryDocValuesConsumer addBinaryField(FieldInfo field, boolean fixedLength, int maxLength) throws IOException {
- return getInstance(field).addBinaryField(field, fixedLength, maxLength);
+ public void addBinaryField(FieldInfo field, Iterable<BytesRef> values) throws IOException {
+ getInstance(field).addBinaryField(field, values);
}
@Override
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValues.java Mon Jan 7 22:29:33 2013
@@ -27,9 +27,6 @@ public abstract class BinaryDocValues {
public abstract int size();
- public abstract boolean isFixedLength();
- public abstract int maxLength();
-
public static class EMPTY extends BinaryDocValues {
private final int size;
@@ -46,15 +43,5 @@ public abstract class BinaryDocValues {
public int size() {
return size;
}
-
- @Override
- public boolean isFixedLength() {
- return true;
- }
-
- @Override
- public int maxLength() {
- return 0;
- }
};
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BytesDVWriter.java Mon Jan 7 22:29:33 2013
@@ -18,8 +18,8 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Iterator;
-import org.apache.lucene.codecs.BinaryDocValuesConsumer;
import org.apache.lucene.codecs.SimpleDVConsumer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefArray;
@@ -37,16 +37,9 @@ class BytesDVWriter extends DocValuesWri
private int addedValues = 0;
private final BytesRef emptyBytesRef = new BytesRef();
- // -2 means not set yet; -1 means length isn't fixed;
- // -otherwise it's the fixed length seen so far:
- int fixedLength = -2;
- int maxLength;
- int totalSize;
-
public BytesDVWriter(FieldInfo fieldInfo, Counter counter) {
this.fieldInfo = fieldInfo;
this.bytesRefArray = new BytesRefArray(counter);
- this.totalSize = 0;
}
public void addValue(int docID, BytesRef value) {
@@ -57,54 +50,60 @@ class BytesDVWriter extends DocValuesWri
// nocommit improve message
throw new IllegalArgumentException("null binaryValue not allowed (field=" + fieldInfo.name + ")");
}
- mergeLength(value.length);
// Fill in any holes:
while(addedValues < docID) {
addedValues++;
bytesRefArray.append(emptyBytesRef);
- mergeLength(0);
}
addedValues++;
bytesRefArray.append(value);
}
- private void mergeLength(int length) {
- if (fixedLength == -2) {
- fixedLength = length;
- } else if (fixedLength != length) {
- fixedLength = -1;
- }
- maxLength = Math.max(maxLength, length);
- totalSize += length;
- }
-
@Override
public void finish(int maxDoc) {
- if (addedValues < maxDoc) {
- mergeLength(0);
- }
}
@Override
public void flush(SegmentWriteState state, SimpleDVConsumer dvConsumer) throws IOException {
- BinaryDocValuesConsumer consumer = dvConsumer.addBinaryField(fieldInfo,
- fixedLength >= 0,
- maxLength);
- final int bufferedDocCount = addedValues;
- BytesRef value = new BytesRef();
- for(int docID=0;docID<bufferedDocCount;docID++) {
- bytesRefArray.get(value, docID);
- consumer.add(value);
- }
final int maxDoc = state.segmentInfo.getDocCount();
- value.length = 0;
- for(int docID=bufferedDocCount;docID<maxDoc;docID++) {
- consumer.add(value);
- }
- consumer.finish();
+
+ dvConsumer.addBinaryField(fieldInfo,
+ new Iterable<BytesRef>() {
+
+ @Override
+ public Iterator<BytesRef> iterator() {
+ return new Iterator<BytesRef>() {
+ BytesRef value = new BytesRef();
+ int upto;
+
+ @Override
+ public boolean hasNext() {
+ return upto < maxDoc;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public BytesRef next() {
+ // nocommit make
+ // mutable Number:
+ if (upto < bytesRefArray.size()) {
+ bytesRefArray.get(value, upto);
+ } else {
+ value.length = 0;
+ }
+ upto++;
+ return value;
+ }
+ };
+ }
+ });
+
reset();
- //System.out.println("FLUSH");
}
public void abort() {
@@ -113,7 +112,5 @@ class BytesDVWriter extends DocValuesWri
private void reset() {
bytesRefArray.clear();
- fixedLength = -2;
- maxLength = 0;
}
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Mon Jan 7 22:29:33 2013
@@ -1378,25 +1378,10 @@ public class CheckIndex {
if (dv == null) {
return;
}
- final boolean fixed = dv.isFixedLength();
- final int maxLength = dv.maxLength();
- boolean fixed2 = true;
- int maxLength2 = -1;
+ // nocommit what else to check ...
BytesRef scratch = new BytesRef();
for (int i = 0; i < reader.maxDoc(); i++) {
dv.get(i, scratch);
- if (maxLength2 == -1) {
- maxLength2 = scratch.length;
- } else {
- fixed2 &= scratch.length == maxLength2;
- maxLength2 = Math.max(maxLength2, scratch.length);
- }
- }
- if (fixed != fixed2) {
- throw new RuntimeException("dv for field: " + fieldName + " reports fixed=" + fixed + " but this is not the case!");
- }
- if (maxLength != maxLength2) {
- throw new RuntimeException("dv for field: " + fieldName + " reports maxLength=" + maxLength + " but this is not the case: " + maxLength2);
}
}
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java Mon Jan 7 22:29:33 2013
@@ -30,6 +30,10 @@ public abstract class SortedDocValues ex
public abstract int getValueCount();
+ public abstract boolean isFixedLength();
+
+ public abstract int maxLength();
+
@Override
public void get(int docID, BytesRef result) {
int ord = getOrd(docID);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Mon Jan 7 22:29:33 2013
@@ -1290,16 +1290,6 @@ class FieldCacheImpl implements FieldCac
bytes.fill(ret, pointer);
}
}
-
- @Override
- public boolean isFixedLength() {
- return isFixedLength;
- }
-
- @Override
- public int maxLength() {
- return maxLength;
- }
}
// TODO: this if DocTermsIndex was already created, we
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/TestDemoDocValue.java Mon Jan 7 22:29:33 2013
@@ -480,7 +480,6 @@ public class TestDemoDocValue extends Lu
IndexReader ireader = DirectoryReader.open(directory); // read-only=true
assert ireader.leaves().size() == 1;
BinaryDocValues dv = ireader.leaves().get(0).reader().getBinaryDocValues("dv");
- assertFalse(dv.isFixedLength());
BytesRef scratch = new BytesRef();
for(int i=0;i<2;i++) {
StoredDocument doc2 = ireader.leaves().get(0).reader().document(i);
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Mon Jan 7 22:29:33 2013
@@ -636,8 +636,6 @@ public class TestDuelingCodecs extends L
BinaryDocValues rightValues = MultiSimpleDocValues.simpleBinaryValues(rightReader, field);
if (leftValues != null && rightValues != null) {
assertEquals(leftValues.size(), rightValues.size());
- assertEquals(leftValues.maxLength(), rightValues.maxLength());
- assertEquals(leftValues.isFixedLength(), rightValues.isFixedLength());
BytesRef scratchLeft = new BytesRef();
BytesRef scratchRight = new BytesRef();
for(int docID=0;docID<leftValues.size();docID++) {
Modified: lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/test/org/apache/lucene/search/TestFieldCache.java Mon Jan 7 22:29:33 2013
@@ -428,46 +428,4 @@ public class TestFieldCache extends Luce
}
assertFalse(failed.get());
}
-
- public void testMaxFixedLength() throws Exception {
- Directory d = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random(), d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
- int fixedLength = random().nextBoolean() ? -1 : _TestUtil.nextInt(random(), 1, 10);
- int numTerms = atLeast(10);
- int actualFixedLength = -2;
- int actualMaxLength = -1;
- for(int termIDX=0;termIDX<numTerms;termIDX++) {
- Document doc = new Document();
- String termString;
- if (fixedLength == -1) {
- termString = _TestUtil.randomRealisticUnicodeString(random());
- } else {
- termString = _TestUtil.randomRealisticUnicodeString(random(), fixedLength, fixedLength);
- }
- BytesRef term = new BytesRef(termString);
- actualMaxLength = Math.max(actualMaxLength, term.length);
- if (actualFixedLength == -2) {
- actualFixedLength = term.length;
- } else if (actualFixedLength != term.length) {
- actualFixedLength = -1;
- }
- doc.add(newField("term", termString, StringField.TYPE_NOT_STORED));
- w.addDocument(doc);
- }
- w.forceMerge(1);
- IndexReader r = w.getReader();
- w.close();
- AtomicReader subR = r.leaves().get(0).reader();
- BinaryDocValues values = FieldCache.DEFAULT.getTerms(subR, "term");
- assertEquals(actualFixedLength >= 0, values.isFixedLength());
- assertEquals(actualMaxLength, values.maxLength());
-
- SortedDocValues sortedValues = FieldCache.DEFAULT.getTermsIndex(subR, "term");
- assertEquals(actualFixedLength >= 0, sortedValues.isFixedLength());
- assertEquals(actualMaxLength, sortedValues.maxLength());
-
- // No field cache insanity because close purges FC entries:
- r.close();
- d.close();
- }
}
Modified: lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java?rev=1430064&r1=1430063&r2=1430064&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java (original)
+++ lucene/dev/branches/lucene4547/lucene/test-framework/src/java/org/apache/lucene/index/MultiSimpleDocValues.java Mon Jan 7 22:29:33 2013
@@ -109,7 +109,6 @@ public class MultiSimpleDocValues {
public static BinaryDocValues simpleBinaryValues(final IndexReader r, final String field) throws IOException {
final List<AtomicReaderContext> leaves = r.leaves();
boolean anyReal = false;
- int maxLength = -1;
for(AtomicReaderContext ctx : leaves) {
BinaryDocValues values = ctx.reader().getBinaryDocValues(field);
@@ -119,14 +118,11 @@ public class MultiSimpleDocValues {
} else {
anyReal = true;
}
-
- maxLength = Math.max(maxLength, values.maxLength());
}
if (!anyReal) {
return null;
} else {
- final int finalMaxLength = maxLength;
return new BinaryDocValues() {
@Override
@@ -150,17 +146,6 @@ public class MultiSimpleDocValues {
public int size() {
return r.maxDoc();
}
-
- @Override
- public boolean isFixedLength() {
- // Harmless lie?
- return false;
- }
-
- @Override
- public int maxLength() {
- return finalMaxLength;
- }
};
}
}