You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/01/06 03:26:37 UTC
svn commit: r1227937 [2/3] - in /lucene/dev/branches/lucene3305: ./
dev-tools/idea/.idea/libraries/ dev-tools/idea/modules/benchmark/
dev-tools/maven/modules/benchmark/ lucene/ lucene/contrib/
lucene/contrib/memory/src/java/org/apache/lucene/index/memo...
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/IndexReader.java Fri Jan 6 02:26:34 2012
@@ -18,8 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.Closeable;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
@@ -31,12 +29,11 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
+import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.*;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.ReaderUtil; // for javadocs
/** IndexReader is an abstract class, providing an interface for accessing an
@@ -730,17 +727,9 @@ public abstract class IndexReader implem
// backward compatible implementation.
// SegmentReader has an efficient implementation.
ensureOpen();
- return norms(field) != null;
+ return normValues(field) != null;
}
- /** Returns the byte-encoded normalization factor for the named field of
- * every document. This is used by the search code to score documents.
- * Returns null if norms were not indexed for this field.
- *
- * @see org.apache.lucene.document.Field#setBoost(float)
- */
- public abstract byte[] norms(String field) throws IOException;
-
/**
* Returns {@link Fields} for this reader.
* This method may return null if the reader has no
@@ -942,101 +931,6 @@ public abstract class IndexReader implem
throw new UnsupportedOperationException("This reader does not support this method.");
}
- /**
- * Prints the filename and size of each file within a given compound file.
- * Add the -extract flag to extract files to the current working directory.
- * In order to make the extracted version of the index work, you have to copy
- * the segments file from the compound index into the directory where the extracted files are stored.
- * @param args Usage: org.apache.lucene.index.IndexReader [-extract] <cfsfile>
- */
- public static void main(String [] args) {
- String filename = null;
- boolean extract = false;
- String dirImpl = null;
-
- int j = 0;
- while(j < args.length) {
- String arg = args[j];
- if ("-extract".equals(arg)) {
- extract = true;
- } else if ("-dir-impl".equals(arg)) {
- if (j == args.length - 1) {
- System.out.println("ERROR: missing value for -dir-impl option");
- System.exit(1);
- }
- j++;
- dirImpl = args[j];
- } else if (filename == null) {
- filename = arg;
- }
- j++;
- }
-
- if (filename == null) {
- System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] [-dir-impl X] <cfsfile>");
- return;
- }
-
- Directory dir = null;
- CompoundFileDirectory cfr = null;
- IOContext context = IOContext.READ;
-
- try {
- File file = new File(filename);
- String dirname = file.getAbsoluteFile().getParent();
- filename = file.getName();
- if (dirImpl == null) {
- dir = FSDirectory.open(new File(dirname));
- } else {
- dir = CommandLineUtil.newFSDirectory(dirImpl, new File(dirname));
- }
-
- cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false);
-
- String [] files = cfr.listAll();
- ArrayUtil.mergeSort(files); // sort the array of filename so that the output is more readable
-
- for (int i = 0; i < files.length; ++i) {
- long len = cfr.fileLength(files[i]);
-
- if (extract) {
- System.out.println("extract " + files[i] + " with " + len + " bytes to local directory...");
- IndexInput ii = cfr.openInput(files[i], context);
-
- FileOutputStream f = new FileOutputStream(files[i]);
-
- // read and write with a small buffer, which is more effective than reading byte by byte
- byte[] buffer = new byte[1024];
- int chunk = buffer.length;
- while(len > 0) {
- final int bufLen = (int) Math.min(chunk, len);
- ii.readBytes(buffer, 0, bufLen);
- f.write(buffer, 0, bufLen);
- len -= bufLen;
- }
-
- f.close();
- ii.close();
- }
- else
- System.out.println(files[i] + ": " + len + " bytes");
- }
- } catch (IOException ioe) {
- ioe.printStackTrace();
- }
- finally {
- try {
- if (dir != null)
- dir.close();
- if (cfr != null)
- cfr.close();
- }
- catch (IOException ioe) {
- ioe.printStackTrace();
- }
- }
- }
-
/** Returns all commit points that exist in the Directory.
* Normally, because the default is {@link
* KeepOnlyLastCommitDeletionPolicy}, there would be only
@@ -1155,6 +1049,8 @@ public abstract class IndexReader implem
* using {@link ReaderUtil#gatherSubReaders} and iterate
* through them yourself. */
public abstract DocValues docValues(String field) throws IOException;
+
+ public abstract DocValues normValues(String field) throws IOException;
private volatile Fields fields;
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java Fri Jan 6 02:26:34 2012
@@ -1,5 +1,7 @@
package org.apache.lucene.index;
+import java.io.IOException;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -18,6 +20,6 @@ package org.apache.lucene.index;
*/
abstract class InvertedDocEndConsumerPerField {
- abstract void finish();
+ abstract void finish() throws IOException;
abstract void abort();
}
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/MultiDocValues.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/MultiDocValues.java Fri Jan 6 02:26:34 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import java.io.IOException;
+import java.lang.reflect.Array;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -33,6 +34,13 @@ import org.apache.lucene.util.ReaderUtil
* @lucene.internal
*/
public class MultiDocValues extends DocValues {
+
+ private static DocValuesPuller DEFAULT_PULLER = new DocValuesPuller();
+ private static final DocValuesPuller NORMS_PULLER = new DocValuesPuller() {
+ public DocValues pull(IndexReader reader, String field) throws IOException {
+ return reader.normValues(field);
+ }
+ };
public static class DocValuesSlice {
public final static DocValuesSlice[] EMPTY_ARRAY = new DocValuesSlice[0];
@@ -46,6 +54,12 @@ public class MultiDocValues extends DocV
this.length = length;
}
}
+
+ private static class DocValuesPuller {
+ public DocValues pull(IndexReader reader, String field) throws IOException {
+ return reader.docValues(field);
+ }
+ }
private DocValuesSlice[] slices;
private int[] starts;
@@ -58,7 +72,6 @@ public class MultiDocValues extends DocV
this.type = promotedType.type();
this.valueSize = promotedType.getValueSize();
}
-
/**
* Returns a single {@link DocValues} instance for this field, merging
* their values on the fly.
@@ -68,15 +81,32 @@ public class MultiDocValues extends DocV
* sub-readers (using {@link Gather}) and iterate through them yourself.
*/
public static DocValues getDocValues(IndexReader r, final String field) throws IOException {
+ return getDocValues(r, field, DEFAULT_PULLER);
+ }
+
+ /**
+ * Returns a single {@link DocValues} instance for this norms field, merging
+ * their values on the fly.
+ *
+ * <p>
+ * <b>NOTE</b>: this is a slow way to access DocValues. It's better to get the
+ * sub-readers (using {@link Gather}) and iterate through them yourself.
+ */
+ public static DocValues getNormDocValues(IndexReader r, final String field) throws IOException {
+ return getDocValues(r, field, NORMS_PULLER);
+ }
+
+
+ private static DocValues getDocValues(IndexReader r, final String field, final DocValuesPuller puller) throws IOException {
final IndexReader[] subs = r.getSequentialSubReaders();
if (subs == null) {
// already an atomic reader
- return r.docValues(field);
+ return puller.pull(r, field);
} else if (subs.length == 0) {
// no fields
return null;
} else if (subs.length == 1) {
- return getDocValues(subs[0], field);
+ return getDocValues(subs[0], field, puller);
} else {
final List<DocValuesSlice> slices = new ArrayList<DocValuesSlice>();
@@ -89,7 +119,7 @@ public class MultiDocValues extends DocV
new ReaderUtil.Gather(r) {
@Override
protected void add(int base, IndexReader r) throws IOException {
- final DocValues d = r.docValues(field);
+ final DocValues d = puller.pull(r, field);
if (d != null) {
TypePromoter incoming = TypePromoter.create(d.type(), d.getValueSize());
promotedType[0] = promotedType[0].promote(incoming);
@@ -195,6 +225,7 @@ public class MultiDocValues extends DocV
private final int[] starts;
private final DocValuesSlice[] slices;
private boolean direct;
+ private Object cachedArray; // cached array if supported
public MultiSource(DocValuesSlice[] slices, int[] starts, boolean direct, Type type) {
super(type);
@@ -243,6 +274,77 @@ public class MultiDocValues extends DocV
final int doc = ensureSource(docID);
return current.getBytes(doc, bytesRef);
}
+
+ @Override
+ public boolean hasArray() {
+ boolean oneRealSource = false;
+ for (DocValuesSlice slice : slices) {
+ try {
+ Source source = slice.docValues.getSource();
+ if (source instanceof EmptySource) {
+ /*
+ * empty source marks a gap in the array skip if we encounter one
+ */
+ continue;
+ }
+ oneRealSource = true;
+ if (!source.hasArray()) {
+ return false;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("load failed", e);
+ }
+ }
+ return oneRealSource;
+ }
+
+ @Override
+ public Object getArray() {
+ if (!hasArray()) {
+ return null;
+ }
+ try {
+ Class<?> componentType = null;
+ Object[] arrays = new Object[slices.length];
+ int numDocs = 0;
+ for (int i = 0; i < slices.length; i++) {
+ DocValuesSlice slice = slices[i];
+ Source source = slice.docValues.getSource();
+ Object array = null;
+ if (!(source instanceof EmptySource)) {
+ // EmptySource is skipped - marks a gap in the array
+ array = source.getArray();
+ }
+ numDocs += slice.length;
+ if (array != null) {
+ if (componentType == null) {
+ componentType = array.getClass().getComponentType();
+ }
+ assert componentType == array.getClass().getComponentType();
+ }
+ arrays[i] = array;
+ }
+ assert componentType != null;
+ synchronized (this) {
+ if (cachedArray != null) {
+ return cachedArray;
+ }
+ final Object globalArray = Array.newInstance(componentType, numDocs);
+
+ for (int i = 0; i < slices.length; i++) {
+ DocValuesSlice slice = slices[i];
+ if (arrays[i] != null) {
+ assert slice.length == Array.getLength(arrays[i]);
+ System.arraycopy(arrays[i], 0, globalArray, slice.start,
+ slice.length);
+ }
+ }
+ return cachedArray = globalArray;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("load failed", e);
+ }
+ }
}
// TODO: this is dup of DocValues.getDefaultSource()?
@@ -269,7 +371,7 @@ public class MultiDocValues extends DocV
}
}
- private static class EmptyFixedSource extends Source {
+ private static class EmptyFixedSource extends EmptySource {
private final int valueSize;
public EmptyFixedSource(Type type, int valueSize) {
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumer.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumer.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumer.java Fri Jan 6 02:26:34 2012
@@ -21,8 +21,12 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Map;
+import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.NormsFormat;
-import org.apache.lucene.codecs.NormsWriter;
+import org.apache.lucene.codecs.PerDocConsumer;
+import org.apache.lucene.document.DocValuesField;
+import org.apache.lucene.index.DocValues.Type;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
// TODO FI: norms could actually be stored as doc store
@@ -33,14 +37,21 @@ import org.apache.lucene.util.IOUtils;
*/
final class NormsConsumer extends InvertedDocEndConsumer {
- final NormsFormat normsFormat;
+ private final NormsFormat normsFormat;
+ private PerDocConsumer consumer;
+ private final DocumentsWriterPerThread dwpt;
public NormsConsumer(DocumentsWriterPerThread dwpt) {
normsFormat = dwpt.codec.normsFormat();
+ this.dwpt = dwpt;
}
@Override
- public void abort() {}
+ public void abort(){
+ if (consumer != null) {
+ consumer.abort();
+ }
+ }
// We only write the _X.nrm file at flush
void files(Collection<String> files) {}
@@ -49,50 +60,39 @@ final class NormsConsumer extends Invert
* not disabled */
@Override
public void flush(Map<FieldInfo,InvertedDocEndConsumerPerField> fieldsToFlush, SegmentWriteState state) throws IOException {
- if (!state.fieldInfos.hasNorms()) {
- return;
- }
-
- NormsWriter normsOut = null;
boolean success = false;
+ boolean anythingFlushed = false;
try {
- normsOut = normsFormat.normsWriter(state);
-
- for (FieldInfo fi : state.fieldInfos) {
- final NormsConsumerPerField toWrite = (NormsConsumerPerField) fieldsToFlush.get(fi);
- int upto = 0;
- // we must check the final value of omitNorms for the fieldinfo, it could have
- // changed for this field since the first time we added it.
- if (!fi.omitNorms && toWrite != null && toWrite.upto > 0) {
- normsOut.startField(fi);
- int docID = 0;
- for (; docID < state.numDocs; docID++) {
- if (upto < toWrite.upto && toWrite.docIDs[upto] == docID) {
- normsOut.writeNorm(toWrite.norms[upto]);
- upto++;
- } else {
- normsOut.writeNorm((byte) 0);
+ if (state.fieldInfos.hasNorms()) {
+ for (FieldInfo fi : state.fieldInfos) {
+ final NormsConsumerPerField toWrite = (NormsConsumerPerField) fieldsToFlush.get(fi);
+ // we must check the final value of omitNorms for the fieldinfo, it could have
+ // changed for this field since the first time we added it.
+ if (!fi.omitNorms) {
+ if (toWrite != null && toWrite.initialized()) {
+ anythingFlushed = true;
+ toWrite.flush(state.numDocs);
+ } else if (fi.isIndexed) {
+ anythingFlushed = true;
+ final DocValuesConsumer valuesConsumer = newConsumer(new PerDocWriteState(state), fi);
+ final DocValuesField value = new DocValuesField("");
+ value.setBytes(new BytesRef(new byte[] {0x00}), Type.BYTES_FIXED_STRAIGHT);
+ valuesConsumer.add(state.numDocs-1, value);
+ valuesConsumer.finish(state.numDocs);
}
}
-
- // we should have consumed every norm
- assert upto == toWrite.upto;
-
- toWrite.reset();
- } else if (fi.isIndexed && !fi.omitNorms) {
- // Fill entire field with default norm:
- normsOut.startField(fi);
- for(;upto<state.numDocs;upto++)
- normsOut.writeNorm((byte) 0);
}
- }
- normsOut.finish(state.numDocs);
+ }
+
success = true;
+ if (!anythingFlushed && consumer != null) {
+ consumer.abort();
+ }
} finally {
if (success) {
- IOUtils.close(normsOut);
+ IOUtils.close(consumer);
} else {
- IOUtils.closeWhileHandlingException(normsOut);
+ IOUtils.closeWhileHandlingException(consumer);
}
}
}
@@ -106,6 +106,16 @@ final class NormsConsumer extends Invert
@Override
InvertedDocEndConsumerPerField addField(DocInverterPerField docInverterPerField,
FieldInfo fieldInfo) {
- return new NormsConsumerPerField(docInverterPerField, fieldInfo);
+ return new NormsConsumerPerField(docInverterPerField, fieldInfo, this);
+ }
+
+ DocValuesConsumer newConsumer(PerDocWriteState perDocWriteState,
+ FieldInfo fieldInfo) throws IOException {
+ if (consumer == null) {
+ consumer = normsFormat.docsConsumer(perDocWriteState);
+ }
+ DocValuesConsumer addValuesField = consumer.addValuesField(
+ Type.BYTES_FIXED_STRAIGHT, fieldInfo);
+ return addValuesField;
}
}
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/NormsConsumerPerField.java Fri Jan 6 02:26:34 2012
@@ -1,5 +1,4 @@
package org.apache.lucene.index;
-
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -16,65 +15,69 @@ package org.apache.lucene.index;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import java.io.IOException;
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.document.DocValuesField;
+import org.apache.lucene.index.DocValues.Type;
import org.apache.lucene.search.similarities.Similarity;
-import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
-/** Taps into DocInverter, as an InvertedDocEndConsumer,
- * which is called at the end of inverting each field. We
- * just look at the length for the field (docState.length)
- * and record the norm. */
-
-final class NormsConsumerPerField extends InvertedDocEndConsumerPerField implements Comparable<NormsConsumerPerField> {
-
- final FieldInfo fieldInfo;
- final DocumentsWriterPerThread.DocState docState;
- final Similarity similarity;
+public class NormsConsumerPerField extends InvertedDocEndConsumerPerField implements Comparable<NormsConsumerPerField> {
+ private final FieldInfo fieldInfo;
+ private final DocumentsWriterPerThread.DocState docState;
+ private final Similarity similarity;
+ private final FieldInvertState fieldState;
+ private DocValuesConsumer consumer;
+ private final DocValuesField value = new DocValuesField("");
+ private final BytesRef spare = new BytesRef(1);
+ private final NormsConsumer parent;
- // Holds all docID/norm pairs we've seen
- int[] docIDs = new int[1];
- byte[] norms = new byte[1];
- int upto;
-
- final FieldInvertState fieldState;
-
- public void reset() {
- // Shrink back if we are overallocated now:
- docIDs = ArrayUtil.shrink(docIDs, upto);
- norms = ArrayUtil.shrink(norms, upto);
- upto = 0;
- }
-
- public NormsConsumerPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) {
+ public NormsConsumerPerField(final DocInverterPerField docInverterPerField, final FieldInfo fieldInfo, NormsConsumer parent) {
this.fieldInfo = fieldInfo;
+ this.parent = parent;
docState = docInverterPerField.docState;
fieldState = docInverterPerField.fieldState;
similarity = docState.similarityProvider.get(fieldInfo.name);
- }
+ spare.length = 1;
+ spare.offset = 0;
- @Override
- void abort() {
- upto = 0;
}
-
+ @Override
public int compareTo(NormsConsumerPerField other) {
return fieldInfo.name.compareTo(other.fieldInfo.name);
}
-
+
@Override
- void finish() {
+ void finish() throws IOException {
if (fieldInfo.isIndexed && !fieldInfo.omitNorms) {
- if (docIDs.length <= upto) {
- assert docIDs.length == upto;
- docIDs = ArrayUtil.grow(docIDs, 1+upto);
- }
- if (norms.length <= upto) {
- assert norms.length == upto;
- norms = ArrayUtil.grow(norms, 1+upto);
- }
- norms[upto] = similarity.computeNorm(fieldState);
- docIDs[upto] = docState.docID;
- upto++;
+ DocValuesConsumer consumer = getConsumer();
+ spare.bytes[0] = similarity.computeNorm(fieldState);
+ value.setBytes(spare, Type.BYTES_FIXED_STRAIGHT);
+ consumer.add(docState.docID, value);
+
+ }
+ }
+
+ void flush(int docCount) throws IOException {
+ assert initialized();
+ consumer.finish(docCount);
+ }
+
+ private DocValuesConsumer getConsumer() throws IOException {
+ if (consumer == null) {
+ consumer = parent.newConsumer(docState.docWriter.newPerDocWriteState(""), fieldInfo);
}
+ return consumer;
+ }
+
+ boolean initialized() {
+ return consumer != null;
}
+
+ @Override
+ void abort() {
+ //
+ }
+
}
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/ParallelReader.java Fri Jan 6 02:26:34 2012
@@ -48,7 +48,7 @@ public class ParallelReader extends Inde
private SortedMap<String,IndexReader> fieldToReader = new TreeMap<String,IndexReader>();
private Map<IndexReader,Collection<String>> readerToFields = new HashMap<IndexReader,Collection<String>>();
private List<IndexReader> storedFieldReaders = new ArrayList<IndexReader>();
- private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
+ private Map<String, DocValues> normsCache = new HashMap<String,DocValues>();
private final ReaderContext topLevelReaderContext = new AtomicReaderContext(this);
private int maxDoc;
private int numDocs;
@@ -337,27 +337,6 @@ public class ParallelReader extends Inde
}
@Override
- public synchronized byte[] norms(String field) throws IOException {
- ensureOpen();
- IndexReader reader = fieldToReader.get(field);
-
- if (reader==null)
- return null;
-
- byte[] bytes = normsCache.get(field);
- if (bytes != null)
- return bytes;
- if (!hasNorms(field))
- return null;
- if (normsCache.containsKey(field)) // cached omitNorms, not missing key
- return null;
-
- bytes = MultiNorms.norms(reader, field);
- normsCache.put(field, bytes);
- return bytes;
- }
-
- @Override
public int docFreq(String field, BytesRef term) throws IOException {
ensureOpen();
IndexReader reader = fieldToReader.get(field);
@@ -427,4 +406,16 @@ public class ParallelReader extends Inde
IndexReader reader = fieldToReader.get(field);
return reader == null ? null : MultiDocValues.getDocValues(reader, field);
}
+
+ // TODO: I suspect this is completely untested!!!!!
+ @Override
+ public synchronized DocValues normValues(String field) throws IOException {
+ DocValues values = normsCache.get(field);
+ if (values == null) {
+ IndexReader reader = fieldToReader.get(field);
+ values = reader == null ? null : MultiDocValues.getNormDocValues(reader, field);
+ normsCache.put(field, values);
+ }
+ return values;
+ }
}
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Fri Jan 6 02:26:34 2012
@@ -25,7 +25,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.NormsReader;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.StoredFieldsReader;
@@ -54,7 +53,7 @@ final class SegmentCoreReaders {
final FieldsProducer fields;
final PerDocProducer perDocProducer;
- final NormsReader norms;
+ final PerDocProducer norms;
final Directory dir;
final Directory cfsDir;
@@ -120,7 +119,7 @@ final class SegmentCoreReaders {
// ask codec for its Norms:
// TODO: since we don't write any norms file if there are no norms,
// kinda jaky to assume the codec handles the case of no norms file at all gracefully?!
- norms = codec.normsFormat().normsReader(cfsDir, si, fieldInfos, context, dir);
+ norms = codec.normsFormat().docsProducer(segmentReadState, dir);
perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState);
final Directory storeDir;
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Fri Jan 6 02:26:34 2012
@@ -841,8 +841,6 @@ public final class SegmentInfos implemen
genOutput.close();
dir.sync(Collections.singleton(IndexFileNames.SEGMENTS_GEN));
}
- } catch (ThreadInterruptedException t) {
- throw t;
} catch (Throwable t) {
// It's OK if we fail to write this file since it's
// used only as one of the retry fallbacks.
@@ -852,6 +850,9 @@ public final class SegmentInfos implemen
// Ignore; this file is only used in a retry
// fallback on init.
}
+ if (t instanceof ThreadInterruptedException) {
+ throw (ThreadInterruptedException) t;
+ }
}
}
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Jan 6 02:26:34 2012
@@ -27,7 +27,6 @@ import java.util.Map;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FieldInfosWriter;
import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.NormsWriter;
import org.apache.lucene.codecs.PerDocConsumer;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.TermVectorsWriter;
@@ -125,8 +124,7 @@ final class SegmentMerger {
mergePerDoc(segmentWriteState);
if (mergeState.fieldInfos.hasNorms()) {
- int numMerged = mergeNorms(segmentWriteState);
- assert numMerged == mergeState.mergedDocCount;
+ mergeNorms(segmentWriteState);
}
if (mergeState.fieldInfos.hasVectors()) {
@@ -379,20 +377,24 @@ final class SegmentMerger {
}
}
}
-
- private int mergeNorms(SegmentWriteState segmentWriteState) throws IOException {
- final NormsWriter writer = codec.normsFormat().normsWriter(segmentWriteState);
-
+
+ private void mergeNorms(SegmentWriteState segmentWriteState) throws IOException {
+ final PerDocConsumer docsConsumer = codec.normsFormat()
+ .docsConsumer(new PerDocWriteState(segmentWriteState));
+ // TODO: remove this check when 3.x indexes are no longer supported
+ // (3.x indexes don't have docvalues)
+ if (docsConsumer == null) {
+ return;
+ }
boolean success = false;
try {
- int numMerged = writer.merge(mergeState);
+ docsConsumer.merge(mergeState);
success = true;
- return numMerged;
} finally {
if (success) {
- IOUtils.close(writer);
+ IOUtils.close(docsConsumer);
} else {
- IOUtils.closeWhileHandlingException(writer);
+ IOUtils.closeWhileHandlingException(docsConsumer);
}
}
}
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SegmentReader.java Fri Jan 6 02:26:34 2012
@@ -18,16 +18,15 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import org.apache.lucene.store.Directory;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
+import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.store.IOContext;
@@ -262,12 +261,6 @@ public final class SegmentReader extends
return fi != null && fi.isIndexed && !fi.omitNorms;
}
- @Override
- public byte[] norms(String field) throws IOException {
- ensureOpen();
- return core.norms.norms(field);
- }
-
/** @lucene.internal */
public TermVectorsReader getTermVectorsReader() {
ensureOpen();
@@ -352,6 +345,17 @@ public final class SegmentReader extends
}
return perDoc.docValues(field);
}
+
+ @Override
+ public DocValues normValues(String field) throws IOException {
+ ensureOpen();
+ final PerDocProducer perDoc = core.norms;
+ if (perDoc == null) {
+ return null;
+ }
+ return perDoc.docValues(field);
+ }
+
/**
* Called when the shared core for this SegmentReader
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/index/SlowMultiReaderWrapper.java Fri Jan 6 02:26:34 2012
@@ -34,7 +34,7 @@ import org.apache.lucene.index.MultiRead
* IndexReader#getSequentialSubReaders}) to emulate an
* atomic reader. This requires implementing the postings
* APIs on-the-fly, using the static methods in {@link
- * MultiFields}, {@link MultiNorms}, {@link MultiDocValues},
+ * MultiFields}, {@link MultiDocValues},
* by stepping through the sub-readers to merge fields/terms,
* appending docs, etc.
*
@@ -53,7 +53,7 @@ import org.apache.lucene.index.MultiRead
public final class SlowMultiReaderWrapper extends FilterIndexReader {
private final ReaderContext readerContext;
- private final Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
+ private final Map<String, DocValues> normsCache = new HashMap<String, DocValues>();
public SlowMultiReaderWrapper(IndexReader other) {
super(other);
@@ -76,7 +76,17 @@ public final class SlowMultiReaderWrappe
ensureOpen();
return MultiDocValues.getDocValues(in, field);
}
-
+
+ @Override
+ public synchronized DocValues normValues(String field) throws IOException {
+ ensureOpen();
+ DocValues values = normsCache.get(field);
+ if (values == null) {
+ values = MultiDocValues.getNormDocValues(in, field);
+ normsCache.put(field, values);
+ }
+ return values;
+ }
@Override
public Bits getLiveDocs() {
ensureOpen();
@@ -87,22 +97,6 @@ public final class SlowMultiReaderWrappe
public IndexReader[] getSequentialSubReaders() {
return null;
}
-
- @Override
- public synchronized byte[] norms(String field) throws IOException {
- ensureOpen();
- byte[] bytes = normsCache.get(field);
- if (bytes != null)
- return bytes;
- if (!hasNorms(field))
- return null;
- if (normsCache.containsKey(field)) // cached omitNorms, not missing key
- return null;
-
- bytes = MultiNorms.norms(in, field);
- normsCache.put(field, bytes);
- return bytes;
- }
@Override
public ReaderContext getTopReaderContext() {
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Fri Jan 6 02:26:34 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.search.similar
import java.io.IOException;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.CollectionStatistics;
@@ -165,7 +166,7 @@ public class BM25Similarity extends Simi
@Override
public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- final byte[] norms = context.reader.norms(fieldName);
+ final DocValues norms = context.reader.normValues(fieldName);
return norms == null
? new ExactBM25DocScorerNoNorms((BM25Stats)stats)
: new ExactBM25DocScorer((BM25Stats)stats, norms);
@@ -173,7 +174,7 @@ public class BM25Similarity extends Simi
@Override
public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
+ return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.normValues(fieldName));
}
private class ExactBM25DocScorer extends ExactDocScorer {
@@ -182,12 +183,12 @@ public class BM25Similarity extends Simi
private final byte[] norms;
private final float[] cache;
- ExactBM25DocScorer(BM25Stats stats, byte norms[]) {
+ ExactBM25DocScorer(BM25Stats stats, DocValues norms) throws IOException {
assert norms != null;
this.stats = stats;
this.weightValue = stats.weight * (k1 + 1); // boost * idf * (k1 + 1)
this.cache = stats.cache;
- this.norms = norms;
+ this.norms = (byte[])norms.getSource().getArray();
}
@Override
@@ -235,11 +236,11 @@ public class BM25Similarity extends Simi
private final byte[] norms;
private final float[] cache;
- SloppyBM25DocScorer(BM25Stats stats, byte norms[]) {
+ SloppyBM25DocScorer(BM25Stats stats, DocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.weight * (k1 + 1);
this.cache = stats.cache;
- this.norms = norms;
+ this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
}
@Override
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java Fri Jan 6 02:26:34 2012
@@ -57,7 +57,7 @@ import org.apache.lucene.util.TermContex
* <a name="indextime"/>
* At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing
* the Similarity implementation to return a per-document byte for the field that will
- * be later accessible via {@link IndexReader#norms(String)}. Lucene makes no assumption
+ * be later accessible via {@link IndexReader#normValues(String)}. Lucene makes no assumption
* about what is in this byte, but it is most useful for encoding length normalization
* information.
* <p>
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Fri Jan 6 02:26:34 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.search.similar
import java.io.IOException;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.CollectionStatistics;
@@ -177,7 +178,7 @@ public abstract class SimilarityBase ext
@Override
public ExactDocScorer exactDocScorer(Stats stats, String fieldName,
AtomicReaderContext context) throws IOException {
- byte norms[] = context.reader.norms(fieldName);
+ DocValues norms = context.reader.normValues(fieldName);
if (stats instanceof MultiSimilarity.MultiStats) {
// a multi term query (e.g. phrase). return the summation,
@@ -196,7 +197,7 @@ public abstract class SimilarityBase ext
@Override
public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName,
AtomicReaderContext context) throws IOException {
- byte norms[] = context.reader.norms(fieldName);
+ DocValues norms = context.reader.normValues(fieldName);
if (stats instanceof MultiSimilarity.MultiStats) {
// a multi term query (e.g. phrase). return the summation,
@@ -274,9 +275,9 @@ public abstract class SimilarityBase ext
private final BasicStats stats;
private final byte[] norms;
- BasicExactDocScorer(BasicStats stats, byte norms[]) {
+ BasicExactDocScorer(BasicStats stats, DocValues norms) throws IOException {
this.stats = stats;
- this.norms = norms;
+ this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
}
@Override
@@ -303,9 +304,9 @@ public abstract class SimilarityBase ext
private final BasicStats stats;
private final byte[] norms;
- BasicSloppyDocScorer(BasicStats stats, byte norms[]) {
+ BasicSloppyDocScorer(BasicStats stats, DocValues norms) throws IOException {
this.stats = stats;
- this.norms = norms;
+ this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
}
@Override
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java Fri Jan 6 02:26:34 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.search.similar
import java.io.IOException;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.CollectionStatistics;
@@ -704,12 +705,12 @@ public abstract class TFIDFSimilarity ex
@Override
public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new ExactTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName));
+ return new ExactTFIDFDocScorer((IDFStats)stats, context.reader.normValues(fieldName));
}
@Override
public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new SloppyTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName));
+ return new SloppyTFIDFDocScorer((IDFStats)stats, context.reader.normValues(fieldName));
}
// TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot.
@@ -721,10 +722,10 @@ public abstract class TFIDFSimilarity ex
private static final int SCORE_CACHE_SIZE = 32;
private float[] scoreCache = new float[SCORE_CACHE_SIZE];
- ExactTFIDFDocScorer(IDFStats stats, byte norms[]) {
+ ExactTFIDFDocScorer(IDFStats stats, DocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.value;
- this.norms = norms;
+ this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
for (int i = 0; i < SCORE_CACHE_SIZE; i++)
scoreCache[i] = tf(i) * weightValue;
}
@@ -750,10 +751,10 @@ public abstract class TFIDFSimilarity ex
private final float weightValue;
private final byte[] norms;
- SloppyTFIDFDocScorer(IDFStats stats, byte norms[]) {
+ SloppyTFIDFDocScorer(IDFStats stats, DocValues norms) throws IOException {
this.stats = stats;
this.weightValue = stats.value;
- this.norms = norms;
+ this.norms = norms == null ? null : (byte[])norms.getSource().getArray();
}
@Override
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java Fri Jan 6 02:26:34 2012
@@ -91,7 +91,7 @@ final class CompoundFileWriter implement
// all entries that are written to a sep. file but not yet moved into CFS
private final Queue<FileEntry> pendingEntries = new LinkedList<FileEntry>();
private boolean closed = false;
- private volatile IndexOutput dataOut;
+ private IndexOutput dataOut;
private final AtomicBoolean outputTaken = new AtomicBoolean(false);
final String entryTableName;
final String dataFileName;
@@ -113,16 +113,25 @@ final class CompoundFileWriter implement
IndexFileNames.stripExtension(name), "",
IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
dataFileName = name;
- boolean success = false;
- try {
- dataOut = directory.createOutput(dataFileName, IOContext.DEFAULT);
- dataOut.writeVInt(FORMAT_CURRENT);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(dataOut);
+
+ }
+
+ private synchronized IndexOutput getOutput() throws IOException {
+ if (dataOut == null) {
+ IndexOutput dataOutput = null;
+ boolean success = false;
+ try {
+ dataOutput = directory.createOutput(dataFileName, IOContext.DEFAULT);
+ dataOutput.writeVInt(FORMAT_CURRENT);
+ dataOut = dataOutput;
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(dataOutput);
+ }
}
- }
+ }
+ return dataOut;
}
/** Returns the directory of the compound file. */
@@ -154,6 +163,7 @@ final class CompoundFileWriter implement
}
closed = true;
// open the compound stream
+ getOutput();
assert dataOut != null;
long finalLength = dataOut.getFilePointer();
assert assertFileLength(finalLength, dataOut);
@@ -246,7 +256,7 @@ final class CompoundFileWriter implement
seenIDs.add(id);
final DirectCFSIndexOutput out;
if (outputTaken.compareAndSet(false, true)) {
- out = new DirectCFSIndexOutput(dataOut, entry, false);
+ out = new DirectCFSIndexOutput(getOutput(), entry, false);
outputLocked = true;
success = true;
} else {
@@ -280,7 +290,7 @@ final class CompoundFileWriter implement
try {
while (!pendingEntries.isEmpty()) {
FileEntry entry = pendingEntries.poll();
- copyFileEntry(dataOut, entry);
+ copyFileEntry(getOutput(), entry);
entries.put(entry.file, entry);
}
} finally {
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/BitVector.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/BitVector.java Fri Jan 6 02:26:34 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -218,6 +219,7 @@ public final class BitVector implements
<code>d</code>, in a format that can be read by the constructor {@link
#BitVector(Directory, String, IOContext)}. */
public final void write(Directory d, String name, IOContext context) throws IOException {
+ assert !(d instanceof CompoundFileDirectory);
IndexOutput output = d.createOutput(name, context);
try {
output.writeInt(-2);
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/IOUtils.java Fri Jan 6 02:26:34 2012
@@ -30,6 +30,8 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import org.apache.lucene.store.Directory;
+
/** This class emulates the new Java 7 "Try-With-Resources" statement.
* Remove once Lucene is on Java 7.
* @lucene.internal */
@@ -318,6 +320,16 @@ public final class IOUtils {
}
}
}
+
+ public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
+ for (String name : files) {
+ try {
+ dir.deleteFile(name);
+ } catch (IOException ignored) {
+ // ignore
+ }
+ }
+ }
}
Modified: lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java Fri Jan 6 02:26:34 2012
@@ -310,17 +310,4 @@ public final class UTF32ToUTF8 {
utf8StateCount++;
return s;
}
-
- public static void main(String[] args) {
- final int startCode = Integer.parseInt(args[0]);
- final int endCode = Integer.parseInt(args[1]);
-
- Automaton a = new Automaton();
- State start = a.getInitialState();
- State end = new State();
- end.setAccept(true);
-
- UTF32ToUTF8 converter = new UTF32ToUTF8();
- converter.convertOneEdge(start, end, startCode, endCode);
- }
}
Modified: lucene/dev/branches/lucene3305/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java Fri Jan 6 02:26:34 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.codecs.preflex
* limitations under the License.
*/
+import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
import org.apache.lucene.util.LuceneTestCase;
@@ -27,7 +28,8 @@ import org.apache.lucene.util.LuceneTest
*/
public class PreFlexRWCodec extends Lucene3xCodec {
private final PostingsFormat postings = new PreFlexRWPostingsFormat();
-
+ private final NormsFormat norms = new PreFlexRWNormsFormat();
+
@Override
public PostingsFormat postingsFormat() {
if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
@@ -36,4 +38,13 @@ public class PreFlexRWCodec extends Luce
return super.postingsFormat();
}
}
+
+ @Override
+ public NormsFormat normsFormat() {
+ if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
+ return norms;
+ } else {
+ return super.normsFormat();
+ }
+ }
}
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearch.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearch.java Fri Jan 6 02:26:34 2012
@@ -24,8 +24,6 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import org.apache.lucene.util.LuceneTestCase;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.lucene.store.*;
import org.apache.lucene.document.*;
@@ -36,11 +34,6 @@ import org.apache.lucene.search.*;
/** JUnit adaptation of an older test case SearchTest. */
public class TestSearch extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run (new TestSuite(TestSearch.class));
- }
-
/** This test performs a number of searches. It also compares output
* of searches using multi-file index segments with single-file
* index segments.
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java Fri Jan 6 02:26:34 2012
@@ -28,18 +28,9 @@ import org.apache.lucene.analysis.*;
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.util.LuceneTestCase;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
public class TestSearchForDuplicates extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run (new TestSuite(TestSearchForDuplicates.class));
- }
-
-
-
static final String PRIORITY_FIELD ="priority";
static final String ID_FIELD ="id";
static final String HIGH_PRIORITY ="high";
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java Fri Jan 6 02:26:34 2012
@@ -20,10 +20,11 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import java.util.Comparator;
+import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.lucene40.values.Bytes;
import org.apache.lucene.codecs.lucene40.values.Floats;
import org.apache.lucene.codecs.lucene40.values.Ints;
-import org.apache.lucene.codecs.lucene40.values.Writer;
+import org.apache.lucene.index.DocValue;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValues.SortedSource;
import org.apache.lucene.index.DocValues.Source;
@@ -58,12 +59,13 @@ public class TestDocValues extends Lucen
public void runTestBytes(final Bytes.Mode mode, final boolean fixedSize)
throws IOException {
-
+ DocValueHolder valueHolder = new DocValueHolder();
+ valueHolder.comp = COMP;
final BytesRef bytesRef = new BytesRef();
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random),
+ DocValuesConsumer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random),
random.nextBoolean());
int maxDoc = 220;
final String[] values = new String[maxDoc];
@@ -79,7 +81,8 @@ public class TestDocValues extends Lucen
values[2 * i] = s;
UnicodeUtil.UTF16toUTF8(s, 0, s.length(), bytesRef);
- w.add(2 * i, bytesRef);
+ valueHolder.bytes = bytesRef;
+ w.add(2 * i, valueHolder);
}
w.finish(maxDoc);
assertEquals(0, trackBytes.get());
@@ -167,12 +170,15 @@ public class TestDocValues extends Lucen
Type.FIXED_INTS_64, Type.FIXED_INTS_64,
Type.FIXED_INTS_64, Type.VAR_INTS, Type.VAR_INTS,
Type.VAR_INTS, };
+ DocValueHolder valueHolder = new DocValueHolder();
for (int i = 0; i < minMax.length; i++) {
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.VAR_INTS, newIOContext(random));
- w.add(0, minMax[i][0]);
- w.add(1, minMax[i][1]);
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.VAR_INTS, newIOContext(random));
+ valueHolder.intValue = minMax[i][0];
+ w.add(0, valueHolder);
+ valueHolder.intValue = minMax[i][1];
+ w.add(1, valueHolder);
w.finish(2);
assertEquals(0, trackBytes.get());
DocValues r = Ints.getValues(dir, "test", 2, Type.VAR_INTS, newIOContext(random));
@@ -200,12 +206,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt8Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
byte[] sourceArray = new byte[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_8, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_8, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, (long) sourceArray[i]);
+ valueHolder.intValue = (long) sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_8, newIOContext(random));
@@ -221,12 +229,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt16Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
short[] sourceArray = new short[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_16, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_16, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, (long) sourceArray[i]);
+ valueHolder.intValue = (long) sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_16, newIOContext(random));
@@ -242,12 +252,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt64Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
long[] sourceArray = new long[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_64, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_64, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, sourceArray[i]);
+ valueHolder.intValue = sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_64, newIOContext(random));
@@ -263,12 +275,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt32Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
int[] sourceArray = new int[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_32, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_32, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, (long) sourceArray[i]);
+ valueHolder.intValue = (long) sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_32, newIOContext(random));
@@ -284,12 +298,14 @@ public class TestDocValues extends Lucen
}
public void testGetFloat32Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
float[] sourceArray = new float[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_32);
+ DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_32);
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, sourceArray[i]);
+ valueHolder.floatValue = sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Floats.getValues(dir, "test", 3, newIOContext(random), Type.FLOAT_32);
@@ -305,12 +321,14 @@ public class TestDocValues extends Lucen
}
public void testGetFloat64Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
double[] sourceArray = new double[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_64);
+ DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_64);
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, sourceArray[i]);
+ valueHolder.floatValue = sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Floats.getValues(dir, "test", 3, newIOContext(random), Type.FLOAT_64);
@@ -326,17 +344,18 @@ public class TestDocValues extends Lucen
}
private void testInts(Type type, int maxBit) throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
long maxV = 1;
final int NUM_VALUES = 333 + random.nextInt(333);
final long[] values = new long[NUM_VALUES];
for (int rx = 1; rx < maxBit; rx++, maxV *= 2) {
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, type, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, type, newIOContext(random));
for (int i = 0; i < NUM_VALUES; i++) {
final long v = random.nextLong() % (1 + maxV);
- values[i] = v;
- w.add(i, v);
+ valueHolder.intValue = values[i] = v;
+ w.add(i, valueHolder);
}
final int additionalDocs = 1 + random.nextInt(9);
w.finish(NUM_VALUES + additionalDocs);
@@ -362,16 +381,17 @@ public class TestDocValues extends Lucen
}
private void runTestFloats(Type type, double delta) throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), type);
+ DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), type);
final int NUM_VALUES = 777 + random.nextInt(777);;
final double[] values = new double[NUM_VALUES];
for (int i = 0; i < NUM_VALUES; i++) {
final double v = type == Type.FLOAT_32 ? random.nextFloat() : random
.nextDouble();
- values[i] = v;
- w.add(i, v);
+ valueHolder.floatValue = values[i] = v;
+ w.add(i, valueHolder);
}
final int additionalValues = 1 + random.nextInt(10);
w.finish(NUM_VALUES + additionalValues);
@@ -411,4 +431,31 @@ public class TestDocValues extends Lucen
return getSource(values).asSortedSource();
}
+ public static class DocValueHolder implements DocValue {
+ BytesRef bytes;
+ long intValue;
+ double floatValue;
+ Comparator<BytesRef> comp;
+ @Override
+ public BytesRef getBytes() {
+ return bytes;
+ }
+
+ @Override
+ public Comparator<BytesRef> bytesComparator() {
+ return comp;
+ }
+
+ @Override
+ public double getFloat() {
+ return floatValue;
+ }
+
+ @Override
+ public long getInt() {
+ return intValue;
+ }
+
+ }
+
}
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDoc.java Fri Jan 6 02:26:34 2012
@@ -26,8 +26,6 @@ import java.io.StringWriter;
import java.util.LinkedList;
import java.util.Collection;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
@@ -45,11 +43,6 @@ import org.apache.lucene.util._TestUtil;
/** JUnit adaptation of an older test case DocTest. */
public class TestDoc extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run (new TestSuite(TestDoc.class));
- }
-
private File workDir;
private File indexDir;
private LinkedList<File> files;
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Fri Jan 6 02:26:34 2012
@@ -26,7 +26,6 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.DocValuesField;
@@ -417,6 +416,124 @@ public class TestDocValuesIndexing exten
w.close();
d.close();
}
+
+ public void testGetArrayNumerics() throws CorruptIndexException, IOException {
+ Directory d = newDirectory();
+ IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ IndexWriter w = new IndexWriter(d, cfg);
+ final int numValues = 50 + atLeast(10);
+ final List<Type> numVariantList = new ArrayList<Type>(NUMERICS);
+ Collections.shuffle(numVariantList, random);
+ for (Type val : numVariantList) {
+ indexValues(w, numValues, val, numVariantList,
+ false, 7);
+ IndexReader r = IndexReader.open(w, true);
+ DocValues docValues = getDocValues(r, val.name());
+ assertNotNull(docValues);
+ // make sure we don't get a direct source since they don't support getArray()
+ Source source = docValues.getSource();
+
+ switch (source.type()) {
+ case FIXED_INTS_8:
+ {
+ assertTrue(source.hasArray());
+ byte[] values = (byte[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((long)values[i], source.getInt(i));
+ }
+ }
+ break;
+ case FIXED_INTS_16:
+ {
+ assertTrue(source.hasArray());
+ short[] values = (short[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((long)values[i], source.getInt(i));
+ }
+ }
+ break;
+ case FIXED_INTS_32:
+ {
+ assertTrue(source.hasArray());
+ int[] values = (int[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((long)values[i], source.getInt(i));
+ }
+ }
+ break;
+ case FIXED_INTS_64:
+ {
+ assertTrue(source.hasArray());
+ long[] values = (long[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals(values[i], source.getInt(i));
+ }
+ }
+ break;
+ case VAR_INTS:
+ assertFalse(source.hasArray());
+ break;
+ case FLOAT_32:
+ {
+ assertTrue(source.hasArray());
+ float[] values = (float[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((double)values[i], source.getFloat(i), 0.0d);
+ }
+ }
+ break;
+ case FLOAT_64:
+ {
+ assertTrue(source.hasArray());
+ double[] values = (double[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals(values[i], source.getFloat(i), 0.0d);
+ }
+ }
+ break;
+ default:
+ fail("unexpected value " + source.type());
+ }
+ r.close();
+ }
+ w.close();
+ d.close();
+ }
+
+ public void testGetArrayBytes() throws CorruptIndexException, IOException {
+ Directory d = newDirectory();
+ IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ IndexWriter w = new IndexWriter(d, cfg);
+ final int numValues = 50 + atLeast(10);
+ // only single byte fixed straight supports getArray()
+ indexValues(w, numValues, Type.BYTES_FIXED_STRAIGHT, null, false, 1);
+ IndexReader r = IndexReader.open(w, true);
+ DocValues docValues = getDocValues(r, Type.BYTES_FIXED_STRAIGHT.name());
+ assertNotNull(docValues);
+ // make sure we don't get a direct source since they don't support
+ // getArray()
+ Source source = docValues.getSource();
+
+ switch (source.type()) {
+ case BYTES_FIXED_STRAIGHT: {
+ BytesRef ref = new BytesRef();
+ assertTrue(source.hasArray());
+ byte[] values = (byte[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ source.getBytes(i, ref);
+ assertEquals(1, ref.length);
+ assertEquals(values[i], ref.bytes[ref.offset]);
+ }
+ }
+ break;
+ default:
+ fail("unexpected value " + source.type());
+ }
+ r.close();
+ w.close();
+ d.close();
+ }
private DocValues getDocValues(IndexReader reader, String field) throws IOException {
return MultiDocValues.getDocValues(reader, field);
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java Fri Jan 6 02:26:34 2012
@@ -462,9 +462,9 @@ public class TestDuelingCodecs extends L
while ((field = fieldsEnum.next()) != null) {
assertEquals(info, leftReader.hasNorms(field), rightReader.hasNorms(field));
if (leftReader.hasNorms(field)) {
- byte leftNorms[] = MultiNorms.norms(leftReader, field);
- byte rightNorms[] = MultiNorms.norms(rightReader, field);
- assertArrayEquals(info, leftNorms, rightNorms);
+ DocValues leftNorms = MultiDocValues.getNormDocValues(leftReader, field);
+ DocValues rightNorms = MultiDocValues.getNormDocValues(rightReader, field);
+ assertDocValues(leftNorms, rightNorms);
}
}
}
@@ -528,15 +528,19 @@ public class TestDuelingCodecs extends L
for (String field : leftValues) {
DocValues leftDocValues = MultiDocValues.getDocValues(leftReader, field);
DocValues rightDocValues = MultiDocValues.getDocValues(rightReader, field);
- assertNotNull(info, leftDocValues);
- assertNotNull(info, rightDocValues);
- assertEquals(info, leftDocValues.type(), rightDocValues.type());
- assertEquals(info, leftDocValues.getValueSize(), rightDocValues.getValueSize());
- assertDocValuesSource(leftDocValues.getDirectSource(), rightDocValues.getDirectSource());
- assertDocValuesSource(leftDocValues.getSource(), rightDocValues.getSource());
+ assertDocValues(leftDocValues, rightDocValues);
}
}
+ public void assertDocValues(DocValues leftDocValues, DocValues rightDocValues) throws Exception {
+ assertNotNull(info, leftDocValues);
+ assertNotNull(info, rightDocValues);
+ assertEquals(info, leftDocValues.type(), rightDocValues.type());
+ assertEquals(info, leftDocValues.getValueSize(), rightDocValues.getValueSize());
+ assertDocValuesSource(leftDocValues.getDirectSource(), rightDocValues.getDirectSource());
+ assertDocValuesSource(leftDocValues.getSource(), rightDocValues.getSource());
+ }
+
/**
* checks source API
*/
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Fri Jan 6 02:26:34 2012
@@ -509,6 +509,7 @@ public class TestIndexReader extends Luc
writer.addDocument(doc);
}
+ // TODO: maybe this can reuse the logic of test dueling codecs?
public static void assertIndexEquals(IndexReader index1, IndexReader index2) throws IOException {
assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs());
assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc());
@@ -531,13 +532,16 @@ public class TestIndexReader extends Luc
it1 = fields1.iterator();
while (it1.hasNext()) {
String curField = it1.next();
- byte[] norms1 = MultiNorms.norms(index1, curField);
- byte[] norms2 = MultiNorms.norms(index2, curField);
+ DocValues norms1 = MultiDocValues.getNormDocValues(index1, curField);
+ DocValues norms2 = MultiDocValues.getNormDocValues(index2, curField);
if (norms1 != null && norms2 != null)
{
- assertEquals(norms1.length, norms2.length);
- for (int i = 0; i < norms1.length; i++) {
- assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", norms1[i], norms2[i]);
+ // todo: generalize this (like TestDuelingCodecs assert)
+ byte[] b1 = (byte[]) norms1.getSource().getArray();
+ byte[] b2 = (byte[]) norms2.getSource().getArray();
+ assertEquals(b1.length, b2.length);
+ for (int i = 0; i < b1.length; i++) {
+ assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", b1[i], b2[i]);
}
}
else
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Fri Jan 6 02:26:34 2012
@@ -75,7 +75,7 @@ public class TestMaxTermFrequency extend
}
public void test() throws Exception {
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
+ byte fooNorms[] = (byte[])MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(expected.get(i).intValue(), fooNorms[i] & 0xff);
}
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestNorms.java Fri Jan 6 02:26:34 2012
@@ -82,11 +82,11 @@ public class TestNorms extends LuceneTes
IndexReader reader = writer.getReader();
writer.close();
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
+ byte fooNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(0, fooNorms[i]);
- byte barNorms[] = MultiNorms.norms(reader, "bar");
+ byte barNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "bar").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(1, barNorms[i]);
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java Fri Jan 6 02:26:34 2012
@@ -289,12 +289,15 @@ public class TestOmitNorms extends Lucen
}
IndexReader ir1 = riw.getReader();
- byte[] norms1 = MultiNorms.norms(ir1, field);
+ // todo: generalize
+ DocValues dv1 = MultiDocValues.getNormDocValues(ir1, field);
+ byte[] norms1 = dv1 == null ? null : (byte[]) dv1.getSource().getArray();
// fully merge and validate MultiNorms against single segment.
riw.forceMerge(1);
IndexReader ir2 = riw.getReader();
- byte[] norms2 = ir2.getSequentialSubReaders()[0].norms(field);
+ DocValues dv2 = ir2.getSequentialSubReaders()[0].normValues(field);
+ byte[] norms2 = dv2 == null ? null : (byte[]) dv2.getSource().getArray();
assertArrayEquals(norms1, norms2);
ir1.close();
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Fri Jan 6 02:26:34 2012
@@ -172,7 +172,7 @@ public class TestSegmentReader extends L
assertEquals(reader.hasNorms(f.name()), !DocHelper.noNorms.containsKey(f.name()));
if (!reader.hasNorms(f.name())) {
// test for norms of null
- byte [] norms = MultiNorms.norms(reader, f.name());
+ DocValues norms = MultiDocValues.getNormDocValues(reader, f.name());
assertNull(norms);
}
}
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java Fri Jan 6 02:26:34 2012
@@ -73,7 +73,7 @@ public class TestUniqueTermCount extends
}
public void test() throws Exception {
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
+ byte fooNorms[] = (byte[])MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(expected.get(i).intValue(), fooNorms[i] & 0xff);
}
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java Fri Jan 6 02:26:34 2012
@@ -30,13 +30,11 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TermContext;
/**
* Tests the use of indexdocvalues in scoring.
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java Fri Jan 6 02:26:34 2012
@@ -24,7 +24,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.MultiNorms;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
@@ -74,8 +74,9 @@ public class TestSimilarityProvider exte
public void testBasics() throws Exception {
// sanity check of norms writer
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
- byte barNorms[] = MultiNorms.norms(reader, "bar");
+ // TODO: generalize
+ byte fooNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
+ byte barNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "bar").getSource().getArray();
for (int i = 0; i < fooNorms.length; i++) {
assertFalse(fooNorms[i] == barNorms[i]);
}
Modified: lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java (original)
+++ lucene/dev/branches/lucene3305/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java Fri Jan 6 02:26:34 2012
@@ -20,16 +20,10 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.lucene.search.DocIdSetIterator;
public class TestSortedVIntList extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run(new TestSuite(TestSortedVIntList.class));
- }
void tstIterator (
SortedVIntList vintList,
Modified: lucene/dev/branches/lucene3305/modules/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/benchmark/build.xml?rev=1227937&r1=1227936&r2=1227937&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/benchmark/build.xml (original)
+++ lucene/dev/branches/lucene3305/modules/benchmark/build.xml Fri Jan 6 02:26:34 2012
@@ -154,6 +154,7 @@
<pathelement path="${analyzers-common.jar}"/>
<pathelement path="${queryparser.jar}"/>
<pathelement path="${facet.jar}"/>
+ <fileset dir="${common.dir}/../modules/analysis/icu/lib" includes="icu4j*.jar"/>
<path refid="base.classpath"/>
<fileset dir="lib">
<include name="**/*.jar"/>