You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/07/14 12:24:20 UTC
svn commit: r1146634 -
/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java
Author: mikemccand
Date: Thu Jul 14 10:24:20 2011
New Revision: 1146634
URL: http://svn.apache.org/viewvc?rev=1146634&view=rev
Log:
LUCENE-3309: switch core to very minimal but expert API for retrieving stored fields, but keep sugar for simple access to load a reconstituted Document
Added:
lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java (with props)
Added: lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java?rev=1146634&view=auto
==============================================================================
--- lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java (added)
+++ lucene/dev/branches/fieldtype/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java Thu Jul 14 10:24:20 2011
@@ -0,0 +1,315 @@
+package org.apache.lucene.document;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldReaderException;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
+
+/** Create this, passing a legacy {@link FieldSelector} to it, then
+ * pass this class to {@link IndexReader#document(int,
+ * StoredFieldVisitor)}, then call {@link #getDocument} to
+ * retrieve the loaded document.
+
+ * <p><b>NOTE</b>: If you use Lazy fields, you should not
+ * access the returned document after the reader has been
+ * closed!
+ */
+
+public class FieldSelectorVisitor extends StoredFieldVisitor {
+
+ private final FieldSelector selector;
+ private final Document doc;
+
+ public FieldSelectorVisitor(FieldSelector selector) {
+ this.selector = selector;
+ doc = new Document();
+ }
+
+ public Document getDocument() {
+ return doc;
+ }
+
+ @Override
+ public boolean binaryField(FieldInfo fieldInfo, IndexInput in, int numBytes) throws IOException {
+ final FieldSelectorResult accept = selector.accept(fieldInfo.name);
+ switch (accept) {
+ case LOAD:
+ case LOAD_AND_BREAK:
+ final byte[] b = new byte[numBytes];
+ in.readBytes(b, 0, b.length);
+ doc.add(new Field(fieldInfo.name, b));
+ return accept != FieldSelectorResult.LOAD;
+ case LAZY_LOAD:
+ case LATENT:
+ addFieldLazy(in, fieldInfo, true, accept == FieldSelectorResult.LAZY_LOAD, numBytes);
+ return false;
+ case SIZE:
+ case SIZE_AND_BREAK:
+ in.seek(in.getFilePointer() + numBytes);
+ addFieldSize(fieldInfo, numBytes);
+ return accept != FieldSelectorResult.SIZE;
+ default:
+ // skip
+ in.seek(in.getFilePointer() + numBytes);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean stringField(FieldInfo fieldInfo, IndexInput in, int numUTF8Bytes) throws IOException {
+ final FieldSelectorResult accept = selector.accept(fieldInfo.name);
+ switch (accept) {
+ case LOAD:
+ case LOAD_AND_BREAK:
+ final byte[] b = new byte[numUTF8Bytes];
+ in.readBytes(b, 0, b.length);
+ Field.TermVector termVector = Field.TermVector.toTermVector(fieldInfo.storeTermVector, fieldInfo.storeOffsetWithTermVector, fieldInfo.storePositionWithTermVector);
+ doc.add(new Field(fieldInfo.name, false, new String(b, "UTF-8"), Field.Store.YES, Field.Index.ANALYZED, termVector));
+ return accept != FieldSelectorResult.LOAD;
+ case LAZY_LOAD:
+ case LATENT:
+ addFieldLazy(in, fieldInfo, false, accept == FieldSelectorResult.LAZY_LOAD, numUTF8Bytes);
+ return false;
+ case SIZE:
+ case SIZE_AND_BREAK:
+ in.seek(in.getFilePointer() + numUTF8Bytes);
+ addFieldSize(fieldInfo, 2*numUTF8Bytes);
+ return accept != FieldSelectorResult.SIZE;
+ default:
+ // skip
+ in.seek(in.getFilePointer() + numUTF8Bytes);
+ return false;
+ }
+ }
+
+ @Override
+ public boolean intField(FieldInfo fieldInfo, int value) throws IOException {
+ return addNumericField(fieldInfo, new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setIntValue(value));
+ }
+
+ @Override
+ public boolean longField(FieldInfo fieldInfo, long value) throws IOException {
+ return addNumericField(fieldInfo, new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setLongValue(value));
+ }
+
+ @Override
+ public boolean floatField(FieldInfo fieldInfo, float value) throws IOException {
+ return addNumericField(fieldInfo, new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setFloatValue(value));
+ }
+
+ @Override
+ public boolean doubleField(FieldInfo fieldInfo, double value) throws IOException {
+ return addNumericField(fieldInfo, new NumericField(fieldInfo.name, Field.Store.YES, fieldInfo.isIndexed).setDoubleValue(value));
+ }
+
+ private boolean addNumericField(FieldInfo fieldInfo, NumericField f) {
+ f.setOmitNorms(fieldInfo.omitNorms);
+ f.setOmitTermFreqAndPositions(fieldInfo.omitTermFreqAndPositions);
+ doc.add(f);
+ final FieldSelectorResult accept = selector.accept(fieldInfo.name);
+ switch (accept) {
+ case LOAD:
+ return false;
+ case LOAD_AND_BREAK:
+ return true;
+ case LAZY_LOAD:
+ case LATENT:
+ return false;
+ case SIZE:
+ return false;
+ case SIZE_AND_BREAK:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ private void addFieldLazy(IndexInput in, FieldInfo fi, boolean binary, boolean cacheResult, int numBytes) throws IOException {
+ final AbstractField f;
+ final long pointer = in.getFilePointer();
+ // Need to move the pointer ahead by toRead positions
+ in.seek(pointer+numBytes);
+ if (binary) {
+ f = new LazyField(in, fi.name, Field.Store.YES, numBytes, pointer, binary, cacheResult);
+ } else {
+ Field.TermVector termVector = Field.TermVector.toTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
+ f = new LazyField(in, fi.name, Field.Store.YES, Field.Index.ANALYZED, termVector, numBytes, pointer, binary, cacheResult);
+ }
+
+ f.setOmitNorms(fi.omitNorms);
+ f.setOmitTermFreqAndPositions(fi.omitTermFreqAndPositions);
+ doc.add(f);
+ }
+
+ // Add the size of field as a byte[] containing the 4 bytes of the integer byte size (high order byte first; char = 2 bytes)
+ // Read just the size -- caller must skip the field content to continue reading fields
+ // Return the size in bytes or chars, depending on field type
+ private void addFieldSize(FieldInfo fi, int numBytes) throws IOException {
+ byte[] sizebytes = new byte[4];
+ sizebytes[0] = (byte) (numBytes>>>24);
+ sizebytes[1] = (byte) (numBytes>>>16);
+ sizebytes[2] = (byte) (numBytes>>> 8);
+ sizebytes[3] = (byte) numBytes ;
+ doc.add(new Field(fi.name, sizebytes));
+ }
+
+ /**
+ * A Lazy field implementation that defers loading of fields until asked for, instead of when the Document is
+ * loaded.
+ */
+ private static class LazyField extends AbstractField {
+ private int toRead;
+ private long pointer;
+ private final boolean cacheResult;
+ private final IndexInput in;
+
+ public LazyField(IndexInput in, String name, Field.Store store, int toRead, long pointer, boolean isBinary, boolean cacheResult) {
+ super(name, store, Field.Index.NO, Field.TermVector.NO);
+ this.in = in;
+ this.toRead = toRead;
+ this.pointer = pointer;
+ this.isBinary = isBinary;
+ this.cacheResult = cacheResult;
+ if (isBinary)
+ binaryLength = toRead;
+ lazy = true;
+ }
+
+ public LazyField(IndexInput in, String name, Field.Store store, Field.Index index, Field.TermVector termVector, int toRead, long pointer, boolean isBinary, boolean cacheResult) {
+ super(name, store, index, termVector);
+ this.in = in;
+ this.toRead = toRead;
+ this.pointer = pointer;
+ this.isBinary = isBinary;
+ this.cacheResult = cacheResult;
+ if (isBinary)
+ binaryLength = toRead;
+ lazy = true;
+ }
+
+ public Number getNumericValue() {
+ return null;
+ }
+
+ public NumericField.DataType getDataType() {
+ return null;
+ }
+
+ private IndexInput localFieldsStream;
+
+ private IndexInput getFieldStream() {
+ if (localFieldsStream == null) {
+ localFieldsStream = (IndexInput) in.clone();
+ }
+ return localFieldsStream;
+ }
+
+ /** The value of the field as a Reader, or null. If null, the String value,
+ * binary value, or TokenStream value is used. Exactly one of stringValue(),
+ * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+ public Reader readerValue() {
+ return null;
+ }
+
+ /** The value of the field as a TokenStream, or null. If null, the Reader value,
+ * String value, or binary value is used. Exactly one of stringValue(),
+ * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+ public TokenStream tokenStreamValue() {
+ return null;
+ }
+
+ /** The value of the field as a String, or null. If null, the Reader value,
+ * binary value, or TokenStream value is used. Exactly one of stringValue(),
+ * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+ synchronized public String stringValue() {
+ if (isBinary)
+ return null;
+ else {
+ if (fieldsData == null) {
+ String result = null;
+ IndexInput localFieldsStream = getFieldStream();
+ try {
+ localFieldsStream.seek(pointer);
+ byte[] bytes = new byte[toRead];
+ localFieldsStream.readBytes(bytes, 0, toRead);
+ result = new String(bytes, "UTF-8");
+ } catch (IOException e) {
+ throw new FieldReaderException(e);
+ }
+ if (cacheResult == true){
+ fieldsData = result;
+ }
+ return result;
+ } else {
+ return (String) fieldsData;
+ }
+ }
+ }
+
+ synchronized private byte[] getBinaryValue(byte[] result) {
+ if (isBinary) {
+ if (fieldsData == null) {
+ // Allocate new buffer if result is null or too small
+ final byte[] b;
+ if (result == null || result.length < toRead)
+ b = new byte[toRead];
+ else
+ b = result;
+
+ IndexInput localFieldsStream = getFieldStream();
+
+ // Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people
+ // since they are already handling this exception when getting the document
+ try {
+ localFieldsStream.seek(pointer);
+ localFieldsStream.readBytes(b, 0, toRead);
+ } catch (IOException e) {
+ throw new FieldReaderException(e);
+ }
+
+ binaryOffset = 0;
+ binaryLength = toRead;
+ if (cacheResult == true){
+ fieldsData = b;
+ }
+ return b;
+ } else {
+ return (byte[]) fieldsData;
+ }
+ } else
+ return null;
+ }
+
+ @Override
+ public BytesRef binaryValue(BytesRef reuse) {
+ final byte[] bytes = getBinaryValue(reuse != null ? reuse.bytes : null);
+ if (bytes != null) {
+ return new BytesRef(bytes, 0, bytes.length);
+ } else {
+ return null;
+ }
+ }
+ }
+}
\ No newline at end of file