You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/11/08 17:51:57 UTC
svn commit: r1199341 - in /lucene/dev/branches/lucene2621:
lucene/contrib/misc/src/java/org/apache/lucene/document/
lucene/contrib/misc/src/test/org/apache/lucene/index/
lucene/contrib/misc/src/test/org/apache/lucene/search/
modules/benchmark/src/java/...
Author: mikemccand
Date: Tue Nov 8 16:51:56 2011
New Revision: 1199341
URL: http://svn.apache.org/viewvc?rev=1199341&view=rev
Log:
LUCENE-2621: cutover fully to StoredFieldVisitor
Added:
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java (with props)
Removed:
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelector.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorResult.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/FieldSelectorVisitor.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/LoadFirstFieldSelector.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/MapFieldSelector.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/SetBasedFieldSelector.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribIndexReader.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribParallelReader.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestLazyBug.java
lucene/dev/branches/lucene2621/lucene/contrib/misc/src/test/org/apache/lucene/search/TestThreadSafe.java
Modified:
lucene/dev/branches/lucene2621/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/DocNameExtractor.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
Added: lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java?rev=1199341&view=auto
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java (added)
+++ lucene/dev/branches/lucene2621/lucene/contrib/misc/src/java/org/apache/lucene/document/LazyDocument.java Tue Nov 8 16:51:56 2011
@@ -0,0 +1,105 @@
+package org.apache.lucene.document;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.BytesRef;
+
+/** Defers actually loading a field's value until you ask
+ * for it. You must not use the returned Field instances
+ * after the provided reader has been closed. */
+
+public class LazyDocument {
+ private IndexReader reader;
+ private final int docID;
+
+ // null until first field is loaded
+ private Document doc;
+
+ private Map<String,LazyField> fields = new HashMap<String,LazyField>();
+
+ public LazyDocument(IndexReader reader, int docID) {
+ this.reader = reader;
+ this.docID = docID;
+ }
+
+ public Field getField(FieldInfo fieldInfo) {
+ LazyField f = fields.get(fieldInfo.name);
+ if (f == null) {
+ final FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(fieldInfo.storeTermVector);
+ ft.setStoreTermVectorPositions(fieldInfo.storePositionWithTermVector);
+ ft.setStoreTermVectorOffsets(fieldInfo.storeOffsetWithTermVector);
+ ft.setStoreTermVectors(fieldInfo.storeTermVector);
+ ft.setIndexed(fieldInfo.isIndexed);
+ ft.setOmitNorms(fieldInfo.omitNorms);
+ ft.setIndexOptions(fieldInfo.indexOptions);
+ f = new LazyField(fieldInfo.name, ft);
+ fields.put(fieldInfo.name, f);
+ }
+ return f;
+ }
+
+ private synchronized Document getDocument() {
+ if (doc == null) {
+ try {
+ doc = reader.document(docID);
+ } catch (IOException ioe) {
+ throw new IllegalStateException("unable to load document", ioe);
+ }
+ reader = null;
+ }
+ return doc;
+ }
+
+ private class LazyField extends Field {
+ public LazyField(String name, FieldType ft) {
+ super(name, ft);
+ }
+
+ @Override
+ public Number numericValue() {
+ return null;
+ }
+
+ @Override
+ public NumericField.DataType numericDataType() {
+ return null;
+ }
+
+ @Override
+ public Reader readerValue() {
+ return null;
+ }
+
+ @Override
+ public String stringValue() {
+ return getDocument().get(name);
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return getDocument().getBinaryValue(name);
+ }
+ }
+}
Modified: lucene/dev/branches/lucene2621/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/DocNameExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/DocNameExtractor.java?rev=1199341&r1=1199340&r2=1199341&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/DocNameExtractor.java (original)
+++ lucene/dev/branches/lucene2621/modules/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/DocNameExtractor.java Tue Nov 8 16:51:56 2011
@@ -58,7 +58,7 @@ public class DocNameExtractor {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
- if (name.isEmpty()) {
+ if (!name.isEmpty()) {
return Status.STOP;
} else if (fieldInfo.name.equals(docNameField)) {
return Status.YES;
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1199341&r1=1199340&r2=1199341&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Tue Nov 8 16:51:56 2011
@@ -17,10 +17,18 @@
package org.apache.solr.search;
+import java.io.IOException;
+import java.net.URL;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.lucene.document.BinaryField;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.LazyDocument;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.document.TextField;
import org.apache.lucene.index.*;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.*;
@@ -47,11 +55,6 @@ import org.apache.solr.update.SolrIndexC
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.net.URL;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
* SolrIndexSearcher adds schema awareness and caching functionality
@@ -388,16 +391,71 @@ public class SolrIndexSearcher extends I
* FieldSelector which loads the specified fields, and load all other
* field lazily.
*/
- static class SetNonLazyFieldSelector implements FieldSelector {
+ // TODO: can we just subclass DocumentStoredFieldVisitor?
+ // need to open up access to its Document...
+ static class SetNonLazyFieldSelector extends StoredFieldVisitor {
private Set<String> fieldsToLoad;
- SetNonLazyFieldSelector(Set<String> toLoad) {
+ final Document doc = new Document();
+ final LazyDocument lazyDoc;
+
+ SetNonLazyFieldSelector(Set<String> toLoad, IndexReader reader, int docID) {
fieldsToLoad = toLoad;
+ lazyDoc = new LazyDocument(reader, docID);
+ }
+
+ public Status needsField(FieldInfo fieldInfo) {
+ if (fieldsToLoad.contains(fieldInfo.name)) {
+ return Status.YES;
+ } else {
+ doc.add(lazyDoc.getField(fieldInfo));
+ return Status.NO;
+ }
+ }
+
+ @Override
+ public void binaryField(FieldInfo fieldInfo, byte[] value, int offset, int length) throws IOException {
+ doc.add(new BinaryField(fieldInfo.name, value));
+ }
+
+ @Override
+ public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+ final FieldType ft = new FieldType(TextField.TYPE_STORED);
+ ft.setStoreTermVectors(fieldInfo.storeTermVector);
+ ft.setStoreTermVectorPositions(fieldInfo.storePositionWithTermVector);
+ ft.setStoreTermVectorOffsets(fieldInfo.storeOffsetWithTermVector);
+ ft.setStoreTermVectors(fieldInfo.storeTermVector);
+ ft.setIndexed(fieldInfo.isIndexed);
+ ft.setOmitNorms(fieldInfo.omitNorms);
+ ft.setIndexOptions(fieldInfo.indexOptions);
+ doc.add(new Field(fieldInfo.name, value, ft));
+ }
+
+ @Override
+ public void intField(FieldInfo fieldInfo, int value) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setIntValue(value));
}
- public FieldSelectorResult accept(String fieldName) {
- if(fieldsToLoad.contains(fieldName))
- return FieldSelectorResult.LOAD;
- else
- return FieldSelectorResult.LAZY_LOAD;
+
+ @Override
+ public void longField(FieldInfo fieldInfo, long value) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setLongValue(value));
+ }
+
+ @Override
+ public void floatField(FieldInfo fieldInfo, float value) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setFloatValue(value));
+ }
+
+ @Override
+ public void doubleField(FieldInfo fieldInfo, double value) {
+ FieldType ft = new FieldType(NumericField.TYPE_STORED);
+ ft.setIndexed(fieldInfo.isIndexed);
+ doc.add(new NumericField(fieldInfo.name, ft).setDoubleValue(value));
}
}
@@ -436,9 +494,9 @@ public class SolrIndexSearcher extends I
if(!enableLazyFieldLoading || fields == null) {
d = getIndexReader().document(i);
} else {
- final FieldSelectorVisitor visitor = new FieldSelectorVisitor(new SetNonLazyFieldSelector(fields));
+ final SetNonLazyFieldSelector visitor = new SetNonLazyFieldSelector(fields, getIndexReader(), i);
getIndexReader().document(i, visitor);
- d = visitor.getDocument();
+ d = visitor.doc;
}
if (documentCache != null) {
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java?rev=1199341&r1=1199340&r2=1199341&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/grouping/distributed/shardresultserializer/TopGroupsResultTransformer.java Tue Nov 8 16:51:56 2011
@@ -18,9 +18,7 @@ package org.apache.solr.search.grouping.
*/
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.FieldSelectorVisitor;
+import org.apache.lucene.document.DocumentStoredFieldVisitor;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
@@ -269,17 +267,9 @@ public class TopGroupsResultTransformer
}
private Document retrieveDocument(final SchemaField uniqueField, int doc) throws IOException {
- FieldSelectorVisitor fieldSelectorVisitor = new FieldSelectorVisitor(new FieldSelector() {
-
- public FieldSelectorResult accept(String fieldName) {
- if (uniqueField.getName().equals(fieldName)) {
- return FieldSelectorResult.LOAD_AND_BREAK;
- }
- return FieldSelectorResult.NO_LOAD;
- }
- });
- rb.req.getSearcher().doc(doc, fieldSelectorVisitor);
- return fieldSelectorVisitor.getDocument();
+ DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(uniqueField.getName());
+ rb.req.getSearcher().doc(doc, visitor);
+ return visitor.getDocument();
}
}