You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/02 20:43:26 UTC
svn commit: r1592005 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/benchmark/
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/
lucene/core/ lucene/core/src/java/org/apache/lucene/document/
lucene/core/src/java/org/apache/luc...
Author: rmuir
Date: Fri May 2 18:43:25 2014
New Revision: 1592005
URL: http://svn.apache.org/r1592005
Log:
LUCENE-5634: Reuse TokenStream instances for string and numeric Fields
Added:
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java
- copied, changed from r1591992, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/Field.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri May 2 18:43:25 2014
@@ -19,6 +19,12 @@ New Features
minimum numeric values from the provided Terms. (Robert Muir, Mike
McCandless)
+Changes in Backwards Compatibility Policy
+
+* LUCENE-5634: Add reuse argument to IndexableField.tokenStream. This
+ can be used by custom fieldtypes, which don't use the Analyzer, but
+ implement their own TokenStream. (Uwe Schindler, Robert Muir)
+
API Changes
* LUCENE-5582: Deprecate IndexOutput.length (just use
@@ -56,6 +62,9 @@ Optimizations
* LUCENE-5591: pass an IOContext with estimated flush size when applying DV
updates. (Shai Erera)
+* LUCENE-5634: IndexWriter reuses TokenStream instances for String and Numeric
+ fields by default. (Uwe Schindler, Shay Banon, Mike McCandless, Robert Muir)
+
Bug fixes
* LUCENE-5600: HttpClientBase did not properly consume a connection if a server
Modified: lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java (original)
+++ lucene/dev/branches/branch_4x/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTokensTask.java Fri May 2 18:43:25 2014
@@ -80,7 +80,7 @@ public class ReadTokensTask extends Perf
continue;
}
- final TokenStream stream = field.tokenStream(analyzer);
+ final TokenStream stream = field.tokenStream(analyzer, null);
// reset the TokenStream to the first token
stream.reset();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/Field.java Fri May 2 18:43:25 2014
@@ -72,8 +72,6 @@ public class Field implements IndexableF
* customize how it's tokenized */
protected TokenStream tokenStream;
- private transient TokenStream internalTokenStream;
-
/**
* Field's boost
* @see #boost()
@@ -492,19 +490,19 @@ public class Field implements IndexableF
}
@Override
- public TokenStream tokenStream(Analyzer analyzer) throws IOException {
+ public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) throws IOException {
if (!fieldType().indexed()) {
return null;
}
final NumericType numericType = fieldType().numericType();
if (numericType != null) {
- if (!(internalTokenStream instanceof NumericTokenStream)) {
+ if (!(reuse instanceof NumericTokenStream && ((NumericTokenStream)reuse).getPrecisionStep() == type.numericPrecisionStep())) {
// lazy init the TokenStream as it is heavy to instantiate
// (attributes,...) if not needed (stored field loading)
- internalTokenStream = new NumericTokenStream(type.numericPrecisionStep());
+ reuse = new NumericTokenStream(type.numericPrecisionStep());
}
- final NumericTokenStream nts = (NumericTokenStream) internalTokenStream;
+ final NumericTokenStream nts = (NumericTokenStream) reuse;
// initialize value in TokenStream
final Number val = (Number) fieldsData;
switch (numericType) {
@@ -523,20 +521,20 @@ public class Field implements IndexableF
default:
throw new AssertionError("Should never get here");
}
- return internalTokenStream;
+ return reuse;
}
if (!fieldType().tokenized()) {
if (stringValue() == null) {
throw new IllegalArgumentException("Non-Tokenized Fields must have a String value");
}
- if (!(internalTokenStream instanceof StringTokenStream)) {
+ if (!(reuse instanceof StringTokenStream)) {
// lazy init the TokenStream as it is heavy to instantiate
// (attributes,...) if not needed (stored field loading)
- internalTokenStream = new StringTokenStream();
+ reuse = new StringTokenStream();
}
- ((StringTokenStream) internalTokenStream).setValue(stringValue());
- return internalTokenStream;
+ ((StringTokenStream) reuse).setValue(stringValue());
+ return reuse;
}
if (tokenStream != null) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Fri May 2 18:43:25 2014
@@ -522,6 +522,9 @@ final class DefaultIndexingChain extends
// Lazy init'd:
NumericDocValuesWriter norms;
+
+ // reused
+ TokenStream tokenStream;
public PerField(FieldInfo fieldInfo, boolean invert) {
this.fieldInfo = fieldInfo;
@@ -585,7 +588,7 @@ final class DefaultIndexingChain extends
*/
boolean aborting = false;
boolean succeededInProcessingField = false;
- try (TokenStream stream = field.tokenStream(docState.analyzer)) {
+ try (TokenStream stream = tokenStream = field.tokenStream(docState.analyzer, tokenStream)) {
// reset the TokenStream to the first token
stream.reset();
invertState.setAttributeSource(stream);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableField.java?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableField.java Fri May 2 18:43:25 2014
@@ -83,9 +83,15 @@ public interface IndexableField {
* implementations should use the given Analyzer to create the TokenStreams.
*
* @param analyzer Analyzer that should be used to create the TokenStreams from
+ * @param reuse TokenStream for a previous instance of this field <b>name</b>. This allows
+ * custom field types (like StringField and NumericField) that do not use
+ * the analyzer to still have good performance. Note: the passed-in type
+ * may be inappropriate, for example if you mix up different types of Fields
+ * for the same field name. So its the responsibility of the implementation to
+ * check.
* @return TokenStream value for indexing the document. Should always return
* a non-null value if the field is to be indexed
* @throws IOException Can be thrown while creating the TokenStream
*/
- public TokenStream tokenStream(Analyzer analyzer) throws IOException;
+ public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) throws IOException;
}
Copied: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java (from r1591992, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java?p2=lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java&r1=1591992&r2=1592005&rev=1592005&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestFieldReuse.java Fri May 2 18:43:25 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.io.Reader;
import java.util.Collections;
import org.apache.lucene.analysis.Analyzer;
@@ -31,6 +32,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
/** test tokenstream reuse by DefaultIndexingChain */
@@ -123,6 +125,26 @@ public class TestFieldReuse extends Base
@Override
public float boost() {
return 1;
+ }
+
+ @Override
+ public BytesRef binaryValue() {
+ return null;
+ }
+
+ @Override
+ public String stringValue() {
+ return null;
+ }
+
+ @Override
+ public Reader readerValue() {
+ return null;
+ }
+
+ @Override
+ public Number numericValue() {
+ return null;
}
}
@@ -131,32 +153,14 @@ public class TestFieldReuse extends Base
IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, null);
IndexWriter iw = new IndexWriter(dir, iwc);
final MyField field1 = new MyField();
- iw.addDocument(new IndexDocument() {
- @Override
- public Iterable<? extends IndexableField> indexableFields() {
- return Collections.singletonList(field1);
- }
- @Override
- public Iterable<StorableField> storableFields() {
- return Collections.emptyList();
- }
- });
+ iw.addDocument(Collections.singletonList(field1));
TokenStream previous = field1.lastReturned;
assertNotNull(previous);
final MyField field2 = new MyField();
- iw.addDocument(new IndexDocument() {
- @Override
- public Iterable<? extends IndexableField> indexableFields() {
- return Collections.singletonList(field2);
- }
- @Override
- public Iterable<StorableField> storableFields() {
- return Collections.emptyList();
- }
- });
+ iw.addDocument(Collections.singletonList(field2));
assertSame(previous, field2.lastSeen);
- iw.shutdown();
+ iw.close();
dir.close();
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri May 2 18:43:25 2014
@@ -1623,7 +1623,7 @@ public class TestIndexWriterExceptions e
}
@Override
- public TokenStream tokenStream(Analyzer analyzer) throws IOException {
+ public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) throws IOException {
return null;
}
});
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Fri May 2 18:43:25 2014
@@ -161,7 +161,7 @@ public class TestIndexableField extends
}
@Override
- public TokenStream tokenStream(Analyzer analyzer) throws IOException {
+ public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) throws IOException {
return readerValue() != null ? analyzer.tokenStream(name(), readerValue()) :
analyzer.tokenStream(name(), new StringReader(stringValue()));
}
@@ -355,7 +355,7 @@ public class TestIndexableField extends
}
@Override
- public TokenStream tokenStream(Analyzer a) {
+ public TokenStream tokenStream(Analyzer a, TokenStream reuse) {
return null;
}
Modified: lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java?rev=1592005&r1=1592004&r2=1592005&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java (original)
+++ lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/document/LazyDocument.java Fri May 2 18:43:25 2014
@@ -191,8 +191,8 @@ public class LazyDocument {
}
@Override
- public TokenStream tokenStream(Analyzer analyzer) throws IOException {
- return getRealValue().tokenStream(analyzer);
+ public TokenStream tokenStream(Analyzer analyzer, TokenStream reuse) throws IOException {
+ return getRealValue().tokenStream(analyzer, reuse);
}
}
}