You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2014/07/05 13:32:32 UTC
svn commit: r1607998 - in /lucene/dev/trunk: lucene/
lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/
lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/
lucene/core/src/java/org/apache/lucene/analysis/ lu...
Author: uschindler
Date: Sat Jul 5 11:32:31 2014
New Revision: 1607998
URL: http://svn.apache.org/r1607998
Log:
LUCENE-5803: Add DelegatingAnalyzerWrapper, an optimized variant of AnalyzerWrapper that doesn't allow to wrap components or readers
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/DelegatingAnalyzerWrapper.java (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Sat Jul 5 11:32:31 2014
@@ -125,6 +125,12 @@ Optimizations
* LUCENE-5797: Optimize norms merging (Adrien Grand, Robert Muir)
+* LUCENE-5803: Add DelegatingAnalyzerWrapper, an optimized variant
+ of AnalyzerWrapper that doesn't allow to wrap components or readers.
+ This wrapper class is the base class of all analyzers that just delegate
+ to another analyzer, e.g. per field name: PerFieldAnalyzerWrapper and
+ Solr's schema support. (Shay Banon, Uwe Schindler, Robert Muir)
+
Test Framework
* LUCENE-5786: Unflushed/ truncated events file (hung testing subprocess).
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java Sat Jul 5 11:32:31 2014
@@ -18,7 +18,7 @@ package org.apache.lucene.analysis.misce
*/
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.AnalyzerWrapper;
+import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
import java.util.Collections;
import java.util.Map;
@@ -48,7 +48,7 @@ import java.util.Map;
* <p>A PerFieldAnalyzerWrapper can be used like any other analyzer, for both indexing
* and query parsing.
*/
-public final class PerFieldAnalyzerWrapper extends AnalyzerWrapper {
+public final class PerFieldAnalyzerWrapper extends DelegatingAnalyzerWrapper {
private final Analyzer defaultAnalyzer;
private final Map<String, Analyzer> fieldAnalyzers;
Modified: lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java Sat Jul 5 11:32:31 2014
@@ -1,14 +1,21 @@
package org.apache.lucene.analysis.miscellaneous;
+import java.io.IOException;
import java.io.Reader;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.AnalyzerWrapper;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockCharFilter;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.Rethrow;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -31,8 +38,8 @@ public class TestPerFieldAnalyzerWrapper
public void testPerField() throws Exception {
String text = "Qwerty";
- Map<String, Analyzer> analyzerPerField = new HashMap<>();
- analyzerPerField.put("special", new SimpleAnalyzer(TEST_VERSION_CURRENT));
+ Map<String,Analyzer> analyzerPerField =
+ Collections.<String,Analyzer>singletonMap("special", new SimpleAnalyzer(TEST_VERSION_CURRENT));
PerFieldAnalyzerWrapper analyzer =
new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), analyzerPerField);
@@ -62,6 +69,63 @@ public class TestPerFieldAnalyzerWrapper
}
}
+ public void testReuseWrapped() throws Exception {
+ final String text = "Qwerty";
+
+ final Analyzer specialAnalyzer = new SimpleAnalyzer(TEST_VERSION_CURRENT);
+ final Analyzer defaultAnalyzer = new WhitespaceAnalyzer(TEST_VERSION_CURRENT);
+
+ TokenStream ts1, ts2, ts3, ts4;
+
+ final PerFieldAnalyzerWrapper wrapper1 = new PerFieldAnalyzerWrapper(defaultAnalyzer,
+ Collections.<String,Analyzer>singletonMap("special", specialAnalyzer));
+
+ // test that the PerFieldWrapper returns the same instance as original Analyzer:
+ ts1 = defaultAnalyzer.tokenStream("something", text);
+ ts2 = wrapper1.tokenStream("something", text);
+ assertSame(ts1, ts2);
+
+ ts1 = specialAnalyzer.tokenStream("special", text);
+ ts2 = wrapper1.tokenStream("special", text);
+ assertSame(ts1, ts2);
+
+ // Wrap with another wrapper, which does *not* extend DelegatingAnalyzerWrapper:
+ final AnalyzerWrapper wrapper2 = new AnalyzerWrapper(wrapper1.getReuseStrategy()) {
+ @Override
+ protected Analyzer getWrappedAnalyzer(String fieldName) {
+ return wrapper1;
+ }
+
+ @Override
+ protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
+ try {
+ assertNotSame(specialAnalyzer.tokenStream("special", text), components.getTokenStream());
+ } catch (IOException e) {
+ Rethrow.rethrow(e);
+ }
+ TokenFilter filter = new ASCIIFoldingFilter(components.getTokenStream());
+ return new TokenStreamComponents(components.getTokenizer(), filter);
+ }
+ };
+ ts3 = wrapper2.tokenStream("special", text);
+ assertNotSame(ts1, ts3);
+ assertTrue(ts3 instanceof ASCIIFoldingFilter);
+ // check that cache did not get corrumpted:
+ ts2 = wrapper1.tokenStream("special", text);
+ assertSame(ts1, ts2);
+
+ // Wrap PerField with another PerField. In that case all TokenStreams returned must be the same:
+ final PerFieldAnalyzerWrapper wrapper3 = new PerFieldAnalyzerWrapper(wrapper1,
+ Collections.<String,Analyzer>singletonMap("moreSpecial", specialAnalyzer));
+ ts1 = specialAnalyzer.tokenStream("special", text);
+ ts2 = wrapper3.tokenStream("special", text);
+ assertSame(ts1, ts2);
+ ts3 = specialAnalyzer.tokenStream("moreSpecial", text);
+ ts4 = wrapper3.tokenStream("moreSpecial", text);
+ assertSame(ts3, ts4);
+ assertSame(ts2, ts3);
+ }
+
public void testCharFilters() throws Exception {
Analyzer a = new Analyzer() {
@Override
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java Sat Jul 5 11:32:31 2014
@@ -33,18 +33,6 @@ import java.io.Reader;
public abstract class AnalyzerWrapper extends Analyzer {
/**
- * Creates a new AnalyzerWrapper. Since the {@link Analyzer.ReuseStrategy} of
- * the wrapped Analyzers are unknown, {@link #PER_FIELD_REUSE_STRATEGY} is assumed.
- * @deprecated Use {@link #AnalyzerWrapper(Analyzer.ReuseStrategy)}
- * and specify a valid {@link Analyzer.ReuseStrategy}, probably retrieved from the
- * wrapped analyzer using {@link #getReuseStrategy()}.
- */
- @Deprecated
- protected AnalyzerWrapper() {
- this(PER_FIELD_REUSE_STRATEGY);
- }
-
- /**
* Creates a new AnalyzerWrapper with the given reuse strategy.
* <p>If you want to wrap a single delegate Analyzer you can probably
* reuse its strategy when instantiating this subclass:
Added: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/DelegatingAnalyzerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/DelegatingAnalyzerWrapper.java?rev=1607998&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/DelegatingAnalyzerWrapper.java (added)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/DelegatingAnalyzerWrapper.java Sat Jul 5 11:32:31 2014
@@ -0,0 +1,92 @@
+package org.apache.lucene.analysis;
+
+import java.io.Reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * An analyzer wrapper, that doesn't allow to wrap components or readers.
+ * By disallowing it, it means that the thread local resources can be delegated
+ * to the delegate analyzer, and not also be allocated on this analyzer.
+ * This wrapper class is the base class of all analyzers that just delegate to
+ * another analyzer, e.g. per field name.
+ *
+ * <p>This solves the problem of per field analyzer wrapper, where it also
+ * maintains a thread local per field token stream components, while it can
+ * safely delegate those and not also hold these data structures, which can
+ * become expensive memory wise.
+ *
+ * <p><b>Please note:</b> This analyzer uses a private {@link Analyzer.ReuseStrategy},
+ * which is returned by {@link #getReuseStrategy()}. This strategy is used when
+ * delegating. If you wrap this analyzer again and reuse this strategy, no
+ * delegation is done and the given fallback is used.
+ */
+public abstract class DelegatingAnalyzerWrapper extends AnalyzerWrapper {
+
+ /**
+ * Constructor.
+ * @param fallbackStrategy is the strategy to use if delegation is not possible
+ * This is to support the common pattern:
+ * {@code new OtherWrapper(thisWrapper.getReuseStrategy())}
+ */
+ protected DelegatingAnalyzerWrapper(ReuseStrategy fallbackStrategy) {
+ super(new DelegatingReuseStrategy(fallbackStrategy));
+ // häckidy-hick-hack, because we cannot call super() with a reference to "this":
+ ((DelegatingReuseStrategy) getReuseStrategy()).wrapper = this;
+ }
+
+ @Override
+ protected final TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
+ return super.wrapComponents(fieldName, components);
+ }
+
+ @Override
+ protected final Reader wrapReader(String fieldName, Reader reader) {
+ return super.wrapReader(fieldName, reader);
+ }
+
+ private static final class DelegatingReuseStrategy extends ReuseStrategy {
+ DelegatingAnalyzerWrapper wrapper;
+ private final ReuseStrategy fallbackStrategy;
+
+ DelegatingReuseStrategy(ReuseStrategy fallbackStrategy) {
+ this.fallbackStrategy = fallbackStrategy;
+ }
+
+ @Override
+ public TokenStreamComponents getReusableComponents(Analyzer analyzer, String fieldName) {
+ if (analyzer == wrapper) {
+ final Analyzer wrappedAnalyzer = wrapper.getWrappedAnalyzer(fieldName);
+ return wrappedAnalyzer.getReuseStrategy().getReusableComponents(wrappedAnalyzer, fieldName);
+ } else {
+ return fallbackStrategy.getReusableComponents(analyzer, fieldName);
+ }
+ }
+
+ @Override
+ public void setReusableComponents(Analyzer analyzer, String fieldName, TokenStreamComponents components) {
+ if (analyzer == wrapper) {
+ final Analyzer wrappedAnalyzer = wrapper.getWrappedAnalyzer(fieldName);
+ wrappedAnalyzer.getReuseStrategy().setReusableComponents(wrappedAnalyzer, fieldName, components);
+ } else {
+ fallbackStrategy.setReusableComponents(analyzer, fieldName, components);
+ }
+ }
+ };
+
+}
\ No newline at end of file
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Sat Jul 5 11:32:31 2014
@@ -274,11 +274,6 @@ public class TestMockAnalyzer extends Ba
}
@Override
- protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
- return components;
- }
-
- @Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
return delegate;
}
@@ -292,7 +287,7 @@ public class TestMockAnalyzer extends Ba
final int positionGap = random().nextInt(1000);
final int offsetGap = random().nextInt(1000);
final Analyzer delegate = new MockAnalyzer(random());
- final Analyzer a = new AnalyzerWrapper(delegate.getReuseStrategy()) {
+ final Analyzer a = new DelegatingAnalyzerWrapper(delegate.getReuseStrategy()) {
@Override
protected Analyzer getWrappedAnalyzer(String fieldName) {
return delegate;
Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java Sat Jul 5 11:32:31 2014
@@ -22,8 +22,8 @@ import java.util.Set;
import java.util.TreeMap;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
@@ -606,7 +606,7 @@ public class FastVectorHighlighterTest e
fieldAnalyzers.put( "field_tripples", new MockAnalyzer( random(), new CharacterRunAutomaton( new RegExp("...").toAutomaton() ), true ) );
fieldAnalyzers.put( "field_sliced", fieldAnalyzers.get( "field" ) );
fieldAnalyzers.put( "field_der_red", fieldAnalyzers.get( "field" ) ); // This is required even though we provide a token stream
- Analyzer analyzer = new AnalyzerWrapper() {
+ Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {
public Analyzer getWrappedAnalyzer(String fieldName) {
return fieldAnalyzers.get( fieldName );
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/IndexSchema.java Sat Jul 5 11:32:31 2014
@@ -18,8 +18,7 @@
package org.apache.solr.schema;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.AnalyzerWrapper;
-import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.analysis.DelegatingAnalyzerWrapper;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
@@ -397,7 +396,7 @@ public class IndexSchema {
return false;
}
- private class SolrIndexAnalyzer extends AnalyzerWrapper {
+ private class SolrIndexAnalyzer extends DelegatingAnalyzerWrapper {
protected final HashMap<String, Analyzer> analyzers;
SolrIndexAnalyzer() {
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java?rev=1607998&r1=1607997&r2=1607998&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java Sat Jul 5 11:32:31 2014
@@ -61,6 +61,7 @@ public class PreAnalyzedField extends Fi
private PreAnalyzedParser parser;
+ private Analyzer analyzer;
@Override
public void init(IndexSchema schema, Map<String, String> args) {
@@ -87,19 +88,19 @@ public class PreAnalyzedField extends Fi
}
args.remove(PARSER_IMPL);
}
- }
-
- @Override
- public Analyzer getIndexAnalyzer() {
- return new SolrAnalyzer() {
-
+ // create Analyzer instance for reuse:
+ analyzer = new SolrAnalyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
return new TokenStreamComponents(new PreAnalyzedTokenizer(parser));
}
-
};
}
+
+ @Override
+ public Analyzer getIndexAnalyzer() {
+ return analyzer;
+ }
@Override
public Analyzer getQueryAnalyzer() {