You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by nk...@apache.org on 2016/02/08 23:36:07 UTC
[32/50] [abbrv] lucene-solr git commit: SOLR-8460: /analysis/field
could throw exceptions for custom attributes.
SOLR-8460: /analysis/field could throw exceptions for custom attributes.
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene_solr_5_4@1724177 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3c173f1f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3c173f1f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3c173f1f
Branch: refs/heads/branch_5_4
Commit: 3c173f1f35823e01c8d5480d75739fbdd425de24
Parents: a83ebdc
Author: Adrien Grand <jp...@apache.org>
Authored: Tue Jan 12 08:58:31 2016 +0000
Committer: Adrien Grand <jp...@apache.org>
Committed: Tue Jan 12 08:58:31 2016 +0000
----------------------------------------------------------------------
solr/CHANGES.txt | 7 ++
.../handler/AnalysisRequestHandlerBase.java | 38 +++++--
.../handler/FieldAnalysisRequestHandler.java | 4 +-
.../FieldAnalysisRequestHandlerTest.java | 112 +++++++++++++++++++
4 files changed, 147 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c173f1f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 93ab6f0..85b9d35 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -460,6 +460,13 @@ Other Changes
* SOLR-8363: Fix check-example-lucene-match-version Ant task and addVersion.py script to
check and update luceneMatchVersion under solr/example/ configs as well logic. (Varun Thacker)
+================== 5.3.2 ==================
+
+Bug Fixes
+----------------------
+
+* SOLR-8460: /analysis/field could throw exceptions for custom attributes. (David Smiley, Uwe Schindler)
+
================== 5.3.1 ==================
Bug Fixes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c173f1f/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
index 09d1324..dd3f9e5 100644
--- a/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
@@ -129,16 +129,17 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
- ListBasedTokenStream listBasedTokenStream = new ListBasedTokenStream(tokens);
+ ListBasedTokenStream listBasedTokenStream = new ListBasedTokenStream(tokenStream, tokens);
for (TokenFilterFactory tokenFilterFactory : filtfacs) {
for (final AttributeSource tok : tokens) {
tok.getAttribute(TokenTrackingAttribute.class).freezeStage();
}
+ // overwrite the vars "tokenStream", "tokens", and "listBasedTokenStream"
tokenStream = tokenFilterFactory.create(listBasedTokenStream);
tokens = analyzeTokenStream(tokenStream);
namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
- listBasedTokenStream = new ListBasedTokenStream(tokens);
+ listBasedTokenStream = new ListBasedTokenStream(listBasedTokenStream, tokens);
}
return namedList;
@@ -190,7 +191,7 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
trackerAtt.setActPosition(position);
tokens.add(tokenStream.cloneAttributes());
}
- tokenStream.end();
+ tokenStream.end(); // TODO should we capture?
} catch (IOException ioe) {
throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
} finally {
@@ -318,7 +319,6 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
}
// ================================================= Inner classes =================================================
-
/**
* TokenStream that iterates over a list of pre-existing Tokens
* @lucene.internal
@@ -330,10 +330,19 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
/**
* Creates a new ListBasedTokenStream which uses the given tokens as its token source.
*
+ * @param attributeSource source of the attribute factory and attribute impls
* @param tokens Source of tokens to be used
*/
- ListBasedTokenStream(List<AttributeSource> tokens) {
+ ListBasedTokenStream(AttributeSource attributeSource, List<AttributeSource> tokens) {
+ super(attributeSource.getAttributeFactory());
this.tokens = tokens;
+ // Make sure all the attributes of the source are here too
+ addAttributes(attributeSource);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
tokenIterator = tokens.iterator();
}
@@ -342,9 +351,9 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
if (tokenIterator.hasNext()) {
clearAttributes();
AttributeSource next = tokenIterator.next();
- Iterator<Class<? extends Attribute>> atts = next.getAttributeClassesIterator();
- while (atts.hasNext()) // make sure all att impls in the token exist here
- addAttribute(atts.next());
+
+ addAttributes(next); // just in case there were delayed attribute additions
+
next.copyTo(this);
return true;
} else {
@@ -352,10 +361,15 @@ public abstract class AnalysisRequestHandlerBase extends RequestHandlerBase {
}
}
- @Override
- public void reset() throws IOException {
- super.reset();
- tokenIterator = tokens.iterator();
+
+ protected void addAttributes(AttributeSource attributeSource) {
+ // note: ideally we wouldn't call addAttributeImpl which is marked internal. But nonetheless it's possible
+ // this method is used by some custom attributes, especially since Solr doesn't provide a way to customize the
+ // AttributeFactory which is the recommended way to choose which classes implement which attributes.
+ Iterator<AttributeImpl> atts = attributeSource.getAttributeImplsIterator();
+ while (atts.hasNext()) {
+ addAttributeImpl(atts.next()); // adds both impl & interfaces
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c173f1f/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
index 1b6e21a..7ec3aac 100644
--- a/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
@@ -207,8 +207,8 @@ public class FieldAnalysisRequestHandler extends AnalysisRequestHandlerBase {
*
* @return NamedList containing the tokens produced by the analyzers of the given field, separated into an index and
* a query group
- */
- private NamedList<NamedList> analyzeValues(FieldAnalysisRequest analysisRequest, FieldType fieldType, String fieldName) {
+ */ // package access for testing
+ NamedList<NamedList> analyzeValues(FieldAnalysisRequest analysisRequest, FieldType fieldType, String fieldName) {
final String queryValue = analysisRequest.getQuery();
final Set<BytesRef> termsToMatch = (queryValue != null && analysisRequest.isShowMatch())
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3c173f1f/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
index e8c8140..496d3a9 100644
--- a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
@@ -17,8 +17,22 @@
package org.apache.solr.handler;
+import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttributeImpl;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeFactory;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+import org.apache.solr.analysis.TokenizerChain;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.AnalysisParams;
import org.apache.solr.common.params.CommonParams;
@@ -27,10 +41,14 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.client.solrj.request.FieldAnalysisRequest;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.CustomAnalyzerStrField;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.TextField;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -433,4 +451,98 @@ public class FieldAnalysisRequestHandlerTest extends AnalysisRequestHandlerTestB
Collections.sort(vals);
assertEquals( "[s, s7, s7w, s7w1+, s9, s9v, s9v2+, sp, spp, spp5+, sv, svk, svk6+]", vals.toString() );
}
+
+ @Test //See SOLR-8460
+ public void testCustomAttribute() throws Exception {
+ FieldAnalysisRequest request = new FieldAnalysisRequest();
+ request.addFieldType("skutype1");
+ request.setFieldValue("hi, 3456-12 a Test");
+ request.setShowMatch(false);
+ FieldType fieldType = new TextField();
+ Analyzer analyzer = new TokenizerChain(
+ new TokenizerFactory(Collections.<String, String>emptyMap()) {
+ @Override
+ public Tokenizer create(AttributeFactory factory) {
+ return new CustomTokenizer(factory);
+ }
+ },
+ new TokenFilterFactory[] {
+ new TokenFilterFactory(Collections.<String, String>emptyMap()) {
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new CustomTokenFilter(input);
+ }
+ }
+ }
+ );
+ fieldType.setIndexAnalyzer(analyzer);
+
+ NamedList<NamedList> result = handler.analyzeValues(request, fieldType, "fieldNameUnused");
+ // just test that we see "900" in the flags attribute here
+ List<NamedList> tokenInfoList = (List<NamedList>) result.findRecursive("index", CustomTokenFilter.class.getName());
+ // '1' from CustomTokenFilter plus 900 from CustomFlagsAttributeImpl.
+ assertEquals(901, tokenInfoList.get(0).get("org.apache.lucene.analysis.tokenattributes.FlagsAttribute#flags"));
+ }
+
+ /** A custom impl of a standard attribute impl; test this instance is used. */
+ public class CustomFlagsAttributeImpl extends FlagsAttributeImpl {
+ @Override
+ public void setFlags(int flags) {
+ super.setFlags(900 + flags);//silly modification
+ }
+ }
+
+ private class CustomTokenizer extends Tokenizer {
+ CharTermAttribute charAtt;
+ FlagsAttribute customAtt;
+ boolean sentOneToken;
+
+ public CustomTokenizer(AttributeFactory factory) {
+ super(factory);
+ addAttributeImpl(new CustomFlagsAttributeImpl());
+ charAtt = addAttribute(CharTermAttribute.class);
+ customAtt = addAttribute(FlagsAttribute.class);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ sentOneToken = false;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (sentOneToken) {
+ return false;
+ }
+ sentOneToken = true;
+ clearAttributes();
+ charAtt.append("firstToken");
+ return true;
+ }
+ }
+
+ private class CustomTokenFilter extends TokenFilter {
+ FlagsAttribute flagAtt;
+
+ public CustomTokenFilter(TokenStream input) {
+ super(input);
+ flagAtt = getAttribute(FlagsAttribute.class);
+ if (flagAtt == null) {
+ throw new IllegalStateException("FlagsAttribute should have been added already");
+ }
+ if (!(flagAtt instanceof CustomFlagsAttributeImpl)) {
+ throw new IllegalStateException("FlagsAttribute should be our custom " + CustomFlagsAttributeImpl.class);
+ }
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ flagAtt.setFlags(1);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
}