You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2015/07/21 19:20:33 UTC
svn commit: r1692174 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/analysis/
solr/core/src/java/org/apache/solr/handler/
solr/core/src/java/org/apache/solr/handler/admin/
solr/core/src/java/org/apache/solr/...
Author: hossman
Date: Tue Jul 21 17:20:33 2015
New Revision: 1692174
URL: http://svn.apache.org/r1692174
Log:
SOLR-7765: Hardened the behavior of TokenizerChain when null arguments are used in constructor. This prevents NPEs in some code paths. (merge r1692170)
Added:
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/collection1/conf/schema-null-charfilters-analyzer.xml
- copied unchanged from r1692170, lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/schema-null-charfilters-analyzer.xml
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java
- copied, changed from r1692170, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Tue Jul 21 17:20:33 2015
@@ -179,6 +179,9 @@ Bug Fixes
* SOLR-7810: map-reduce contrib script to set classpath for convenience refers to example
rather than server. (Mark Miller)
+* SOLR-7765: Hardened the behavior of TokenizerChain when null arguments are used in constructor.
+ This prevents NPEs in some code paths. (Konstantin Gribov, hossman)
+
Optimizations
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java Tue Jul 21 17:20:33 2015
@@ -29,22 +29,47 @@ import java.io.Reader;
* create a TokenStream.
*/
public final class TokenizerChain extends SolrAnalyzer {
+ private static final CharFilterFactory[] EMPTY_CHAR_FITLERS = new CharFilterFactory[0];
+ private static final TokenFilterFactory[] EMPTY_TOKEN_FITLERS = new TokenFilterFactory[0];
+
final private CharFilterFactory[] charFilters;
final private TokenizerFactory tokenizer;
final private TokenFilterFactory[] filters;
+ /**
+ * Creates a new TokenizerChain w/o any CharFilterFactories.
+ *
+ * @param tokenizer Factory for the Tokenizer to use, must not be null.
+ * @param filters Factories for the TokenFilters to use - if null, will be treated as if empty.
+ */
public TokenizerChain(TokenizerFactory tokenizer, TokenFilterFactory[] filters) {
this(null,tokenizer,filters);
}
+ /**
+ * Creates a new TokenizerChain.
+ *
+ * @param charFilters Factories for the CharFilters to use, if any - if null, will be treated as if empty.
+ * @param tokenizer Factory for the Tokenizer to use, must not be null.
+ * @param filters Factories for the TokenFilters to use if any- if null, will be treated as if empty.
+ */
public TokenizerChain(CharFilterFactory[] charFilters, TokenizerFactory tokenizer, TokenFilterFactory[] filters) {
+ charFilters = null == charFilters ? EMPTY_CHAR_FITLERS : charFilters;
+ filters = null == filters ? EMPTY_TOKEN_FITLERS : filters;
+ if (null == tokenizer) {
+ throw new NullPointerException("TokenizerFactory must not be null");
+ }
+
this.charFilters = charFilters;
this.tokenizer = tokenizer;
this.filters = filters;
}
+ /** @return array of CharFilterFactories, may be empty but never null */
public CharFilterFactory[] getCharFilterFactories() { return charFilters; }
+ /** @return the TokenizerFactory in use, will never be null */
public TokenizerFactory getTokenizerFactory() { return tokenizer; }
+ /** @return array of TokenFilterFactories, may be empty but never null */
public TokenFilterFactory[] getTokenFilterFactories() { return filters; }
@Override
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java Tue Jul 21 17:20:33 2015
@@ -104,7 +104,7 @@ public abstract class AnalysisRequestHan
NamedList<Object> namedList = new NamedList<>();
- if( cfiltfacs != null ){
+ if (0 < cfiltfacs.length) {
String source = value;
for(CharFilterFactory cfiltfac : cfiltfacs ){
Reader reader = new StringReader(source);
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Tue Jul 21 17:20:33 2015
@@ -497,15 +497,15 @@ public class LukeRequestHandler extends
TokenizerChain tchain = (TokenizerChain)analyzer;
CharFilterFactory[] cfiltfacs = tchain.getCharFilterFactories();
- SimpleOrderedMap<Map<String, Object>> cfilters = new SimpleOrderedMap<>();
- for (CharFilterFactory cfiltfac : cfiltfacs) {
- Map<String, Object> tok = new HashMap<>();
- String className = cfiltfac.getClass().getName();
- tok.put("className", className);
- tok.put("args", cfiltfac.getOriginalArgs());
- cfilters.add(className.substring(className.lastIndexOf('.')+1), tok);
- }
- if (cfilters.size() > 0) {
+ if (0 < cfiltfacs.length) {
+ SimpleOrderedMap<Map<String, Object>> cfilters = new SimpleOrderedMap<>();
+ for (CharFilterFactory cfiltfac : cfiltfacs) {
+ Map<String, Object> tok = new HashMap<>();
+ String className = cfiltfac.getClass().getName();
+ tok.put("className", className);
+ tok.put("args", cfiltfac.getOriginalArgs());
+ cfilters.add(className.substring(className.lastIndexOf('.')+1), tok);
+ }
aninfo.add("charFilters", cfilters);
}
@@ -516,15 +516,15 @@ public class LukeRequestHandler extends
aninfo.add("tokenizer", tokenizer);
TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
- SimpleOrderedMap<Map<String, Object>> filters = new SimpleOrderedMap<>();
- for (TokenFilterFactory filtfac : filtfacs) {
- Map<String, Object> tok = new HashMap<>();
- String className = filtfac.getClass().getName();
- tok.put("className", className);
- tok.put("args", filtfac.getOriginalArgs());
- filters.add(className.substring(className.lastIndexOf('.')+1), tok);
- }
- if (filters.size() > 0) {
+ if (0 < filtfacs.length) {
+ SimpleOrderedMap<Map<String, Object>> filters = new SimpleOrderedMap<>();
+ for (TokenFilterFactory filtfac : filtfacs) {
+ Map<String, Object> tok = new HashMap<>();
+ String className = filtfac.getClass().getName();
+ tok.put("className", className);
+ tok.put("args", filtfac.getOriginalArgs());
+ filters.add(className.substring(className.lastIndexOf('.')+1), tok);
+ }
aninfo.add("filters", filters);
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java Tue Jul 21 17:20:33 2015
@@ -883,7 +883,7 @@ public abstract class FieldType extends
Map<String,String> factoryArgs;
TokenizerChain tokenizerChain = (TokenizerChain)analyzer;
CharFilterFactory[] charFilterFactories = tokenizerChain.getCharFilterFactories();
- if (null != charFilterFactories && charFilterFactories.length > 0) {
+ if (0 < charFilterFactories.length) {
List<SimpleOrderedMap<Object>> charFilterProps = new ArrayList<>();
for (CharFilterFactory charFilterFactory : charFilterFactories) {
SimpleOrderedMap<Object> props = new SimpleOrderedMap<>();
@@ -927,7 +927,7 @@ public abstract class FieldType extends
analyzerProps.add(TOKENIZER, tokenizerProps);
TokenFilterFactory[] filterFactories = tokenizerChain.getTokenFilterFactories();
- if (null != filterFactories && filterFactories.length > 0) {
+ if (0 < filterFactories.length) {
List<SimpleOrderedMap<Object>> filterProps = new ArrayList<>();
for (TokenFilterFactory filterFactory : filterFactories) {
SimpleOrderedMap<Object> props = new SimpleOrderedMap<>();
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldTypePluginLoader.java Tue Jul 21 17:20:33 2015
@@ -175,10 +175,8 @@ public final class FieldTypePluginLoader
MultiTermChainBuilder builder = new MultiTermChainBuilder();
CharFilterFactory[] charFactories = tc.getCharFilterFactories();
- if (charFactories != null) {
- for (CharFilterFactory fact : charFactories) {
- builder.add(fact);
- }
+ for (CharFilterFactory fact : charFactories) {
+ builder.add(fact);
}
builder.add(tc.getTokenizerFactory());
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchema.java Tue Jul 21 17:20:33 2015
@@ -1285,20 +1285,18 @@ public final class ManagedIndexSchema ex
*/
protected void informResourceLoaderAwareObjectsInChain(TokenizerChain chain) {
CharFilterFactory[] charFilters = chain.getCharFilterFactories();
- if (charFilters != null) {
- for (CharFilterFactory next : charFilters) {
- if (next instanceof ResourceLoaderAware) {
- try {
- ((ResourceLoaderAware) next).inform(loader);
- } catch (IOException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
- }
+ for (CharFilterFactory next : charFilters) {
+ if (next instanceof ResourceLoaderAware) {
+ try {
+ ((ResourceLoaderAware) next).inform(loader);
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
}
- }
}
TokenizerFactory tokenizerFactory = chain.getTokenizerFactory();
- if (tokenizerFactory != null && tokenizerFactory instanceof ResourceLoaderAware) {
+ if (tokenizerFactory instanceof ResourceLoaderAware) {
try {
((ResourceLoaderAware) tokenizerFactory).inform(loader);
} catch (IOException e) {
@@ -1307,14 +1305,12 @@ public final class ManagedIndexSchema ex
}
TokenFilterFactory[] filters = chain.getTokenFilterFactories();
- if (filters != null) {
- for (TokenFilterFactory next : filters) {
- if (next instanceof ResourceLoaderAware) {
- try {
- ((ResourceLoaderAware) next).inform(loader);
- } catch (IOException e) {
- throw new SolrException(ErrorCode.SERVER_ERROR, e);
- }
+ for (TokenFilterFactory next : filters) {
+ if (next instanceof ResourceLoaderAware) {
+ try {
+ ((ResourceLoaderAware) next).inform(loader);
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
}
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/handler/admin/LukeRequestHandlerTest.java Tue Jul 21 17:20:33 2015
@@ -20,6 +20,7 @@ package org.apache.solr.handler.admin;
import org.apache.solr.common.luke.FieldFlag;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.CustomAnalyzerStrField; // jdoc
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.TestHarness;
import org.junit.Before;
@@ -198,6 +199,27 @@ public class LukeRequestHandlerTest exte
}
}
+ /** @see CustomAnalyzerStrField */
+ public void testNullFactories() throws Exception {
+ deleteCore();
+ initCore("solrconfig.xml", "schema-null-charfilters-analyzer.xml");
+
+ try {
+ assertQ(req("qt", "/admin/luke", "show", "schema")
+ , "//lst[@name='custom_tc_string']/lst[@name='indexAnalyzer']"
+ , "//lst[@name='custom_tc_string']/lst[@name='queryAnalyzer']"
+ , "0=count(//lst[@name='custom_tc_string']/lst[@name='indexAnalyzer']/lst[@name='filters'])"
+ , "0=count(//lst[@name='custom_tc_string']/lst[@name='queryAnalyzer']/lst[@name='filters'])"
+ , "0=count(//lst[@name='custom_tc_string']/lst[@name='indexAnalyzer']/lst[@name='charFilters'])"
+ , "0=count(//lst[@name='custom_tc_string']/lst[@name='queryAnalyzer']/lst[@name='charFilters'])"
+ );
+ } finally {
+ // Put back the configuration expected by the rest of the tests in this suite
+ deleteCore();
+ initCore("solrconfig.xml", "schema12.xml");
+ }
+ }
+
public void testCopyFieldLists() throws Exception {
SolrQueryRequest req = req("qt", "/admin/luke", "show", "schema");
Copied: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java (from r1692170, lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java?p2=lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java&p1=lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java&r1=1692170&r2=1692174&rev=1692174&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/CustomAnalyzerStrField.java Tue Jul 21 17:20:33 2015
@@ -44,13 +44,13 @@ public class CustomAnalyzerStrField exte
// two arg constructor
Analyzer a2 = new TokenizerChain
- (new KeywordTokenizerFactory(new HashMap<>()),
+ (new KeywordTokenizerFactory(new HashMap<String,String>()),
r.nextBoolean() ? null : new TokenFilterFactory[0]);
// three arg constructor
Analyzer a3 = new TokenizerChain
(r.nextBoolean() ? null : new CharFilterFactory[0],
- new KeywordTokenizerFactory(new HashMap<>()),
+ new KeywordTokenizerFactory(new HashMap<String,String>()),
r.nextBoolean() ? null : new TokenFilterFactory[0]);
if (r.nextBoolean()) {
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java?rev=1692174&r1=1692173&r2=1692174&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/schema/MultiTermTest.java Tue Jul 21 17:20:33 2015
@@ -88,7 +88,7 @@ public class MultiTermTest extends SolrT
assertTrue((factory instanceof ASCIIFoldingFilterFactory) || (factory instanceof LowerCaseFilterFactory));
}
- assertTrue(tc.getCharFilterFactories() == null);
+ assertTrue(tc.getCharFilterFactories().length == 0);
}
}