You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/19 17:52:57 UTC
svn commit: r1559524 - in /lucene/dev/branches/lucene5376/lucene:
analysis/common/src/java/org/apache/lucene/analysis/util/
server/src/java/org/apache/lucene/server/
server/src/java/org/apache/lucene/server/handlers/
server/src/java/org/apache/lucene/s...
Author: mikemccand
Date: Sun Jan 19 16:52:57 2014
New Revision: 1559524
URL: http://svn.apache.org/r1559524
Log:
LUCENE-5376: cutover to analysis factories to create tokenizer and token filters; still need char filters & whole analyzers
Added:
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java (with props)
Modified:
lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java
lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java
lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java Sun Jan 19 16:52:57 2014
@@ -17,13 +17,14 @@ package org.apache.lucene.analysis.util;
* limitations under the License.
*/
+import java.lang.reflect.InvocationTargetException;
import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.Locale;
import java.util.Map;
-import java.util.LinkedHashMap;
-import java.util.Set;
import java.util.ServiceConfigurationError;
+import java.util.Set;
import org.apache.lucene.util.SPIClassIterator;
@@ -108,6 +109,18 @@ final class AnalysisSPILoader<S extends
final Class<? extends S> service = lookupClass(name);
try {
return service.getConstructor(Map.class).newInstance(args);
+ } catch (InvocationTargetException ite) {
+ // nocommit ... trying to throw the "original" IAE,
+ // but is this correct/safe?
+ Throwable t = ite;
+ while (t.getCause() != null) {
+ if (t.getCause() instanceof IllegalArgumentException) {
+ throw (IllegalArgumentException) t.getCause();
+ }
+ t = t.getCause();
+ }
+ throw new IllegalArgumentException("SPI class of type "+clazz.getName()+" with name '"+name+"' cannot be instantiated. " +
+ "This is likely due to a misconfiguration of the java class '" + service.getName() + "': ", ite);
} catch (Exception e) {
throw new IllegalArgumentException("SPI class of type "+clazz.getName()+" with name '"+name+"' cannot be instantiated. " +
"This is likely due to a misconfiguration of the java class '" + service.getName() + "': ", e);
Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java Sun Jan 19 16:52:57 2014
@@ -24,7 +24,7 @@ package org.apache.lucene.analysis.util;
* @lucene.experimental
*/
public interface MultiTermAwareComponent {
- /** Returns an analysis component to handle analysis if multi-term queries.
+ /** Returns an analysis component to handle analysis of multi-term queries.
* The returned component must be a TokenizerFactory, TokenFilterFactory or CharFilterFactory.
*/
public AbstractAnalysisFactory getMultiTermComponent();
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java Sun Jan 19 16:52:57 2014
@@ -47,6 +47,8 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.AnalyzerWrapper;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.util.FilesystemResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoader;
import org.apache.lucene.document.Document;
import org.apache.lucene.expressions.Bindings;
import org.apache.lucene.facet.FacetsConfig;
@@ -138,6 +140,9 @@ public class IndexState implements Close
/** Creates directories */
DirectoryFactory df = DirectoryFactory.get("FSDirectory");
+ /** Loads all resources for analysis components */
+ public final ResourceLoader resourceLoader;
+
/** Where all index state is saved */
public final File rootDir;
@@ -402,6 +407,16 @@ public class IndexState implements Close
this.globalState = globalState;
this.name = name;
this.rootDir = rootDir;
+ if (rootDir != null) {
+ if (!rootDir.exists()) {
+ rootDir.mkdirs();
+ }
+ this.resourceLoader = new FilesystemResourceLoader(rootDir);
+ } else {
+ // nocommit can/should we make a DirectoryResourceLoader?
+ this.resourceLoader = null;
+ }
+
this.doCreate = doCreate;
if (doCreate == false) {
initSaveLoadState();
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java Sun Jan 19 16:52:57 2014
@@ -64,7 +64,7 @@ public class AnalysisHandler extends Han
final String text = r.getString("text");
// TODO: allow passing field name in, and we use its analyzer?
- final Analyzer a = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "analyzer");
+ final Analyzer a = RegisterFieldHandler.getAnalyzer(state, r, "analyzer");
return new FinishRequest() {
@Override
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java Sun Jan 19 16:52:57 2014
@@ -173,11 +173,13 @@ public class BuildSuggestHandler extends
r = pr.r;
Analyzer indexAnalyzer;
Analyzer queryAnalyzer;
+ // nocommit allow passing a field name, and we use that
+ // field name's analyzer?
if (r.hasParam("analyzer")) {
- indexAnalyzer = queryAnalyzer = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "analyzer");
+ indexAnalyzer = queryAnalyzer = RegisterFieldHandler.getAnalyzer(state, r, "analyzer");
} else {
- indexAnalyzer = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "indexAnalyzer");
- queryAnalyzer = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "queryAnalyzer");
+ indexAnalyzer = RegisterFieldHandler.getAnalyzer(state, r, "indexAnalyzer");
+ queryAnalyzer = RegisterFieldHandler.getAnalyzer(state, r, "queryAnalyzer");
}
if (indexAnalyzer == null) {
r.fail("analyzer", "analyzer or indexAnalyzer must be specified");
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java Sun Jan 19 16:52:57 2014
@@ -70,7 +70,7 @@ public class CreateIndexHandler extends
try {
globalState.createIndex(indexName, rootDir);
} catch (IllegalArgumentException iae) {
- r.fail("invalid indexName \"" + indexName + "\": " + iae.toString());
+ r.fail("invalid indexName \"" + indexName + "\": " + iae.toString(), iae);
}
return "{}";
Added: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java?rev=1559524&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java Sun Jan 19 16:52:57 2014
@@ -0,0 +1,72 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.util.ResourceLoader;
+
+/** Holds "files" in RAM, and falls back to delegate if the
+ * resource isn't in RAM. This is used for analysis
+ * components that specify their "files" as strings in
+ * RAM. */
+
+class RAMResourceLoaderWrapper implements ResourceLoader {
+
+ private final Map<String,String> files = new HashMap<String,String>();
+ private final ResourceLoader delegate;
+
+ /** Sole constructor. */
+ public RAMResourceLoaderWrapper(ResourceLoader delegate) {
+ this.delegate = delegate;
+ }
+
+ public void add(String name, String contents) {
+ if (files.containsKey(name)) {
+ throw new IllegalArgumentException("name \"" + name + "\" was already added");
+ }
+ files.put(name, contents);
+ }
+
+ @Override
+ public InputStream openResource(String resource) throws IOException {
+ String value = files.get(resource);
+ if (value != null) {
+ return new ByteArrayInputStream(value.getBytes("UTF-8"));
+ } else if (delegate == null) {
+ throw new FileNotFoundException("resource \"" + resource + "\" is not found");
+ } else {
+ return delegate.openResource(resource);
+ }
+ }
+
+ @Override
+ public <T> T newInstance(String cname, Class<T> expectedType) {
+ return delegate.newInstance(cname, expectedType);
+ }
+
+ @Override
+ public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
+ return delegate.findClass(cname, expectedType);
+ }
+}
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java Sun Jan 19 16:52:57 2014
@@ -23,6 +23,7 @@ import java.io.StringReader;
import java.text.Collator;
import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -64,8 +65,13 @@ import org.apache.lucene.analysis.standa
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.synonym.SynonymFilter;
+import org.apache.lucene.analysis.synonym.SynonymFilterFactory;
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.collation.CollationKeyAnalyzer;
@@ -88,9 +94,11 @@ import org.apache.lucene.server.GlobalSt
import org.apache.lucene.server.IndexState;
import org.apache.lucene.server.params.*;
import org.apache.lucene.server.params.PolyType.PolyEntry;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.Version;
+import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
import net.minidev.json.JSONValue;
import com.ibm.icu.lang.UCharacter;
@@ -237,53 +245,32 @@ public class RegisterFieldHandler extend
new Param("positionIncrementGap", "How many positions to insert between separate values in a multi-valued field", new IntType(), 0),
new Param("offsetGap", "How many offsets to insert between separate values in a multi-valued field", new IntType(), 1),
new Param("tokenizer", "Tokenizer class (for a custom analysis chain).",
- new StructType(
- new Param("class",
- "Tokenizer class",
- new PolyType(Tokenizer.class,
- new PolyEntry("WhitespaceTokenizer", "A WhitespaceTokenizer is a tokenizer that divides text at whitespace (see @lucene:analyzers-common:org.apache.lucene.analysis.core.WhitespaceTokenizer)", new StructType()),
- new PolyEntry("StandardTokenizer", "A grammar-based tokenizer constructed with JFlex (see @lucene:analyzers-common:org.apache.lucene.analysis.standard.StandardTokenizer)",
- new Param("maxTokenLength", "Max length of each token", new IntType(), StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH)),
- new PolyEntry("PatternTokenizer", "This tokenizer uses regex pattern matching to construct distinct tokens for the input stream (see @lucene:analyzers-common:org.apache.lucene.analysis.pattern.PatternTokenizer)",
- new Param("pattern", "Regular expression pattern", new StringType()),
- new Param("group", "Group index for the tokens (-1 to do 'split')", new IntType(), -1)),
- new PolyEntry("ICUTokenizer", "Breaks text into words according to UAX #29: Unicode Text Segmentation (http://www.unicode.org/reports/tr29",
- new Param("rules", "Customize the tokenization per-script",
- new ListType(
- new StructType(
- new Param("script", "Script", new StringType()),
- new Param("rules", "Rules", new StringType()))))))))),
+ new OrType(new StringType(), new StructType())),
+ // nocommit somehow tap into TokenizerFactory.availableTokenizers
new Param("tokenFilters", "Optional list of TokenFilters to apply after the Tokenizer",
new ListType(
- new StructType(
- new Param("class", "TokenFilter class",
- new PolyType(TokenFilter.class,
- new PolyEntry("ArabicStemFilter", "A TokenFilter that applies ArabicStemmer to stem Arabic words. (see @lucene:analyzers-common:org.apache.lucene.analysis.ar.ArabicStemFilter)"),
- new PolyEntry("StandardFilter", "Normalizes tokens extracted with StandardTokenizer. (see @lucene:analyzers-common:org.apache.lucene.analysis.standard.StandardFilter)"),
- new PolyEntry("EnglishPossessiveFilter", "TokenFilter that removes possessives (trailing 's) from words (see @lucene:analyzers-common:org.apache.lucene.analysis.en.EnglishPossessiveFilter)"),
- new PolyEntry("PorterStemFilter", "Transforms the token stream as per the Porter stemming algorithm (see @lucene:analyzers-common:org.apache.lucene.analysis.en.PorterStemFilter)"),
- new PolyEntry("SuggestStopFilter", "Like {@link StopFilter} except it will not remove the last token if that token was not followed by some token separator.",
- new Param("stopWords", "Stop words to remove during analysis",
- new ListType(new StringType()), DEFAULT_ENGLISH_STOP_WORDS)),
- new PolyEntry("EnglishMinimalStemFilter", "A TokenFilter that applies EnglishMinimalStemmer to stem English words (see @lucene:analyzers-common:org.apache.lucene.analysis.en.EnglishMinimalStemFilter)"),
- new PolyEntry("SetKeywordMarkerFilter", "Marks terms as keywords via the KeywordAttribute (see @lucene:analyzers-common:org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter)", new Param("keyWords", "List of tokens to mark as keywords", new ListType(new StringType()))),
- new PolyEntry("StopFilter", "Removes stop words from a token stream (see @lucene:analyzers-common:org.apache.lucene.analysis.core.StopFilter)",
- new Param("stopWords", "Stop words to remove during analysis",
- new ListType(new StringType()), DEFAULT_ENGLISH_STOP_WORDS)),
- new PolyEntry("SynonymFilter", "Matches single- or multi-word synonyms and injects or replaces the match with a corresponding synonym",
- //new Param("synonymFile", "Local file to load synonyms from", new StringType()), // nocommit TODO
- new Param("ignoreCase", "True if matching should be case insensitive", new BooleanType(), true),
- new Param("analyzer", "Analyzer to use to tokenize synonym inputs", ANALYZER_TYPE_WRAP),
- new Param("synonyms", "Specify synonyms inline (instead of synonymFile)",
- new ListType(
- new StructType(
- new Param("input", "String or list of strings with input token(s) to match", new OrType(new ListType(new StringType()), new StringType())),
- // TODO: allow more than one token on the output?
- new Param("output", "Single token to replace the matched tokens with", new StringType()),
- new Param("replace", "True if the input tokens should be replaced with the output token; false if the input tokens should be preserved and the output token overlaid", new BooleanType(), true))))),
- new PolyEntry("LowerCaseFilter", "Normalizes token text to lower case (see @lucene:analyzers-common:org.apache.lucene.analysis.core.LowerCaseFilter)", new StructType())))))),
+ new OrType(new StringType(), new StructType(new Param("class", "TokenFilter class name", new StringType()))))),
MATCH_VERSION_PARAM);
+ static StructType SYNONYM_FILTER_TYPE = new StructType(
+ new Param("ignoreCase", "True if matching should be case insensitive", new BooleanType(), true),
+ new Param("analyzer", "Analyzer to use to tokenize synonym inputs", ANALYZER_TYPE_WRAP),
+ new Param("synonyms", "Synonyms",
+ new ListType(
+ new StructType(
+ new Param("input", "String or list of strings with input token(s) to match", new OrType(new ListType(new StringType()), new StringType())),
+ // TODO: allow more than one token on the output?
+ new Param("output", "Single token to replace the matched tokens with", new StringType()),
+ new Param("replace", "True if the input tokens should be replaced with the output token; false if the input tokens should be preserved and the output token overlaid", new BooleanType(), true)))));
+
+ static StructType ICU_TOKENIZER_TYPE = new StructType(
+ new Param("cjkAsWords", "??", new BooleanType(), true),
+ new Param("rules", "Customize the tokenization per-script",
+ new ListType(
+ new StructType(
+ new Param("script", "Script", new StringType()),
+ new Param("rules", "Rules", new StringType())))));
+
static {
ANALYZER_TYPE_WRAP.set(ANALYZER_TYPE);
}
@@ -415,7 +402,7 @@ public class RegisterFieldHandler extend
return new FieldDef(name, null, "virtual", null, null, null, true, null, null, null, false, null, values);
}
- private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,FieldDef> pendingFieldDefs, String name, JSONObject o) {
+ private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,FieldDef> pendingFieldDefs, String name, JSONObject o) throws IOException {
// This way f.fail reports which field name was problematic:
Request f = new Request(r, name, o, FIELD_TYPE);
@@ -613,12 +600,12 @@ public class RegisterFieldHandler extend
Analyzer indexAnalyzer;
Analyzer searchAnalyzer;
- Analyzer analyzer = getAnalyzer(state.matchVersion, f, "analyzer");
+ Analyzer analyzer = getAnalyzer(state, f, "analyzer");
if (analyzer != null) {
indexAnalyzer = searchAnalyzer = analyzer;
} else {
- indexAnalyzer = getAnalyzer(state.matchVersion, f, "indexAnalyzer");
- searchAnalyzer = getAnalyzer(state.matchVersion, f, "searchAnalyzer");
+ indexAnalyzer = getAnalyzer(state, f, "indexAnalyzer");
+ searchAnalyzer = getAnalyzer(state, f, "searchAnalyzer");
}
if (type.equals("text") && ft.indexed()) {
@@ -733,38 +720,21 @@ public class RegisterFieldHandler extend
return words;
}
- final static Pattern COMMENTS_PATTERN = Pattern.compile("#.*$", Pattern.MULTILINE);
-
- static TokenStreamComponents buildCustomAnalysisChain(Version matchVersion, Request chain) {
-
- Request t = chain.getStruct("tokenizer");
+ static TokenizerFactory buildICUTokenizerFactory(JSONObject params) {
+
+ boolean cjkAsWords;
- Request.PolyResult pr = t.getPoly("class");
+ final BreakIterator breakers[];
- // nocommit charFilters
+ if (params != null) {
+
+ // nocommit don't pass null, null:
+ Request icuRequest = new Request(null, null, params, ICU_TOKENIZER_TYPE);
+ cjkAsWords = icuRequest.getBoolean("cjkAsWords");
- Tokenizer tokenizer;
- // nocommit use analysis factories
- if (pr.name.equals("StandardTokenizer")) {
- tokenizer = new StandardTokenizer(matchVersion);
- ((StandardTokenizer) tokenizer).setMaxTokenLength(pr.r.getInt("maxTokenLength"));
- } else if (pr.name.equals("WhitespaceTokenizer")) {
- tokenizer = new WhitespaceTokenizer(matchVersion);
- } else if (pr.name.equals("PatternTokenizer")) {
- Pattern p;
- try {
- p = Pattern.compile(pr.r.getString("pattern"));
- } catch (PatternSyntaxException pse) {
- pr.r.fail("pattern", "failed to compile Pattern", pse);
- // Dead code but compiler disagrees:
- p = null;
- }
- tokenizer = new PatternTokenizer(p, pr.r.getInt("group"));
- } else if (pr.name.equals("ICUTokenizer")) {
- final BreakIterator breakers[];
- if (pr.r.hasParam("rules")) {
+ if (icuRequest.hasParam("rules")) {
breakers = new BreakIterator[UScript.CODE_LIMIT];
- for(Object o : pr.r.getList("rules")) {
+ for(Object o : icuRequest.getList("rules")) {
Request r2 = (Request) o;
String script = r2.getString("script");
String rules = r2.getString("rules");
@@ -786,9 +756,12 @@ public class RegisterFieldHandler extend
} else {
breakers = null;
}
+ } else {
+ cjkAsWords = true;
+ breakers = null;
+ }
- // nocommit: what is this doing, just passing true...
- ICUTokenizerConfig config = new DefaultICUTokenizerConfig(true) {
+ final ICUTokenizerConfig config = new DefaultICUTokenizerConfig(true) {
@Override
public BreakIterator getBreakIterator(int script) {
@@ -802,116 +775,283 @@ public class RegisterFieldHandler extend
// TODO: we could also allow codes->types mapping
};
- tokenizer = new ICUTokenizer(config);
+ return new TokenizerFactory(Collections.<String,String>emptyMap()) {
+
+ @Override
+ public Tokenizer create(AttributeFactory factory) {
+ return new ICUTokenizer(factory, config);
+ };
+ };
+ }
+
+ static TokenFilterFactory buildSynonymFilterFactory(IndexState state, Request r) throws IOException {
+
+ Analyzer a = getAnalyzer(state, r, "analyzer");
+ if (a == null) {
+ r.fail("analyzer", "analyzer is required");
+ }
+
+ final SynonymMap synonymMap = parseSynonyms(r.getList("synonyms"), a);
+
+ final boolean ignoreCase = r.getBoolean("ignoreCase");
+
+ // nocommit expand? dedup?
+ return new TokenFilterFactory(Collections.<String,String>emptyMap()) {
+ @Override
+ public TokenFilter create(TokenStream input) {
+ return new SynonymFilter(input, synonymMap, ignoreCase);
+ }
+ };
+ }
+
+ final static Pattern COMMENTS_PATTERN = Pattern.compile("#.*$", Pattern.MULTILINE);
+
+ static Analyzer buildCustomAnalyzer(IndexState state, Version matchVersion, Request chain) throws IOException {
+
+ // nocommit what is MultiTermAwareComponent?
+
+ // nocommit charFilters
+
+ // Build TokenizerFactory:
+ String className;
+ JSONObject t;
+ if (chain.isString("tokenizer")) {
+ className = chain.getString("tokenizer");
+ t = null;
} else {
- // BUG
- tokenizer = null;
- assert false;
+ Object o = chain.getAndRemoveRaw("tokenizer");
+ if ((o instanceof JSONObject) == false) {
+ chain.fail("tokenizer", "must be string or struct; got: " + o.getClass());
+ }
+ t = (JSONObject) o;
+
+ o = t.get("class");
+ if ((o instanceof String) == false) {
+ chain.fail("tokenizer", "class must be string; got: " + o.getClass());
+ }
+
+ className = (String) o;
}
- TokenStream last = tokenizer;
+ TokenizerFactory tokenizerFactory;
+ if (className.toLowerCase(Locale.ROOT).equals("icu")) {
+ tokenizerFactory = buildICUTokenizerFactory(t);
+ } else {
+ Map<String,String> factoryArgs = new HashMap<String,String>();
+ // nocommit how to allow the SPI name and separately
+ // also a fully qualified class name ...
+ factoryArgs.put("class", className);
+ factoryArgs.put("luceneMatchVersion", matchVersion.toString());
+ if (t != null) {
+ for(Map.Entry<String,Object> ent : t.entrySet()) {
+ factoryArgs.put(ent.getKey(), ent.getValue().toString());
+ }
+ }
+
+ try {
+ tokenizerFactory = TokenizerFactory.forName(className, factoryArgs);
+ } catch (IllegalArgumentException iae) {
+ chain.fail("tokenizer", "failed to create TokenizerFactory for class \"" + className + "\": " + iae, iae);
+
+ // Dead code but compiler disagrees:
+ tokenizerFactory = null;
+ }
+
+ if (tokenizerFactory instanceof ResourceLoaderAware) {
+ ((ResourceLoaderAware) tokenizerFactory).inform(state.resourceLoader);
+ }
+ }
+
+ // Build TokenFilters
+ List<TokenFilterFactory> tokenFilterFactories;
if (chain.hasParam("tokenFilters")) {
+ tokenFilterFactories = new ArrayList<TokenFilterFactory>();
for(Object o : chain.getList("tokenFilters")) {
- Request sub = (Request) o;
- pr = sub.getPoly("class");
- // nocommit use analysis factories
- if (pr.name.equals("StandardFilter")) {
- last = new StandardFilter(matchVersion, last);
- } else if (pr.name.equals("EnglishPossessiveFilter")) {
- last = new EnglishPossessiveFilter(matchVersion, last);
- } else if (pr.name.equals("PorterStemFilter")) {
- last = new PorterStemFilter(last);
- } else if (pr.name.equals("ArabicStemFilter")) {
- last = new ArabicStemFilter(last);
- } else if (pr.name.equals("EnglishMinimalStemFilter")) {
- last = new EnglishMinimalStemFilter(last);
- } else if (pr.name.equals("LowerCaseFilter")) {
- last = new LowerCaseFilter(matchVersion, last);
- } else if (pr.name.equals("SetKeywordMarkerFilter")) {
- CharArraySet set = new CharArraySet(matchVersion, toStringList(pr.r.getList("keyWords")), false);
- last = new SetKeywordMarkerFilter(last, set);
- } else if (pr.name.equals("StopFilter")) {
- CharArraySet set = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
- last = new StopFilter(matchVersion, last, set);
- } else if (pr.name.equals("SuggestStopFilter")) {
- CharArraySet set = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
- last = new SuggestStopFilter(last, set);
- } else if (pr.name.equals("SynonymFilter")) {
- Analyzer a = getAnalyzer(matchVersion, pr.r, "analyzer");
- if (a == null) {
- pr.r.fail("analyzer", "analyzer is required");
+ String paramName = "tokenFilters[" + tokenFilterFactories.size() + "]";
+
+ Request sub;
+ JSONObject subParams;
+ if (o instanceof String) {
+ className = (String) o;
+ sub = null;
+ subParams = null;
+ } else {
+ if ((o instanceof Request) == false) {
+ // nocommit make sure test hits this
+ chain.fail(paramName, "each filter must be string or struct; got: " + o.getClass());
}
+ sub = (Request) o;
+ subParams = sub.getRawParams();
- try {
- SynonymMap.Parser parser = new SynonymMap.Parser(true, a) {
- @Override
- public void parse(Reader in) throws IOException {
- // nocommit move parsing in here!
- };
- };
-
- CharsRef scratch = new CharsRef();
- CharsRef scratchOutput = new CharsRef();
- for(Object o2 : pr.r.getList("synonyms")) {
- Request syn = (Request) o2;
- boolean replace = syn.getBoolean("replace");
- CharsRef output = new CharsRef(syn.getString("output"));
- if (!syn.isString("input")) {
- for(Object o3 : syn.getList("input")) {
- parser.add(parser.analyze((String) o3, scratch),
- output,
- !replace);
+ className = sub.getString("class");
+ }
+
+ TokenFilterFactory tokenFilterFactory;
+
+ if (className.toLowerCase(Locale.ROOT).equals("synonym")) {
+ if (sub == null) {
+ chain.fail(paramName, "no synonyms were specified");
+ }
+ tokenFilterFactory = buildSynonymFilterFactory(state, new Request(chain, paramName, sub.getRawParams(), SYNONYM_FILTER_TYPE));
+ } else {
+
+ Map<String,String> factoryArgs = new HashMap<String,String>();
+
+ // nocommit how to allow the SPI name and separately
+ // also a fully qualified class name ...
+ factoryArgs.put("class", className);
+ factoryArgs.put("luceneMatchVersion", matchVersion.toString());
+
+ ResourceLoader resources = state.resourceLoader;
+ RAMResourceLoaderWrapper ramResources = null;
+
+
+ if (sub != null) {
+
+ for(Map.Entry<String,Object> ent : subParams.entrySet()) {
+ String argName = ent.getKey();
+ Object argValue = ent.getValue();
+
+ // Messy: allow components that expect files for
+ // things like stopword lists, keywords, to come
+ // in as inlined string values. We "hijack" any
+ // argument name ending in FileContents and make a RAM
+ // file out of it:
+ String argString;
+
+ if (argName.endsWith("FileContents")) {
+ if (ramResources == null) {
+ ramResources = new RAMResourceLoaderWrapper(resources);
+ resources = ramResources;
}
+
+ String value;
+ if (argValue instanceof String) {
+ value = (String) argValue;
+ } else if (argValue instanceof JSONArray) {
+ StringBuilder b = new StringBuilder();
+ for(Object v : (JSONArray) argValue) {
+ if ((v instanceof String) == false) {
+ sub.fail(argName, "array must contain strings; got: " + v.getClass().getSimpleName());
+ }
+ b.append((String) v);
+ b.append('\n');
+ }
+ value = b.toString();
+ } else {
+ sub.fail(argName, "must be a String or JSONArray; got: " + argValue.getClass().getSimpleName());
+
+ // Dead code but compiler disagrees:
+ value = null;
+ }
+
+ argName = argName.substring(0, argName.length()-12);
+ ramResources.add(argName, value);
+ argString = argName;
} else {
- parser.add(parser.analyze(syn.getString("input"), scratch),
- output,
- !replace);
+ argString = argValue.toString();
}
+
+ factoryArgs.put(argName, argString);
}
- last = new SynonymFilter(last, parser.build(), pr.r.getBoolean("ignoreCase"));
- } catch (IOException ioe) {
- throw new RuntimeException(ioe);
+
+ // Clear all bindings from the incoming request,
+ // so that they are not seen as unused by the
+ // server. If any params are really unused, the
+ // analysis factory should throw its own
+ // IllegalArgumentException:
+ subParams.clear();
+ }
+
+ try {
+ tokenFilterFactory = TokenFilterFactory.forName(className, factoryArgs);
+ } catch (IllegalArgumentException iae) {
+ chain.fail("tokenizer", "failed to create TokenFilterFactory for class \"" + className + "\": " + iae, iae);
+
+ // Dead code but compiler disagrees:
+ tokenFilterFactory = null;
+ }
+
+ if (tokenFilterFactory instanceof ResourceLoaderAware) {
+ ((ResourceLoaderAware) tokenFilterFactory).inform(resources);
}
- } else {
- assert false: "unrecognized: " + pr.name;
}
+
+ tokenFilterFactories.add(tokenFilterFactory);
}
+ } else {
+ tokenFilterFactories = null;
}
- return new TokenStreamComponents(tokenizer, last);
+ return new CustomAnalyzer(tokenizerFactory, tokenFilterFactories,
+ chain.getInt("positionIncrementGap"),
+ chain.getInt("offsetGap"));
+ }
+
+ private static SynonymMap parseSynonyms(List<Object> syns, Analyzer a) {
+ try {
+ // nocommit this is awkward! I just want to use Parser's
+ // analyze utility method...
+ SynonymMap.Parser parser = new SynonymMap.Parser(true, a) {
+ @Override
+ public void parse(Reader in) throws IOException {
+ // nocommit move parsing in here!
+ };
+ };
+
+ CharsRef scratch = new CharsRef();
+ CharsRef scratchOutput = new CharsRef();
+ for(Object o2 : syns) {
+ Request syn = (Request) o2;
+ boolean replace = syn.getBoolean("replace");
+ CharsRef output = new CharsRef(syn.getString("output"));
+ if (syn.isString("input") == false) {
+ for(Object o3 : syn.getList("input")) {
+ parser.add(parser.analyze((String) o3, scratch),
+ output,
+ !replace);
+ }
+ } else {
+ parser.add(parser.analyze(syn.getString("input"), scratch),
+ output,
+ !replace);
+ }
+ }
+ return parser.build();
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
}
private static class CustomAnalyzer extends Analyzer {
- private final String json;
- private final Version matchVersion;
- private int positionIncrementGap;
- private int offsetGap;
-
- public CustomAnalyzer(Version matchVersion, String json) {
- this.matchVersion = matchVersion;
- this.json = json;
+ private final int posIncGap;
+ private final int offsetGap;
+ private final TokenizerFactory tokenizerFactory;
+ private final List<TokenFilterFactory> tokenFilterFactories;
+
+ public CustomAnalyzer(TokenizerFactory tokenizerFactory, List<TokenFilterFactory> tokenFilterFactories, int posIncGap, int offsetGap) {
+ this.tokenizerFactory = tokenizerFactory;
+ this.tokenFilterFactories = tokenFilterFactories;
+ this.posIncGap = posIncGap;
+ this.offsetGap = offsetGap;
}
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- JSONObject o;
- try {
- o = (JSONObject) JSONValue.parseStrict(json);
- } catch (net.minidev.json.parser.ParseException pe) {
- // BUG
- throw new RuntimeException(pe);
- }
- Request r = new Request(null, fieldName, o, (StructType) ANALYZER_TYPE);
- positionIncrementGap = r.getInt("positionIncrementGap");
- offsetGap = r.getInt("offsetGap");
- return buildCustomAnalysisChain(matchVersion,
- r);
+ Tokenizer tokenizer = tokenizerFactory.create();
+ TokenStream last = tokenizer;
+ if (tokenFilterFactories != null) {
+ for(TokenFilterFactory factory : tokenFilterFactories) {
+ last = factory.create(last);
+ }
+ }
+ return new TokenStreamComponents(tokenizer, last);
}
@Override
public int getPositionIncrementGap(String fieldName) {
- return positionIncrementGap;
+ return posIncGap;
}
@Override
@@ -920,7 +1060,7 @@ public class RegisterFieldHandler extend
}
}
- static Analyzer getAnalyzer(Version matchVersionGlobal, Request f, String name) {
+ static Analyzer getAnalyzer(IndexState state, Request f, String name) throws IOException {
Analyzer analyzer;
if (f.hasParam(name)) {
Request a = f.getStruct(name);
@@ -930,10 +1070,11 @@ public class RegisterFieldHandler extend
if (a.hasParam("matchVersion")) {
matchVersion = getVersion(a.getEnum("matchVersion"));
} else {
- matchVersion = matchVersionGlobal;
+ matchVersion = state.matchVersion;
}
if (a.hasParam("class")) {
+ // nocommit just lookup via CP:
// Predefined analyzer class:
Request.PolyResult pr = a.getPoly("class");
// TODO: try to "share" a single instance of
@@ -1021,11 +1162,7 @@ public class RegisterFieldHandler extend
analyzer = null;
}
} else if (a.hasParam("tokenizer")) {
- a.getInt("positionIncrementGap");
- a.getInt("offsetGap");
- // Ensures the args are all correct:
- buildCustomAnalysisChain(matchVersion, a);
- analyzer = new CustomAnalyzer(matchVersion, jsonOrig);
+ analyzer = buildCustomAnalyzer(state, matchVersion, a);
} else {
f.fail(name, "either class or tokenizer/tokenFilters are required");
analyzer = null;
Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java Sun Jan 19 16:52:57 2014
@@ -404,6 +404,13 @@ public class Request {
return params.get(name);
}
+ /** Returns the raw object, and removes the binding. */
+ public Object getAndRemoveRaw(String name) {
+ Object o = params.get(name);
+ params.remove(name);
+ return o;
+ }
+
/** Retrieve a struct parameter. This can be called
* multiple times for a given parameter name. */
public Request getStruct(String name) {
@@ -612,6 +619,12 @@ public class Request {
}
/** Throws a {@link RequestFailedException} with the
+ * provided parameter and message. */
+ public void fail(String message, Throwable cause) {
+ fail(null, message, cause);
+ }
+
+ /** Throws a {@link RequestFailedException} with the
* provided parameter and message and original cause. */
public void fail(String param, String reason, Throwable cause) {
StringBuilder sb = new StringBuilder();
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java Sun Jan 19 16:52:57 2014
@@ -42,42 +42,56 @@ public class TestAnalysis extends Server
}
public void testCustomAnalysisChain() throws Exception {
- //send("{body: {type: text, analyzer: {tokenizer: StandardTokenizer, tokenFilters: [LowerCaseFilter]}}}", "registerFields");
- //send("{queryParser: {class: classic, defaultField: body}}", "settings");
- JSONObject o = send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: StandardTokenizer, tokenFilters: [LowerCaseFilter]}}");
- assertEquals("here is some text", justTokens(o));
+ send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: Standard, tokenFilters: [LowerCase]}}");
+ assertEquals("here is some text", justTokens());
- o = send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: StandardTokenizer}}");
- assertEquals("Here is some text", justTokens(o));
+ send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: Standard}}");
+ assertEquals("Here is some text", justTokens());
- o = send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: {class: StandardTokenizer, maxTokenLength: 2}, tokenFilters: [LowerCaseFilter]}}");
- assertEquals("is", justTokens(o));
-
- // test maxTokenLength
+ send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: {class: Standard, maxTokenLength: 2}, tokenFilters: [LowerCase]}}");
+ assertEquals("is", justTokens());
}
public void testPatternTokenizer() throws Exception {
- JSONObject o = send("analyze", "{text: 'Here is \\'some\\' text', analyzer: {tokenizer: {class: PatternTokenizer, pattern: \"\\'([^\\']+)\\'\", group: 1}}}");
- assertEquals("some", justTokens(o));
+ send("analyze", "{text: 'Here is \\'some\\' text', analyzer: {tokenizer: {class: Pattern, pattern: \"\\'([^\\']+)\\'\", group: 1}}}");
+ assertEquals("some", justTokens());
+ }
+
+ public void testExtraArgs() throws Exception {
+ assertFailsWith("analyze", "{text: 'Here is \\'some\\' text', analyzer: {tokenizer: {class: Pattern, pattern: \"\\'([^\\']+)\\'\", group: 1, bad: 14}}}", "analyze > analyzer > tokenizer: failed to create TokenizerFactory for class \"Pattern\": java.lang.IllegalArgumentException: Unknown parameters: {bad=14}");
}
- public void testSetKeywordMarkerFilter() throws Exception {
+ public void testKeywordMarkerFilter() throws Exception {
// No KWMarkerFilter, dogs is stemmed:
- JSONObject o = send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: StandardTokenizer, tokenFilters: [EnglishPossessiveFilter, LowerCaseFilter, StopFilter, EnglishMinimalStemFilter]}}");
- assertEquals("here some dog", justTokens(o));
+ send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, Stop, EnglishMinimalStem]}}");
+ assertEquals("here some dog", justTokens());
- // KWMarkerFilter protects dogs:
- o = send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: StandardTokenizer, tokenFilters: [EnglishPossessiveFilter, LowerCaseFilter, StopFilter, {class: SetKeywordMarkerFilter, keyWords:[dogs]}, EnglishMinimalStemFilter]}}");
- assertEquals("here some dogs", justTokens(o));
+ // Use KWMarkerFilter with protectedFileContents to protect dogs:
+ send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, Stop, {class: KeywordMarker, protectedFileContents:[dogs]}, EnglishMinimalStem]}}");
+ assertEquals("here some dogs", justTokens());
+
+ // Use KWMarkerFilter with pattern to protect dogs:
+ send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, Stop, {class: KeywordMarker, pattern: dogs}, EnglishMinimalStem]}}");
+ assertEquals("here some dogs", justTokens());
}
public void testEnglishAnalyzer() throws Exception {
- JSONObject o = send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer}}");
- assertEquals("dog go run", justTokens(o));
+ send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer}}");
+ assertEquals("dog go run", justTokens());
// This time protecting dogs from stemming:
- o = send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer, stemExclusionSet: [dogs]}}");
- assertEquals("dogs go run", justTokens(o));
+ send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer, stemExclusionSet: [dogs]}}");
+ assertEquals("dogs go run", justTokens());
+ }
+
+ public void testStopFilter() throws Exception {
+ // Uses default (english) stop words:
+ send("analyze", "{text: 'the dogs go running', analyzer: {tokenizer: Whitespace, tokenFilters: [Stop]}}");
+ assertEquals("dogs go running", justTokens());
+
+ // This time making only running a stop word:
+ send("analyze", "{text: 'the dogs go running', analyzer: {tokenizer: Whitespace, tokenFilters: [{class: Stop, wordsFileContents: [running]}]}}");
+ assertEquals("the dogs go", justTokens());
}
public void testPositionIncrementGap() throws Exception {
@@ -85,32 +99,34 @@ public class TestAnalysis extends Server
_TestUtil.rmDir(new File("posinc"));
send("createIndex", "{rootDir: posinc}");
send("settings", "{directory: RAMDirectory}");
- send("registerFields", "{fields: {author1: {type: text, analyzer: {tokenizer: WhitespaceTokenizer}, multiValued: true}, author2: {type: text, analyzer: {tokenizer: WhitespaceTokenizer, positionIncrementGap: 1}, multiValued: true}}}");
+ send("registerFields", "{fields: {author1: {type: text, analyzer: {tokenizer: Whitespace}, multiValued: true}, author2: {type: text, analyzer: {tokenizer: Whitespace, positionIncrementGap: 1}, multiValued: true}}}");
send("startIndex");
- long gen = getLong(send("addDocument", "{fields: {author1: [bob, smith], author2: [bob, smith]}}"), "indexGen");
+ send("addDocument", "{fields: {author1: [bob, smith], author2: [bob, smith]}}");
+ long gen = getLong("indexGen");
// This one matches because the two values act like they
// were just concatenated:
- JSONObject result = send("search", String.format(Locale.ROOT, "{queryText: 'author1: \"bob smith\"', searcher: {indexGen: %d}}", gen));
- assertEquals(1, getInt(result, "hits.length"));
+ send("search", String.format(Locale.ROOT, "{queryText: 'author1: \"bob smith\"', searcher: {indexGen: %d}}", gen));
+ assertEquals(1, getInt("hits.length"));
// This one doesn't match because a hole is inserted
// between the two values:
- result = send("search", String.format(Locale.ROOT, "{queryText: 'author2: \"bob smith\"', searcher: {indexGen: %d}}", gen));
- assertEquals(0, getInt(result, "hits.length"));
+ send("search", String.format(Locale.ROOT, "{queryText: 'author2: \"bob smith\"', searcher: {indexGen: %d}}", gen));
+ assertEquals(0, getInt("hits.length"));
send("stopIndex");
send("deleteIndex");
}
+ // nocommit test loading syns from "file" too
public void testSynonymFilter() throws Exception {
- JSONObject o = send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter, {class: SynonymFilter, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns'}]}]}}");
- assertEquals("dns/0 is/1 complex/2", tokensAndPositions(o));
+ send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase, {class: Synonym, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns'}]}]}}");
+ assertEquals("dns/0 is/1 complex/2", tokensAndPositions());
- o = send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter, {class: SynonymFilter, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns', replace: false}]}]}}");
- assertEquals("domain/0 dns/0:3 name/1 service/2 is/3 complex/4", tokensAndPositions(o));
+ send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase, {class: Synonym, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns', replace: false}]}]}}");
+ assertEquals("domain/0 dns/0:3 name/1 service/2 is/3 complex/4", tokensAndPositions());
- o = send("analyze", "{text: 'mother knows best', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter, {class: SynonymFilter, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: ['mother', 'mommy'], output: 'mom'}]}]}}");
- assertEquals("mom/0 knows/1 best/2", tokensAndPositions(o));
+ send("analyze", "{text: 'mother knows best', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase, {class: Synonym, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: ['mother', 'mommy'], output: 'mom'}]}]}}");
+ assertEquals("mom/0 knows/1 best/2", tokensAndPositions());
}
String ONLY_WHITESPACE_RULES = "\\n!!forward;\\n" +
@@ -132,13 +148,18 @@ public class TestAnalysis extends Server
"$NonWhitespace+ {1};";
public void testICUTokenizer() throws Exception {
- JSONObject o = send("analyze", "{text: 'domain-name service is complex', analyzer: {tokenizer: {class: ICUTokenizer, rules: [{script: Latn, rules: \"" + ONLY_WHITESPACE_RULES + "\"}]}}}");
- assertEquals("domain-name/0 service/1 is/2 complex/3", tokensAndPositions(o));
+ send("analyze", "{text: 'domain-name service is complex', analyzer: {tokenizer: {class: ICU, rules: [{script: Latn, rules: \"" + ONLY_WHITESPACE_RULES + "\"}]}}}");
+ assertEquals("domain-name/0 service/1 is/2 complex/3", tokensAndPositions());
+ }
+
+ public void testSpanishLightStem() throws Exception {
+ send("analyze", "{text: 'las lomitas', analyzer: {tokenizer: Standard, tokenFilters: [SpanishLightStem]}}");
+ assertEquals("las/0 lomit/1", tokensAndPositions());
}
- private String justTokens(JSONObject o) {
+ private String justTokens() {
StringBuilder sb = new StringBuilder();
- for(Object _o : (JSONArray) o.get("tokens")) {
+ for(Object _o : (JSONArray) lastResult.get("tokens")) {
JSONObject token = (JSONObject) _o;
if (sb.length() > 0) {
sb.append(' ');
@@ -148,9 +169,9 @@ public class TestAnalysis extends Server
return sb.toString();
}
- private String tokensAndPositions(JSONObject o) {
+ private String tokensAndPositions() {
StringBuilder sb = new StringBuilder();
- for(Object _o : (JSONArray) o.get("tokens")) {
+ for(Object _o : (JSONArray) lastResult.get("tokens")) {
JSONObject token = (JSONObject) _o;
if (sb.length() > 0) {
sb.append(' ');
Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java Sun Jan 19 16:52:57 2014
@@ -61,7 +61,7 @@ public class TestSuggest extends ServerB
out.write("5\u001fthe time is now\u001ffoobar\n");
out.close();
- JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'AnalyzingSuggester', suggestName: 'suggest', indexAnalyzer: EnglishAnalyzer, queryAnalyzer: {tokenizer: StandardTokenizer, tokenFilters: [EnglishPossessiveFilter,LowerCaseFilter,PorterStemFilter]]}}");
+ JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'AnalyzingSuggester', suggestName: 'suggest', indexAnalyzer: EnglishAnalyzer, queryAnalyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, PorterStem]]}}");
assertEquals(5, result.get("count"));
//commit();
@@ -81,9 +81,6 @@ public class TestSuggest extends ServerB
assertEquals(5, get(result, "results[2].weight"));
assertEquals("foobar", get(result, "results[2].payload"));
- // ForkLastTokenFilter allows 'the' to match
- // 'theories'; without it (and StopKeywordFilter) the
- // would be dropped:
result = send("suggestLookup", "{text: 'the', suggestName: 'suggest'}");
assertEquals(1, get(result, "results.length"));
@@ -107,7 +104,7 @@ public class TestSuggest extends ServerB
out.write("15\u001flove lost\u001ffoobar\n");
out.close();
- JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: InfixSuggester, suggestName: suggest2, analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+ JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: InfixSuggester, suggestName: suggest2, analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
assertEquals(1, result.get("count"));
//commit();
@@ -145,7 +142,7 @@ public class TestSuggest extends ServerB
out.write("15\u001flove lost\u001ffoobar\n");
out.close();
- JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'FuzzySuggester', suggestName: 'suggest3', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+ JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'FuzzySuggester', suggestName: 'suggest3', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
assertEquals(1, result.get("count"));
//commit();
@@ -177,7 +174,7 @@ public class TestSuggest extends ServerB
send("addDocument", "{fields: {text: 'the cat meows', weight: 1, payload: 'payload1'}}");
long indexGen = getLong(send("addDocument", "{fields: {text: 'the dog barks', weight: 2, payload: 'payload2'}}"), "indexGen");
- JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightField: weight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+ JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightField: weight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
// nocommit count isn't returned for stored fields source:
//assertEquals(2, result.get("count"));
@@ -213,7 +210,7 @@ public class TestSuggest extends ServerB
send("addDocument", "{fields: {text: 'the cat meows', negWeight: -1, payload: 'payload1'}}");
long indexGen = getLong(send("addDocument", "{fields: {text: 'the dog barks', negWeight: -2, payload: 'payload2'}}"), "indexGen");
- JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightExpression: -negWeight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+ JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightExpression: -negWeight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
// nocommit count isn't returned for stored fields source:
//assertEquals(2, result.get("count"));