You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/01/19 17:52:57 UTC

svn commit: r1559524 - in /lucene/dev/branches/lucene5376/lucene: analysis/common/src/java/org/apache/lucene/analysis/util/ server/src/java/org/apache/lucene/server/ server/src/java/org/apache/lucene/server/handlers/ server/src/java/org/apache/lucene/s...

Author: mikemccand
Date: Sun Jan 19 16:52:57 2014
New Revision: 1559524

URL: http://svn.apache.org/r1559524
Log:
LUCENE-5376: cutover to analysis factories to create tokenizer and token filters; still need char filters & whole analyzers

Added:
    lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java   (with props)
Modified:
    lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
    lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java
    lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java
    lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java
    lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java
    lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java
    lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java
    lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java
    lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java
    lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java

Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AnalysisSPILoader.java Sun Jan 19 16:52:57 2014
@@ -17,13 +17,14 @@ package org.apache.lucene.analysis.util;
  * limitations under the License.
  */
 
+import java.lang.reflect.InvocationTargetException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
-import java.util.LinkedHashMap;
-import java.util.Set;
 import java.util.ServiceConfigurationError;
+import java.util.Set;
 
 import org.apache.lucene.util.SPIClassIterator;
 
@@ -108,6 +109,18 @@ final class AnalysisSPILoader<S extends 
     final Class<? extends S> service = lookupClass(name);
     try {
       return service.getConstructor(Map.class).newInstance(args);
+    } catch (InvocationTargetException ite) {
+      // nocommit ... trying to throw the "original" IAE,
+      // but is this correct/safe?
+      Throwable t = ite;
+      while (t.getCause() != null) {
+        if (t.getCause() instanceof IllegalArgumentException) {
+          throw (IllegalArgumentException) t.getCause();
+        }
+        t = t.getCause();
+      }
+      throw new IllegalArgumentException("SPI class of type "+clazz.getName()+" with name '"+name+"' cannot be instantiated. " +
+            "This is likely due to a misconfiguration of the java class '" + service.getName() + "': ", ite);
     } catch (Exception e) {
       throw new IllegalArgumentException("SPI class of type "+clazz.getName()+" with name '"+name+"' cannot be instantiated. " +
             "This is likely due to a misconfiguration of the java class '" + service.getName() + "': ", e);

Modified: lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java (original)
+++ lucene/dev/branches/lucene5376/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/MultiTermAwareComponent.java Sun Jan 19 16:52:57 2014
@@ -24,7 +24,7 @@ package org.apache.lucene.analysis.util;
  * @lucene.experimental
  */
 public interface MultiTermAwareComponent {
-  /** Returns an analysis component to handle analysis if multi-term queries.
+  /** Returns an analysis component to handle analysis of multi-term queries.
    * The returned component must be a TokenizerFactory, TokenFilterFactory or CharFilterFactory.
    */
   public AbstractAnalysisFactory getMultiTermComponent();

Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/IndexState.java Sun Jan 19 16:52:57 2014
@@ -47,6 +47,8 @@ import java.util.regex.Pattern;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.AnalyzerWrapper;
 import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.util.FilesystemResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoader;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.expressions.Bindings;
 import org.apache.lucene.facet.FacetsConfig;
@@ -138,6 +140,9 @@ public class IndexState implements Close
   /** Creates directories */
   DirectoryFactory df = DirectoryFactory.get("FSDirectory");
 
+  /** Loads all resources for analysis components */
+  public final ResourceLoader resourceLoader;
+
   /** Where all index state is saved */
   public final File rootDir;
 
@@ -402,6 +407,16 @@ public class IndexState implements Close
     this.globalState = globalState;
     this.name = name;
     this.rootDir = rootDir;
+    if (rootDir != null) {
+      if (!rootDir.exists()) {
+        rootDir.mkdirs();
+      }
+      this.resourceLoader = new FilesystemResourceLoader(rootDir);
+    } else {
+      // nocommit can/should we make a DirectoryResourceLoader?
+      this.resourceLoader = null;
+    }
+
     this.doCreate = doCreate;
     if (doCreate == false) {
       initSaveLoadState();

Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/AnalysisHandler.java Sun Jan 19 16:52:57 2014
@@ -64,7 +64,7 @@ public class AnalysisHandler extends Han
     final String text = r.getString("text");
 
     // TODO: allow passing field name in, and we use its analyzer?
-    final Analyzer a = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "analyzer");
+    final Analyzer a = RegisterFieldHandler.getAnalyzer(state, r, "analyzer");
 
     return new FinishRequest() {
       @Override

Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/BuildSuggestHandler.java Sun Jan 19 16:52:57 2014
@@ -173,11 +173,13 @@ public class BuildSuggestHandler extends
     r = pr.r;
     Analyzer indexAnalyzer;
     Analyzer queryAnalyzer;
+    // nocommit allow passing a field name, and we use that
+    // field name's analyzer?
     if (r.hasParam("analyzer")) {
-      indexAnalyzer = queryAnalyzer = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "analyzer");
+      indexAnalyzer = queryAnalyzer = RegisterFieldHandler.getAnalyzer(state, r, "analyzer");
     } else {
-      indexAnalyzer = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "indexAnalyzer");
-      queryAnalyzer = RegisterFieldHandler.getAnalyzer(state.matchVersion, r, "queryAnalyzer");
+      indexAnalyzer = RegisterFieldHandler.getAnalyzer(state, r, "indexAnalyzer");
+      queryAnalyzer = RegisterFieldHandler.getAnalyzer(state, r, "queryAnalyzer");
     }
     if (indexAnalyzer == null) {
       r.fail("analyzer", "analyzer or indexAnalyzer must be specified");

Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/CreateIndexHandler.java Sun Jan 19 16:52:57 2014
@@ -70,7 +70,7 @@ public class CreateIndexHandler extends 
         try {
           globalState.createIndex(indexName, rootDir);
         } catch (IllegalArgumentException iae) {
-          r.fail("invalid indexName \"" + indexName + "\": " + iae.toString());
+          r.fail("invalid indexName \"" + indexName + "\": " + iae.toString(), iae);
         }
 
         return "{}";

Added: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java?rev=1559524&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java (added)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java Sun Jan 19 16:52:57 2014
@@ -0,0 +1,72 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.util.ResourceLoader;
+
+/** Holds "files" in RAM, and falls back to delegate if the
+ *  resource isn't in RAM.  This is used for analysis
+ *  components that specify their "files" as strings in
+ *  RAM. */
+
+class RAMResourceLoaderWrapper implements ResourceLoader {
+
+  private final Map<String,String> files = new HashMap<String,String>();
+  private final ResourceLoader delegate;
+
+  /** Sole constructor. */
+  public RAMResourceLoaderWrapper(ResourceLoader delegate) {
+    this.delegate = delegate;
+  }
+
+  public void add(String name, String contents) {
+    if (files.containsKey(name)) {
+      throw new IllegalArgumentException("name \"" + name + "\" was already added");
+    }
+    files.put(name, contents);
+  }
+
+  @Override
+  public InputStream openResource(String resource) throws IOException {
+    String value = files.get(resource);
+    if (value != null) {
+      return new ByteArrayInputStream(value.getBytes("UTF-8"));
+    } else if (delegate == null) {
+      throw new FileNotFoundException("resource \"" + resource + "\" is not found");
+    } else {
+      return delegate.openResource(resource);
+    }
+  }
+
+  @Override
+  public <T> T newInstance(String cname, Class<T> expectedType) {
+    return delegate.newInstance(cname, expectedType);
+  }
+
+  @Override
+  public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
+    return delegate.findClass(cname, expectedType);
+  }
+}

Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java Sun Jan 19 16:52:57 2014
@@ -23,6 +23,7 @@ import java.io.StringReader;
 import java.text.Collator;
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -64,8 +65,13 @@ import org.apache.lucene.analysis.standa
 import org.apache.lucene.analysis.standard.StandardFilter;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.synonym.SynonymFilter;
+import org.apache.lucene.analysis.synonym.SynonymFilterFactory;
 import org.apache.lucene.analysis.synonym.SynonymMap;
 import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
 import org.apache.lucene.codecs.DocValuesFormat;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.collation.CollationKeyAnalyzer;
@@ -88,9 +94,11 @@ import org.apache.lucene.server.GlobalSt
 import org.apache.lucene.server.IndexState;
 import org.apache.lucene.server.params.*;
 import org.apache.lucene.server.params.PolyType.PolyEntry;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
 import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.NumericUtils;
 import org.apache.lucene.util.Version;
+import net.minidev.json.JSONArray;
 import net.minidev.json.JSONObject;
 import net.minidev.json.JSONValue;
 import com.ibm.icu.lang.UCharacter;
@@ -237,53 +245,32 @@ public class RegisterFieldHandler extend
                    new Param("positionIncrementGap", "How many positions to insert between separate values in a multi-valued field", new IntType(), 0),
                    new Param("offsetGap", "How many offsets to insert between separate values in a multi-valued field", new IntType(), 1),
                    new Param("tokenizer", "Tokenizer class (for a custom analysis chain).",
-                             new StructType(
-                                 new Param("class",
-                                           "Tokenizer class",
-                                           new PolyType(Tokenizer.class,
-                                                        new PolyEntry("WhitespaceTokenizer", "A WhitespaceTokenizer is a tokenizer that divides text at whitespace (see @lucene:analyzers-common:org.apache.lucene.analysis.core.WhitespaceTokenizer)", new StructType()),
-                                                        new PolyEntry("StandardTokenizer", "A grammar-based tokenizer constructed with JFlex (see @lucene:analyzers-common:org.apache.lucene.analysis.standard.StandardTokenizer)",
-                                                            new Param("maxTokenLength", "Max length of each token", new IntType(), StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH)),
-                                                        new PolyEntry("PatternTokenizer", "This tokenizer uses regex pattern matching to construct distinct tokens for the input stream (see @lucene:analyzers-common:org.apache.lucene.analysis.pattern.PatternTokenizer)",
-                                                            new Param("pattern", "Regular expression pattern", new StringType()),
-                                                                      new Param("group", "Group index for the tokens (-1 to do 'split')", new IntType(), -1)),
-                                                        new PolyEntry("ICUTokenizer", "Breaks text into words according to UAX #29: Unicode Text Segmentation (http://www.unicode.org/reports/tr29",
-                                                            new Param("rules", "Customize the tokenization per-script",
-                                                               new ListType(
-                                                                   new StructType(
-                                                                       new Param("script", "Script", new StringType()),
-                                                                       new Param("rules", "Rules", new StringType()))))))))),
+                             new OrType(new StringType(), new StructType())),
+                             // nocommit somehow tap into TokenizerFactory.availableTokenizers
                    new Param("tokenFilters", "Optional list of TokenFilters to apply after the Tokenizer",
                              new ListType(
-                                 new StructType(
-                                     new Param("class", "TokenFilter class",
-                                         new PolyType(TokenFilter.class,
-                                                      new PolyEntry("ArabicStemFilter", "A TokenFilter that applies ArabicStemmer to stem Arabic words. (see @lucene:analyzers-common:org.apache.lucene.analysis.ar.ArabicStemFilter)"),
-                                                      new PolyEntry("StandardFilter", "Normalizes tokens extracted with StandardTokenizer. (see @lucene:analyzers-common:org.apache.lucene.analysis.standard.StandardFilter)"),
-                                                      new PolyEntry("EnglishPossessiveFilter", "TokenFilter that removes possessives (trailing 's) from words (see @lucene:analyzers-common:org.apache.lucene.analysis.en.EnglishPossessiveFilter)"),
-                                                      new PolyEntry("PorterStemFilter", "Transforms the token stream as per the Porter stemming algorithm (see @lucene:analyzers-common:org.apache.lucene.analysis.en.PorterStemFilter)"),
-                                                      new PolyEntry("SuggestStopFilter", "Like {@link StopFilter} except it will not remove the last token if that token was not followed by some token separator.",
-                                                                    new Param("stopWords", "Stop words to remove during analysis",
-                                                                        new ListType(new StringType()), DEFAULT_ENGLISH_STOP_WORDS)),
-                                                      new PolyEntry("EnglishMinimalStemFilter", "A TokenFilter that applies EnglishMinimalStemmer to stem English words (see @lucene:analyzers-common:org.apache.lucene.analysis.en.EnglishMinimalStemFilter)"),
-                                                      new PolyEntry("SetKeywordMarkerFilter", "Marks terms as keywords via the KeywordAttribute (see @lucene:analyzers-common:org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter)", new Param("keyWords", "List of tokens to mark as keywords", new ListType(new StringType()))),
-                                                      new PolyEntry("StopFilter", "Removes stop words from a token stream (see @lucene:analyzers-common:org.apache.lucene.analysis.core.StopFilter)",
-                                                          new Param("stopWords", "Stop words to remove during analysis",
-                                                                    new ListType(new StringType()), DEFAULT_ENGLISH_STOP_WORDS)),
-                                                      new PolyEntry("SynonymFilter", "Matches single- or multi-word synonyms and injects or replaces the match with a corresponding synonym",
-                                                          //new Param("synonymFile", "Local file to load synonyms from", new StringType()),   // nocommit TODO
-                                                          new Param("ignoreCase", "True if matching should be case insensitive", new BooleanType(), true),
-                                                          new Param("analyzer", "Analyzer to use to tokenize synonym inputs", ANALYZER_TYPE_WRAP),
-                                                          new Param("synonyms", "Specify synonyms inline (instead of synonymFile)",
-                                                              new ListType(
-                                                                  new StructType(
-                                                                      new Param("input", "String or list of strings with input token(s) to match", new OrType(new ListType(new StringType()), new StringType())),
-                                                                      // TODO: allow more than one token on the output?
-                                                                      new Param("output", "Single token to replace the matched tokens with", new StringType()),
-                                                                      new Param("replace", "True if the input tokens should be replaced with the output token; false if the input tokens should be preserved and the output token overlaid", new BooleanType(), true))))),
-                                                      new PolyEntry("LowerCaseFilter", "Normalizes token text to lower case (see @lucene:analyzers-common:org.apache.lucene.analysis.core.LowerCaseFilter)", new StructType())))))),
+                                 new OrType(new StringType(), new StructType(new Param("class", "TokenFilter class name", new StringType()))))),
                    MATCH_VERSION_PARAM);
 
+  static StructType SYNONYM_FILTER_TYPE = new StructType(
+                                              new Param("ignoreCase", "True if matching should be case insensitive", new BooleanType(), true),
+                                              new Param("analyzer", "Analyzer to use to tokenize synonym inputs", ANALYZER_TYPE_WRAP),
+                                              new Param("synonyms", "Synonyms",
+                                                  new ListType(
+                                                      new StructType(
+                                                          new Param("input", "String or list of strings with input token(s) to match", new OrType(new ListType(new StringType()), new StringType())),
+                                                          // TODO: allow more than one token on the output?
+                                                          new Param("output", "Single token to replace the matched tokens with", new StringType()),
+                                                          new Param("replace", "True if the input tokens should be replaced with the output token; false if the input tokens should be preserved and the output token overlaid", new BooleanType(), true)))));
+
+  static StructType ICU_TOKENIZER_TYPE = new StructType(
+                                             new Param("cjkAsWords", "??", new BooleanType(), true),
+                                             new Param("rules", "Customize the tokenization per-script",
+                                                 new ListType(
+                                                     new StructType(
+                                                         new Param("script", "Script", new StringType()),
+                                                         new Param("rules", "Rules", new StringType())))));
+
   static {
     ANALYZER_TYPE_WRAP.set(ANALYZER_TYPE);
   }
@@ -415,7 +402,7 @@ public class RegisterFieldHandler extend
     return new FieldDef(name, null, "virtual", null, null, null, true, null, null, null, false, null, values);
   }
 
-  private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,FieldDef> pendingFieldDefs, String name, JSONObject o) {
+  private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,FieldDef> pendingFieldDefs, String name, JSONObject o) throws IOException {
 
     // This way f.fail reports which field name was problematic:
     Request f = new Request(r, name, o, FIELD_TYPE);
@@ -613,12 +600,12 @@ public class RegisterFieldHandler extend
 
     Analyzer indexAnalyzer;
     Analyzer searchAnalyzer;
-    Analyzer analyzer = getAnalyzer(state.matchVersion, f, "analyzer");
+    Analyzer analyzer = getAnalyzer(state, f, "analyzer");
     if (analyzer != null) {
       indexAnalyzer = searchAnalyzer = analyzer;
     } else {
-      indexAnalyzer = getAnalyzer(state.matchVersion, f, "indexAnalyzer");
-      searchAnalyzer = getAnalyzer(state.matchVersion, f, "searchAnalyzer");
+      indexAnalyzer = getAnalyzer(state, f, "indexAnalyzer");
+      searchAnalyzer = getAnalyzer(state, f, "searchAnalyzer");
     }
 
     if (type.equals("text") && ft.indexed()) {
@@ -733,38 +720,21 @@ public class RegisterFieldHandler extend
     return words;
   }
 
-  final static Pattern COMMENTS_PATTERN = Pattern.compile("#.*$", Pattern.MULTILINE);
-
-  static TokenStreamComponents buildCustomAnalysisChain(Version matchVersion, Request chain) {
-
-    Request t = chain.getStruct("tokenizer");
+  static TokenizerFactory buildICUTokenizerFactory(JSONObject params) {
+      
+    boolean cjkAsWords;
 
-    Request.PolyResult pr = t.getPoly("class");
+    final BreakIterator breakers[];
 
-    // nocommit charFilters
+    if (params != null) {
+            
+      // nocommit don't pass null, null:
+      Request icuRequest = new Request(null, null, params, ICU_TOKENIZER_TYPE);
+      cjkAsWords = icuRequest.getBoolean("cjkAsWords");
 
-    Tokenizer tokenizer;
-    // nocommit use analysis factories
-    if (pr.name.equals("StandardTokenizer")) {
-      tokenizer = new StandardTokenizer(matchVersion);
-      ((StandardTokenizer) tokenizer).setMaxTokenLength(pr.r.getInt("maxTokenLength"));
-    } else if (pr.name.equals("WhitespaceTokenizer")) {
-      tokenizer = new WhitespaceTokenizer(matchVersion);
-    } else if (pr.name.equals("PatternTokenizer")) {
-      Pattern p;
-      try {
-        p = Pattern.compile(pr.r.getString("pattern"));
-      } catch (PatternSyntaxException pse) {
-        pr.r.fail("pattern", "failed to compile Pattern", pse);
-        // Dead code but compiler disagrees:
-        p = null;
-      }
-      tokenizer = new PatternTokenizer(p, pr.r.getInt("group"));
-    } else if (pr.name.equals("ICUTokenizer")) {
-      final BreakIterator breakers[];
-      if (pr.r.hasParam("rules")) {
+      if (icuRequest.hasParam("rules")) {
         breakers = new BreakIterator[UScript.CODE_LIMIT];
-        for(Object o : pr.r.getList("rules")) {
+        for(Object o : icuRequest.getList("rules")) {
           Request r2 = (Request) o;
           String script = r2.getString("script");
           String rules = r2.getString("rules");
@@ -786,9 +756,12 @@ public class RegisterFieldHandler extend
       } else {
         breakers = null;
       }
+    } else {
+      cjkAsWords = true;
+      breakers = null;
+    }
 
-      // nocommit: what is this doing, just passing true...
-      ICUTokenizerConfig config = new DefaultICUTokenizerConfig(true) {
+    final ICUTokenizerConfig config = new DefaultICUTokenizerConfig(true) {
         
         @Override
         public BreakIterator getBreakIterator(int script) {
@@ -802,116 +775,283 @@ public class RegisterFieldHandler extend
         // TODO: we could also allow codes->types mapping
       };
 
-      tokenizer = new ICUTokenizer(config);
+    return new TokenizerFactory(Collections.<String,String>emptyMap()) {
+      
+      @Override
+      public Tokenizer create(AttributeFactory factory) {
+        return new ICUTokenizer(factory, config);
+      };
+    };
+  }
+
+  static TokenFilterFactory buildSynonymFilterFactory(IndexState state, Request r) throws IOException {
+
+    Analyzer a = getAnalyzer(state, r, "analyzer");
+    if (a == null) {
+      r.fail("analyzer", "analyzer is required");
+    }
+
+    final SynonymMap synonymMap = parseSynonyms(r.getList("synonyms"), a);
+
+    final boolean ignoreCase = r.getBoolean("ignoreCase");
+
+    // nocommit expand?  dedup?
 
+    return new TokenFilterFactory(Collections.<String,String>emptyMap()) {
+      @Override
+      public TokenFilter create(TokenStream input) {
+        return new SynonymFilter(input, synonymMap, ignoreCase);
+      }
+    };
+  }
+
+  final static Pattern COMMENTS_PATTERN = Pattern.compile("#.*$", Pattern.MULTILINE);
+
+  static Analyzer buildCustomAnalyzer(IndexState state, Version matchVersion, Request chain) throws IOException {
+
+    // nocommit what is MultiTermAwareComponent?
+
+    // nocommit charFilters
+
+    // Build TokenizerFactory:
+    String className;
+    JSONObject t;
+    if (chain.isString("tokenizer")) {
+      className = chain.getString("tokenizer");
+      t = null;
     } else {
-      // BUG
-      tokenizer = null;
-      assert false;
+      Object o = chain.getAndRemoveRaw("tokenizer");
+      if ((o instanceof JSONObject) == false) {
+        chain.fail("tokenizer", "must be string or struct; got: " + o.getClass());
+      }
+      t = (JSONObject) o;
+
+      o = t.get("class");
+      if ((o instanceof String) == false) {
+        chain.fail("tokenizer", "class must be string; got: " + o.getClass());
+      }
+      
+      className = (String) o;
     }
 
-    TokenStream last = tokenizer;
+    TokenizerFactory tokenizerFactory;
+    if (className.toLowerCase(Locale.ROOT).equals("icu")) {
+      tokenizerFactory = buildICUTokenizerFactory(t);
+    } else {
+      Map<String,String> factoryArgs = new HashMap<String,String>();
+      // nocommit how to allow the SPI name and separately
+      // also a fully qualified class name ...
+      factoryArgs.put("class", className);
+      factoryArgs.put("luceneMatchVersion", matchVersion.toString());
+      if (t != null) {
+        for(Map.Entry<String,Object> ent : t.entrySet()) {
+          factoryArgs.put(ent.getKey(), ent.getValue().toString());
+        }
+      }
+
+      try {
+        tokenizerFactory = TokenizerFactory.forName(className, factoryArgs);
+      } catch (IllegalArgumentException iae) {
+        chain.fail("tokenizer", "failed to create TokenizerFactory for class \"" + className + "\": " + iae, iae);
+
+        // Dead code but compiler disagrees:
+        tokenizerFactory = null;
+      }
+
+      if (tokenizerFactory instanceof ResourceLoaderAware) {
+        ((ResourceLoaderAware) tokenizerFactory).inform(state.resourceLoader);
+      }
+    }
+
+    // Build TokenFilters
+    List<TokenFilterFactory> tokenFilterFactories;
     if (chain.hasParam("tokenFilters")) {
+      tokenFilterFactories = new ArrayList<TokenFilterFactory>();
       for(Object o : chain.getList("tokenFilters")) {
-        Request sub = (Request) o;
-        pr = sub.getPoly("class");
-        // nocommit use analysis factories
-        if (pr.name.equals("StandardFilter")) {
-          last = new StandardFilter(matchVersion, last);
-        } else if (pr.name.equals("EnglishPossessiveFilter")) {
-          last = new EnglishPossessiveFilter(matchVersion, last);
-        } else if (pr.name.equals("PorterStemFilter")) {
-          last = new PorterStemFilter(last);
-        } else if (pr.name.equals("ArabicStemFilter")) {
-          last = new ArabicStemFilter(last);
-        } else if (pr.name.equals("EnglishMinimalStemFilter")) {
-          last = new EnglishMinimalStemFilter(last);
-        } else if (pr.name.equals("LowerCaseFilter")) {
-          last = new LowerCaseFilter(matchVersion, last);
-        } else if (pr.name.equals("SetKeywordMarkerFilter")) {
-          CharArraySet set = new CharArraySet(matchVersion, toStringList(pr.r.getList("keyWords")), false);
-          last = new SetKeywordMarkerFilter(last, set);
-        } else if (pr.name.equals("StopFilter")) {
-          CharArraySet set = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
-          last = new StopFilter(matchVersion, last, set);
-        } else if (pr.name.equals("SuggestStopFilter")) {
-          CharArraySet set = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
-          last = new SuggestStopFilter(last, set);
-        } else if (pr.name.equals("SynonymFilter")) {
-          Analyzer a = getAnalyzer(matchVersion, pr.r, "analyzer");
-          if (a == null) {
-            pr.r.fail("analyzer", "analyzer is required");
+        String paramName = "tokenFilters[" + tokenFilterFactories.size() + "]";
+
+        Request sub;
+        JSONObject subParams;
+        if (o instanceof String) {
+          className = (String) o;
+          sub = null;
+          subParams = null;
+        } else {
+          if ((o instanceof Request) == false) {
+            // nocommit make sure test hits this
+            chain.fail(paramName, "each filter must be string or struct; got: " + o.getClass());
           }
+          sub = (Request) o;
+          subParams = sub.getRawParams();
 
-          try {
-            SynonymMap.Parser parser = new SynonymMap.Parser(true, a) {
-                @Override
-                public void parse(Reader in) throws IOException {
-                  // nocommit move parsing in here!
-                };
-              };
-
-            CharsRef scratch = new CharsRef();
-            CharsRef scratchOutput = new CharsRef();
-            for(Object o2 : pr.r.getList("synonyms")) {
-              Request syn = (Request) o2;
-              boolean replace = syn.getBoolean("replace");
-              CharsRef output = new CharsRef(syn.getString("output"));
-              if (!syn.isString("input")) {
-                for(Object o3 : syn.getList("input")) {
-                  parser.add(parser.analyze((String) o3, scratch),
-                             output,
-                             !replace);
+          className = sub.getString("class");
+        }
+
+        TokenFilterFactory tokenFilterFactory;
+
+        if (className.toLowerCase(Locale.ROOT).equals("synonym")) {
+          if (sub == null) {
+            chain.fail(paramName, "no synonyms were specified");
+          }
+          tokenFilterFactory = buildSynonymFilterFactory(state, new Request(chain, paramName, sub.getRawParams(), SYNONYM_FILTER_TYPE));
+        } else {
+
+          Map<String,String> factoryArgs = new HashMap<String,String>();
+
+          // nocommit how to allow the SPI name and separately
+          // also a fully qualified class name ...
+          factoryArgs.put("class", className);
+          factoryArgs.put("luceneMatchVersion", matchVersion.toString());
+
+          ResourceLoader resources = state.resourceLoader;
+          RAMResourceLoaderWrapper ramResources = null;
+
+
+          if (sub != null) {
+
+            for(Map.Entry<String,Object> ent : subParams.entrySet()) {
+              String argName = ent.getKey();
+              Object argValue = ent.getValue();
+
+              // Messy: allow components that expect files for
+              // things like stopword lists, keywords, to come
+              // in as inlined string values.  We "hijack" any
+              // argument name ending in FileContents and make a RAM
+              // file out of it:
+              String argString;
+
+              if (argName.endsWith("FileContents")) {
+                if (ramResources == null) {
+                  ramResources = new RAMResourceLoaderWrapper(resources);
+                  resources = ramResources;
                 }
+
+                String value;
+                if (argValue instanceof String) {
+                  value = (String) argValue;
+                } else if (argValue instanceof JSONArray) {
+                  StringBuilder b = new StringBuilder();
+                  for(Object v : (JSONArray) argValue) {
+                    if ((v instanceof String) == false) {
+                      sub.fail(argName, "array must contain strings; got: " + v.getClass().getSimpleName());
+                    }
+                    b.append((String) v);
+                    b.append('\n');
+                  }
+                  value = b.toString();
+                } else {
+                  sub.fail(argName, "must be a String or JSONArray; got: " + argValue.getClass().getSimpleName());
+
+                  // Dead code but compiler disagrees:
+                  value = null;
+                }
+
+                argName = argName.substring(0, argName.length()-12);
+                ramResources.add(argName, value);
+                argString = argName;
               } else {
-                parser.add(parser.analyze(syn.getString("input"), scratch),
-                           output,
-                           !replace);
+                argString = argValue.toString();
               }
+
+              factoryArgs.put(argName, argString);
             }
-            last = new SynonymFilter(last, parser.build(), pr.r.getBoolean("ignoreCase"));
-          } catch (IOException ioe) {
-            throw new RuntimeException(ioe);
+
+            // Clear all bindings from the incoming request,
+            // so that they are not seen as unused by the
+            // server.  If any params are really unused, the
+            // analysis factory should throw its own
+            // IllegalArgumentException:
+            subParams.clear();
+          }
+
+          try {
+            tokenFilterFactory = TokenFilterFactory.forName(className, factoryArgs);
+          } catch (IllegalArgumentException iae) {
+            chain.fail("tokenizer", "failed to create TokenFilterFactory for class \"" + className + "\": " + iae, iae);
+
+            // Dead code but compiler disagrees:
+            tokenFilterFactory = null;
+          }
+
+          if (tokenFilterFactory instanceof ResourceLoaderAware) {
+            ((ResourceLoaderAware) tokenFilterFactory).inform(resources);
           }
-        } else {
-          assert false: "unrecognized: " + pr.name;
         }
+
+        tokenFilterFactories.add(tokenFilterFactory);
       }
+    } else {
+      tokenFilterFactories = null;
     }
 
-    return new TokenStreamComponents(tokenizer, last);
+    return new CustomAnalyzer(tokenizerFactory, tokenFilterFactories,
+                              chain.getInt("positionIncrementGap"),
+                              chain.getInt("offsetGap"));
+  }
+
+  private static SynonymMap parseSynonyms(List<Object> syns, Analyzer a) {
+    try {
+      // nocommit this is awkward!  I just want to use Parser's
+      // analyze utility method...
+      SynonymMap.Parser parser = new SynonymMap.Parser(true, a) {
+          @Override
+          public void parse(Reader in) throws IOException {
+            // nocommit move parsing in here!
+          };
+        };
+
+      CharsRef scratch = new CharsRef();
+      CharsRef scratchOutput = new CharsRef();
+      for(Object o2 : syns) {
+        Request syn = (Request) o2;
+        boolean replace = syn.getBoolean("replace");
+        CharsRef output = new CharsRef(syn.getString("output"));
+        if (syn.isString("input") == false) {
+          for(Object o3 : syn.getList("input")) {
+            parser.add(parser.analyze((String) o3, scratch),
+                       output,
+                       !replace);
+          }
+        } else {
+          parser.add(parser.analyze(syn.getString("input"), scratch),
+                     output,
+                     !replace);
+        }
+      }
+      return parser.build();
+    } catch (IOException ioe) {
+      throw new RuntimeException(ioe);
+    }
   }
 
   private static class CustomAnalyzer extends Analyzer {
-    private final String json;
-    private final Version matchVersion;
-    private int positionIncrementGap;
-    private int offsetGap;
-
-    public CustomAnalyzer(Version matchVersion, String json) {
-      this.matchVersion = matchVersion;
-      this.json = json;
+    private final int posIncGap;
+    private final int offsetGap;
+    private final TokenizerFactory tokenizerFactory;
+    private final List<TokenFilterFactory> tokenFilterFactories;
+
+    public CustomAnalyzer(TokenizerFactory tokenizerFactory, List<TokenFilterFactory> tokenFilterFactories, int posIncGap, int offsetGap) {
+      this.tokenizerFactory = tokenizerFactory;
+      this.tokenFilterFactories = tokenFilterFactories;
+      this.posIncGap = posIncGap;
+      this.offsetGap = offsetGap;
     }
 
     @Override
     protected TokenStreamComponents createComponents(String fieldName) {
-      JSONObject o;
-      try {
-        o = (JSONObject) JSONValue.parseStrict(json);
-      } catch (net.minidev.json.parser.ParseException pe) {
-        // BUG
-        throw new RuntimeException(pe);
-      }
-      Request r = new Request(null, fieldName, o, (StructType) ANALYZER_TYPE);
-      positionIncrementGap = r.getInt("positionIncrementGap");
-      offsetGap = r.getInt("offsetGap");
-      return buildCustomAnalysisChain(matchVersion,
-                                      r);
+      Tokenizer tokenizer = tokenizerFactory.create();
+      TokenStream last = tokenizer;
+      if (tokenFilterFactories != null) {
+        for(TokenFilterFactory factory : tokenFilterFactories) {
+          last = factory.create(last);
+        }
+      }
+      return new TokenStreamComponents(tokenizer, last);
     }
 
     @Override
     public int getPositionIncrementGap(String fieldName) {
-      return positionIncrementGap;
+      return posIncGap;
     }
 
     @Override
@@ -920,7 +1060,7 @@ public class RegisterFieldHandler extend
     }
   }
 
-  static Analyzer getAnalyzer(Version matchVersionGlobal, Request f, String name) {
+  static Analyzer getAnalyzer(IndexState state, Request f, String name) throws IOException {
     Analyzer analyzer;
     if (f.hasParam(name)) {
       Request a = f.getStruct(name);
@@ -930,10 +1070,11 @@ public class RegisterFieldHandler extend
       if (a.hasParam("matchVersion")) {
         matchVersion = getVersion(a.getEnum("matchVersion"));
       } else {
-        matchVersion = matchVersionGlobal;
+        matchVersion = state.matchVersion;
       }
 
       if (a.hasParam("class")) {
+        // nocommit just lookup via CP:
         // Predefined analyzer class:
         Request.PolyResult pr = a.getPoly("class");
         // TODO: try to "share" a single instance of
@@ -1021,11 +1162,7 @@ public class RegisterFieldHandler extend
           analyzer = null;
         }
       } else if (a.hasParam("tokenizer")) {
-        a.getInt("positionIncrementGap");
-        a.getInt("offsetGap");
-        // Ensures the args are all correct:
-        buildCustomAnalysisChain(matchVersion, a);
-        analyzer = new CustomAnalyzer(matchVersion, jsonOrig);
+        analyzer = buildCustomAnalyzer(state, matchVersion, a);
       } else {
         f.fail(name, "either class or tokenizer/tokenFilters are required");
         analyzer = null;

Modified: lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/java/org/apache/lucene/server/params/Request.java Sun Jan 19 16:52:57 2014
@@ -404,6 +404,13 @@ public class Request {
     return params.get(name);
   }
 
+  /** Returns the raw object, and removes the binding. */
+  public Object getAndRemoveRaw(String name) {
+    Object o = params.get(name);
+    params.remove(name);
+    return o;
+  }
+
   /** Retrieve a struct parameter.  This can be called
    *  multiple times for a given parameter name. */
   public Request getStruct(String name) {
@@ -612,6 +619,12 @@ public class Request {
   }
 
   /** Throws a {@link RequestFailedException} with the
+   *  provided parameter and message. */
+  public void fail(String message, Throwable cause) {
+    fail(null, message, cause);
+  }
+
+  /** Throws a {@link RequestFailedException} with the
    *  provided parameter and message and original cause. */
   public void fail(String param, String reason, Throwable cause) {
     StringBuilder sb = new StringBuilder();

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestAnalysis.java Sun Jan 19 16:52:57 2014
@@ -42,42 +42,56 @@ public class TestAnalysis extends Server
   }
 
   public void testCustomAnalysisChain() throws Exception {
-    //send("{body: {type: text, analyzer: {tokenizer: StandardTokenizer, tokenFilters: [LowerCaseFilter]}}}", "registerFields");
-    //send("{queryParser: {class: classic, defaultField: body}}", "settings");
-    JSONObject o = send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: StandardTokenizer, tokenFilters: [LowerCaseFilter]}}");
-    assertEquals("here is some text", justTokens(o));
+    send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: Standard, tokenFilters: [LowerCase]}}");
+    assertEquals("here is some text", justTokens());
 
-    o = send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: StandardTokenizer}}");
-    assertEquals("Here is some text", justTokens(o));
+    send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: Standard}}");
+    assertEquals("Here is some text", justTokens());
 
-    o = send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: {class: StandardTokenizer, maxTokenLength: 2}, tokenFilters: [LowerCaseFilter]}}");
-    assertEquals("is", justTokens(o));
-
-    // test maxTokenLength
+    send("analyze", "{text: 'Here is some text', analyzer: {tokenizer: {class: Standard, maxTokenLength: 2}, tokenFilters: [LowerCase]}}");
+    assertEquals("is", justTokens());
   }
 
   public void testPatternTokenizer() throws Exception {
-    JSONObject o = send("analyze", "{text: 'Here is \\'some\\' text', analyzer: {tokenizer: {class: PatternTokenizer, pattern: \"\\'([^\\']+)\\'\", group: 1}}}");
-    assertEquals("some", justTokens(o));
+    send("analyze", "{text: 'Here is \\'some\\' text', analyzer: {tokenizer: {class: Pattern, pattern: \"\\'([^\\']+)\\'\", group: 1}}}");
+    assertEquals("some", justTokens());
+  }
+
+  public void testExtraArgs() throws Exception {
+    assertFailsWith("analyze", "{text: 'Here is \\'some\\' text', analyzer: {tokenizer: {class: Pattern, pattern: \"\\'([^\\']+)\\'\", group: 1, bad: 14}}}", "analyze > analyzer > tokenizer: failed to create TokenizerFactory for class \"Pattern\": java.lang.IllegalArgumentException: Unknown parameters: {bad=14}");
   }
 
-  public void testSetKeywordMarkerFilter() throws Exception {
+  public void testKeywordMarkerFilter() throws Exception {
     // No KWMarkerFilter, dogs is stemmed:
-    JSONObject o = send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: StandardTokenizer, tokenFilters: [EnglishPossessiveFilter, LowerCaseFilter, StopFilter, EnglishMinimalStemFilter]}}");
-    assertEquals("here some dog", justTokens(o));
+    send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, Stop, EnglishMinimalStem]}}");
+    assertEquals("here some dog", justTokens());
 
-    // KWMarkerFilter protects dogs:
-    o = send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: StandardTokenizer, tokenFilters: [EnglishPossessiveFilter, LowerCaseFilter, StopFilter, {class: SetKeywordMarkerFilter, keyWords:[dogs]}, EnglishMinimalStemFilter]}}");
-    assertEquals("here some dogs", justTokens(o));
+    // Use KWMarkerFilter with protectedFileContents to protect dogs:
+    send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, Stop, {class: KeywordMarker, protectedFileContents:[dogs]}, EnglishMinimalStem]}}");
+    assertEquals("here some dogs", justTokens());
+
+    // Use KWMarkerFilter with pattern to protect dogs:
+    send("analyze", "{text: 'Here is some dogs', analyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, Stop, {class: KeywordMarker, pattern: dogs}, EnglishMinimalStem]}}");
+    assertEquals("here some dogs", justTokens());
   }
 
   public void testEnglishAnalyzer() throws Exception {
-    JSONObject o = send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer}}");
-    assertEquals("dog go run", justTokens(o));
+    send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer}}");
+    assertEquals("dog go run", justTokens());
 
     // This time protecting dogs from stemming:
-    o = send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer, stemExclusionSet: [dogs]}}");
-    assertEquals("dogs go run", justTokens(o));
+    send("analyze", "{text: 'dogs go running', analyzer: {class: EnglishAnalyzer, stemExclusionSet: [dogs]}}");
+    assertEquals("dogs go run", justTokens());
+  }
+
+  public void testStopFilter() throws Exception {
+    // Uses default (english) stop words:
+    send("analyze", "{text: 'the dogs go running', analyzer: {tokenizer: Whitespace, tokenFilters: [Stop]}}");
+    assertEquals("dogs go running", justTokens());
+
+    // This time making only running a stop word:
+    send("analyze", "{text: 'the dogs go running', analyzer: {tokenizer: Whitespace, tokenFilters: [{class: Stop, wordsFileContents: [running]}]}}");
+    assertEquals("the dogs go", justTokens());
   }
 
   public void testPositionIncrementGap() throws Exception {
@@ -85,32 +99,34 @@ public class TestAnalysis extends Server
     _TestUtil.rmDir(new File("posinc"));
     send("createIndex", "{rootDir: posinc}");
     send("settings", "{directory: RAMDirectory}");
-    send("registerFields", "{fields: {author1: {type: text, analyzer: {tokenizer: WhitespaceTokenizer}, multiValued: true}, author2: {type: text, analyzer: {tokenizer: WhitespaceTokenizer, positionIncrementGap: 1}, multiValued: true}}}");
+    send("registerFields", "{fields: {author1: {type: text, analyzer: {tokenizer: Whitespace}, multiValued: true}, author2: {type: text, analyzer: {tokenizer: Whitespace, positionIncrementGap: 1}, multiValued: true}}}");
     send("startIndex");
-    long gen = getLong(send("addDocument", "{fields: {author1: [bob, smith], author2: [bob, smith]}}"), "indexGen");
+    send("addDocument", "{fields: {author1: [bob, smith], author2: [bob, smith]}}");
+    long gen = getLong("indexGen");
 
     // This one matches because the two values act like they
     // were just concatenated:
-    JSONObject result = send("search", String.format(Locale.ROOT, "{queryText: 'author1: \"bob smith\"', searcher: {indexGen: %d}}", gen));
-    assertEquals(1, getInt(result, "hits.length"));
+    send("search", String.format(Locale.ROOT, "{queryText: 'author1: \"bob smith\"', searcher: {indexGen: %d}}", gen));
+    assertEquals(1, getInt("hits.length"));
 
     // This one doesn't match because a hole is inserted
     // between the two values:
-    result = send("search", String.format(Locale.ROOT, "{queryText: 'author2: \"bob smith\"', searcher: {indexGen: %d}}", gen));
-    assertEquals(0, getInt(result, "hits.length"));
+    send("search", String.format(Locale.ROOT, "{queryText: 'author2: \"bob smith\"', searcher: {indexGen: %d}}", gen));
+    assertEquals(0, getInt("hits.length"));
     send("stopIndex");
     send("deleteIndex");
   }
 
+  // nocommit test loading syns from "file" too
   public void testSynonymFilter() throws Exception {
-    JSONObject o = send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter, {class: SynonymFilter, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns'}]}]}}");
-    assertEquals("dns/0 is/1 complex/2", tokensAndPositions(o));
+    send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase, {class: Synonym, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns'}]}]}}");
+    assertEquals("dns/0 is/1 complex/2", tokensAndPositions());
 
-    o = send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter, {class: SynonymFilter, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns', replace: false}]}]}}");
-    assertEquals("domain/0 dns/0:3 name/1 service/2 is/3 complex/4", tokensAndPositions(o));
+    send("analyze", "{text: 'domain name service is complex', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase, {class: Synonym, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: 'domain name service', output: 'dns', replace: false}]}]}}");
+    assertEquals("domain/0 dns/0:3 name/1 service/2 is/3 complex/4", tokensAndPositions());
 
-    o = send("analyze", "{text: 'mother knows best', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter, {class: SynonymFilter, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: ['mother', 'mommy'], output: 'mom'}]}]}}");
-    assertEquals("mom/0 knows/1 best/2", tokensAndPositions(o));
+    send("analyze", "{text: 'mother knows best', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase, {class: Synonym, ignoreCase: true, analyzer: WhitespaceAnalyzer, synonyms: [{input: ['mother', 'mommy'], output: 'mom'}]}]}}");
+    assertEquals("mom/0 knows/1 best/2", tokensAndPositions());
   }
 
   String ONLY_WHITESPACE_RULES = "\\n!!forward;\\n" + 
@@ -132,13 +148,18 @@ public class TestAnalysis extends Server
     "$NonWhitespace+   {1};";
 
   public void testICUTokenizer() throws Exception {
-    JSONObject o = send("analyze", "{text: 'domain-name service is complex', analyzer: {tokenizer: {class: ICUTokenizer, rules: [{script: Latn, rules: \"" + ONLY_WHITESPACE_RULES + "\"}]}}}");
-    assertEquals("domain-name/0 service/1 is/2 complex/3", tokensAndPositions(o));
+    send("analyze", "{text: 'domain-name service is complex', analyzer: {tokenizer: {class: ICU, rules: [{script: Latn, rules: \"" + ONLY_WHITESPACE_RULES + "\"}]}}}");
+    assertEquals("domain-name/0 service/1 is/2 complex/3", tokensAndPositions());
+  }
+
+  public void testSpanishLightStem() throws Exception {
+    send("analyze", "{text: 'las lomitas', analyzer: {tokenizer: Standard, tokenFilters: [SpanishLightStem]}}");
+    assertEquals("las/0 lomit/1", tokensAndPositions());
   }
 
-  private String justTokens(JSONObject o) {
+  private String justTokens() {
     StringBuilder sb = new StringBuilder();
-    for(Object _o : (JSONArray) o.get("tokens")) {
+    for(Object _o : (JSONArray) lastResult.get("tokens")) {
       JSONObject token = (JSONObject) _o;
       if (sb.length() > 0) {
         sb.append(' ');
@@ -148,9 +169,9 @@ public class TestAnalysis extends Server
     return sb.toString();
   }
 
-  private String tokensAndPositions(JSONObject o) {
+  private String tokensAndPositions() {
     StringBuilder sb = new StringBuilder();
-    for(Object _o : (JSONArray) o.get("tokens")) {
+    for(Object _o : (JSONArray) lastResult.get("tokens")) {
       JSONObject token = (JSONObject) _o;
       if (sb.length() > 0) {
         sb.append(' ');

Modified: lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java?rev=1559524&r1=1559523&r2=1559524&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/server/src/test/org/apache/lucene/server/TestSuggest.java Sun Jan 19 16:52:57 2014
@@ -61,7 +61,7 @@ public class TestSuggest extends ServerB
     out.write("5\u001fthe time is now\u001ffoobar\n");
     out.close();
 
-    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'AnalyzingSuggester', suggestName: 'suggest', indexAnalyzer: EnglishAnalyzer, queryAnalyzer: {tokenizer: StandardTokenizer, tokenFilters: [EnglishPossessiveFilter,LowerCaseFilter,PorterStemFilter]]}}");
+    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'AnalyzingSuggester', suggestName: 'suggest', indexAnalyzer: EnglishAnalyzer, queryAnalyzer: {tokenizer: Standard, tokenFilters: [EnglishPossessive, LowerCase, PorterStem]]}}");
     assertEquals(5, result.get("count"));
     //commit();
 
@@ -81,9 +81,6 @@ public class TestSuggest extends ServerB
       assertEquals(5, get(result, "results[2].weight"));
       assertEquals("foobar", get(result, "results[2].payload"));
 
-      // ForkLastTokenFilter allows 'the' to match
-      // 'theories'; without it (and StopKeywordFilter) the
-      // would be dropped:
       result = send("suggestLookup", "{text: 'the', suggestName: 'suggest'}");
       assertEquals(1, get(result, "results.length"));
 
@@ -107,7 +104,7 @@ public class TestSuggest extends ServerB
     out.write("15\u001flove lost\u001ffoobar\n");
     out.close();
 
-    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: InfixSuggester, suggestName: suggest2, analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: InfixSuggester, suggestName: suggest2, analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
     assertEquals(1, result.get("count"));
     //commit();
 
@@ -145,7 +142,7 @@ public class TestSuggest extends ServerB
     out.write("15\u001flove lost\u001ffoobar\n");
     out.close();
 
-    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'FuzzySuggester', suggestName: 'suggest3', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+    JSONObject result = send("buildSuggest", "{source: {localFile: '" + tempFile.getAbsolutePath() + "'}, class: 'FuzzySuggester', suggestName: 'suggest3', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
     assertEquals(1, result.get("count"));
     //commit();
 
@@ -177,7 +174,7 @@ public class TestSuggest extends ServerB
     send("addDocument", "{fields: {text: 'the cat meows', weight: 1, payload: 'payload1'}}");
     long indexGen = getLong(send("addDocument", "{fields: {text: 'the dog barks', weight: 2, payload: 'payload2'}}"), "indexGen");
 
-    JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightField: weight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+    JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightField: weight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
     // nocommit count isn't returned for stored fields source:
     //assertEquals(2, result.get("count"));
 
@@ -213,7 +210,7 @@ public class TestSuggest extends ServerB
     send("addDocument", "{fields: {text: 'the cat meows', negWeight: -1, payload: 'payload1'}}");
     long indexGen = getLong(send("addDocument", "{fields: {text: 'the dog barks', negWeight: -2, payload: 'payload2'}}"), "indexGen");
 
-    JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightExpression: -negWeight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: WhitespaceTokenizer, tokenFilters: [LowerCaseFilter]}}");
+    JSONObject result = send("buildSuggest", "{source: {searcher: {indexGen: " + indexGen + "}, suggestField: text, weightExpression: -negWeight, payloadField: payload}, class: 'AnalyzingSuggester', suggestName: 'suggest', analyzer: {tokenizer: Whitespace, tokenFilters: [LowerCase]}}");
     // nocommit count isn't returned for stored fields source:
     //assertEquals(2, result.get("count"));