You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/03/16 19:11:10 UTC

svn commit: r1578133 [5/11] - in /lucene/dev/branches/lucene5376_2/lucene: ./ analysis/common/src/java/org/apache/lucene/analysis/charfilter/ analysis/common/src/java/org/apache/lucene/analysis/pattern/ analysis/common/src/java/org/apache/lucene/analys...

Added: lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/GetCommitUserDataHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/GetCommitUserDataHandler.java?rev=1578133&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/GetCommitUserDataHandler.java (added)
+++ lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/GetCommitUserDataHandler.java Sun Mar 16 18:11:07 2014
@@ -0,0 +1,95 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.facet.taxonomy.SearcherTaxonomyManager.SearcherAndTaxonomy;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.server.FinishRequest;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.params.*;
+import net.minidev.json.JSONObject;
+
+/** Handles {@code getCommitUserData}. */
+public class GetCommitUserDataHandler extends Handler {
+
+  private static StructType TYPE = new StructType(
+                                       new Param("indexName", "Index name", new StringType()),
+                                       new Param("searcher", "Specific searcher version to use for retrieving the commit userData; if this is missing, the current IndexWriter's commitData is returned.",
+                                                 SearchHandler.SEARCHER_VERSION_TYPE));
+
+  /** Sole constructor. */
+  public GetCommitUserDataHandler(GlobalState state) {
+    super(state);
+  }
+
+  @Override
+  public StructType getType() {
+    return TYPE;
+  }
+
+  @Override
+  public String getTopDoc() {
+    return "Gets the custom user data in the index, previously set with setCommitUserData.";
+  }
+  
+  @Override
+  public FinishRequest handle(final IndexState state, final Request r, Map<String,List<String>> params) throws Exception {
+
+    return new FinishRequest() {
+      @Override
+      public String finish() throws IOException, InterruptedException {
+        long searcherVersion;
+        Map<String,String> userData;
+        if (r.hasParam("searcher")) {
+          // Specific searcher version:
+          SearcherAndTaxonomy s = SearchHandler.getSearcherAndTaxonomy(r, state, null);
+          try {
+            DirectoryReader dr = (DirectoryReader) s.searcher.getIndexReader();
+            searcherVersion = dr.getVersion();
+            IndexCommit commit = dr.getIndexCommit();
+            userData = commit.getUserData();
+          } finally {
+            state.manager.release(s);
+          }
+
+        } else {
+          // Just use current IndexWriter:
+          searcherVersion = -1;
+          state.verifyStarted(r);
+          userData = state.writer.getIndexWriter().getCommitData();
+        }
+
+        JSONObject result = new JSONObject();
+        result.put("searcher", searcherVersion);
+        JSONObject data = new JSONObject();
+        result.put("userData", data);
+        for(Map.Entry<String,String> ent : userData.entrySet()) {
+          data.put(ent.getKey(), ent.getValue());
+        }
+        return data.toString();
+      }
+    };
+  }
+}

Added: lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/Handler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/Handler.java?rev=1578133&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/Handler.java (added)
+++ lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/Handler.java Sun Mar 16 18:11:07 2014
@@ -0,0 +1,89 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.lucene.server.FinishRequest;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.PreHandle;
+import org.apache.lucene.server.params.*;
+
+// nocommit needs ChannelHandlerContext too?
+
+/** Handles one method. */
+
+public abstract class Handler {
+
+  /** Processes request into a {@link FinishRequest}, which is then
+   *  invoked to actually make changes.  We do this two-step
+   *  process so that we can fail if there are unhandled
+   *  params, without having made any changes to the
+   *  index.  When this returns, it must have retrieved all
+   *  parameters it needs to use from the provided {@link
+   *  Request}. */
+  public abstract FinishRequest handle(IndexState state, Request request, Map<String,List<String>> params) throws Exception;
+
+  /** Returns the {@link StructType} describing the
+   *  parameters this method accepts. */
+  public abstract StructType getType();
+
+  /** Returns the brief summary documentation for this
+   *  method (English). */
+  public abstract String getTopDoc();
+
+  /** The {@link GlobalState} instance. */
+  protected final GlobalState globalState;
+
+  /** True if this handler requires indexName. */
+  public boolean requiresIndexName = true;
+
+  // nocommit nuke this; the handlers can get globalState
+  // via IndexState?  hmm, except for those handlers that
+  // don't take an indexName (e.g. createIndex).
+
+  /** Sole constructor. */
+  protected Handler(GlobalState globalState) {
+    this.globalState = globalState;
+  }
+
+  /** True if this handler uses streaming (e.g. the bulk
+   *  indexing APIs). */
+  public boolean doStream() {
+    return false;
+  }
+
+  /** Invoked for handlers that use streaming. */
+  public String handleStreamed(Reader reader, Map<String,List<String>> params) throws Exception {
+    throw new UnsupportedOperationException();
+  }
+
+  /** For plugins: list of {@link PreHandle}rs, so a plugin
+   *  can modify the request before the default handler
+   *  sees it. */
+  public final List<PreHandle> preHandlers = new CopyOnWriteArrayList<PreHandle>();
+
+  /** Add a pre-handler for this method. */
+  public synchronized void addPreHandle(PreHandle h) {
+    preHandlers.add(h);
+  }
+}

Added: lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveSettingsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveSettingsHandler.java?rev=1578133&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveSettingsHandler.java (added)
+++ lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveSettingsHandler.java Sun Mar 16 18:11:07 2014
@@ -0,0 +1,82 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.server.FinishRequest;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.params.*;
+
+/** For changing index settings that can be changed while
+ *  the index is running. */
+public class LiveSettingsHandler extends Handler {
+
+  /** The parameters this handler accepts. */
+  public final static StructType TYPE =
+    new StructType(
+        new Param("indexName", "The index", new StringType()),
+        new Param("maxRefreshSec", "Longest time to wait before reopening IndexSearcher (i.e., periodic background reopen).", new FloatType(), 1.0f),
+        new Param("minRefreshSec", "Shortest time to wait before reopening IndexSearcher (i.e., when a search is waiting for a specific indexGen).", new FloatType(), .05f),
+        new Param("maxSearcherAgeSec", "Non-current searchers older than this are pruned.", new FloatType(), 60.0f),
+        new Param("index.ramBufferSizeMB", "Size (in MB) of IndexWriter's RAM buffer.", new FloatType(), 16.0f)
+                   );
+
+  @Override
+  public StructType getType() {
+    return TYPE;
+  }
+
+  @Override
+  public String getTopDoc() {
+    return "Change global offline or online settings for this index.";
+  }
+
+  /** Sole constructor. */
+  public LiveSettingsHandler(GlobalState state) {
+    super(state);
+  }
+
+  @Override
+  public FinishRequest handle(final IndexState state, Request r, Map<String,List<String>> params) throws Exception {
+
+    // TODO: should this be done inside finish?  Ie, so it's
+    // "all or no change"?
+    if (r.hasParam("maxRefreshSec")) {
+      state.setMaxRefreshSec(r.getFloat("maxRefreshSec"));
+    }
+    if (r.hasParam("minRefreshSec")) {
+      state.setMinRefreshSec(r.getFloat("minRefreshSec"));
+    }
+    if (r.hasParam("maxSearcherAgeSec")) {
+      state.setMaxSearcherAgeSec(r.getFloat("maxSearcherAgeSec"));
+    }
+    if (r.hasParam("index.ramBufferSizeMB")) {
+      state.setIndexRAMBufferSizeMB(r.getFloat("index.ramBufferSizeMB"));
+    }
+
+    return new FinishRequest() {
+      @Override
+      public String finish() {
+        return state.getLiveSettingsJSON();
+      }
+    };
+  }
+}

Added: lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveValuesHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveValuesHandler.java?rev=1578133&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveValuesHandler.java (added)
+++ lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/LiveValuesHandler.java Sun Mar 16 18:11:07 2014
@@ -0,0 +1,84 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.server.FieldDef;
+import org.apache.lucene.server.FinishRequest;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.StringLiveFieldValues;
+import org.apache.lucene.server.params.ListType;
+import org.apache.lucene.server.params.Param;
+import org.apache.lucene.server.params.Request;
+import org.apache.lucene.server.params.StringType;
+import org.apache.lucene.server.params.StructType;
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+
+/** Handles {@code liveValues}. */
+public class LiveValuesHandler extends Handler {
+  private final static StructType TYPE = new StructType (
+                                             new Param("indexName", "Index name", new StringType()),
+                                             new Param("ids", "List of ids to retrieve", new ListType(new StringType())),
+                                             new Param("field", "Which field value to look up", new StringType()));
+  @Override
+  public String getTopDoc() {
+    return "Lookup live field values.";
+  }
+
+  @Override
+  public StructType getType() {
+    return TYPE;
+  }
+
+  /** Sole constructor. */
+  public LiveValuesHandler(GlobalState state) {
+    super(state);
+  }
+
+  @Override
+  public FinishRequest handle(final IndexState state, final Request r, Map<String,List<String>> params) throws Exception {
+    FieldDef fd = state.getField(r, "field");
+
+    if (fd.liveValuesIDField == null) {
+      r.fail("field", "field \"" + fd.liveValuesIDField + "\" was not registered with liveValues");
+    }
+
+    final List<Object> idValues = r.getList("ids");
+
+    final StringLiveFieldValues lv = state.liveFieldValues.get(fd.name);
+    assert lv != null;
+
+    return new FinishRequest() {
+      @Override
+      public String finish() throws IOException {
+        JSONObject result = new JSONObject();
+        JSONArray arr = new JSONArray();
+        result.put("values", arr);
+        for(Object o : idValues) {
+          arr.add(lv.get((String) o));
+        }
+        return result.toString();
+      }
+    };
+  }  
+}

Added: lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java?rev=1578133&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java (added)
+++ lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RAMResourceLoaderWrapper.java Sun Mar 16 18:11:07 2014
@@ -0,0 +1,74 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.util.ResourceLoader;
+
+/** Holds "files" in RAM, and falls back to delegate if the
+ *  resource isn't in RAM.  This is used for analysis
+ *  components that specify their "files" as strings inlined
+ *  in the JSON (e.g. list of stopwords for a stop filter). */
+
+// nocommit hacky ... e.g. how are these "files" gc'd after loading/parsing
+
+class RAMResourceLoaderWrapper implements ResourceLoader {
+
+  private final Map<String,String> files = new HashMap<String,String>();
+  private final ResourceLoader delegate;
+
+  /** Sole constructor. */
+  public RAMResourceLoaderWrapper(ResourceLoader delegate) {
+    this.delegate = delegate;
+  }
+
+  public void add(String name, String contents) {
+    if (files.containsKey(name)) {
+      throw new IllegalArgumentException("name \"" + name + "\" was already added");
+    }
+    files.put(name, contents);
+  }
+
+  @Override
+  public InputStream openResource(String resource) throws IOException {
+    String value = files.get(resource);
+    if (value != null) {
+      return new ByteArrayInputStream(value.getBytes("UTF-8"));
+    } else if (delegate == null) {
+      throw new FileNotFoundException("resource \"" + resource + "\" is not found");
+    } else {
+      return delegate.openResource(resource);
+    }
+  }
+
+  @Override
+  public <T> T newInstance(String cname, Class<T> expectedType) {
+    return delegate.newInstance(cname, expectedType);
+  }
+
+  @Override
+  public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
+    return delegate.findClass(cname, expectedType);
+  }
+}

Added: lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java?rev=1578133&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java (added)
+++ lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/RegisterFieldHandler.java Sun Mar 16 18:11:07 2014
@@ -0,0 +1,1371 @@
+package org.apache.lucene.server.handlers;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.text.Collator;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharFilter;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.ar.ArabicAnalyzer;
+import org.apache.lucene.analysis.ar.ArabicStemFilter;
+import org.apache.lucene.analysis.bg.BulgarianAnalyzer;
+import org.apache.lucene.analysis.br.BrazilianAnalyzer;
+import org.apache.lucene.analysis.ca.CatalanAnalyzer;
+import org.apache.lucene.analysis.cjk.CJKAnalyzer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.de.GermanAnalyzer;
+import org.apache.lucene.analysis.en.EnglishAnalyzer;
+import org.apache.lucene.analysis.en.EnglishMinimalStemFilter;
+import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
+import org.apache.lucene.analysis.en.PorterStemFilter;
+import org.apache.lucene.analysis.eu.BasqueAnalyzer;
+import org.apache.lucene.analysis.hy.ArmenianAnalyzer;
+import org.apache.lucene.analysis.icu.segmentation.DefaultICUTokenizerConfig;
+import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
+import org.apache.lucene.analysis.icu.segmentation.ICUTokenizerConfig;
+import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
+import org.apache.lucene.analysis.pattern.PatternTokenizer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.synonym.SynonymFilter;
+import org.apache.lucene.analysis.synonym.SynonymFilterFactory;
+import org.apache.lucene.analysis.synonym.SynonymMap;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.CharFilterFactory;
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.collation.CollationKeyAnalyzer;
+import org.apache.lucene.document.FieldType.NumericType;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.expressions.Expression;
+import org.apache.lucene.expressions.js.JavascriptCompiler;
+import org.apache.lucene.facet.FacetsConfig;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.queries.function.ValueSource;
+import org.apache.lucene.search.similarities.BM25Similarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.suggest.analyzing.SuggestStopFilter;
+import org.apache.lucene.server.FieldDef;
+import org.apache.lucene.server.FieldDefBindings;
+import org.apache.lucene.server.FinishRequest;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.params.*;
+import org.apache.lucene.server.params.PolyType.PolyEntry;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.Version;
+import net.minidev.json.JSONArray;
+import net.minidev.json.JSONObject;
+import net.minidev.json.JSONValue;
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.lang.UProperty;
+import com.ibm.icu.lang.UScript;
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.text.RuleBasedBreakIterator;
+
+/** Handles {@code registerFields}. */
+public class RegisterFieldHandler extends Handler {
+
+  private final static List<Object> DEFAULT_ENGLISH_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : EnglishAnalyzer.getDefaultStopSet()) {
+      DEFAULT_ENGLISH_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_ARABIC_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : ArabicAnalyzer.getDefaultStopSet()) {
+      DEFAULT_ARABIC_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_ARMENIAN_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : ArmenianAnalyzer.getDefaultStopSet()) {
+      DEFAULT_ARMENIAN_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_BASQUE_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : BasqueAnalyzer.getDefaultStopSet()) {
+      DEFAULT_BASQUE_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_BRAZILIAN_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : BrazilianAnalyzer.getDefaultStopSet()) {
+      DEFAULT_BRAZILIAN_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_BULGARIAN_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : BulgarianAnalyzer.getDefaultStopSet()) {
+      DEFAULT_BULGARIAN_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_CJK_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : CJKAnalyzer.getDefaultStopSet()) {
+      DEFAULT_CJK_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_CATALAN_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : CatalanAnalyzer.getDefaultStopSet()) {
+      DEFAULT_CATALAN_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  private final static List<Object> DEFAULT_GERMAN_STOP_WORDS = new ArrayList<Object>();
+  static {
+    for(Object o : GermanAnalyzer.getDefaultStopSet()) {
+      DEFAULT_GERMAN_STOP_WORDS.add(new String((char[]) o));
+    }
+  }
+
+  // Breaks the recursion:
+  private final static WrapType ANALYZER_TYPE_WRAP = new WrapType();
+
+  /** Type to accept lucene versions. */
+  public final static Param MATCH_VERSION_PARAM = new Param("matchVersion", "Lucene version to match.", new EnumType("LUCENE_40", "LUCENE_40",
+                                                                                                                     "LUCENE_41", "LUCENE_41",
+                                                                                                                     "LUCENE_42", "LUCENE_42",
+                                                                                                                     "LUCENE_43", "LUCENE_43",
+                                                                                                                     "LUCENE_44", "LUCENE_44",
+                                                                                                                     "LUCENE_45", "LUCENE_45",
+                                                                                                                     "LUCENE_46", "LUCENE_46"));
+
+  /** Analyzer type. */
+  final static Type ANALYZER_TYPE =
+    new StructType(
+                   new Param("class",
+                             "An existing Analyzer class.  Use either this, or define your own analysis chain by setting tokenizer and tokenFilter.",
+                             new PolyType(Analyzer.class,
+                                          new PolyEntry("ArabicAnalyzer", "Analyzer for Arabic (see @lucene:analyzers-common:org.apache.lucene.analysis.ar.ArabicAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis",
+                                                                  new ListType(new StringType()), DEFAULT_ARABIC_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("ArmenianAnalyzer", "Analyzer for Armenian. (see @lucene:analyzers-common:org.apache.lucene.analysis.hy.ArmenianAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis",
+                                                                  new ListType(new StringType()), DEFAULT_ARMENIAN_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("BasqueAnalyzer", "Analyzer for Basque. (see @lucene:analyzers-common:org.apache.lucene.analysis.eu.BasqueAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis",
+                                                                  new ListType(new StringType()), DEFAULT_BASQUE_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("BrazilianAnalyzer", "Analyzer for Brazilian Portuguese language (see @lucene:analyzers-common:org.apache.lucene.analysis.br.BrazilianAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis",
+                                                                  new ListType(new StringType()), DEFAULT_BRAZILIAN_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("BulgarianAnalyzer", "Analyzer for Bulgarian (see @lucene:analyzers-common:org.apache.lucene.analysis.bg.BulgarianAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis",
+                                                                  new ListType(new StringType()), DEFAULT_BULGARIAN_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("CatalanAnalyzer", "Analyzer for Catalan (see @lucene:analyzers-common:org.apache.lucene.analysis.ca.CatalanAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis",
+                                                                  new ListType(new StringType()), DEFAULT_BULGARIAN_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("CJKAnalyzer", "An Analyzer that tokenizes text with StandardTokenizer, normalizes content with CJKWidthFilter, folds case with LowerCaseFilter, forms bigrams of CJK with CJKBigramFilter, and filters stopwords with StopFilter (see @lucene:analyzers-common:org.apache.lucene.analysis.cjk.CJKAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis",
+                                                                  new ListType(new StringType()), DEFAULT_CJK_STOP_WORDS)),
+                                          new PolyEntry("CollationKeyAnalyzer", "<p> Configures KeywordTokenizer with CollationAttributeFactory (see @lucene:analyzers-common:org.apache.lucene.collation.CollationKeyAnalyzer)",
+                                                        new Param("locale", "Locale", SearchHandler.LOCALE_TYPE)),
+                                          new PolyEntry("EnglishAnalyzer", "Analyzer for English. (see @lucene:analyzers-common:org.apache.lucene.analysis.en.EnglishAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis.",
+                                                                  new ListType(new StringType()), DEFAULT_ENGLISH_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("GermanAnalyzer", "Analyzer for German language (see @lucene:analyzers-common:org.apache.lucene.analysis.de.GermanAnalyzer)",
+                                                        new Param("stopWords", "Stop words to remove during analysis.",
+                                                                  new ListType(new StringType()), DEFAULT_GERMAN_STOP_WORDS),
+                                                        new Param("stemExclusionSet", "A set of terms not to be stemmed",
+                                                                  new ListType(new StringType()))),
+                                          new PolyEntry("StandardAnalyzer", "Filters StandardTokenizer with StandardFilter, LowerCaseFilter and StopFilter, using a list of English stop words (see @lucene:analyzers-common:org.apache.lucene.analysis.standard.StandardAnalyzer)",
+                                                        new Param("maxTokenLength", "Max token length.", new IntType(), StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH),
+                                                        new Param("stopWords", "Stop words to remove during analysis.",
+                                                                  new ListType(new StringType()), DEFAULT_ENGLISH_STOP_WORDS)),
+                                          new PolyEntry("WhitespaceAnalyzer", "An Analyzer that uses WhitespaceTokenizer (see @lucene:analyzers-common:org.apache.lucene.analysis.core.WhitespaceAnalyzer)")),
+                             "StandardAnalyzer"),
+                   new Param("positionIncrementGap", "How many positions to insert between separate values in a multi-valued field", new IntType(), 0),
+                   new Param("offsetGap", "How many offsets to insert between separate values in a multi-valued field", new IntType(), 1),
+                   new Param("tokenizer", "Tokenizer class (for a custom analysis chain).",
+                             new OrType(new StringType(), new StructType(new Param("class", "Tokenizer short name (e.g, 'Whitespace')", new StringType())))),
+                             // nocommit somehow tap into TokenizerFactory.availableTokenizers
+                   new Param("tokenFilters", "Optional chain of TokenFilters to apply after the Tokenizer",
+                             new ListType(
+                                 new OrType(new StringType(), new StructType(new Param("class", "TokenFilter short name (e.g. 'Stop')", new StringType()))))),
+                   new Param("charFilters", "Optional chain of CharFilters to apply beforethe Tokenizer",
+                             new ListType(
+                                 new OrType(new StringType(), new StructType(new Param("class", "CharFilter short name", new StringType()))))),
+                   MATCH_VERSION_PARAM);
+
+  static StructType SYNONYM_FILTER_TYPE = new StructType(
+                                              new Param("ignoreCase", "True if matching should be case insensitive", new BooleanType(), true),
+                                              new Param("analyzer", "Analyzer to use to tokenize synonym inputs", ANALYZER_TYPE_WRAP),
+                                              new Param("synonyms", "Synonyms",
+                                                  /** nocommit syn filter: maybe the simpler groups / aliases format? */
+                                                  new ListType(
+                                                      new StructType(
+                                                          new Param("input", "String or list of strings with input token(s) to match", new OrType(new ListType(new StringType()), new StringType())),
+                                                          // TODO: allow more than one token on the output?
+                                                          new Param("output", "Single token to replace the matched tokens with", new StringType()),
+                                                          new Param("replace", "True if the input tokens should be replaced with the output token; false if the input tokens should be preserved and the output token overlaid", new BooleanType(), true)))));
+
+  static StructType ICU_TOKENIZER_TYPE = new StructType(
+                                             new Param("cjkAsWords", "??", new BooleanType(), true),
+                                             new Param("rules", "Customize the tokenization per-script",
+                                                 new ListType(
+                                                     new StructType(
+                                                         new Param("script", "Script", new StringType()),
+                                                         new Param("rules", "Rules", new StringType())))));
+
+  static {
+    ANALYZER_TYPE_WRAP.set(ANALYZER_TYPE);
+  }
+
+  private final static StructType FIELD_TYPE =
+    new StructType(
+        new Param("type", "Type of the value.",
+                  new EnumType("text", "Text that's tokenized and indexed, with the index-time analyzer.",
+                               "atom", "Text that's indexed as a single token, with DOCS_ONLY and omitting norms.",
+                               "boolean", "Boolean value.",
+                               "float", "Float value.",
+                               "double", "Double value.",
+                               "int", "Int value.",
+                               "long", "Long value.",
+                               // nocommit name this "dynamic" instead of "virtual"?
+                               "virtual", "Virtual field defined with a JavaScript expression.",
+                               // nocommit need tests for internal:
+                               "internal", "Internal field, currently only for holding indexed facets data.")),
+        new Param("search", "True if the value should be available for searching (or numeric range searching, for a numeric field).", new BooleanType(), false),
+        new Param("tokenize", "True if the value should be tokenized.", new BooleanType(), true),
+        new Param("store", "True if the value should be stored.", new BooleanType(), false),
+        new Param("multiValued", "True if this field may sometimes have more than one value.", new BooleanType(), false),
+        new Param("highlight", "True if the value should be indexed for highlighting.", new BooleanType(), false),
+        new Param("postingsFormat", "Which PostingsFormat should be used to index this field.",
+                  new StringType(), "Lucene41"),
+        new Param("docValuesFormat", "Which DocValuesFormat should be used to index this field.",
+                  new StringType(), "Lucene45"),
+        new Param("sort", "True if the value should be indexed into doc values for sorting.", new BooleanType(), false),
+        new Param("group", "True if the value should be indexed into doc values for grouping.", new BooleanType(), false),
+        new Param("facet", "Whether this field should index facets, and how.",
+                  new EnumType("no", "No facets are indexed.",
+                               "flat", "Facets are indexed with no hierarchy.",
+                               "hierarchy", "Facets are indexed and are hierarchical.",
+                               "numericRange", "Compute facet counts for custom numeric ranges",
+                               "sortedSetDocValues", "Use SortedSetDocValuesFacetCounts, which must be flat but don't require a taxonomy index"),
+                  "no"),
+        new Param("facetIndexFieldName",
+                  "Which underlying Lucene index field is used to hold any indexed taxonomy or sorted set doc values facets",
+                  new StringType(), FacetsConfig.DEFAULT_INDEX_FIELD_NAME),
+        new Param("storeDocValues", "Whether to index the value into doc values.", new BooleanType(), false),
+        new Param("liveValues", "Enable live values for this field: whenever this field is retrieved during a search, the live (most recetly added) value will always be returned; set this to the field name of your id (primary key) field.  Uses @lucene:core:org.apache.lucene.index.LiveFieldValues under the hood.", new StringType()),
+        new Param("numericPrecisionStep", "If the value is numeric, what precision step to use during indexing.", new IntType(), NumericUtils.PRECISION_STEP_DEFAULT), // nocommit is 16 better?
+        new Param("omitNorms", "True if norms are omitted.", new BooleanType(), false),
+        new Param("analyzer", "Analyzer to use for this field during indexing and searching.", ANALYZER_TYPE),
+        new Param("indexAnalyzer", "Analyzer to use for this field during indexing.", ANALYZER_TYPE),
+        new Param("searchAnalyzer", "Analyzer to use for this field during searching.", ANALYZER_TYPE),
+        new Param("indexOptions", "How the tokens should be indexed.",
+                  new EnumType("docs", "Index only doc ids (for binary search).",
+                               "docsFreqs", "Index doc ids and term frequencies.",
+                               "docsFreqsPositions", "Index doc ids, term frequences and positions.",
+                               "docsFreqsPositionsOffsets", "Index doc ids, term frequencies, positions and offsets."),
+                  "docsFreqsPositions"),
+        new Param("expression", "The JavaScript expression defining a virtual field's value (only used with type=virtual).", new StringType()),
+        new Param("termVectors", "Whether/how term vectors should be indexed.",
+                  new EnumType("terms", "Index terms and freqs only.",
+                               "termsPositions", "Index terms, freqs and positions.",
+                               "termsPpositionsOffsets", "Index terms, freqs, positions and offsets.",
+                               "termsPositionsdOoffsetsPayloads", "Index terms, freqs, positions, offsets and payloads."
+                               )),
+        new Param("similarity", "Which Similarity implementation to use for this field.",
+                  new StructType(
+                                 new Param("class",
+                                           "Which Similarity class to use.",
+                                           new PolyType(Similarity.class,
+                                               new PolyEntry("DefaultSimilarity", "Expert: Default scoring implementation. (see @lucene:core:org.apache.lucene.search.similarities.DefaultSimilarity)", new StructType()),
+                                               new PolyEntry("BM25Similarity", "BM25 Similarity (see @lucene:core:org.apache.lucene.search.similarities.BM25Similarity)",
+                                                   new StructType(new Param("k1", "Controls non-linear term frequency normalization (saturation).", new FloatType(), 1.2f),
+                                                                  new Param("b", "Controls to what degree document length normalizes tf values.", new FloatType(), 0.75f)))),
+                                           "DefaultSimilarity")))
+                   );
+
+  /** Method type. */
+  public final static StructType TYPE =
+    new StructType(
+        new Param("indexName", "Index name", new StringType()),
+        new Param("fields", "New fields to register",
+            new StructType(new Param("*", "Register this field name with the provided type.  Note that the field name must be of the form [a-zA-Z_][a-zA-Z_0-9]*.  You can register multiple fields in one request.", FIELD_TYPE))));
+
+  /** Sole constructor. */
+  public RegisterFieldHandler(GlobalState state) {
+    super(state);
+  }
+
+  @Override
+  public String getTopDoc() {
+    return "Registers one or more fields.  Fields must be registered before they can be added in a document (via @addDocument).  Pass a struct whose keys are the fields names to register and whose values define the type for that field.  Any number of fields may be registered in a single request, and once a field is registered it cannot be changed (write-once).  <p>This returns the full set of fields currently registered.";
+  }
+
+  @Override
+  public StructType getType() {
+    return TYPE;
+  }
+
+  private FieldDef parseOneVirtualFieldType(Request r, IndexState state, Map<String,FieldDef> pendingFieldDefs, String name, JSONObject o) {
+    String exprString = r.getString("expression");
+    Expression expr;
+
+    try {
+      expr = JavascriptCompiler.compile(exprString);
+    } catch (ParseException pe) {
+      // Static error (e.g. bad JavaScript syntax):
+      r.fail("expression", "could not parse expression: " + pe, pe);
+
+      // Dead code but compiler disagrees:
+      expr = null;
+    } catch (IllegalArgumentException iae) {
+      // Static error (e.g. bad JavaScript syntax):
+      r.fail("expression", "could not parse expression: " + iae, iae);
+
+      // Dead code but compiler disagrees:
+      expr = null;
+    }
+
+    Map<String,FieldDef> allFields = new HashMap<String,FieldDef>(state.getAllFields());
+    allFields.putAll(pendingFieldDefs);
+
+    ValueSource values;
+    try {
+      values = expr.getValueSource(new FieldDefBindings(allFields));
+    } catch (RuntimeException re) {
+      // Dynamic error (e.g. referred to a field that
+      // doesn't exist):
+      r.fail("expression", "could not evaluate expression: " + re, re);
+
+      // Dead code but compiler disagrees:
+      values = null;
+    }
+
+    return new FieldDef(name, null, "virtual", null, null, null, true, null, null, null, false, null, values);
+  }
+
+  private FieldDef parseOneFieldType(Request r, IndexState state, Map<String,FieldDef> pendingFieldDefs, String name, JSONObject o) throws IOException {
+
+    // This way f.fail reports which field name was problematic:
+    Request f = new Request(r, name, o, FIELD_TYPE);
+
+    String type = f.getEnum("type");
+    if (type.equals("virtual")) {
+      return parseOneVirtualFieldType(f, state, pendingFieldDefs, name, o);
+    }
+
+    FieldType ft = new FieldType();
+
+    // nocommit why do we have storeDocValues?  it's too low level?
+    boolean dv = f.getBoolean("storeDocValues");
+    boolean sorted = f.getBoolean("sort");
+    boolean grouped = f.getBoolean("group");
+
+    // TODO: current we only highlight using
+    // PostingsHighlighter; if we enable others (that use
+    // term vectors), we need to fix this so app specifies
+    // up front which highlighter(s) it wants to use at
+    // search time:
+    boolean highlighted = f.getBoolean("highlight");
+
+    if (highlighted) {
+      if (type.equals("text") == false && type.equals("atom") == false) {
+        f.fail("highlight", "only type=text or type=atom fields can have highlight=true");
+      }
+    }
+
+    boolean multiValued = f.getBoolean("multiValued");
+    if (multiValued) {
+      if (sorted) {
+        f.fail("multiValued", "cannot sort on multiValued fields");
+      }
+      if (grouped) {
+        f.fail("multiValued", "cannot group on multiValued fields");
+      }
+    }      
+
+    if (type.equals("text")) {
+      if (sorted) {
+        f.fail("sort", "cannot sort text fields; use atom instead");
+      }
+      ft.setIndexed(true);
+      ft.setTokenized(true);
+      if (grouped) {
+        ft.setDocValueType(DocValuesType.SORTED);
+      } else if (dv) {
+        ft.setDocValueType(DocValuesType.BINARY);
+      }
+      if (highlighted) {
+        ft.setStored(true);
+        ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+      }
+    } else if (type.equals("atom")) {
+      if (f.hasParam("analyzer")) {
+        f.fail("analyzer", "no analyzer allowed with atom (it's hardwired to KeywordAnalyzer internally)");
+      }
+      ft.setIndexed(true);
+      ft.setIndexOptions(IndexOptions.DOCS_ONLY);
+      ft.setOmitNorms(true);
+      ft.setTokenized(false);
+      if (sorted || grouped) {
+        ft.setDocValueType(DocValuesType.SORTED);
+      } else if (grouped || dv) {
+        ft.setDocValueType(DocValuesType.BINARY);
+      }
+    } else if (type.equals("boolean")) {
+      if (dv || sorted || grouped) {
+        ft.setDocValueType(DocValuesType.NUMERIC);
+      }
+    } else if (type.equals("long")) {
+      if (dv || sorted || grouped) {
+        ft.setDocValueType(DocValuesType.NUMERIC);
+      }
+    } else if (type.equals("int")) {
+      if (dv || sorted || grouped) {
+        ft.setDocValueType(DocValuesType.NUMERIC);
+      }
+    } else if (type.equals("double")) {
+      if (dv || sorted || grouped) {
+        ft.setDocValueType(DocValuesType.NUMERIC);
+      }
+    } else if (type.equals("float")) {
+      if (dv || sorted || grouped) {
+        ft.setDocValueType(DocValuesType.NUMERIC);
+      }
+    } else {
+      assert false;
+    }
+
+    if (f.hasParam("store")) {
+      ft.setStored(f.getBoolean("store"));
+      if (!ft.stored() && highlighted) {
+        f.fail("store", "store=false is not allowed when highlight=true");
+      }
+    }
+
+    if (f.hasParam("search")) {
+      ft.setIndexed(f.getBoolean("search"));
+    }
+
+    if (f.hasParam("analyzer") && !ft.indexed()) {
+      f.fail("analyzer", "no analyzer allowed when search=false");
+    }
+
+    if (type.equals("text") || type.equals("atom")) {
+
+      if (ft.indexed()) {
+        if (f.hasParam("tokenize")) {
+          ft.setTokenized(f.getBoolean("tokenize"));
+        }
+        if (f.hasParam("omitNorms")) {
+          ft.setOmitNorms(f.getBoolean("omitNorms"));
+        }
+
+        if (f.hasParam("termVectors")) {
+          String tv = f.getString("termVectors");
+          if (tv.equals("terms")) {
+            ft.setStoreTermVectors(true);
+          } else if (tv.equals("termsPositions")) {
+            ft.setStoreTermVectors(true);
+            ft.setStoreTermVectorPositions(true);
+          } else if (tv.equals("termsPositionsOffsets")) {
+            ft.setStoreTermVectors(true);
+            ft.setStoreTermVectorPositions(true);
+            ft.setStoreTermVectorOffsets(true);
+          } else if (tv.equals("termsPositionsOffsetsPayloads")) {
+            ft.setStoreTermVectors(true);
+            ft.setStoreTermVectorPositions(true);
+            ft.setStoreTermVectorOffsets(true);
+            ft.setStoreTermVectorPayloads(true);
+          } else {
+            assert false;
+          }
+        }
+
+        if (f.hasParam("indexOptions")) {
+          String io = f.getString("indexOptions");
+          if (io.equals("docs")) {
+            ft.setIndexOptions(IndexOptions.DOCS_ONLY);
+          } else if (io.equals("docsFreqs")) {
+            ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+          } else if (io.equals("docsFreqsPositions")) {
+            ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+          } else if (io.equals("docsFreqsPositionsAndOffsets")) {
+            ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+          } else {
+            assert false;
+          }
+        }
+      }
+    } else if (type.equals("boolean")) {
+      ft.setIndexed(true);
+      ft.setOmitNorms(true);
+      ft.setTokenized(false);
+      ft.setIndexOptions(IndexOptions.DOCS_ONLY);
+    } else if (ft.indexed()) {
+      ft.setNumericPrecisionStep(f.getInt("numericPrecisionStep"));
+      if (type.equals("float")) {
+        ft.setNumericType(NumericType.FLOAT);
+      } else if (type.equals("double")) {
+        ft.setNumericType(NumericType.DOUBLE);
+      } else if (type.equals("long")) {
+        ft.setNumericType(NumericType.LONG);
+      } else {
+        assert type.equals("int"): "type=" + type;
+        ft.setNumericType(NumericType.INT);
+      }
+    }
+
+    String pf = f.getString("postingsFormat");
+    if (PostingsFormat.forName(pf) == null) {
+      f.fail("postingsFormat", "unrecognized postingsFormat \"" + pf + "\"");
+    }
+    String dvf = f.getString("docValuesFormat");
+    if (DocValuesFormat.forName(dvf) == null) {
+      f.fail("docValuesFormat", "unrecognized docValuesFormat \"" + dvf + "\"");
+    }
+
+    Similarity sim;
+    if (f.hasParam("similarity")) {
+      Request s = f.getStruct("similarity");
+      Request.PolyResult pr = s.getPoly("class");
+      if (pr.name.equals("DefaultSimilarity")) {
+        sim = new DefaultSimilarity();
+      } else if (pr.name.equals("BM25Similarity")) {
+        sim = new BM25Similarity(pr.r.getFloat("k1"), pr.r.getFloat("b"));
+      } else {
+        assert false;
+        sim = null;
+      }
+    } else {
+      sim = new DefaultSimilarity();
+    }
+
+    Analyzer indexAnalyzer;
+    Analyzer searchAnalyzer;
+    Analyzer analyzer = getAnalyzer(state, f, "analyzer");
+    if (analyzer != null) {
+      indexAnalyzer = searchAnalyzer = analyzer;
+    } else {
+      indexAnalyzer = getAnalyzer(state, f, "indexAnalyzer");
+      searchAnalyzer = getAnalyzer(state, f, "searchAnalyzer");
+    }
+
+    if (type.equals("text") && ft.indexed()) {
+      if (indexAnalyzer == null) {
+        f.fail("indexAnalyzer", "either analyzer or indexAnalyzer must be specified for an indexed text field");
+      }
+      if (searchAnalyzer == null) {
+        f.fail("searchAnalyzer", "either analyzer or searchAnalyzer must be specified for an indexed text field");
+      }
+    }
+
+    if (indexAnalyzer == null) {
+      indexAnalyzer = dummyAnalyzer;
+    }
+
+    if (searchAnalyzer == null) {
+      searchAnalyzer = searchAnalyzer;
+    }
+
+    String liveValuesIDField;
+    if (f.hasParam("liveValues")) {
+      // nocommit: sort of silly you cannot register id & live
+      // fields in same request...
+      FieldDef idField = state.getField(f, "liveValues");
+      liveValuesIDField = idField.name;
+      if (!type.equals("atom") && !type.equals("atom")) {
+        f.fail("liveValues", "only type=atom or type=text fields may have liveValues enabled");
+      }
+      if (multiValued) {
+        f.fail("liveValues", "liveValues fields must not be multiValued");
+      }
+      if (!ft.stored()) {
+        f.fail("liveValues", "this field is not stored");
+      }
+      if (!idField.fieldType.stored()) {
+        f.fail("liveValues", "id field \"" + liveValuesIDField + "\" is not stored");
+      }
+      if (idField.multiValued) {
+        f.fail("liveValues", "id field \"" + liveValuesIDField + "\" must not be multiValued");
+      }
+      if (!idField.valueType.equals("atom") && !idField.valueType.equals("text")) {
+        f.fail("liveValues", "id field \"" + liveValuesIDField + "\" must have type=atom or type=text");
+      }
+      // TODO: we could relax this, since
+      // PostingsHighlighter lets you pull from "private"
+      // source:
+      if (highlighted) {
+        f.fail("liveValues", "cannot highlight live fields");
+      }
+    } else {
+      liveValuesIDField = null;
+    }
+
+    String facet = f.getEnum("facet");
+    if (facet.equals("hierarchy")) {
+      if (highlighted) {
+        f.fail("facet", "facet=hierarchy fields cannot have highlight=true");
+      }
+      if (ft.indexed()) {
+        f.fail("facet", "facet=hierarchy fields cannot have search=true");
+      }
+      if (ft.stored()) {
+        f.fail("facet", "facet=hierarchy fields cannot have store=true");
+      }
+    } else if (facet.equals("numericRange")) {
+      if (!type.equals("long") && !type.equals("int") && !type.equals("float") && !type.equals("double")) {
+        f.fail("facet", "numericRange facets only applies to numeric types");
+      }
+      if (!ft.indexed()) {
+        f.fail("search", "facet=numericRange fields must have search=true");
+      }
+      // We index the field as NumericField, for drill-down, and store doc values, for dynamic facet counting
+      ft.setDocValueType(DocValuesType.NUMERIC);
+    } else if (facet.equals("no")) {
+      if (ft.indexed() == false && ft.stored() == false && ft.docValueType() == null) {
+        f.fail("field does nothing: it's neither searched, stored, sorted, grouped, highlighted nor faceted");
+      }
+    }
+
+    ft.freeze();
+
+    if (facet.equals("no") == false && facet.equals("numericRange") == false) {
+      // hierarchy, float or sortedSetDocValues
+      if (facet.equals("hierarchy")) {
+        state.facetsConfig.setHierarchical(name, true);
+      }
+      if (multiValued) {
+        state.facetsConfig.setMultiValued(name, true);
+      }
+      state.facetsConfig.setIndexFieldName(name, f.getString("facetIndexFieldName"));
+    }
+
+    // nocommit facetsConfig.setRequireDimCount
+
+    return new FieldDef(name, ft, type, facet, pf, dvf, multiValued, sim, indexAnalyzer, searchAnalyzer, highlighted, liveValuesIDField, null);
+  }
+
+  /** Messy: we need this for indexed-but-not-tokenized
+   *  fields, solely for .getOffsetGap I think. */
+  public final static Analyzer dummyAnalyzer = new Analyzer() {
+      @Override
+      protected TokenStreamComponents createComponents(String fieldName) {
+        throw new UnsupportedOperationException();
+      }
+    };
+
+  private static List<String> toStringList(List<Object> l) {
+    List<String> words = new ArrayList<String>();
+    for(Object o : l) {
+      words.add((String) o);
+    }
+    return words;
+  }
+
+  final static Pattern COMMENTS_PATTERN = Pattern.compile("#.*$", Pattern.MULTILINE);
+
+  static TokenizerFactory buildICUTokenizerFactory(Request sub) {
+      
+    boolean cjkAsWords;
+
+    final BreakIterator breakers[];
+
+    if (sub != null) {
+      Request icuRequest = new Request(sub, ICU_TOKENIZER_TYPE);
+            
+      cjkAsWords = icuRequest.getBoolean("cjkAsWords");
+
+      if (icuRequest.hasParam("rules")) {
+        breakers = new BreakIterator[UScript.CODE_LIMIT];
+        for(Object o : icuRequest.getList("rules")) {
+          Request r2 = (Request) o;
+          String script = r2.getString("script");
+          String rules = r2.getString("rules");
+          rules = COMMENTS_PATTERN.matcher(rules).replaceAll("");
+          int code;
+          try {
+            code = UCharacter.getPropertyValueEnum(UProperty.SCRIPT, script);
+          } catch (IllegalArgumentException iae) {
+            r2.fail("script", "failed to parse as script code: " + iae.getMessage());
+            // Dead code but compiler disagrees:
+            code = -1;
+          }
+          try {
+            breakers[code] = new RuleBasedBreakIterator(rules);
+          } catch (IllegalArgumentException iae) {
+            r2.fail("rules", "failed to parse rules: " + iae.getMessage());
+          }
+        }
+      } else {
+        breakers = null;
+      }
+    } else {
+      cjkAsWords = true;
+      breakers = null;
+    }
+
+    final ICUTokenizerConfig config = new DefaultICUTokenizerConfig(true) {
+        
+        @Override
+        public BreakIterator getBreakIterator(int script) {
+          if (breakers[script] != null) {
+            return (BreakIterator) breakers[script].clone();
+          } else {
+            return super.getBreakIterator(script);
+          }
+        }
+
+        // TODO: we could also allow codes->types mapping
+      };
+
+    return new TokenizerFactory(Collections.<String,String>emptyMap()) {
+      
+      @Override
+      public Tokenizer create(AttributeFactory factory) {
+        return new ICUTokenizer(factory, config);
+      };
+    };
+  }
+
+  // nocommit can we use SynonmyFilterFactory???  we could
+  // make a custom Reader subclass that sneaks the
+  // JSONObject "through", and then a custom
+  // SynFilter.Parser that grabs that JSONObject?
+
+  static TokenFilterFactory buildSynonymFilterFactory(IndexState state, Request r) throws IOException {
+
+    Analyzer a = getAnalyzer(state, r, "analyzer");
+    if (a == null) {
+      r.fail("analyzer", "analyzer is required");
+    }
+
+    final SynonymMap synonymMap = parseSynonyms(r.getList("synonyms"), a);
+
+    final boolean ignoreCase = r.getBoolean("ignoreCase");
+
+    // nocommit expand?  dedup?
+
+    return new TokenFilterFactory(Collections.<String,String>emptyMap()) {
+      @Override
+      public TokenFilter create(TokenStream input) {
+        return new SynonymFilter(input, synonymMap, ignoreCase);
+      }
+    };
+  }
+
+  private static List<CharFilterFactory> parseCharFilters(IndexState state, Version matchVersion, Request chain) throws IOException {
+    List<CharFilterFactory> charFilters;
+    if (chain.hasParam("charFilters")) {
+      charFilters = new ArrayList<CharFilterFactory>();
+      for(Object o : chain.getList("charFilters")) {
+        String paramName = "charFilters[" + charFilters.size() + "]";
+        Request sub;
+        String className;
+        if (o instanceof String) {
+          className = (String) o;
+          sub = null;
+        } else {
+          // The type already validated this:
+          assert o instanceof Request;
+          sub = (Request) o;
+          className = sub.getString("class");
+        }
+
+        TwoThings<Map<String,String>,ResourceLoader> things = parseArgsAndResources(state,
+                                                                                    className,
+                                                                                    matchVersion,
+                                                                                    sub);
+
+        CharFilterFactory factory;
+        try {
+          factory = CharFilterFactory.forName(className, things.a);
+        } catch (IllegalArgumentException iae) {
+          chain.fail(paramName, "failed to create CharFilterFactory for class \"" + className + "\": " + iae, iae);
+
+          // Dead code but compiler disagrees:
+          factory = null;
+        }
+
+        if (factory instanceof ResourceLoaderAware) {
+          ((ResourceLoaderAware) factory).inform(things.b);
+        }
+
+        charFilters.add(factory);
+      }
+    } else {
+      charFilters = null;
+    }
+
+    return charFilters;
+  }
+
+  private static final class TwoThings<A,B> {
+    public final A a;
+    public final B b;
+
+    public TwoThings(A a, B b) {
+      this.a = a;
+      this.b = b;
+    }
+  };
+
+  /** Parses the arguments for an analysis factory
+   *  component, but also detects any argument name of the
+   *  form xxxFileContents and puts its value into a "fake"
+   *  (RAM) file, leaving xxx referring to that file.  This
+   *  way any existing component expecting to load a
+   *  resource from a "file" will (hacky) work. */
+  private static TwoThings<Map<String,String>,ResourceLoader> parseArgsAndResources(IndexState state,
+                                                                                    String className,
+                                                                                    Version matchVersion,
+                                                                                    Request sub) {
+    
+    Map<String,String> factoryArgs = new HashMap<String,String>();
+
+    // nocommit how to allow the SPI name and separately
+    // also a fully qualified class name ...
+    factoryArgs.put("class", className);
+    factoryArgs.put("luceneMatchVersion", matchVersion.toString());
+
+    ResourceLoader resources = state.resourceLoader;
+    RAMResourceLoaderWrapper ramResources = null;
+
+    if (sub != null) {
+      for(Map.Entry<String,Object> ent : sub.getRawParams().entrySet()) {
+        String argName = ent.getKey();
+        Object argValue = ent.getValue();
+
+        // Messy / impedance mismatch: allow
+        // components that expect files for things
+        // like stopword lists, keywords, to come in
+        // as inlined string values.  We "hijack" any
+        // argument name ending in FileContents and
+        // make a RAM file out of it:
+        String argString;
+
+        if (argName.endsWith("FileContents")) {
+          if (ramResources == null) {
+            ramResources = new RAMResourceLoaderWrapper(resources);
+            resources = ramResources;
+          }
+
+          String value;
+          if (argValue instanceof String) {
+            value = (String) argValue;
+          } else if (argValue instanceof JSONArray) {
+            // Each element in the array is mapped to
+            // one line in the file
+            StringBuilder b = new StringBuilder();
+            for(Object v : (JSONArray) argValue) {
+              if ((v instanceof String) == false) {
+                sub.failWrongClass(argName, "array must contain strings", v);
+              }
+              b.append((String) v);
+              b.append('\n');
+            }
+            value = b.toString();
+          } else {
+            sub.failWrongClass(argName, "must be a String or Array", argValue);
+
+            // Dead code but compiler disagrees:
+            value = null;
+          }
+
+          argName = argName.substring(0, argName.length()-12);
+          ramResources.add(argName, value);
+          argString = argName;
+        } else {
+          argString = argValue.toString();
+        }
+
+        factoryArgs.put(argName, argString);
+      }
+
+      // Clear all bindings from the incoming request,
+      // so that they are not seen as unused by the
+      // server.  If any params are really unused, the
+      // analysis factory should throw its own
+      // IllegalArgumentException:
+      sub.clearParams();
+    }
+
+    return new TwoThings<Map<String,String>,ResourceLoader>(factoryArgs, resources);
+  }
+
+  static TokenizerFactory parseTokenizer(IndexState state, Version matchVersion, Request chain) throws IOException {
+    // Build TokenizerFactory:
+    String className;
+    Request sub;
+    if (chain.isString("tokenizer")) {
+      className = chain.getString("tokenizer");
+      sub = null;
+    } else {
+      sub = chain.getStruct("tokenizer");
+
+      className = sub.getString("class");
+    }
+
+    TokenizerFactory tokenizerFactory;
+    if (className.toLowerCase(Locale.ROOT).equals("icu")) {
+      tokenizerFactory = buildICUTokenizerFactory(sub);
+    } else {
+
+      TwoThings<Map<String,String>,ResourceLoader> things = parseArgsAndResources(state, className, matchVersion, sub);
+
+      try {
+        tokenizerFactory = TokenizerFactory.forName(className, things.a);
+      } catch (IllegalArgumentException iae) {
+        chain.fail("tokenizer", "failed to create TokenizerFactory for class \"" + className + "\": " + iae, iae);
+
+        // Dead code but compiler disagrees:
+        tokenizerFactory = null;
+      }
+
+      if (tokenizerFactory instanceof ResourceLoaderAware) {
+        // nocommit need test case that requires a
+        // xxxFileContents to a Tokenizer
+        ((ResourceLoaderAware) tokenizerFactory).inform(things.b);
+      }
+    }
+
+    return tokenizerFactory;
+  }
+
+  static List<TokenFilterFactory> parseTokenFilters(IndexState state, Version matchVersion, Request chain) throws IOException {
+
+    // Build TokenFilters
+    List<TokenFilterFactory> tokenFilterFactories;
+    if (chain.hasParam("tokenFilters")) {
+      tokenFilterFactories = new ArrayList<TokenFilterFactory>();
+      for(Object o : chain.getList("tokenFilters")) {
+        String paramName = "tokenFilters[" + tokenFilterFactories.size() + "]";
+        String className;
+        Request sub;
+        if (o instanceof String) {
+          className = (String) o;
+          sub = null;
+        } else {
+          // The type already validated this:
+          assert o instanceof Request;
+          sub = (Request) o;
+
+          className = sub.getString("class");
+        }
+
+        TokenFilterFactory tokenFilterFactory;
+
+        if (className.toLowerCase(Locale.ROOT).equals("synonym")) {
+          if (sub == null) {
+            chain.fail(paramName, "no synonyms were specified");
+          }
+          tokenFilterFactory = buildSynonymFilterFactory(state, new Request(chain, paramName, sub.getRawParams(), SYNONYM_FILTER_TYPE));
+        } else {
+          TwoThings<Map<String,String>,ResourceLoader> things = parseArgsAndResources(state, className, matchVersion, sub);
+          
+          try {
+            tokenFilterFactory = TokenFilterFactory.forName(className, things.a);
+          } catch (IllegalArgumentException iae) {
+            chain.fail("tokenizer", "failed to create TokenFilterFactory for class \"" + className + "\": " + iae, iae);
+
+            // Dead code but compiler disagrees:
+            tokenFilterFactory = null;
+          }
+
+          if (tokenFilterFactory instanceof ResourceLoaderAware) {
+            ((ResourceLoaderAware) tokenFilterFactory).inform(things.b);
+          }
+        }
+
+        tokenFilterFactories.add(tokenFilterFactory);
+      }
+    } else {
+      tokenFilterFactories = null;
+    }
+
+    return tokenFilterFactories;
+  }
+
+  static Analyzer buildCustomAnalyzer(IndexState state, Version matchVersion, Request chain) throws IOException {
+
+    // nocommit what is MultiTermAwareComponent?
+
+    return new CustomAnalyzer(parseCharFilters(state, matchVersion, chain),
+                              parseTokenizer(state, matchVersion, chain),
+                              parseTokenFilters(state, matchVersion, chain),
+                              chain.getInt("positionIncrementGap"),
+                              chain.getInt("offsetGap"));
+  }
+
+  private static SynonymMap parseSynonyms(List<Object> syns, Analyzer a) {
+    try {
+      // nocommit this is awkward!  I just want to use Parser's
+      // analyze utility method... if the Parser could just take the JSONObject...
+      SynonymMap.Parser parser = new SynonymMap.Parser(true, a) {
+          @Override
+          public void parse(Reader in) throws IOException {
+            // nocommit move parsing in here!
+          };
+        };
+
+      CharsRef scratch = new CharsRef();
+      CharsRef scratchOutput = new CharsRef();
+      for(Object o2 : syns) {
+        Request syn = (Request) o2;
+        boolean replace = syn.getBoolean("replace");
+        CharsRef output = new CharsRef(syn.getString("output"));
+        if (syn.isString("input") == false) {
+          for(Object o3 : syn.getList("input")) {
+            parser.add(parser.analyze((String) o3, scratch),
+                       output,
+                       !replace);
+          }
+        } else {
+          parser.add(parser.analyze(syn.getString("input"), scratch),
+                     output,
+                     !replace);
+        }
+      }
+      return parser.build();
+    } catch (IOException ioe) {
+      throw new RuntimeException(ioe);
+    }
+  }
+
+  /** An analyzer based on the custom charFilter, tokenizer,
+   *  tokenFilters factory chains specified when the field
+   *  was registered. */
+  private static class CustomAnalyzer extends Analyzer {
+    private final int posIncGap;
+    private final int offsetGap;
+    private final TokenizerFactory tokenizerFactory;
+    private final List<TokenFilterFactory> tokenFilterFactories;
+    private final List<CharFilterFactory> charFilterFactories;
+
+    public CustomAnalyzer(List<CharFilterFactory> charFilterFactories,
+                          TokenizerFactory tokenizerFactory,
+                          List<TokenFilterFactory> tokenFilterFactories,
+                          int posIncGap, int offsetGap) {
+      this.charFilterFactories = charFilterFactories;
+      this.tokenizerFactory = tokenizerFactory;
+      this.tokenFilterFactories = tokenFilterFactories;
+      this.posIncGap = posIncGap;
+      this.offsetGap = offsetGap;
+    }
+
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName) {
+      Tokenizer tokenizer = tokenizerFactory.create();
+      TokenStream last = tokenizer;
+      if (tokenFilterFactories != null) {
+        for(TokenFilterFactory factory : tokenFilterFactories) {
+          last = factory.create(last);
+        }
+      }
+      return new TokenStreamComponents(tokenizer, last);
+    }
+
+    @Override
+    protected Reader initReader(String fieldName, Reader reader) {
+      Reader result = reader;
+      if (charFilterFactories != null) {
+        for(CharFilterFactory factory : charFilterFactories) {
+          result = factory.create(result);
+        }
+      }
+      return result;
+    }
+
+    @Override
+    public int getPositionIncrementGap(String fieldName) {
+      return posIncGap;
+    }
+
+    @Override
+    public int getOffsetGap(String fieldName) {
+      return offsetGap;
+    }
+  }
+
+  static Analyzer getAnalyzer(IndexState state, Request f, String name) throws IOException {
+    Analyzer analyzer;
+    if (f.hasParam(name)) {
+      Request a = f.getStruct(name);
+      String jsonOrig = a.toString();
+
+      Version matchVersion;
+      if (a.hasParam("matchVersion")) {
+        matchVersion = getVersion(a.getEnum("matchVersion"));
+      } else {
+        matchVersion = state.matchVersion;
+      }
+
+      if (a.hasParam("class")) {
+        // nocommit just lookup via CP:
+        // Predefined analyzer class:
+        Request.PolyResult pr = a.getPoly("class");
+        // TODO: try to "share" a single instance of
+        // each?  Analyzer can be costly!
+        if (pr.name.equals("StandardAnalyzer")) {
+          CharArraySet set = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          StandardAnalyzer sa = new StandardAnalyzer(matchVersion, set);
+          analyzer = sa;
+          sa.setMaxTokenLength(pr.r.getInt("maxTokenLength"));
+        } else if (pr.name.equals("EnglishAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new EnglishAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new EnglishAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("GermanAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new GermanAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new GermanAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("ArabicAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new ArabicAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new ArabicAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("ArmenianAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new ArmenianAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new ArmenianAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("BasqueAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new BasqueAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new BasqueAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("BrazilianAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new BrazilianAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new BrazilianAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("BulgarianAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new BulgarianAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new BulgarianAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("CatalanAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          if (pr.r.hasParam("stemExclusionSet")) {
+            CharArraySet stemExclusions = new CharArraySet(matchVersion, toStringList(pr.r.getList("stemExclusionSet")), false);
+            analyzer = new CatalanAnalyzer(matchVersion, stopWords, stemExclusions);
+          } else {
+            analyzer = new CatalanAnalyzer(matchVersion, stopWords);
+          }
+        } else if (pr.name.equals("CJKAnalyzer")) {
+          CharArraySet stopWords = new CharArraySet(matchVersion, toStringList(pr.r.getList("stopWords")), false);
+          analyzer = new CJKAnalyzer(matchVersion, stopWords);
+        } else if (pr.name.equals("CollationKeyAnalyzer")) {
+          Locale locale = SearchHandler.getLocale(pr.r.getStruct("locale"));
+          analyzer = new CollationKeyAnalyzer(matchVersion, Collator.getInstance(locale));
+        } else if (pr.name.equals("WhitespaceAnalyzer")) {
+          analyzer = new WhitespaceAnalyzer(matchVersion);
+        } else {
+          f.fail("class", "unrecognized analyzer class \"" + pr.name + "\"");
+          // Dead code but compiler disagrees:
+          analyzer = null;
+        }
+      } else if (a.hasParam("tokenizer")) {
+        analyzer = buildCustomAnalyzer(state, matchVersion, a);
+      } else {
+        f.fail(name, "either class or tokenizer/tokenFilters are required");
+        analyzer = null;
+      }
+    } else {
+      analyzer = null;
+    }
+
+    return analyzer;
+  }
+
+  /** Parses a Lucene version constant. */
+  @SuppressWarnings("deprecation")
+  public static Version getVersion(String v) {
+    if (v.equals("LUCENE_40")) {
+      return Version.LUCENE_40;
+    } else if (v.equals("LUCENE_41")) {
+      return Version.LUCENE_41;
+    } else if (v.equals("LUCENE_42")) {
+      return Version.LUCENE_42;
+    } else if (v.equals("LUCENE_43")) {
+      return Version.LUCENE_43;
+    } else if (v.equals("LUCENE_44")) {
+      return Version.LUCENE_44;
+    } else if (v.equals("LUCENE_45")) {
+      return Version.LUCENE_45;
+    } else if (v.equals("LUCENE_46")) {
+      return Version.LUCENE_46;
+    } else {
+      throw new IllegalArgumentException("unhandled version " + v);
+    }
+  }
+
+  @Override
+  public FinishRequest handle(final IndexState state, Request r, Map<String,List<String>> params) throws Exception {
+
+    assert state != null;
+
+    final Map<String,FieldDef> pendingFieldDefs = new HashMap<String,FieldDef>();
+    final Map<String,String> saveStates = new HashMap<String,String>();
+
+    if (r.hasParam("fields")) {
+      r = r.getStruct("fields");
+
+      Set<String> seen = new HashSet<String>();
+
+      // We make two passes.  In the first pass, we do the
+      // "real" fields, and second pass does the virtual
+      // fields, so that any fields the virtual field
+      // references are guaranteed to exist, in a single
+      // request (or, from the saved json):
+      for(int pass=0;pass<2;pass++) {
+        Iterator<Map.Entry<String,Object>> it = r.getParams();
+        while(it.hasNext()) {
+          Map.Entry<String,Object> ent = it.next();
+          String fieldName = ent.getKey();
+
+          if (pass == 1 && seen.contains(fieldName)) {
+            continue;
+          }
+
+          if (!(ent.getValue() instanceof JSONObject)) {
+            r.fail("field \"" + fieldName + "\": expected object containing the field type but got: " + ent.getValue());
+          }
+
+          if (pass == 0 && "virtual".equals(((JSONObject) ent.getValue()).get("type"))) {
+            // Do this on 2nd pass so the field it refers to will be registered even if it's a single request
+            continue;
+          }
+
+          if (!IndexState.isSimpleName(fieldName)) {
+            r.fail("invalid field name \"" + fieldName + "\": must be [a-zA-Z_][a-zA-Z0-9]*");
+          }
+
+          if (fieldName.endsWith("_boost")) {
+            r.fail("invalid field name \"" + fieldName + "\": field names cannot end with _boost");
+          }
+
+          if (seen.contains(fieldName)) {
+            throw new IllegalArgumentException("field \"" + fieldName + "\" appears at least twice in this request");
+          }
+      
+          seen.add(fieldName);
+
+          JSONObject fd = (JSONObject) ent.getValue();
+
+          saveStates.put(fieldName, fd.toString());
+
+          pendingFieldDefs.put(fieldName, parseOneFieldType(r, state, pendingFieldDefs, fieldName, fd));
+        }
+      }
+    }
+
+    return new FinishRequest() {
+      @Override
+      public String finish() throws IOException {
+        for(Map.Entry<String,FieldDef> ent : pendingFieldDefs.entrySet()) {
+          // Silly: we need JSONObject.clone...
+          JSONObject o;
+          try {
+            o = (JSONObject) JSONValue.parseStrict(saveStates.get(ent.getKey()));
+          } catch (net.minidev.json.parser.ParseException pe) {
+            // BUG
+            assert false;
+            throw new RuntimeException(pe);
+          }
+
+          state.addField(ent.getValue(), o);
+        }
+
+        return state.getAllFieldsJSON();
+      }
+    };
+  }
+}

Added: lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/ReleaseSnapshotHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/ReleaseSnapshotHandler.java?rev=1578133&view=auto
==============================================================================
--- lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/ReleaseSnapshotHandler.java (added)
+++ lucene/dev/branches/lucene5376_2/lucene/server/src/java/org/apache/lucene/server/handlers/ReleaseSnapshotHandler.java Sun Mar 16 18:11:07 2014
@@ -0,0 +1,78 @@
+package org.apache.lucene.server.handlers;
+
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.server.FinishRequest;
+import org.apache.lucene.server.GlobalState;
+import org.apache.lucene.server.IndexState;
+import org.apache.lucene.server.params.Param; 
+import org.apache.lucene.server.params.Request; 
+import org.apache.lucene.server.params.StringType; 
+import org.apache.lucene.server.params.StructType; 
+
+/** Handles {@code releaseSnapshot}. */
+public class ReleaseSnapshotHandler extends Handler {
+
+  final static StructType TYPE = new StructType(  
+                               new Param("indexName", "Index Name", new StringType()),
+                               new Param("id", "The id for this snapshot; this must have been previously created via @createSnapshot.", new StringType()));
+
+  @Override
+  public String getTopDoc() {
+    return "Releases a snapshot previously created with @createSnapshot.";
+  }
+
+  @Override
+  public StructType getType() {
+    return TYPE;
+  }
+
+  /** Sole constructor. */
+  public ReleaseSnapshotHandler(GlobalState state) {
+    super(state);
+  }
+
+  @Override
+  public FinishRequest handle(final IndexState state, final Request r, Map<String,List<String>> params) throws Exception {
+
+    final IndexState.Gens gens = new IndexState.Gens(r, "id");
+
+    return new FinishRequest() {
+      @Override
+      public String finish() throws IOException {
+
+        // SearcherLifetimeManager pruning thread will drop
+        // the searcher (if it's old enough) next time it
+        // wakes up:
+        state.snapshots.release(gens.indexGen);
+        state.writer.getIndexWriter().deleteUnusedFiles();
+        state.snapshotGenToVersion.remove(gens.indexGen);
+
+        state.taxoSnapshots.release(gens.taxoGen);
+        state.taxoInternalWriter.deleteUnusedFiles();
+        state.decRef(gens.stateGen);
+
+        return "{}";
+      }
+    };
+  }
+}
\ No newline at end of file