You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/11/12 10:15:38 UTC

svn commit: r1034304 [3/7] - in /lucene/dev/branches/docvalues: ./ lucene/ lucene/contrib/ lucene/contrib/benchmark/conf/ lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ lucene/contrib/benchmark/src/test/org/apache/lucene/be...

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Fri Nov 12 09:15:30 2010
@@ -28,13 +28,19 @@ import org.apache.lucene.index.DocsAndPo
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.index.FieldInfos;
 import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.UnicodeUtil;
 
 import java.io.IOException;
 import java.util.Comparator;
 import java.util.Map;
+import java.util.Set;
 import java.util.HashMap;
+import java.util.TreeMap;
+import java.util.SortedMap;
+import java.util.Iterator;
 
 class SimpleTextFieldsReader extends FieldsProducer {
 
@@ -79,7 +85,7 @@ class SimpleTextFieldsReader extends Fie
   private class SimpleTextFieldsEnum extends FieldsEnum {
     private final IndexInput in;
     private final BytesRef scratch = new BytesRef(10);
-    private boolean omitTF;
+    private String current;
 
     public SimpleTextFieldsEnum() {
       this.in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
@@ -90,11 +96,12 @@ class SimpleTextFieldsReader extends Fie
       while(true) {
         readLine(in, scratch);
         if (scratch.equals(END)) {
+          current = null;
           return null;
         }
         if (scratch.startsWith(FIELD)) {
           String field = StringHelper.intern(new String(scratch.bytes, scratch.offset + FIELD.length, scratch.length - FIELD.length, "UTF-8"));
-          omitTF = fieldInfos.fieldInfo(field).omitTermFreqAndPositions;
+          current = field;
           return field;
         }
       }
@@ -102,7 +109,7 @@ class SimpleTextFieldsReader extends Fie
 
     @Override
     public TermsEnum terms() throws IOException {
-      return new SimpleTextTermsEnum(in.getFilePointer(), omitTF);
+      return SimpleTextFieldsReader.this.terms(current).iterator();
     }
 
     @Override
@@ -116,21 +123,42 @@ class SimpleTextFieldsReader extends Fie
     private final IndexInput in;
     private final boolean omitTF;
     private BytesRef current;
-    private final long fieldStart;
-    private final BytesRef scratch = new BytesRef(10);
-    private final BytesRef scratch2 = new BytesRef(10);
     private int docFreq;
     private long docsStart;
     private boolean ended;
+    private final TreeMap<BytesRef,TermData> allTerms;
+    private Iterator<Map.Entry<BytesRef,TermData>> iter;
 
-    public SimpleTextTermsEnum(long offset, boolean omitTF) throws IOException {
+    public SimpleTextTermsEnum(TreeMap<BytesRef,TermData> allTerms, boolean omitTF) throws IOException {
       this.in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
-      this.in.seek(offset);
+      this.allTerms = allTerms;
       this.omitTF = omitTF;
-      fieldStart = offset;
+      iter = allTerms.entrySet().iterator();
     }
 
     public SeekStatus seek(BytesRef text, boolean useCache /* ignored */) throws IOException {
+      
+      final SortedMap<BytesRef,TermData> tailMap = allTerms.tailMap(text);
+
+      if (tailMap.isEmpty()) {
+        current = null;
+        return SeekStatus.END;
+      } else {
+        current = tailMap.firstKey();
+        final TermData td = tailMap.get(current);
+        docsStart = td.docsStart;
+        docFreq = td.docFreq;
+        iter = tailMap.entrySet().iterator();
+        assert iter.hasNext();
+        iter.next();
+        if (current.equals(text)) {
+          return SeekStatus.FOUND;
+        } else {
+          return SeekStatus.NOT_FOUND;
+        }
+      }
+
+      /*
       if (current != null) {
         final int cmp = current.compareTo(text);
         if (cmp == 0) {
@@ -160,6 +188,7 @@ class SimpleTextFieldsReader extends Fie
       current = null;
       ended = true;
       return SeekStatus.END;
+      */
     }
 
     @Override
@@ -169,6 +198,20 @@ class SimpleTextFieldsReader extends Fie
     @Override
     public BytesRef next() throws IOException {
       assert !ended;
+
+      if (iter.hasNext()) {
+        Map.Entry<BytesRef,TermData> ent = iter.next();
+        current = ent.getKey();
+        TermData td = ent.getValue();
+        docFreq = td.docFreq;
+        docsStart = td.docsStart;
+        return current;
+      } else {
+        current = null;
+        return null;
+      }
+
+      /*
       readLine(in, scratch);
       if (scratch.equals(END) || scratch.startsWith(FIELD)) {
         ended = true;
@@ -199,6 +242,7 @@ class SimpleTextFieldsReader extends Fie
         in.seek(lineStart);
         return current;
       }
+      */
     }
 
     @Override
@@ -261,7 +305,8 @@ class SimpleTextFieldsReader extends Fie
     private int tf;
     private Bits skipDocs;
     private final BytesRef scratch = new BytesRef(10);
-
+    private final UnicodeUtil.UTF16Result scratchUTF16 = new UnicodeUtil.UTF16Result();
+    
     public SimpleTextDocsEnum() {
       this.inStart = SimpleTextFieldsReader.this.in;
       this.in = (IndexInput) this.inStart.clone();
@@ -309,7 +354,8 @@ class SimpleTextFieldsReader extends Fie
             }
             return docID;
           }
-          docID = Integer.parseInt(new String(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length));
+          UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length, scratchUTF16);
+          docID = ArrayUtil.parseInt(scratchUTF16.result, 0, scratchUTF16.length);
           termFreq = 0;
           first = false;
         } else if (scratch.startsWith(POS)) {
@@ -346,6 +392,8 @@ class SimpleTextFieldsReader extends Fie
     private Bits skipDocs;
     private final BytesRef scratch = new BytesRef(10);
     private final BytesRef scratch2 = new BytesRef(10);
+    private final UnicodeUtil.UTF16Result scratchUTF16 = new UnicodeUtil.UTF16Result();
+    private final UnicodeUtil.UTF16Result scratchUTF16_2 = new UnicodeUtil.UTF16Result();
     private BytesRef payload;
     private long nextDocStart;
 
@@ -388,7 +436,8 @@ class SimpleTextFieldsReader extends Fie
             in.seek(posStart);
             return docID;
           }
-          docID = Integer.parseInt(new String(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length));
+          UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+DOC.length, scratch.length-DOC.length, scratchUTF16);
+          docID = ArrayUtil.parseInt(scratchUTF16.result, 0, scratchUTF16.length);
           tf = 0;
           posStart = in.getFilePointer();
           first = false;
@@ -419,7 +468,8 @@ class SimpleTextFieldsReader extends Fie
     public int nextPosition() throws IOException {
       readLine(in, scratch);
       assert scratch.startsWith(POS): "got line=" + scratch.utf8ToString();
-      final int pos = Integer.parseInt(new String(scratch.bytes, scratch.offset+POS.length, scratch.length-POS.length));
+      UnicodeUtil.UTF8toUTF16(scratch.bytes, scratch.offset+POS.length, scratch.length-POS.length, scratchUTF16_2);
+      final int pos = ArrayUtil.parseInt(scratchUTF16_2.result, 0, scratchUTF16_2.length);
       final long fp = in.getFilePointer();
       readLine(in, scratch);
       if (scratch.startsWith(PAYLOAD)) {
@@ -454,20 +504,70 @@ class SimpleTextFieldsReader extends Fie
     }
   }
 
+  static class TermData {
+    public long docsStart;
+    public int docFreq;
+
+    public TermData(long docsStart, int docFreq) {
+      this.docsStart = docsStart;
+      this.docFreq = docFreq;
+    }
+  }
+
   private class SimpleTextTerms extends Terms {
     private final String field;
     private final long termsStart;
     private final boolean omitTF;
 
-    public SimpleTextTerms(String field, long termsStart) {
+    // NOTE: horribly, horribly RAM consuming, but then
+    // SimpleText should never be used in production
+    private final TreeMap<BytesRef,TermData> allTerms = new TreeMap<BytesRef,TermData>();
+
+    private final BytesRef scratch = new BytesRef(10);
+
+    public SimpleTextTerms(String field, long termsStart) throws IOException {
       this.field = StringHelper.intern(field);
       this.termsStart = termsStart;
       omitTF = fieldInfos.fieldInfo(field).omitTermFreqAndPositions;
+      loadTerms();
+    }
+
+    private void loadTerms() throws IOException {
+      IndexInput in = (IndexInput) SimpleTextFieldsReader.this.in.clone();
+      in.seek(termsStart);
+      final BytesRef lastTerm = new BytesRef(10);
+      long lastDocsStart = -1;
+      int docFreq = 0;
+      while(true) {
+        readLine(in, scratch);
+        if (scratch.equals(END) || scratch.startsWith(FIELD)) {
+          if (lastDocsStart != -1) {
+            allTerms.put(new BytesRef(lastTerm),
+                         new TermData(lastDocsStart, docFreq));
+          }
+          break;
+        } else if (scratch.startsWith(DOC)) {
+          docFreq++;
+        } else if (scratch.startsWith(TERM)) {
+          if (lastDocsStart != -1) {
+            allTerms.put(new BytesRef(lastTerm),
+                         new TermData(lastDocsStart, docFreq));
+          }
+          lastDocsStart = in.getFilePointer();
+          final int len = scratch.length - TERM.length;
+          if (len > lastTerm.length) {
+            lastTerm.grow(len);
+          }
+          System.arraycopy(scratch.bytes, TERM.length, lastTerm.bytes, 0, len);
+          lastTerm.length = len;
+          docFreq = 0;
+        }
+      }
     }
 
     @Override
     public TermsEnum iterator() throws IOException {
-      return new SimpleTextTermsEnum(termsStart, omitTF);
+      return new SimpleTextTermsEnum(allTerms, omitTF);
     }
 
     @Override

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java Fri Nov 12 09:15:30 2010
@@ -1045,7 +1045,7 @@ public abstract class QueryParserBase {
 
     String termImage=discardEscapeChar(term.image);
     if (wildcard) {
-      q = getWildcardQuery(qfield, termImage);
+      q = getWildcardQuery(qfield, term.image);
     } else if (prefix) {
       q = getPrefixQuery(qfield,
           discardEscapeChar(term.image.substring

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Fri Nov 12 09:15:30 2010
@@ -55,7 +55,7 @@ public final class FuzzyTermsEnum extend
     attributes().addAttribute(MultiTermQuery.BoostAttribute.class);
   
   private final MultiTermQuery.MaxNonCompetitiveBoostAttribute maxBoostAtt;
-  private final Priv.LevenshteinAutomataAttribute dfaAtt;
+  private final LevenshteinAutomataAttribute dfaAtt;
   
   private float bottom;
   private BytesRef bottomTerm;
@@ -110,7 +110,7 @@ public final class FuzzyTermsEnum extend
     for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp))
            termText[j++] = cp = utf16.codePointAt(i);
     this.termLength = termText.length;
-    this.dfaAtt = atts.addAttribute(Priv.LevenshteinAutomataAttribute.class);
+    this.dfaAtt = atts.addAttribute(LevenshteinAutomataAttribute.class);
 
     //The prefix could be longer than the word.
     //It's kind of silly though.  It means we must match the entire word.
@@ -552,49 +552,44 @@ public final class FuzzyTermsEnum extend
     return scale_factor;
   }
   
-  // Wrapper class to hide the attribute from outside!
-  private static final class Priv {
-  
-    /** @lucene.internal */
-    public static interface LevenshteinAutomataAttribute extends Attribute {
-      public List<ByteRunAutomaton> automata();
-    }
+  /** @lucene.internal */
+  public static interface LevenshteinAutomataAttribute extends Attribute {
+    public List<ByteRunAutomaton> automata();
+  }
     
-    /** @lucene.internal */
-    public static final class LevenshteinAutomataAttributeImpl extends AttributeImpl implements LevenshteinAutomataAttribute {
-      private final List<ByteRunAutomaton> automata = new ArrayList<ByteRunAutomaton>();
+  /** @lucene.internal */
+  public static final class LevenshteinAutomataAttributeImpl extends AttributeImpl implements LevenshteinAutomataAttribute {
+    private final List<ByteRunAutomaton> automata = new ArrayList<ByteRunAutomaton>();
       
-      public List<ByteRunAutomaton> automata() {
-        return automata;
-      }
+    public List<ByteRunAutomaton> automata() {
+      return automata;
+    }
 
-      @Override
-      public void clear() {
-        automata.clear();
-      }
+    @Override
+    public void clear() {
+      automata.clear();
+    }
 
-      @Override
-      public int hashCode() {
-        return automata.hashCode();
-      }
+    @Override
+    public int hashCode() {
+      return automata.hashCode();
+    }
 
-      @Override
-      public boolean equals(Object other) {
-        if (this == other)
-          return true;
-        if (!(other instanceof LevenshteinAutomataAttributeImpl))
-          return false;
-        return automata.equals(((LevenshteinAutomataAttributeImpl) other).automata);
-      }
+    @Override
+    public boolean equals(Object other) {
+      if (this == other)
+        return true;
+      if (!(other instanceof LevenshteinAutomataAttributeImpl))
+        return false;
+      return automata.equals(((LevenshteinAutomataAttributeImpl) other).automata);
+    }
 
-      @Override
-      public void copyTo(AttributeImpl target) {
-        final List<ByteRunAutomaton> targetAutomata =
-          ((LevenshteinAutomataAttribute) target).automata();
-        targetAutomata.clear();
-        targetAutomata.addAll(automata);
-      }
+    @Override
+    public void copyTo(AttributeImpl target) {
+      final List<ByteRunAutomaton> targetAutomata =
+        ((LevenshteinAutomataAttribute) target).automata();
+      targetAutomata.clear();
+      targetAutomata.addAll(automata);
     }
-    
   }
 }

Propchange: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
 /lucene/dev/branches/preflexfixes/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:967125-979432
-/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:1021634-1034284
 /lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:824912-931101
 /lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:909334,948516
 /lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:924483-924731,924781,925176-925462

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/search/WildcardQuery.java Fri Nov 12 09:15:30 2010
@@ -45,6 +45,9 @@ public class WildcardQuery extends Autom
   /** Char equality with support for wildcards */
   public static final char WILDCARD_CHAR = '?';
 
+  /** Escape character */
+  public static final char WILDCARD_ESCAPE = '\\';
+  
   /**
    * Constructs a query for terms matching <code>term</code>. 
    */
@@ -56,6 +59,7 @@ public class WildcardQuery extends Autom
    * Convert Lucene wildcard syntax into an automaton.
    * @lucene.internal
    */
+  @SuppressWarnings("fallthrough")
   public static Automaton toAutomaton(Term wildcardquery) {
     List<Automaton> automata = new ArrayList<Automaton>();
     
@@ -63,6 +67,7 @@ public class WildcardQuery extends Autom
     
     for (int i = 0; i < wildcardText.length();) {
       final int c = wildcardText.codePointAt(i);
+      int length = Character.charCount(c);
       switch(c) {
         case WILDCARD_STRING: 
           automata.add(BasicAutomata.makeAnyString());
@@ -70,10 +75,18 @@ public class WildcardQuery extends Autom
         case WILDCARD_CHAR:
           automata.add(BasicAutomata.makeAnyChar());
           break;
+        case WILDCARD_ESCAPE:
+          // add the next codepoint instead, if it exists
+          if (i + length < wildcardText.length()) {
+            final int nextChar = wildcardText.codePointAt(i + length);
+            length += Character.charCount(nextChar);
+            automata.add(BasicAutomata.makeChar(nextChar));
+            break;
+          } // else fallthru, lenient parsing with a trailing \
         default:
           automata.add(BasicAutomata.makeChar(c));
       }
-      i += Character.charCount(c);
+      i += length;
     }
     
     return BasicOperations.concatenate(automata);

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/Directory.java Fri Nov 12 09:15:30 2010
@@ -197,6 +197,11 @@ public abstract class Directory implemen
       return this.toString();
   }
 
+  @Override
+  public String toString() {
+    return super.toString() + " lockFactory=" + getLockFactory();
+  }
+
   /**
    * Copies the file <i>src</i> to {@link Directory} <i>to</i> under the new
    * file name <i>dest</i>.

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/store/FSDirectory.java Fri Nov 12 09:15:30 2010
@@ -398,7 +398,7 @@ public abstract class FSDirectory extend
   /** For debug output. */
   @Override
   public String toString() {
-    return this.getClass().getName() + "@" + directory;
+    return this.getClass().getName() + "@" + directory + " lockFactory=" + getLockFactory();
   }
 
   /**

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/BitUtil.java Fri Nov 12 09:15:30 2010
@@ -20,7 +20,9 @@ package org.apache.lucene.util; // from 
 /**  A variety of high efficiency bit twiddling routines.
  * @lucene.internal
  */
-public class BitUtil {
+public final class BitUtil {
+
+  private BitUtil() {} // no instance
 
   /** Returns the number of bits set in the long */
   public static int pop(long x) {

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/CodecUtil.java Fri Nov 12 09:15:30 2010
@@ -31,6 +31,8 @@ import java.io.IOException;
  */
 
 public final class CodecUtil {
+  private CodecUtil() {} // no instance
+
   private final static int CODEC_MAGIC = 0x3fd76c17;
 
   public static IndexOutput writeHeader(IndexOutput out, String codec, int version)

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/English.java Fri Nov 12 09:15:30 2010
@@ -20,7 +20,9 @@ package org.apache.lucene.util;
 /**
  * @lucene.internal
  */ 
-public class English {
+public final class English {
+
+  private English() {} // no instance
 
   public static String longToEnglish(long i) {
     StringBuilder result = new StringBuilder();

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IOUtils.java Fri Nov 12 09:15:30 2010
@@ -23,7 +23,10 @@ import java.io.IOException;
 import org.apache.lucene.store.DataOutput;
 
 /** @lucene.internal */
-public class IOUtils {
+public final class IOUtils {
+
+  private IOUtils() {} // no instance
+
   /**
    * <p>Closes all given <tt>Closeable</tt>s, suppressing all thrown exceptions. Some of the <tt>Closeable</tt>s
    * may be null, they are ignored. After everything is closed, method either throws <tt>priorException</tt>,

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/IndexableBinaryStringTools.java Fri Nov 12 09:15:30 2010
@@ -53,7 +53,7 @@ import java.nio.ByteBuffer;
  *
  * @lucene.experimental
  */
-public class IndexableBinaryStringTools {
+public final class IndexableBinaryStringTools {
 
   private static final CodingCase[] CODING_CASES = {
     // CodingCase(int initialShift, int finalShift)

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ReaderUtil.java Fri Nov 12 09:15:30 2010
@@ -28,7 +28,9 @@ import org.apache.lucene.index.IndexRead
  *
  * @lucene.internal
  */
-public class ReaderUtil {
+public final class ReaderUtil {
+
+  private ReaderUtil() {} // no instance
 
   public static class Slice {
     public static final Slice[] EMPTY_ARRAY = new Slice[0];

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/ToStringUtils.java Fri Nov 12 09:15:30 2010
@@ -20,7 +20,10 @@ package org.apache.lucene.util;
 /**
  * Helper methods to ease implementing {@link Object#toString()}.
  */
-public class ToStringUtils {
+public final class ToStringUtils {
+
+  private ToStringUtils() {} // no instance
+
   /**
    * for printing boost only if not 1.0
    */

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/UnicodeUtil.java Fri Nov 12 09:15:30 2010
@@ -93,7 +93,9 @@ package org.apache.lucene.util;
  * @lucene.internal
  */
 
-final public class UnicodeUtil {
+public final class UnicodeUtil {
+
+  private UnicodeUtil() {} // no instance
 
   public static final int UNI_SUR_HIGH_START = 0xD800;
   public static final int UNI_SUR_HIGH_END = 0xDBFF;

Modified: lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/java/org/apache/lucene/util/automaton/SpecialOperations.java Fri Nov 12 09:15:30 2010
@@ -178,7 +178,7 @@ final public class SpecialOperations {
    * Reverses the language of the given (non-singleton) automaton while returning
    * the set of new initial states.
    */
-  static Set<State> reverse(Automaton a) {
+  public static Set<State> reverse(Automaton a) {
     a.expandSingleton();
     // reverse all edges
     HashMap<State, HashSet<Transition>> m = new HashMap<State, HashSet<Transition>>();

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Fri Nov 12 09:15:30 2010
@@ -73,7 +73,10 @@ public class TestExternalCodecs extends 
   //   - good improvement would be to write through to disk,
   //     and then load into ram from disk
   public static class RAMOnlyCodec extends Codec {
-
+    
+    public RAMOnlyCodec() {
+      name = "RamOnly";
+    }
     // Postings state:
     static class RAMPostings extends FieldsProducer {
       final Map<String,RAMField> fieldToTerms = new TreeMap<String,RAMField>();
@@ -482,6 +485,7 @@ public class TestExternalCodecs extends 
     @Override
     public FieldsProducer fieldsProducer(SegmentReadState readState)
       throws IOException {
+    
       return state.get(readState.segmentInfo.name);
     }
 
@@ -495,20 +499,10 @@ public class TestExternalCodecs extends 
   }
 
   public static class MyCodecs extends CodecProvider {
-    PerFieldCodecWrapper perField;
-
     MyCodecs() {
       Codec ram = new RAMOnlyCodec();
-      Codec pulsing = new PulsingReverseTermsCodec();
-      perField = new PerFieldCodecWrapper(ram);
-      perField.add("field2", pulsing);
-      perField.add("id", pulsing);
-      register(perField);
-    }
-    
-    @Override
-    public Codec getWriter(SegmentWriteState state) {
-      return perField;
+      register(ram);
+      setDefaultFieldCodec(ram.name);
     }
   }
 
@@ -631,20 +625,28 @@ public class TestExternalCodecs extends 
   // whose term sort is backwards unicode code point, and
   // storing "field1" as a custom entirely-in-RAM codec
   public void testPerFieldCodec() throws Exception {
+    CodecProvider provider = new MyCodecs();
+    Codec pulsing = new PulsingReverseTermsCodec();
+    provider.register(pulsing);
+    
     
     final int NUM_DOCS = 173;
     Directory dir = newDirectory();
     IndexWriter w = new IndexWriter(dir,
-                                    newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, true)).setCodecProvider(new MyCodecs()));
+                                    newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, true, true)).setCodecProvider(provider));
 
     w.setMergeFactor(3);
     Document doc = new Document();
     // uses default codec:
     doc.add(newField("field1", "this field uses the standard codec as the test", Field.Store.NO, Field.Index.ANALYZED));
     // uses pulsing codec:
-    doc.add(newField("field2", "this field uses the pulsing codec as the test", Field.Store.NO, Field.Index.ANALYZED));
+    Field field2 = newField("field2", "this field uses the pulsing codec as the test", Field.Store.NO, Field.Index.ANALYZED);
+    provider.setFieldCodec(field2.name(), pulsing.name);
+    doc.add(field2);
     
     Field idField = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED);
+    provider.setFieldCodec(idField.name(), pulsing.name);
+
     doc.add(idField);
     for(int i=0;i<NUM_DOCS;i++) {
       idField.setValue(""+i);

Propchange: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
 /lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/document/TestDateTools.java:967125-979432
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:1021634-1034284
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestDateTools.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:829134,829881,831036,896850,909334,948516

Propchange: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
 /lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:967125-979432
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:1021634-1034284
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/document/TestNumberTools.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:829134,829881,831036,896850,909334,948516

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/RandomIndexWriter.java Fri Nov 12 09:15:30 2010
@@ -42,6 +42,7 @@ public class RandomIndexWriter implement
   private final Random r;
   int docCount;
   int flushAt;
+  private boolean getReaderCalled;
 
   // Randomly calls Thread.yield so we mixup thread scheduling
   private static final class MockIndexWriter extends IndexWriter {
@@ -123,10 +124,13 @@ public class RandomIndexWriter implement
   }
 
   public IndexReader getReader() throws IOException {
+    getReaderCalled = true;
+    if (r.nextInt(4) == 2)
+      w.optimize();
     // If we are writing with PreFlexRW, force a full
     // IndexReader.open so terms are sorted in codepoint
     // order during searching:
-    if (!w.codecs.getWriter(null).name.equals("PreFlex") && r.nextBoolean()) {
+    if (!w.codecs.getDefaultFieldCodec().equals("PreFlex") && r.nextBoolean()) {
       if (LuceneTestCase.VERBOSE) {
         System.out.println("RIW.getReader: use NRT reader");
       }
@@ -141,7 +145,9 @@ public class RandomIndexWriter implement
   }
 
   public void close() throws IOException {
-    if (r.nextInt(4) == 2) {
+    // if someone isn't using getReader() API, we want to be sure to
+    // maybeOptimize since presumably they might open a reader on the dir.
+    if (getReaderCalled == false && r.nextInt(4) == 2) {
       w.optimize();
     }
     w.close();

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Nov 12 09:15:30 2010
@@ -18,20 +18,26 @@ package org.apache.lucene.index;
  */
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 
-import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
+import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
+import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.store.AlreadyClosedException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util._TestUtil;
 
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.PhraseQuery;
-
 public class TestAddIndexes extends LuceneTestCase {
   
   public void testSimpleCase() throws IOException {
@@ -563,4 +569,365 @@ public class TestAddIndexes extends Luce
     dir2.close();
   }
 
+  // TODO: these are also in TestIndexWriter... add a simple doc-writing method
+  // like this to LuceneTestCase?
+  private void addDoc(IndexWriter writer) throws IOException
+  {
+      Document doc = new Document();
+      doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+      writer.addDocument(doc);
+  }
+  
+  private abstract class RunAddIndexesThreads {
+
+    Directory dir, dir2;
+    final static int NUM_INIT_DOCS = 17;
+    IndexWriter writer2;
+    final List<Throwable> failures = new ArrayList<Throwable>();
+    volatile boolean didClose;
+    final IndexReader[] readers;
+    final int NUM_COPY;
+    final static int NUM_THREADS = 5;
+    final Thread[] threads = new Thread[NUM_THREADS];
+
+    public RunAddIndexesThreads(int numCopy) throws Throwable {
+      NUM_COPY = numCopy;
+      dir = new MockDirectoryWrapper(new RAMDirectory());
+      IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
+          TEST_VERSION_CURRENT, new MockAnalyzer())
+          .setMaxBufferedDocs(2));
+      for (int i = 0; i < NUM_INIT_DOCS; i++)
+        addDoc(writer);
+      writer.close();
+
+      dir2 = new MockDirectoryWrapper(new RAMDirectory());
+      writer2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+      writer2.commit();
+
+      readers = new IndexReader[NUM_COPY];
+      for(int i=0;i<NUM_COPY;i++)
+        readers[i] = IndexReader.open(dir, true);
+    }
+
+    void launchThreads(final int numIter) {
+
+      for(int i=0;i<NUM_THREADS;i++) {
+        threads[i] = new Thread() {
+            @Override
+            public void run() {
+              try {
+
+                final Directory[] dirs = new Directory[NUM_COPY];
+                for(int k=0;k<NUM_COPY;k++)
+                  dirs[k] = new MockDirectoryWrapper(new RAMDirectory(dir));
+
+                int j=0;
+
+                while(true) {
+                  // System.out.println(Thread.currentThread().getName() + ": iter j=" + j);
+                  if (numIter > 0 && j == numIter)
+                    break;
+                  doBody(j++, dirs);
+                }
+              } catch (Throwable t) {
+                handle(t);
+              }
+            }
+          };
+      }
+
+      for(int i=0;i<NUM_THREADS;i++)
+        threads[i].start();
+    }
+
+    void joinThreads() throws Exception {
+      for(int i=0;i<NUM_THREADS;i++)
+        threads[i].join();
+    }
+
+    void close(boolean doWait) throws Throwable {
+      didClose = true;
+      writer2.close(doWait);
+    }
+
+    void closeDir() throws Throwable {
+      for(int i=0;i<NUM_COPY;i++)
+        readers[i].close();
+      dir2.close();
+    }
+
+    abstract void doBody(int j, Directory[] dirs) throws Throwable;
+    abstract void handle(Throwable t);
+  }
+
+  private class CommitAndAddIndexes extends RunAddIndexesThreads {
+    public CommitAndAddIndexes(int numCopy) throws Throwable {
+      super(numCopy);
+    }
+
+    @Override
+    void handle(Throwable t) {
+      t.printStackTrace(System.out);
+      synchronized(failures) {
+        failures.add(t);
+      }
+    }
+
+    @Override
+    void doBody(int j, Directory[] dirs) throws Throwable {
+      switch(j%5) {
+      case 0:
+        writer2.addIndexes(dirs);
+        writer2.optimize();
+        break;
+      case 1:
+        writer2.addIndexes(dirs);
+        break;
+      case 2:
+        writer2.addIndexes(readers);
+        break;
+      case 3:
+        writer2.addIndexes(dirs);
+        writer2.maybeMerge();
+        break;
+      case 4:
+        writer2.commit();
+      }
+    }
+  }
+  
+  // LUCENE-1335: test simultaneous addIndexes & commits
+  // from multiple threads
+  public void testAddIndexesWithThreads() throws Throwable {
+
+    final int NUM_ITER = 15;
+    final int NUM_COPY = 3;
+    CommitAndAddIndexes c = new CommitAndAddIndexes(NUM_COPY);
+    c.launchThreads(NUM_ITER);
+
+    for(int i=0;i<100;i++)
+      addDoc(c.writer2);
+
+    c.joinThreads();
+
+    int expectedNumDocs = 100+NUM_COPY*(4*NUM_ITER/5)*RunAddIndexesThreads.NUM_THREADS*RunAddIndexesThreads.NUM_INIT_DOCS;
+    assertEquals(expectedNumDocs, c.writer2.numDocs());
+
+    c.close(true);
+
+    assertTrue(c.failures.size() == 0);
+
+    _TestUtil.checkIndex(c.dir2);
+
+    IndexReader reader = IndexReader.open(c.dir2, true);
+    assertEquals(expectedNumDocs, reader.numDocs());
+    reader.close();
+
+    c.closeDir();
+  }
+
+  private class CommitAndAddIndexes2 extends CommitAndAddIndexes {
+    public CommitAndAddIndexes2(int numCopy) throws Throwable {
+      super(numCopy);
+    }
+
+    @Override
+    void handle(Throwable t) {
+      if (!(t instanceof AlreadyClosedException) && !(t instanceof NullPointerException)) {
+        t.printStackTrace(System.out);
+        synchronized(failures) {
+          failures.add(t);
+        }
+      }
+    }
+  }
+
+  // LUCENE-1335: test simultaneous addIndexes & close
+  public void testAddIndexesWithClose() throws Throwable {
+    final int NUM_COPY = 3;
+    CommitAndAddIndexes2 c = new CommitAndAddIndexes2(NUM_COPY);
+    //c.writer2.setInfoStream(System.out);
+    c.launchThreads(-1);
+
+    // Close w/o first stopping/joining the threads
+    c.close(true);
+    //c.writer2.close();
+
+    c.joinThreads();
+
+    _TestUtil.checkIndex(c.dir2);
+
+    c.closeDir();
+
+    assertTrue(c.failures.size() == 0);
+  }
+
+  private class CommitAndAddIndexes3 extends RunAddIndexesThreads {
+    public CommitAndAddIndexes3(int numCopy) throws Throwable {
+      super(numCopy);
+    }
+
+    @Override
+    void doBody(int j, Directory[] dirs) throws Throwable {
+      switch(j%5) {
+      case 0:
+        writer2.addIndexes(dirs);
+        writer2.optimize();
+        break;
+      case 1:
+        writer2.addIndexes(dirs);
+        break;
+      case 2:
+        writer2.addIndexes(readers);
+        break;
+      case 3:
+        writer2.optimize();
+        break;
+      case 4:
+        writer2.commit();
+      }
+    }
+
+    @Override
+    void handle(Throwable t) {
+      boolean report = true;
+
+      if (t instanceof AlreadyClosedException || t instanceof MergePolicy.MergeAbortedException || t instanceof NullPointerException) {
+        report = !didClose;
+      } else if (t instanceof IOException)  {
+        Throwable t2 = t.getCause();
+        if (t2 instanceof MergePolicy.MergeAbortedException) {
+          report = !didClose;
+        }
+      }
+      if (report) {
+        t.printStackTrace(System.out);
+        synchronized(failures) {
+          failures.add(t);
+        }
+      }
+    }
+  }
+
+  // LUCENE-1335: test simultaneous addIndexes & close
+  public void testAddIndexesWithCloseNoWait() throws Throwable {
+
+    final int NUM_COPY = 50;
+    CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
+    c.launchThreads(-1);
+
+    Thread.sleep(500);
+
+    // Close w/o first stopping/joining the threads
+    c.close(false);
+
+    c.joinThreads();
+
+    _TestUtil.checkIndex(c.dir2);
+
+    c.closeDir();
+
+    assertTrue(c.failures.size() == 0);
+  }
+
+  // LUCENE-1335: test simultaneous addIndexes & close
+  public void testAddIndexesWithRollback() throws Throwable {
+    
+    final int NUM_COPY = 50;
+    CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
+    c.launchThreads(-1);
+
+    Thread.sleep(500);
+
+    // Close w/o first stopping/joining the threads
+    c.didClose = true;
+    c.writer2.rollback();
+
+    c.joinThreads();
+
+    _TestUtil.checkIndex(c.dir2);
+
+    c.closeDir();
+
+    assertTrue(c.failures.size() == 0);
+  }
+  
+  private void addDocs3(IndexWriter writer, int numDocs) throws IOException {
+    for (int i = 0; i < numDocs; i++) {
+      Document doc = new Document();
+      doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+      doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED));
+      writer.addDocument(doc);
+    }
+  }
+
+  public void testSimpleCaseCustomCodecProvider() throws IOException {
+    // main directory
+    Directory dir = newDirectory();
+    // two auxiliary directories
+    Directory aux = newDirectory();
+    Directory aux2 = newDirectory();
+    CodecProvider provider = new MockCodecProvider();
+    IndexWriter writer = null;
+
+    writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+        new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setCodecProvider(
+        provider));
+    // add 100 documents
+    addDocs3(writer, 100);
+    assertEquals(100, writer.maxDoc());
+    writer.commit();
+    writer.close();
+    _TestUtil.checkIndex(dir, provider);
+
+    writer = newWriter(aux, newIndexWriterConfig(TEST_VERSION_CURRENT,
+        new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setCodecProvider(
+        provider));
+    ((LogMergePolicy) writer.getConfig().getMergePolicy())
+        .setUseCompoundFile(false); // use one without a compound file
+    ((LogMergePolicy) writer.getConfig().getMergePolicy())
+        .setUseCompoundDocStore(false); // use one without a compound file
+    // add 40 documents in separate files
+    addDocs(writer, 40);
+    assertEquals(40, writer.maxDoc());
+    writer.commit();
+    writer.close();
+
+    writer = newWriter(aux2, newIndexWriterConfig(TEST_VERSION_CURRENT,
+        new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setCodecProvider(
+        provider));
+    // add 40 documents in compound files
+    addDocs2(writer, 50);
+    assertEquals(50, writer.maxDoc());
+    writer.commit();
+    writer.close();
+
+    // test doc count before segments are merged
+    writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+        new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setCodecProvider(
+        provider));
+    assertEquals(100, writer.maxDoc());
+    writer.addIndexes(new Directory[] { aux, aux2 });
+    assertEquals(190, writer.maxDoc());
+    writer.close();
+    _TestUtil.checkIndex(dir, provider);
+
+    dir.close();
+    aux.close();
+    aux2.close();
+  }
+
+  public static class MockCodecProvider extends CodecProvider {
+    public MockCodecProvider() {
+      StandardCodec standardCodec = new StandardCodec();
+      SimpleTextCodec simpleTextCodec = new SimpleTextCodec();
+      MockSepCodec mockSepCodec = new MockSepCodec();
+      register(standardCodec);
+      register(mockSepCodec);
+      register(simpleTextCodec);
+      setFieldCodec("id", simpleTextCodec.name);
+      setFieldCodec("content", mockSepCodec.name);
+    }
+  }
+  
 }

Propchange: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Nov 12 09:15:30 2010
@@ -1,6 +1,6 @@
-/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614
+/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:943137,949730,957490,960490,961612,979161,980654,982195,987811,988512,1025544,1026614,1034080
 /lucene/dev/branches/preflexfixes/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:967125-979432
-/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:1021634-1029001
+/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:1021634-1034284
 /lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:824912-931101
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
 /lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:829134,829881,831036,896850,909334,948516

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestCodecs.java Fri Nov 12 09:15:30 2010
@@ -25,7 +25,6 @@ import org.apache.lucene.analysis.MockAn
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.FieldsProducer;
@@ -271,10 +270,10 @@ public class TestCodecs extends LuceneTe
 
     final Directory dir = newDirectory();
     this.write(fieldInfos, dir, fields);
-    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, CodecProvider.getDefault().getWriter(null));
+    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()));
     si.setHasProx(false);
 
-    final FieldsProducer reader = si.getCodec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
+    final FieldsProducer reader = si.getCodecInfo().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
 
     final FieldsEnum fieldsEnum = reader.iterator();
     assertNotNull(fieldsEnum.next());
@@ -319,9 +318,9 @@ public class TestCodecs extends LuceneTe
     final Directory dir = newDirectory();
 
     this.write(fieldInfos, dir, fields);
-    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, CodecProvider.getDefault().getWriter(null));
+    final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, -1, SEGMENT, false, true, SegmentCodecs.build(fieldInfos, CodecProvider.getDefault()));
 
-    final FieldsProducer terms = si.getCodec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
+    final FieldsProducer terms = si.getCodecInfo().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 1024, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR));
 
     final Verify[] threads = new Verify[NUM_TEST_THREADS-1];
     for(int i=0;i<NUM_TEST_THREADS-1;i++) {
@@ -402,13 +401,9 @@ public class TestCodecs extends LuceneTe
 
     protected MockSepCodecs() {
       this.register(new MockSepCodec());
+      this.setDefaultFieldCodec("MockSep");
     }
-
-    @Override
-    public Codec getWriter(final SegmentWriteState state) {
-      return this.lookup("MockSep");
-    }
-
+    
   }
 
   private class Verify extends Thread {
@@ -611,11 +606,10 @@ public class TestCodecs extends LuceneTe
   private void write(final FieldInfos fieldInfos, final Directory dir, final FieldData[] fields) throws Throwable {
 
     final int termIndexInterval = this.nextInt(13, 27);
+    final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, CodecProvider.getDefault());
+    final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval, codecInfo);
 
-    final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, null, 10000, 10000, termIndexInterval,
-                                                    CodecProvider.getDefault());
-
-    final FieldsConsumer consumer = state.codec.fieldsConsumer(state);
+    final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
     Arrays.sort(fields);
     for (final FieldData field : fields) {
       field.write(consumer);

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestDoc.java Fri Nov 12 09:15:30 2010
@@ -197,7 +197,7 @@ public class TestDoc extends LuceneTestC
       merger.closeReaders();
       
       final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
-                                               useCompoundFile, -1, null, false, merger.hasProx(), merger.getCodec());
+                                               useCompoundFile, -1, null, false, merger.hasProx(), merger.getSegmentCodecs());
       
       if (useCompoundFile) {
         List<String> filesToDelete = merger.createCompoundFile(merged + ".cfs", info);

Modified: lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1034304&r1=1034303&r2=1034304&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/docvalues/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Fri Nov 12 09:15:30 2010
@@ -1813,4 +1813,29 @@ public class TestIndexReader extends Luc
     r.close();
     dir.close();
   }
+  
+  // LUCENE-2753
+  public void testListCommits() throws Exception {
+    Directory dir = newDirectory();
+    SnapshotDeletionPolicy sdp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( 
+        TEST_VERSION_CURRENT, null).setIndexDeletionPolicy(sdp));
+    writer.addDocument(new Document());
+    writer.commit();
+    sdp.snapshot("c1");
+    writer.addDocument(new Document());
+    writer.commit();
+    sdp.snapshot("c2");
+    writer.addDocument(new Document());
+    writer.commit();
+    sdp.snapshot("c3");
+    writer.close();
+    long currentGen = 0;
+    for (IndexCommit ic : IndexReader.listCommits(dir)) {
+      assertTrue("currentGen=" + currentGen + " commitGen=" + ic.getGeneration(), currentGen < ic.getGeneration());
+      currentGen = ic.getGeneration();
+    }
+    dir.close();
+  }
+  
 }