You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/01/30 18:56:43 UTC

svn commit: r1237850 [3/4] - in /lucene/dev/branches/lucene3661: ./ lucene/ lucene/contrib/ lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ lucene/contrib/misc/ lucene/contrib/misc/src/java/ lucene/contrib/misc/src/java/or...

Modified: lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java Mon Jan 30 17:56:40 2012
@@ -89,11 +89,11 @@ public class TestFSTs extends LuceneTest
     return br;
   }
 
-  private static IntsRef toIntsRef(String s, int inputMode) {
+  static IntsRef toIntsRef(String s, int inputMode) {
     return toIntsRef(s, inputMode, new IntsRef(10));
   }
 
-  private static IntsRef toIntsRef(String s, int inputMode, IntsRef ir) {
+  static IntsRef toIntsRef(String s, int inputMode, IntsRef ir) {
     if (inputMode == 0) {
       // utf8
       return toIntsRef(new BytesRef(s), ir);
@@ -103,7 +103,7 @@ public class TestFSTs extends LuceneTest
     }
   }
 
-  private static IntsRef toIntsRefUTF32(String s, IntsRef ir) {
+  static IntsRef toIntsRefUTF32(String s, IntsRef ir) {
     final int charLength = s.length();
     int charIdx = 0;
     int intIdx = 0;
@@ -120,7 +120,7 @@ public class TestFSTs extends LuceneTest
     return ir;
   }
 
-  private static IntsRef toIntsRef(BytesRef br, IntsRef ir) {
+  static IntsRef toIntsRef(BytesRef br, IntsRef ir) {
     if (br.length > ir.ints.length) {
       ir.grow(br.length);
     }
@@ -172,7 +172,7 @@ public class TestFSTs extends LuceneTest
         final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
         final List<FSTTester.InputOutput<Long>> pairs = new ArrayList<FSTTester.InputOutput<Long>>(terms2.length);
         for(int idx=0;idx<terms2.length;idx++) {
-          pairs.add(new FSTTester.InputOutput<Long>(terms2[idx], outputs.get(idx)));
+          pairs.add(new FSTTester.InputOutput<Long>(terms2[idx], (long) idx));
         }
         final FST<Long> fst = new FSTTester<Long>(random, dir, inputMode, pairs, outputs, true).doTest(0, 0, false);
         assertNotNull(fst);
@@ -230,7 +230,7 @@ public class TestFSTs extends LuceneTest
       final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
       final List<FSTTester.InputOutput<Long>> pairs = new ArrayList<FSTTester.InputOutput<Long>>(terms.length);
       for(int idx=0;idx<terms.length;idx++) {
-        pairs.add(new FSTTester.InputOutput<Long>(terms[idx], outputs.get(idx)));
+        pairs.add(new FSTTester.InputOutput<Long>(terms[idx], (long) idx));
       }
       new FSTTester<Long>(random, dir, inputMode, pairs, outputs, true).doTest();
     }
@@ -244,7 +244,7 @@ public class TestFSTs extends LuceneTest
       for(int idx=0;idx<terms.length;idx++) {
         final long value = lastOutput + _TestUtil.nextInt(random, 1, 1000);
         lastOutput = value;
-        pairs.add(new FSTTester.InputOutput<Long>(terms[idx], outputs.get(value)));
+        pairs.add(new FSTTester.InputOutput<Long>(terms[idx], value));
       }
       new FSTTester<Long>(random, dir, inputMode, pairs, outputs, doShare).doTest();
     }
@@ -254,7 +254,7 @@ public class TestFSTs extends LuceneTest
       final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(random.nextBoolean());
       final List<FSTTester.InputOutput<Long>> pairs = new ArrayList<FSTTester.InputOutput<Long>>(terms.length);
       for(int idx=0;idx<terms.length;idx++) {
-        pairs.add(new FSTTester.InputOutput<Long>(terms[idx], outputs.get(random.nextLong()) & Long.MAX_VALUE));
+        pairs.add(new FSTTester.InputOutput<Long>(terms[idx], random.nextLong() & Long.MAX_VALUE));
       }
       new FSTTester<Long>(random, dir, inputMode, pairs, outputs, false).doTest();
     }
@@ -270,8 +270,7 @@ public class TestFSTs extends LuceneTest
         final long value = lastOutput + _TestUtil.nextInt(random, 1, 1000);
         lastOutput = value;
         pairs.add(new FSTTester.InputOutput<PairOutputs.Pair<Long,Long>>(terms[idx],
-                                                                         outputs.get(o1.get(idx),
-                                                                                     o2.get(value))));
+                                                                         outputs.newPair((long) idx, value)));
       }
       new FSTTester<PairOutputs.Pair<Long,Long>>(random, dir, inputMode, pairs, outputs, false).doTest();
     }
@@ -393,6 +392,7 @@ public class TestFSTs extends LuceneTest
       final FST.Arc<T> arc = fst.getFirstArc(new FST.Arc<T>());
       final T NO_OUTPUT = fst.outputs.getNoOutput();
       T output = NO_OUTPUT;
+      final FST.BytesReader fstReader = fst.getBytesReader(0);
 
       for(int i=0;i<=term.length;i++) {
         final int label;
@@ -401,8 +401,9 @@ public class TestFSTs extends LuceneTest
         } else {
           label = term.ints[term.offset+i];
         }
-        //System.out.println("   loop i=" + i + " label=" + label + " output=" + fst.outputs.outputToString(output) + " curArc: target=" + arc.target + " isFinal?=" + arc.isFinal());
-        if (fst.findTargetArc(label, arc, arc) == null) {
+        // System.out.println("   loop i=" + i + " label=" + label + " output=" + fst.outputs.outputToString(output) + " curArc: target=" + arc.target + " isFinal?=" + arc.isFinal());
+        if (fst.findTargetArc(label, arc, arc, fstReader) == null) {
+          // System.out.println("    not found");
           if (prefixLength != null) {
             prefixLength[0] = i;
             return output;
@@ -462,16 +463,19 @@ public class TestFSTs extends LuceneTest
 
     FST<T> doTest(int prune1, int prune2, boolean allowRandomSuffixSharing) throws IOException {
       if (VERBOSE) {
-        System.out.println("TEST: prune1=" + prune1 + " prune2=" + prune2);
+        System.out.println("\nTEST: prune1=" + prune1 + " prune2=" + prune2);
       }
 
+      final boolean willRewrite = random.nextBoolean();
+
       final Builder<T> builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4,
                                                 prune1, prune2,
                                                 prune1==0 && prune2==0,
                                                 allowRandomSuffixSharing ? random.nextBoolean() : true,
                                                 allowRandomSuffixSharing ? _TestUtil.nextInt(random, 1, 10) : Integer.MAX_VALUE,
                                                 outputs,
-                                                null);
+                                                null,
+                                                willRewrite);
 
       for(InputOutput<T> pair : pairs) {
         if (pair.output instanceof UpToTwoPositiveIntOutputs.TwoLongs) {
@@ -486,7 +490,7 @@ public class TestFSTs extends LuceneTest
       }
       FST<T> fst = builder.finish();
 
-      if (random.nextBoolean() && fst != null) {        
+      if (random.nextBoolean() && fst != null && !willRewrite) {
         TestFSTs t = new TestFSTs();
         IOContext context = t.newIOContext(random);
         IndexOutput out = dir.createOutput("fst.bin", context);
@@ -522,6 +526,21 @@ public class TestFSTs extends LuceneTest
         verifyPruned(inputMode, fst, prune1, prune2);
       }
 
+      if (willRewrite && fst != null) {
+        if (VERBOSE) {
+          System.out.println("TEST: now rewrite");
+        }
+        final FST<T> packed = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000));
+        if (VERBOSE) {
+          System.out.println("TEST: now verify packed FST");
+        }
+        if (prune1 == 0 && prune2 == 0) {
+          verifyUnPruned(inputMode, packed);
+        } else {
+          verifyPruned(inputMode, packed, prune1, prune2);
+        }
+      }
+
       return fst;
     }
 
@@ -638,7 +657,7 @@ public class TestFSTs extends LuceneTest
       num = atLeast(100);
       for(int iter=0;iter<num;iter++) {
         if (VERBOSE) {
-          System.out.println("TEST: iter=" + iter);
+          System.out.println("  iter=" + iter);
         }
         if (random.nextBoolean()) {
           // seek to term that doesn't exist:
@@ -866,7 +885,15 @@ public class TestFSTs extends LuceneTest
             prefixes.put(IntsRef.deepCopyOf(scratch), cmo);
           } else {
             cmo.count++;
-            cmo.output = outputs.common(cmo.output, pair.output);
+            T output1 = cmo.output;
+            if (output1.equals(outputs.getNoOutput())) {
+              output1 = outputs.getNoOutput();
+            }
+            T output2 = pair.output;
+            if (output2.equals(outputs.getNoOutput())) {
+              output2 = outputs.getNoOutput();
+            }
+            cmo.output = outputs.common(output1, output2);
           }
           if (idx == pair.input.length) {
             cmo.isFinal = true;
@@ -992,7 +1019,7 @@ public class TestFSTs extends LuceneTest
 
   public void testRandomWords() throws IOException {
     testRandomWords(1000, atLeast(2));
-    //testRandomWords(20, 100);
+    //testRandomWords(100, 1);
   }
 
   String inputModeToString(int mode) {
@@ -1055,50 +1082,6 @@ public class TestFSTs extends LuceneTest
     }
   }
 
-  // NOTE: this test shows a case where our current builder
-  // fails to produce minimal FST:
-  /*
-  public void test3() throws Exception {
-    final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
-    Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
-    IntsRef scratchIntsRef = new IntsRef();
-    builder.add(Util.toIntsRef(new BytesRef("aa$"), scratchIntsRef), outputs.get(0));
-    builder.add(Util.toIntsRef(new BytesRef("aab$"), scratchIntsRef), 1L);
-    builder.add(Util.toIntsRef(new BytesRef("bbb$"), scratchIntsRef), 2L);
-    final FST<Long> fst = builder.finish();
-    //System.out.println("NODES " + fst.getNodeCount() + " ARCS " + fst.getArcCount());
-    // NOTE: we produce 7 nodes today
-    assertEquals(6, fst.getNodeCount());
-    // NOTE: we produce 8 arcs today
-    assertEquals(7, fst.getNodeCount());
-    //Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
-    //Util.toDot(fst, w, false, false);
-    //w.close();
-  }
-  */
-
-  // NOTE: this test shows a case where our current builder
-  // fails to produce minimal FST:
-  /*
-  public void test4() throws Exception {
-    final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
-    Builder<BytesRef> builder = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, outputs);
-    IntsRef scratchIntsRef = new IntsRef();
-    builder.add(Util.toIntsRef(new BytesRef("aa$"), scratchIntsRef), outputs.getNoOutput());
-    builder.add(Util.toIntsRef(new BytesRef("aab$"), scratchIntsRef), new BytesRef("1"));
-    builder.add(Util.toIntsRef(new BytesRef("bbb$"), scratchIntsRef), new BytesRef("11"));
-    final FST<BytesRef> fst = builder.finish();
-    //System.out.println("NODES " + fst.getNodeCount() + " ARCS " + fst.getArcCount());
-    // NOTE: we produce 7 nodes today
-    assertEquals(6, fst.getNodeCount());
-    // NOTE: we produce 8 arcs today
-    assertEquals(7, fst.getNodeCount());
-    //Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8");
-    //Util.toDot(fst, w, false, false);
-    //w.close();
-  }
-  */
-
   // Build FST for all unique terms in the test line docs
   // file, up until a time limit
   public void testRealTerms() throws Exception {
@@ -1126,7 +1109,10 @@ public class TestFSTs extends LuceneTest
     IndexReader r = IndexReader.open(writer, true);
     writer.close();
     final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(random.nextBoolean());
-    Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
+
+    final boolean doRewrite = random.nextBoolean();
+
+    Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doRewrite);
 
     boolean storeOrd = random.nextBoolean();
     if (VERBOSE) {
@@ -1162,59 +1148,69 @@ public class TestFSTs extends LuceneTest
         } else {
           output = termsEnum.docFreq();
         }
-        builder.add(Util.toIntsRef(term, scratchIntsRef), outputs.get(output));
+        builder.add(Util.toIntsRef(term, scratchIntsRef), (long) output);
         ord++;
         if (VERBOSE && ord % 100000 == 0 && LuceneTestCase.TEST_NIGHTLY) {
           System.out.println(ord + " terms...");
         }
       }
-      final FST<Long> fst = builder.finish();
+      FST<Long> fst = builder.finish();
       if (VERBOSE) {
         System.out.println("FST: " + docCount + " docs; " + ord + " terms; " + fst.getNodeCount() + " nodes; " + fst.getArcCount() + " arcs;" + " " + fst.sizeInBytes() + " bytes");
       }
 
       if (ord > 0) {
-        // Now confirm BytesRefFSTEnum and TermsEnum act the
-        // same:
-        final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst);
-        int num = atLeast(1000);
-        for(int iter=0;iter<num;iter++) {
-          final BytesRef randomTerm = new BytesRef(getRandomString());
-        
-          if (VERBOSE) {
-            System.out.println("TEST: seek non-exist " + randomTerm.utf8ToString() + " " + randomTerm);
+        for(int rewriteIter=0;rewriteIter<2;rewriteIter++) {
+          if (rewriteIter == 1) {
+            if (doRewrite) {
+              // Verify again, with packed FST:
+              fst = fst.pack(_TestUtil.nextInt(random, 1, 10), _TestUtil.nextInt(random, 0, 10000000));
+            } else {
+              break;
+            }
           }
+          // Now confirm BytesRefFSTEnum and TermsEnum act the
+          // same:
+          final BytesRefFSTEnum<Long> fstEnum = new BytesRefFSTEnum<Long>(fst);
+          int num = atLeast(1000);
+          for(int iter=0;iter<num;iter++) {
+            final BytesRef randomTerm = new BytesRef(getRandomString());
+        
+            if (VERBOSE) {
+              System.out.println("TEST: seek non-exist " + randomTerm.utf8ToString() + " " + randomTerm);
+            }
 
-          final TermsEnum.SeekStatus seekResult = termsEnum.seekCeil(randomTerm);
-          final BytesRefFSTEnum.InputOutput fstSeekResult = fstEnum.seekCeil(randomTerm);
+            final TermsEnum.SeekStatus seekResult = termsEnum.seekCeil(randomTerm);
+            final BytesRefFSTEnum.InputOutput fstSeekResult = fstEnum.seekCeil(randomTerm);
 
-          if (seekResult == TermsEnum.SeekStatus.END) {
-            assertNull("got " + (fstSeekResult == null ? "null" : fstSeekResult.input.utf8ToString()) + " but expected null", fstSeekResult);
-          } else {
-            assertSame(termsEnum, fstEnum, storeOrd);
-            for(int nextIter=0;nextIter<10;nextIter++) {
-              if (VERBOSE) {
-                System.out.println("TEST: next");
-                if (storeOrd) {
-                  System.out.println("  ord=" + termsEnum.ord());
-                }
-              }
-              if (termsEnum.next() != null) {
-                if (VERBOSE) {
-                  System.out.println("  term=" + termsEnum.term().utf8ToString());
-                }
-                assertNotNull(fstEnum.next());
-                assertSame(termsEnum, fstEnum, storeOrd);
-              } else {
+            if (seekResult == TermsEnum.SeekStatus.END) {
+              assertNull("got " + (fstSeekResult == null ? "null" : fstSeekResult.input.utf8ToString()) + " but expected null", fstSeekResult);
+            } else {
+              assertSame(termsEnum, fstEnum, storeOrd);
+              for(int nextIter=0;nextIter<10;nextIter++) {
                 if (VERBOSE) {
-                  System.out.println("  end!");
+                  System.out.println("TEST: next");
+                  if (storeOrd) {
+                    System.out.println("  ord=" + termsEnum.ord());
+                  }
                 }
-                BytesRefFSTEnum.InputOutput<Long> nextResult = fstEnum.next();
-                if (nextResult != null) {
-                  System.out.println("expected null but got: input=" + nextResult.input.utf8ToString() + " output=" + outputs.outputToString(nextResult.output));
-                  fail();
+                if (termsEnum.next() != null) {
+                  if (VERBOSE) {
+                    System.out.println("  term=" + termsEnum.term().utf8ToString());
+                  }
+                  assertNotNull(fstEnum.next());
+                  assertSame(termsEnum, fstEnum, storeOrd);
+                } else {
+                  if (VERBOSE) {
+                    System.out.println("  end!");
+                  }
+                  BytesRefFSTEnum.InputOutput<Long> nextResult = fstEnum.next();
+                  if (nextResult != null) {
+                    System.out.println("expected null but got: input=" + nextResult.input.utf8ToString() + " output=" + outputs.outputToString(nextResult.output));
+                    fail();
+                  }
+                  break;
                 }
-                break;
               }
             }
           }
@@ -1248,14 +1244,17 @@ public class TestFSTs extends LuceneTest
     private int inputMode;
     private final Outputs<T> outputs;
     private final Builder<T> builder;
+    private final boolean doPack;
 
-    public VisitTerms(String dirOut, String wordsFileIn, int inputMode, int prune, Outputs<T> outputs) {
+    public VisitTerms(String dirOut, String wordsFileIn, int inputMode, int prune, Outputs<T> outputs, boolean doPack, boolean noArcArrays) {
       this.dirOut = dirOut;
       this.wordsFileIn = wordsFileIn;
       this.inputMode = inputMode;
       this.outputs = outputs;
-      
-      builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs, null);
+      this.doPack = doPack;
+
+      builder = new Builder<T>(inputMode == 0 ? FST.INPUT_TYPE.BYTE1 : FST.INPUT_TYPE.BYTE4, 0, prune, prune == 0, true, Integer.MAX_VALUE, outputs, null, doPack);
+      builder.setAllowArrayArcs(!noArcArrays);
     }
 
     protected abstract T getOutput(IntsRef input, int ord) throws IOException;
@@ -1287,14 +1286,15 @@ public class TestFSTs extends LuceneTest
         }
 
         assert builder.getTermCount() == ord;
-        final FST<T> fst = builder.finish();
+        FST<T> fst = builder.finish();
         if (fst == null) {
           System.out.println("FST was fully pruned!");
           System.exit(0);
         }
 
-        if (dirOut == null)
+        if (dirOut == null) {
           return;
+        }
 
         System.out.println(ord + " terms; " + fst.getNodeCount() + " nodes; " + fst.getArcCount() + " arcs; " + fst.getArcWithOutputCount() + " arcs w/ output; tot size " + fst.sizeInBytes());
         if (fst.getNodeCount() < 100) {
@@ -1304,11 +1304,16 @@ public class TestFSTs extends LuceneTest
           System.out.println("Wrote FST to out.dot");
         }
 
+        if (doPack) {
+          System.out.println("Pack...");
+          fst = fst.pack(4, 100000000);
+          System.out.println("New size " + fst.sizeInBytes() + " bytes");
+        }
+        
         Directory dir = FSDirectory.open(new File(dirOut));
         IndexOutput out = dir.createOutput("fst.bin", IOContext.DEFAULT);
         fst.save(out);
         out.close();
-
         System.out.println("Saved FST to fst.bin.");
 
         if (!verify) {
@@ -1317,37 +1322,42 @@ public class TestFSTs extends LuceneTest
 
         System.out.println("\nNow verify...");
 
-        is.close();
-        is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), "UTF-8"), 65536);
-
-        ord = 0;
-        tStart = System.currentTimeMillis();
         while(true) {
-          String w = is.readLine();
-          if (w == null) {
-            break;
-          }
-          toIntsRef(w, inputMode, intsRef);
-          T expected = getOutput(intsRef, ord);
-          T actual = Util.get(fst, intsRef);
-          if (actual == null) {
-            throw new RuntimeException("unexpected null output on input=" + w);
-          }
-          if (!actual.equals(expected)) {
-            throw new RuntimeException("wrong output (got " + outputs.outputToString(actual) + " but expected " + outputs.outputToString(expected) + ") on input=" + w);
-          }
+          is.close();
+          is = new BufferedReader(new InputStreamReader(new FileInputStream(wordsFileIn), "UTF-8"), 65536);
 
-          ord++;
-          if (ord % 500000 == 0) {
-            System.out.println(((System.currentTimeMillis()-tStart)/1000.0) + "s: " + ord + "...");
-          }
-          if (ord >= limit) {
-            break;
+          ord = 0;
+          tStart = System.currentTimeMillis();
+          while(true) {
+            String w = is.readLine();
+            if (w == null) {
+              break;
+            }
+            toIntsRef(w, inputMode, intsRef);
+            T expected = getOutput(intsRef, ord);
+            T actual = Util.get(fst, intsRef);
+            if (actual == null) {
+              throw new RuntimeException("unexpected null output on input=" + w);
+            }
+            if (!actual.equals(expected)) {
+              throw new RuntimeException("wrong output (got " + outputs.outputToString(actual) + " but expected " + outputs.outputToString(expected) + ") on input=" + w);
+            }
+
+            ord++;
+            if (ord % 500000 == 0) {
+              System.out.println(((System.currentTimeMillis()-tStart)/1000.0) + "s: " + ord + "...");
+            }
+            if (ord >= limit) {
+              break;
+            }
           }
-        }
 
-        double totSec = ((System.currentTimeMillis() - tStart)/1000.0);
-        System.out.println("Verify took " + totSec + " sec + (" + (int) ((totSec*1000000000/ord)) + " nsec per lookup)");
+          double totSec = ((System.currentTimeMillis() - tStart)/1000.0);
+          System.out.println("Verify took " + totSec + " sec + (" + (int) ((totSec*1000000000/ord)) + " nsec per lookup)");
+
+          // NOTE: comment out to profile lookup...
+          break;
+        }
 
       } finally {
         is.close();
@@ -1355,7 +1365,7 @@ public class TestFSTs extends LuceneTest
     }
   }
 
-  // java -cp build/classes/test:build/classes/test-framework:build/classes/java:lib/junit-4.7.jar org.apache.lucene.util.automaton.fst.TestFSTs /x/tmp/allTerms3.txt out
+  // java -cp build/classes/test:build/classes/test-framework:build/classes/java:lib/junit-4.7.jar org.apache.lucene.util.fst.TestFSTs /x/tmp/allTerms3.txt out
   public static void main(String[] args) throws IOException {
     int prune = 0;
     int limit = Integer.MAX_VALUE;
@@ -1363,7 +1373,8 @@ public class TestFSTs extends LuceneTest
     boolean storeOrds = false;
     boolean storeDocFreqs = false;
     boolean verify = true;
-    
+    boolean doPack = false;
+    boolean noArcArrays = false;
     String wordsFileIn = null;
     String dirOut = null;
 
@@ -1381,10 +1392,14 @@ public class TestFSTs extends LuceneTest
         inputMode = 1;
       } else if (args[idx].equals("-docFreq")) {
         storeDocFreqs = true;
+      } else if (args[idx].equals("-noArcArrays")) {
+        noArcArrays = true;
       } else if (args[idx].equals("-ords")) {
         storeOrds = true;
       } else if (args[idx].equals("-noverify")) {
         verify = false;
+      } else if (args[idx].equals("-pack")) {
+        doPack = true;
       } else if (args[idx].startsWith("-")) {
         System.err.println("Unrecognized option: " + args[idx]);
         System.exit(-1);
@@ -1413,44 +1428,44 @@ public class TestFSTs extends LuceneTest
       final PositiveIntOutputs o1 = PositiveIntOutputs.getSingleton(true);
       final PositiveIntOutputs o2 = PositiveIntOutputs.getSingleton(false);
       final PairOutputs<Long,Long> outputs = new PairOutputs<Long,Long>(o1, o2);
-      new VisitTerms<PairOutputs.Pair<Long,Long>>(dirOut, wordsFileIn, inputMode, prune, outputs) {
+      new VisitTerms<PairOutputs.Pair<Long,Long>>(dirOut, wordsFileIn, inputMode, prune, outputs, doPack, noArcArrays) {
         Random rand;
         @Override
         public PairOutputs.Pair<Long,Long> getOutput(IntsRef input, int ord) {
           if (ord == 0) {
             rand = new Random(17);
           }
-          return new PairOutputs.Pair<Long,Long>(o1.get(ord),
-                                                 o2.get(_TestUtil.nextInt(rand, 1, 5000)));
+          return outputs.newPair((long) ord,
+                                 (long) _TestUtil.nextInt(rand, 1, 5000));
         }
       }.run(limit, verify);
     } else if (storeOrds) {
       // Store only ords
       final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
-      new VisitTerms<Long>(dirOut, wordsFileIn, inputMode, prune, outputs) {
+      new VisitTerms<Long>(dirOut, wordsFileIn, inputMode, prune, outputs, doPack, noArcArrays) {
         @Override
         public Long getOutput(IntsRef input, int ord) {
-          return outputs.get(ord);
+          return (long) ord;
         }
       }.run(limit, verify);
     } else if (storeDocFreqs) {
       // Store only docFreq
       final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(false);
-      new VisitTerms<Long>(dirOut, wordsFileIn, inputMode, prune, outputs) {
+      new VisitTerms<Long>(dirOut, wordsFileIn, inputMode, prune, outputs, doPack, noArcArrays) {
         Random rand;
         @Override
         public Long getOutput(IntsRef input, int ord) {
           if (ord == 0) {
             rand = new Random(17);
           }
-          return outputs.get(_TestUtil.nextInt(rand, 1, 5000));
+          return (long) _TestUtil.nextInt(rand, 1, 5000);
         }
       }.run(limit, verify);
     } else {
       // Store nothing
       final NoOutputs outputs = NoOutputs.getSingleton();
       final Object NO_OUTPUT = outputs.getNoOutput();
-      new VisitTerms<Object>(dirOut, wordsFileIn, inputMode, prune, outputs) {
+      new VisitTerms<Object>(dirOut, wordsFileIn, inputMode, prune, outputs, doPack, noArcArrays) {
         @Override
         public Object getOutput(IntsRef input, int ord) {
           return NO_OUTPUT;
@@ -1468,6 +1483,46 @@ public class TestFSTs extends LuceneTest
     assertNull(fstEnum.seekCeil(new BytesRef("foobaz")));
   }
 
+  /*
+  public void testTrivial() throws Exception {
+
+    // Get outputs -- passing true means FST will share
+    // (delta code) the outputs.  This should result in
+    // smaller FST if the outputs grow monotonically.  But
+    // if numbers are "random", false should give smaller
+    // final size:
+    final NoOutputs outputs = NoOutputs.getSingleton();
+
+    String[] strings = new String[] {"station", "commotion", "elation", "elastic", "plastic", "stop", "ftop", "ftation", "stat"};
+
+    final Builder<Object> builder = new Builder<Object>(FST.INPUT_TYPE.BYTE1,
+                                                        0, 0,
+                                                        true,
+                                                        true,
+                                                        Integer.MAX_VALUE,
+                                                        outputs,
+                                                        null,
+                                                        true);
+    Arrays.sort(strings);
+    final IntsRef scratch = new IntsRef();
+    for(String s : strings) {
+      builder.add(Util.toIntsRef(new BytesRef(s), scratch), outputs.getNoOutput());
+    }
+    final FST<Object> fst = builder.finish();
+    System.out.println("DOT before rewrite");
+    Writer w = new OutputStreamWriter(new FileOutputStream("/mnt/scratch/before.dot"));
+    Util.toDot(fst, w, false, false);
+    w.close();
+
+    final FST<Object> rewrite = new FST<Object>(fst, 1, 100);
+
+    System.out.println("DOT after rewrite");
+    w = new OutputStreamWriter(new FileOutputStream("/mnt/scratch/after.dot"));
+    Util.toDot(rewrite, w, false, false);
+    w.close();
+  }
+  */
+
   public void testSimple() throws Exception {
 
     // Get outputs -- passing true means FST will share
@@ -1484,9 +1539,9 @@ public class TestFSTs extends LuceneTest
     final BytesRef b = new BytesRef("b");
     final BytesRef c = new BytesRef("c");
 
-    builder.add(Util.toIntsRef(a, new IntsRef()), outputs.get(17));
-    builder.add(Util.toIntsRef(b, new IntsRef()), outputs.get(42));
-    builder.add(Util.toIntsRef(c, new IntsRef()), outputs.get(13824324872317238L));
+    builder.add(Util.toIntsRef(a, new IntsRef()), 17L);
+    builder.add(Util.toIntsRef(b, new IntsRef()), 42L);
+    builder.add(Util.toIntsRef(c, new IntsRef()), 13824324872317238L);
 
     final FST<Long> fst = builder.finish();
 
@@ -1795,11 +1850,11 @@ public class TestFSTs extends LuceneTest
   public void testFinalOutputOnEndState() throws Exception {
     final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
 
-    final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, null);
-    builder.add(Util.toUTF32("stat", new IntsRef()), outputs.get(17));
-    builder.add(Util.toUTF32("station", new IntsRef()), outputs.get(10));
+    final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE4, 2, 0, true, true, Integer.MAX_VALUE, outputs, null, random.nextBoolean());
+    builder.add(Util.toUTF32("stat", new IntsRef()), 17L);
+    builder.add(Util.toUTF32("station", new IntsRef()), 10L);
     final FST<Long> fst = builder.finish();
-    //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
+    //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot"));
     StringWriter w = new StringWriter();
     Util.toDot(fst, w, false, false);
     w.close();
@@ -1809,8 +1864,8 @@ public class TestFSTs extends LuceneTest
 
   public void testInternalFinalState() throws Exception {
     final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
-
-    final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null);
+    final boolean willRewrite = random.nextBoolean();
+    final Builder<Long> builder = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, willRewrite);
     builder.add(Util.toIntsRef(new BytesRef("stat"), new IntsRef()), outputs.getNoOutput());
     builder.add(Util.toIntsRef(new BytesRef("station"), new IntsRef()), outputs.getNoOutput());
     final FST<Long> fst = builder.finish();
@@ -1819,17 +1874,23 @@ public class TestFSTs extends LuceneTest
     Util.toDot(fst, w, false, false);
     w.close();
     //System.out.println(w.toString());
-    assertTrue(w.toString().indexOf("6 [shape=doublecircle") != -1);
+    final String expected;
+    if (willRewrite) {
+      expected = "4 -> 3 [label=\"t\" style=\"bold\"";
+    } else {
+      expected = "8 -> 6 [label=\"t\" style=\"bold\"";
+    }
+    assertTrue(w.toString().indexOf(expected) != -1);
   }
 
   // Make sure raw FST can differentiate between final vs
   // non-final end nodes
-  public void testNonFinalStopNodes() throws Exception {
+  public void testNonFinalStopNode() throws Exception {
     final PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true);
     final Long nothing = outputs.getNoOutput();
     final Builder<Long> b = new Builder<Long>(FST.INPUT_TYPE.BYTE1, outputs);
 
-    final FST<Long> fst = new FST<Long>(FST.INPUT_TYPE.BYTE1, outputs);
+    final FST<Long> fst = new FST<Long>(FST.INPUT_TYPE.BYTE1, outputs, false);
 
     final Builder.UnCompiledNode<Long> rootNode = new Builder.UnCompiledNode<Long>(b, 0);
 
@@ -1839,8 +1900,8 @@ public class TestFSTs extends LuceneTest
       node.isFinal = true;
       rootNode.addArc('a', node);
       final Builder.CompiledNode frozen = new Builder.CompiledNode();
-      frozen.address = fst.addNode(node);
-      rootNode.arcs[0].nextFinalOutput = outputs.get(17);
+      frozen.node = fst.addNode(node);
+      rootNode.arcs[0].nextFinalOutput = 17L;
       rootNode.arcs[0].isFinal = true;
       rootNode.arcs[0].output = nothing;
       rootNode.arcs[0].target = frozen;
@@ -1851,13 +1912,18 @@ public class TestFSTs extends LuceneTest
       final Builder.UnCompiledNode<Long> node = new Builder.UnCompiledNode<Long>(b, 0);
       rootNode.addArc('b', node);
       final Builder.CompiledNode frozen = new Builder.CompiledNode();
-      frozen.address = fst.addNode(node);
+      frozen.node = fst.addNode(node);
       rootNode.arcs[1].nextFinalOutput = nothing;
-      rootNode.arcs[1].output = outputs.get(42);
+      rootNode.arcs[1].output = 42L;
       rootNode.arcs[1].target = frozen;
     }
 
     fst.finish(fst.addNode(rootNode));
+
+    StringWriter w = new StringWriter();
+    //Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot"));
+    Util.toDot(fst, w, false, false);
+    w.close();
     
     checkStopNodes(fst, outputs);
 

Modified: lucene/dev/branches/lucene3661/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/branches/lucene3661/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Mon Jan 30 17:56:40 2012
@@ -226,6 +226,9 @@ public final class SynonymFilter extends
 
   private final FST<BytesRef> fst;
 
+  private final FST.BytesReader fstReader;
+
+
   private final BytesRef scratchBytes = new BytesRef();
   private final CharsRef scratchChars = new CharsRef();
 
@@ -241,7 +244,7 @@ public final class SynonymFilter extends
     this.synonyms = synonyms;
     this.ignoreCase = ignoreCase;
     this.fst = synonyms.fst;
-
+    this.fstReader = fst.getBytesReader(0);
     if (fst == null) {
       throw new IllegalArgumentException("fst must be non-null");
     }
@@ -366,7 +369,7 @@ public final class SynonymFilter extends
       int bufUpto = 0;
       while(bufUpto < bufferLen) {
         final int codePoint = Character.codePointAt(buffer, bufUpto, bufferLen);
-        if (fst.findTargetArc(ignoreCase ? Character.toLowerCase(codePoint) : codePoint, scratchArc, scratchArc) == null) {
+        if (fst.findTargetArc(ignoreCase ? Character.toLowerCase(codePoint) : codePoint, scratchArc, scratchArc, fstReader) == null) {
           //System.out.println("    stop");
           break byToken;
         }
@@ -388,7 +391,7 @@ public final class SynonymFilter extends
 
       // See if the FST wants to continue matching (ie, needs to
       // see the next input token):
-      if (fst.findTargetArc(SynonymMap.WORD_SEPARATOR, scratchArc, scratchArc) == null) {
+      if (fst.findTargetArc(SynonymMap.WORD_SEPARATOR, scratchArc, scratchArc, fstReader) == null) {
         // No further rules can match here; we're done
         // searching for matching rules starting at the
         // current input position.

Modified: lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/TokenInfoFST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/TokenInfoFST.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/TokenInfoFST.java (original)
+++ lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/TokenInfoFST.java Mon Jan 30 17:56:40 2012
@@ -47,16 +47,17 @@ public final class TokenInfoFST {
     FST.Arc<Long> firstArc = new FST.Arc<Long>();
     fst.getFirstArc(firstArc);
     FST.Arc<Long> arc = new FST.Arc<Long>();
+    final FST.BytesReader fstReader = fst.getBytesReader(0);
     // TODO: jump to 3040, readNextRealArc to ceiling? (just be careful we don't add bugs)
     for (int i = 0; i < rootCache.length; i++) {
-      if (fst.findTargetArc(0x3040 + i, firstArc, arc) != null) {
+      if (fst.findTargetArc(0x3040 + i, firstArc, arc, fstReader) != null) {
         rootCache[i] = new FST.Arc<Long>().copyFrom(arc);
       }
     }
     return rootCache;
   }
   
-  public FST.Arc<Long> findTargetArc(int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache) throws IOException {
+  public FST.Arc<Long> findTargetArc(int ch, FST.Arc<Long> follow, FST.Arc<Long> arc, boolean useCache, FST.BytesReader fstReader) throws IOException {
     if (useCache && ch >= 0x3040 && ch <= cacheCeiling) {
       assert ch != FST.END_LABEL;
       final Arc<Long> result = rootCache[ch - 0x3040];
@@ -67,13 +68,17 @@ public final class TokenInfoFST {
         return arc;
       }
     } else {
-      return fst.findTargetArc(ch, follow, arc);
+      return fst.findTargetArc(ch, follow, arc, fstReader);
     }
   }
   
   public Arc<Long> getFirstArc(FST.Arc<Long> arc) {
     return fst.getFirstArc(arc);
   }
+
+  public FST.BytesReader getBytesReader(int pos) {
+    return fst.getBytesReader(pos);
+  }
   
   /** @lucene.internal for testing only */
   FST<Long> getInternalFST() {

Modified: lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java (original)
+++ lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java Mon Jan 30 17:56:40 2012
@@ -113,7 +113,7 @@ public final class UserDictionary implem
       for (int i = 0; i < token.length(); i++) {
         scratch.ints[i] = (int) token.charAt(i);
       }
-      fstBuilder.add(scratch, fstOutput.get(ord));
+      fstBuilder.add(scratch, ord);
       segmentations.add(wordIdAndLength);
       ord++;
     }
@@ -134,6 +134,8 @@ public final class UserDictionary implem
     TreeMap<Integer, int[]> result = new TreeMap<Integer, int[]>(); // index, [length, length...]
     boolean found = false; // true if we found any results
 
+    final FST.BytesReader fstReader = fst.getBytesReader(0);
+
     FST.Arc<Long> arc = new FST.Arc<Long>();
     int end = off + len;
     for (int startOffset = off; startOffset < end; startOffset++) {
@@ -142,7 +144,7 @@ public final class UserDictionary implem
       int remaining = end - startOffset;
       for (int i = 0; i < remaining; i++) {
         int ch = chars[startOffset+i];
-        if (fst.findTargetArc(ch, arc, arc, i == 0) == null) {
+        if (fst.findTargetArc(ch, arc, arc, i == 0, fstReader) == null) {
           break; // continue to next position
         }
         output += arc.output.intValue();

Modified: lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java (original)
+++ lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java Mon Jan 30 17:56:40 2012
@@ -35,7 +35,7 @@ import org.apache.lucene.util.fst.FST;
 public class Viterbi {
   
   private final TokenInfoFST fst;
-  
+
   private final TokenInfoDictionary dictionary;
   
   private final UnknownDictionary unkDictionary;
@@ -214,6 +214,8 @@ public class Viterbi {
     ViterbiNode bosNode = new ViterbiNode(-1, BOS, 0, BOS.length, 0, 0, 0, -1, Type.KNOWN);
     addToArrays(bosNode, 0, 1, startIndexArr, endIndexArr, startSizeArr, endSizeArr);
     
+    final FST.BytesReader fstReader = fst.getBytesReader(0);
+
     // Process user dictionary;
     if (useUserDictionary) {
       processUserDictionary(text, offset, length, startIndexArr, endIndexArr, startSizeArr, endSizeArr);
@@ -238,7 +240,7 @@ public class Viterbi {
       for (int endIndex = 1; endIndex < suffixLength + 1; endIndex++) {
         int ch = text[suffixStart + endIndex - 1];
         
-        if (fst.findTargetArc(ch, arc, arc, endIndex == 1) == null) {
+        if (fst.findTargetArc(ch, arc, arc, endIndex == 1, fstReader) == null) {
           break; // continue to next position
         }
         output += arc.output.intValue();

Modified: lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary$fst.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary%24fst.dat?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
Binary files - no diff available.

Modified: lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java (original)
+++ lucene/dev/branches/lucene3661/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java Mon Jan 30 17:56:40 2012
@@ -131,7 +131,7 @@ public class TokenInfoDictionaryBuilder 
     System.out.println("  encode...");
 
     PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton(true);
-    Builder<Long> fstBuilder = new Builder<Long>(FST.INPUT_TYPE.BYTE2, fstOutput);
+    Builder<Long> fstBuilder = new Builder<Long>(FST.INPUT_TYPE.BYTE2, 0, 0, true, true, Integer.MAX_VALUE, fstOutput, null, true);
     IntsRef scratch = new IntsRef();
     long ord = -1; // first ord will be 0
     String lastValue = null;
@@ -155,13 +155,14 @@ public class TokenInfoDictionaryBuilder 
         for (int i = 0; i < token.length(); i++) {
           scratch.ints[i] = (int) token.charAt(i);
         }
-        fstBuilder.add(scratch, fstOutput.get(ord));
+        fstBuilder.add(scratch, ord);
       }
       dictionary.addMapping((int)ord, offset);
       offset = next;
     }
     
-    FST<Long> fst = fstBuilder.finish();
+    final FST<Long> fst = fstBuilder.finish().pack(2, 100000);
+    
     System.out.print("  " + fst.getNodeCount() + " nodes, " + fst.getArcCount() + " arcs, " + fst.sizeInBytes() + " bytes...  ");
     dictionary.setFST(fst);
     System.out.println(" done");

Modified: lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java (original)
+++ lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java Mon Jan 30 17:56:40 2012
@@ -329,8 +329,11 @@ public class FSTCompletion {
   private boolean descendWithPrefix(Arc<Object> arc, BytesRef utf8)
       throws IOException {
     final int max = utf8.offset + utf8.length;
+    // Cannot save as instance var since multiple threads
+    // can use FSTCompletion at once...
+    final FST.BytesReader fstReader = automaton.getBytesReader(0);
     for (int i = utf8.offset; i < max; i++) {
-      if (automaton.findTargetArc(utf8.bytes[i] & 0xff, arc, arc) == null) {
+      if (automaton.findTargetArc(utf8.bytes[i] & 0xff, arc, arc, fstReader) == null) {
         // No matching prefixes, return an empty result.
         return false;
       }

Modified: lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java (original)
+++ lucene/dev/branches/lucene3661/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java Mon Jan 30 17:56:40 2012
@@ -234,7 +234,7 @@ public class FSTCompletionBuilder {
     final Object empty = outputs.getNoOutput();
     final Builder<Object> builder = new Builder<Object>(
         FST.INPUT_TYPE.BYTE1, 0, 0, true, true, 
-        shareMaxTailLength, outputs, null);
+        shareMaxTailLength, outputs, null, false);
     
     BytesRef scratch = new BytesRef();
     final IntsRef scratchIntsRef = new IntsRef();

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/cloud/Overseer.java Mon Jan 30 17:56:40 2012
@@ -176,20 +176,24 @@ public class Overseer implements NodeSta
         return false;
       }
       /**
-       * Try to assign core to the cluster
+       * Try to assign core to the cluster. 
        * @throws KeeperException 
        * @throws InterruptedException 
        */
       private CloudState updateState(CloudState state, String nodeName, CoreState coreState) throws KeeperException, InterruptedException {
         String collection = coreState.getCollectionName();
         String zkCoreNodeName = coreState.getCoreNodeName();
-        
-          String shardId;
-          if (coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP) == null) {
-            shardId = AssignShard.assignShard(collection, state);
-          } else {
-            shardId = coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP);
-          }
+
+        // use the provided non null shardId
+        String shardId = coreState.getProperties().get(ZkStateReader.SHARD_ID_PROP);
+        if(shardId==null) {
+          //use shardId from CloudState
+          shardId = getAssignedId(state, nodeName, coreState);
+        }
+        if(shardId==null) {
+          //request new shardId 
+          shardId = AssignShard.assignShard(collection, state);
+        }
           
           Map<String,String> props = new HashMap<String,String>();
           for (Entry<String,String> entry : coreState.getProperties().entrySet()) {
@@ -209,6 +213,23 @@ public class Overseer implements NodeSta
           CloudState newCloudState = updateSlice(state, collection, slice);
           return newCloudState;
       }
+
+      /*
+       * Return an already assigned id or null if not assigned
+       */
+      private String getAssignedId(final CloudState state, final String nodeName,
+          final CoreState coreState) {
+        final String key = coreState.getProperties().get(ZkStateReader.NODE_NAME_PROP) + "_" +  coreState.getProperties().get(ZkStateReader.CORE_NAME_PROP);
+        Map<String, Slice> slices = state.getSlices(coreState.getCollectionName());
+        if (slices != null) {
+          for (Slice slice : slices.values()) {
+            if (slice.getShards().get(key) != null) {
+              return slice.getName();
+            }
+          }
+        }
+        return null;
+      }
       
       private CloudState updateSlice(CloudState state, String collection, Slice slice) {
         
@@ -480,6 +501,7 @@ public class Overseer implements NodeSta
     Set<String> downNodes = complement(oldLiveNodes, liveNodes);
     for(String node: downNodes) {
       NodeStateWatcher watcher = nodeStateWatches.remove(node);
+      log.debug("Removed NodeStateWatcher for node:" + node);
     }
   }
   
@@ -491,6 +513,7 @@ public class Overseer implements NodeSta
         if (!nodeStateWatches.containsKey(nodeName)) {
           zkCmdExecutor.ensureExists(path, zkClient);
           nodeStateWatches.put(nodeName, new NodeStateWatcher(zkClient, nodeName, path, this));
+          log.debug("Added NodeStateWatcher for node " + nodeName);
         } else {
           log.debug("watch already added");
         }

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/IndexDeletionPolicyWrapper.java Mon Jan 30 17:56:40 2012
@@ -19,6 +19,7 @@ package org.apache.solr.core;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.store.Directory;
+import org.apache.solr.update.SolrIndexWriter;
 
 import java.io.IOException;
 import java.util.*;
@@ -65,13 +66,13 @@ public class IndexDeletionPolicyWrapper 
   /**
    * Set the duration for which commit point is to be reserved by the deletion policy.
    *
-   * @param indexVersion version of the commit point to be reserved
+   * @param indexGen gen of the commit point to be reserved
    * @param reserveTime  time in milliseconds for which the commit point is to be reserved
    */
-  public void setReserveDuration(Long indexVersion, long reserveTime) {
+  public void setReserveDuration(Long indexGen, long reserveTime) {
     long timeToSet = System.currentTimeMillis() + reserveTime;
     for(;;) {
-      Long previousTime = reserves.put(indexVersion, timeToSet);
+      Long previousTime = reserves.put(indexGen, timeToSet);
 
       // this is the common success case: the older time didn't exist, or
       // came before the new time.
@@ -103,19 +104,19 @@ public class IndexDeletionPolicyWrapper 
   /** Permanently prevent this commit point from being deleted.
    * A counter is used to allow a commit point to be correctly saved and released
    * multiple times. */
-  public synchronized void saveCommitPoint(Long indexCommitVersion) {
-    AtomicInteger reserveCount = savedCommits.get(indexCommitVersion);
+  public synchronized void saveCommitPoint(Long indexCommitGen) {
+    AtomicInteger reserveCount = savedCommits.get(indexCommitGen);
     if (reserveCount == null) reserveCount = new AtomicInteger();
     reserveCount.incrementAndGet();
-    savedCommits.put(indexCommitVersion, reserveCount);
+    savedCommits.put(indexCommitGen, reserveCount);
   }
 
   /** Release a previously saved commit point */
-  public synchronized void releaseCommitPoint(Long indexCommitVersion) {
-    AtomicInteger reserveCount = savedCommits.get(indexCommitVersion);
+  public synchronized void releaseCommitPoint(Long indexCommitGen) {
+    AtomicInteger reserveCount = savedCommits.get(indexCommitGen);
     if (reserveCount == null) return;// this should not happen
     if (reserveCount.decrementAndGet() <= 0) {
-      savedCommits.remove(indexCommitVersion);
+      savedCommits.remove(indexCommitGen);
     }
   }
 
@@ -165,10 +166,10 @@ public class IndexDeletionPolicyWrapper 
 
     @Override
     public void delete() {
-      Long version = delegate.getVersion();
-      Long reserve = reserves.get(version);
+      Long gen = delegate.getGeneration();
+      Long reserve = reserves.get(gen);
       if (reserve != null && System.currentTimeMillis() < reserve) return;
-      if(savedCommits.containsKey(version)) return;
+      if(savedCommits.containsKey(gen)) return;
       delegate.delete();
     }
 
@@ -188,11 +189,6 @@ public class IndexDeletionPolicyWrapper 
     }
 
     @Override
-    public long getVersion() {
-      return delegate.getVersion();
-    }
-
-    @Override
     public long getGeneration() {
       return delegate.getGeneration();
     }
@@ -203,22 +199,17 @@ public class IndexDeletionPolicyWrapper 
     }
 
     @Override
-    public long getTimestamp() throws IOException {
-      return delegate.getTimestamp();
-    }
-
-    @Override
     public Map getUserData() throws IOException {
       return delegate.getUserData();
     }    
   }
 
   /**
-   * @param version the version of the commit point
+   * @param gen the gen of the commit point
    * @return a commit point corresponding to the given version
    */
-  public IndexCommit getCommitPoint(Long version) {
-    return solrVersionVsCommits.get(version);
+  public IndexCommit getCommitPoint(Long gen) {
+    return solrVersionVsCommits.get(gen);
   }
 
   /**
@@ -236,10 +227,20 @@ public class IndexDeletionPolicyWrapper 
     Map<Long, IndexCommit> map = new ConcurrentHashMap<Long, IndexCommit>();
     for (IndexCommitWrapper wrapper : list) {
       if (!wrapper.isDeleted())
-        map.put(wrapper.getVersion(), wrapper.delegate);
+        map.put(wrapper.delegate.getGeneration(), wrapper.delegate);
     }
     solrVersionVsCommits = map;
     latestCommit = ((list.get(list.size() - 1)).delegate);
   }
+
+  public static long getCommitTimestamp(IndexCommit commit) throws IOException {
+    final Map<String,String> commitData = commit.getUserData();
+    String commitTime = commitData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
+    if (commitTime != null) {
+      return Long.parseLong(commitTime);
+    } else {
+      return 0;
+    }
+  }
 }
 

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/core/SolrDeletionPolicy.java Mon Jan 30 17:56:40 2012
@@ -87,7 +87,6 @@ public class SolrDeletionPolicy implemen
       }
 
       sb.append(",segFN=").append(commit.getSegmentsFileName());
-      sb.append(",version=").append(commit.getVersion());
       sb.append(",generation=").append(commit.getGeneration());
       sb.append(",filenames=").append(commit.getFileNames());
     } catch (Exception e) {
@@ -133,7 +132,7 @@ public class SolrDeletionPolicy implemen
     synchronized (this) {
       long maxCommitAgeTimeStamp = -1L;
       IndexCommit newest = commits.get(commits.size() - 1);
-      log.info("newest commit = " + newest.getVersion());
+      log.info("newest commit = " + newest.getGeneration());
 
       int singleSegKept = (newest.getSegmentCount() == 1) ? 1 : 0;
       int totalKept = 1;
@@ -149,7 +148,7 @@ public class SolrDeletionPolicy implemen
               DateMathParser dmp = new DateMathParser(DateField.UTC, Locale.US);
               maxCommitAgeTimeStamp = dmp.parseMath(maxCommitAge).getTime();
             }
-            if (commit.getTimestamp() < maxCommitAgeTimeStamp) {
+            if (IndexDeletionPolicyWrapper.getCommitTimestamp(commit) < maxCommitAgeTimeStamp) {
               commit.delete();
               continue;
             }
@@ -191,8 +190,6 @@ public class SolrDeletionPolicy implemen
 
     sb.append('/');
     sb.append(commit.getGeneration());
-    sb.append('_');
-    sb.append(commit.getVersion());
     return sb.toString();
   }
 

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java Mon Jan 30 17:56:40 2012
@@ -60,6 +60,7 @@ import org.apache.solr.request.SolrQuery
 import org.apache.solr.response.BinaryQueryResponseWriter;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.update.SolrIndexWriter;
 import org.apache.solr.util.NumberUtils;
 import org.apache.solr.util.RefCounted;
 import org.apache.solr.util.plugin.SolrCoreAware;
@@ -139,8 +140,8 @@ public class ReplicationHandler extends 
         // in a catastrophic failure, but will result in the client getting an empty file list for
         // the CMD_GET_FILE_LIST command.
         //
-        core.getDeletionPolicy().setReserveDuration(commitPoint.getVersion(), reserveCommitDuration);
-        rsp.add(CMD_INDEX_VERSION, commitPoint.getVersion());
+        core.getDeletionPolicy().setReserveDuration(commitPoint.getGeneration(), reserveCommitDuration);
+        rsp.add(CMD_INDEX_VERSION, core.getDeletionPolicy().getCommitTimestamp(commitPoint));
         rsp.add(GENERATION, commitPoint.getGeneration());
       } else {
         // This happens when replication is not configured to happen after startup and no commit/optimize
@@ -219,7 +220,7 @@ public class ReplicationHandler extends 
     for (IndexCommit c : commits.values()) {
       try {
         NamedList<Object> nl = new NamedList<Object>();
-        nl.add("indexVersion", c.getVersion());
+        nl.add("indexVersion", core.getDeletionPolicy().getCommitTimestamp(c));
         nl.add(GENERATION, c.getGeneration());
         nl.add(CMD_GET_FILE_LIST, c.getFileNames());
         l.add(nl);
@@ -332,21 +333,21 @@ public class ReplicationHandler extends 
 
   @SuppressWarnings("unchecked")
   private void getFileList(SolrParams solrParams, SolrQueryResponse rsp) {
-    String v = solrParams.get(CMD_INDEX_VERSION);
+    String v = solrParams.get(GENERATION);
     if (v == null) {
-      rsp.add("status", "no indexversion specified");
+      rsp.add("status", "no index generation specified");
       return;
     }
-    long version = Long.parseLong(v);
-    IndexCommit commit = core.getDeletionPolicy().getCommitPoint(version);
+    long gen = Long.parseLong(v);
+    IndexCommit commit = core.getDeletionPolicy().getCommitPoint(gen);
  
     //System.out.println("ask for files for gen:" + commit.getGeneration() + core.getCoreDescriptor().getCoreContainer().getZkController().getNodeName());
     if (commit == null) {
-      rsp.add("status", "invalid indexversion");
+      rsp.add("status", "invalid index generation");
       return;
     }
     // reserve the indexcommit for sometime
-    core.getDeletionPolicy().setReserveDuration(version, reserveCommitDuration);
+    core.getDeletionPolicy().setReserveDuration(gen, reserveCommitDuration);
     List<Map<String, Object>> result = new ArrayList<Map<String, Object>>();
     try {
       //get all the files in the commit
@@ -359,10 +360,10 @@ public class ReplicationHandler extends 
         result.add(fileMeta);
       }
     } catch (IOException e) {
-      rsp.add("status", "unable to get file names for given indexversion");
+      rsp.add("status", "unable to get file names for given index generation");
       rsp.add("exception", e);
-      LOG.warn("Unable to get file names for indexCommit version: "
-               + version, e);
+      LOG.warn("Unable to get file names for indexCommit generation: "
+               + gen, e);
     }
     rsp.add(CMD_GET_FILE_LIST, result);
     if (confFileNameAlias.size() < 1)
@@ -487,8 +488,13 @@ public class ReplicationHandler extends 
     long version[] = new long[2];
     RefCounted<SolrIndexSearcher> searcher = core.getSearcher();
     try {
-      version[0] = searcher.get().getIndexReader().getIndexCommit().getVersion();
-      version[1] = searcher.get().getIndexReader().getIndexCommit().getGeneration();
+      final IndexCommit commit = searcher.get().getIndexReader().getIndexCommit();
+      final Map<String,String> commitData = commit.getUserData();
+      String commitTime = commitData.get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
+      if (commitTime != null) {
+        version[0] = Long.parseLong(commitTime);
+      }
+      version[1] = commit.getGeneration();
     } catch (IOException e) {
       LOG.warn("Unable to get index version : ", e);
     } finally {
@@ -574,7 +580,6 @@ public class ReplicationHandler extends 
     }
 
     if (isMaster && commit != null) {
-      master.add("replicatableIndexVersion", commit.getVersion());
       master.add("replicatableGeneration", commit.getGeneration());
     }
 
@@ -846,7 +851,7 @@ public class ReplicationHandler extends 
                 Collection<IndexCommit> commits = IndexReader.listCommits(reader.directory());
                 for (IndexCommit ic : commits) {
                   if(ic.getSegmentCount() == 1){
-                    if(indexCommitPoint == null || indexCommitPoint.getVersion() < ic.getVersion()) indexCommitPoint = ic;
+                    if(indexCommitPoint == null || indexCommitPoint.getGeneration() < ic.getGeneration()) indexCommitPoint = ic;
                   }
                 }
               } else{
@@ -857,7 +862,7 @@ public class ReplicationHandler extends 
               // always saves the last commit point (and the last optimized commit point, if needed)
               /***
               if(indexCommitPoint != null){
-                core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getVersion());
+                core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getGeneration());
               }
               ***/
             }
@@ -958,10 +963,10 @@ public class ReplicationHandler extends 
           // always saves the last commit point (and the last optimized commit point, if needed)
           /***
           if (indexCommitPoint != null) {
-            core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getVersion());
+            core.getDeletionPolicy().saveCommitPoint(indexCommitPoint.getGeneration());
           }
           if(oldCommitPoint != null){
-            core.getDeletionPolicy().releaseCommitPoint(oldCommitPoint.getVersion());
+            core.getDeletionPolicy().releaseCommitPoint(oldCommitPoint.getGeneration());
           }
           ***/
         }
@@ -989,7 +994,7 @@ public class ReplicationHandler extends 
 
     private FastOutputStream fos;
 
-    private Long indexVersion;
+    private Long indexGen;
     private IndexDeletionPolicyWrapper delPolicy;
 
     public FileStream(SolrParams solrParams) {
@@ -1004,8 +1009,8 @@ public class ReplicationHandler extends 
       String sLen = params.get(LEN);
       String compress = params.get(COMPRESSION);
       String sChecksum = params.get(CHECKSUM);
-      String sindexVersion = params.get(CMD_INDEX_VERSION);
-      if (sindexVersion != null) indexVersion = Long.parseLong(sindexVersion);
+      String sGen = params.get(GENERATION);
+      if (sGen != null) indexGen = Long.parseLong(sGen);
       if (Boolean.parseBoolean(compress)) {
         fos = new FastOutputStream(new DeflaterOutputStream(out));
       } else {
@@ -1063,9 +1068,9 @@ public class ReplicationHandler extends 
             }
             fos.write(buf, 0, (int) bytesRead);
             fos.flush();
-            if (indexVersion != null && (packetsWritten % 5 == 0)) {
+            if (indexGen != null && (packetsWritten % 5 == 0)) {
               //after every 5 packets reserve the commitpoint for some time
-              delPolicy.setReserveDuration(indexVersion, reserveCommitDuration);
+              delPolicy.setReserveDuration(indexGen, reserveCommitDuration);
             }
             packetsWritten++;
           }

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapPuller.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapPuller.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapPuller.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapPuller.java Mon Jan 30 17:56:40 2012
@@ -28,6 +28,7 @@ import org.apache.solr.common.util.FileU
 import org.apache.solr.common.util.JavaBinCodec;
 import org.apache.solr.common.util.NamedList;
 import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.IndexDeletionPolicyWrapper;
 import static org.apache.solr.handler.ReplicationHandler.*;
 
 import org.apache.solr.request.LocalSolrQueryRequest;
@@ -210,10 +211,10 @@ public class SnapPuller {
   /**
    * Fetches the list of files in a given index commit point
    */
-  void fetchFileList(long version) throws IOException {
+  void fetchFileList(long gen) throws IOException {
     PostMethod post = new PostMethod(masterUrl);
     post.addParameter(COMMAND, CMD_GET_FILE_LIST);
-    post.addParameter(CMD_INDEX_VERSION, String.valueOf(version));
+    post.addParameter(GENERATION, String.valueOf(gen));
     post.addParameter("wt", "javabin");
 
     @SuppressWarnings("unchecked")
@@ -225,7 +226,7 @@ public class SnapPuller {
       filesToDownload = Collections.synchronizedList(f);
     else {
       filesToDownload = Collections.emptyList();
-      LOG.error("No files to download for indexversion: "+ version);
+      LOG.error("No files to download for index generation: "+ gen);
     }
 
     f = nl.get(CONF_FILES);
@@ -274,7 +275,7 @@ public class SnapPuller {
       }
       
       if (latestVersion == 0L) {
-        if (force && commit.getVersion() != 0) {
+        if (force && commit.getGeneration() != 0) {
           // since we won't get the files for an empty index,
           // we just clear ours and commit
           core.getUpdateHandler().getSolrCoreState().getIndexWriter(core).deleteAll();
@@ -288,17 +289,17 @@ public class SnapPuller {
         return true;
       }
       
-      if (!force && commit.getVersion() == latestVersion && commit.getGeneration() == latestGeneration) {
+      if (!force && IndexDeletionPolicyWrapper.getCommitTimestamp(commit) == latestVersion) {
         //master and slave are already in sync just return
         LOG.info("Slave in sync with master.");
         successfulInstall = true;
         return true;
       }
-      LOG.info("Master's version: " + latestVersion + ", generation: " + latestGeneration);
-      LOG.info("Slave's version: " + commit.getVersion() + ", generation: " + commit.getGeneration());
+      LOG.info("Master's generation: " + latestGeneration);
+      LOG.info("Slave's generation: " + commit.getGeneration());
       LOG.info("Starting replication process");
       // get the list of files first
-      fetchFileList(latestVersion);
+      fetchFileList(latestGeneration);
       // this can happen if the commit point is deleted before we fetch the file list.
       if(filesToDownload.isEmpty()) return false;
       LOG.info("Number of files in latest index in master: " + filesToDownload.size());
@@ -309,7 +310,7 @@ public class SnapPuller {
       filesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
       // if the generateion of master is older than that of the slave , it means they are not compatible to be copied
       // then a new index direcory to be created and all the files need to be copied
-      boolean isFullCopyNeeded = commit.getVersion() >= latestVersion || force;
+      boolean isFullCopyNeeded = IndexDeletionPolicyWrapper.getCommitTimestamp(commit) >= latestVersion || force;
       File tmpIndexDir = createTempindexDir(core);
       if (isIndexStale())
         isFullCopyNeeded = true;
@@ -318,11 +319,11 @@ public class SnapPuller {
       File indexDir = null ;
       try {
         indexDir = new File(core.getIndexDir());
-        downloadIndexFiles(isFullCopyNeeded, tmpIndexDir, latestVersion);
+        downloadIndexFiles(isFullCopyNeeded, tmpIndexDir, latestGeneration);
         LOG.info("Total time taken for download : " + ((System.currentTimeMillis() - replicationStartTime) / 1000) + " secs");
         Collection<Map<String, Object>> modifiedConfFiles = getModifiedConfFiles(confFilesToDownload);
         if (!modifiedConfFiles.isEmpty()) {
-          downloadConfFiles(confFilesToDownload, latestVersion);
+          downloadConfFiles(confFilesToDownload, latestGeneration);
           if (isFullCopyNeeded) {
             successfulInstall = modifyIndexProps(tmpIndexDir.getName());
             deleteTmpIdxDir =  false;
@@ -530,7 +531,7 @@ public class SnapPuller {
     }.start();
   }
 
-  private void downloadConfFiles(List<Map<String, Object>> confFilesToDownload, long latestVersion) throws Exception {
+  private void downloadConfFiles(List<Map<String, Object>> confFilesToDownload, long latestGeneration) throws Exception {
     LOG.info("Starting download of configuration files from master: " + confFilesToDownload);
     confFilesDownloaded = Collections.synchronizedList(new ArrayList<Map<String, Object>>());
     File tmpconfDir = new File(solrCore.getResourceLoader().getConfigDir(), "conf." + getDateAsStr(new Date()));
@@ -542,7 +543,7 @@ public class SnapPuller {
       }
       for (Map<String, Object> file : confFilesToDownload) {
         String saveAs = (String) (file.get(ALIAS) == null ? file.get(NAME) : file.get(ALIAS));
-        fileFetcher = new FileFetcher(tmpconfDir, file, saveAs, true, latestVersion);
+        fileFetcher = new FileFetcher(tmpconfDir, file, saveAs, true, latestGeneration);
         currentFile = file;
         fileFetcher.fetchFile();
         confFilesDownloaded.add(new HashMap<String, Object>(file));
@@ -561,13 +562,13 @@ public class SnapPuller {
    *
    * @param downloadCompleteIndex is it a fresh index copy
    * @param tmpIdxDir               the directory to which files need to be downloadeed to
-   * @param latestVersion         the version number
+   * @param latestGeneration         the version number
    */
-  private void downloadIndexFiles(boolean downloadCompleteIndex, File tmpIdxDir, long latestVersion) throws Exception {
+  private void downloadIndexFiles(boolean downloadCompleteIndex, File tmpIdxDir, long latestGeneration) throws Exception {
     for (Map<String, Object> file : filesToDownload) {
       File localIndexFile = new File(solrCore.getIndexDir(), (String) file.get(NAME));
       if (!localIndexFile.exists() || downloadCompleteIndex) {
-        fileFetcher = new FileFetcher(tmpIdxDir, file, (String) file.get(NAME), false, latestVersion);
+        fileFetcher = new FileFetcher(tmpIdxDir, file, (String) file.get(NAME), false, latestGeneration);
         currentFile = file;
         fileFetcher.fetchFile();
         filesDownloaded.add(new HashMap<String, Object>(file));
@@ -578,7 +579,7 @@ public class SnapPuller {
   }
 
   /**
-   * All the files which are common between master and slave must have same timestamp and size else we assume they are
+   * All the files which are common between master and slave must have same size else we assume they are
    * not compatible (stale).
    *
    * @return true if the index stale and we need to download a fresh copy, false otherwise.
@@ -896,10 +897,10 @@ public class SnapPuller {
 
     private boolean aborted = false;
 
-    private Long indexVersion;
+    private Long indexGen;
 
     FileFetcher(File dir, Map<String, Object> fileDetails, String saveAs,
-                boolean isConf, long latestVersion) throws IOException {
+                boolean isConf, long latestGen) throws IOException {
       this.copy2Dir = dir;
       this.fileName = (String) fileDetails.get(NAME);
       this.size = (Long) fileDetails.get(SIZE);
@@ -908,7 +909,7 @@ public class SnapPuller {
       if(fileDetails.get(LAST_MODIFIED) != null){
         lastmodified = (Long)fileDetails.get(LAST_MODIFIED);
       }
-      indexVersion = latestVersion;
+      indexGen = latestGen;
 
       this.file = new File(copy2Dir, saveAs);
       
@@ -1077,7 +1078,7 @@ public class SnapPuller {
       //the method is command=filecontent
       post.addParameter(COMMAND, CMD_GET_FILE);
       //add the version to download. This is used to reserve the download
-      post.addParameter(CMD_INDEX_VERSION, indexVersion.toString());
+      post.addParameter(GENERATION, indexGen.toString());
       if (isConf) {
         //set cf instead of file for config file
         post.addParameter(CONF_FILE_SHORT, fileName);

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapShooter.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapShooter.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/SnapShooter.java Mon Jan 30 17:56:40 2012
@@ -70,7 +70,7 @@ public class SnapShooter {
   }
 
   void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, final ReplicationHandler replicationHandler) {
-    replicationHandler.core.getDeletionPolicy().saveCommitPoint(indexCommit.getVersion());
+    replicationHandler.core.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration());
 
     new Thread() {
       @Override
@@ -112,7 +112,7 @@ public class SnapShooter {
       LOG.error("Exception while creating snapshot", e);
       details.add("snapShootException", e.getMessage());
     } finally {
-      replicationHandler.core.getDeletionPolicy().releaseCommitPoint(indexCommit.getVersion());   
+      replicationHandler.core.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
       replicationHandler.snapShootDetails = details;
       if (lock != null) {
         try {

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Mon Jan 30 17:56:40 2012
@@ -48,6 +48,7 @@ import org.apache.solr.handler.RequestHa
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.response.SolrQueryResponse;
 import org.apache.solr.schema.FieldType;
+import org.apache.solr.update.SolrIndexWriter;
 import org.apache.solr.schema.IndexSchema;
 import org.apache.solr.schema.SchemaField;
 import org.apache.solr.search.SolrIndexSearcher;
@@ -562,7 +563,10 @@ public class LukeRequestHandler extends 
     indexInfo.add("current", reader.isCurrent() );
     indexInfo.add("hasDeletions", reader.hasDeletions() );
     indexInfo.add("directory", dir );
-    indexInfo.add("lastModified", new Date(IndexReader.lastModified(dir)) );
+    String s = reader.getIndexCommit().getUserData().get(SolrIndexWriter.COMMIT_TIME_MSEC_KEY);
+    if (s != null) {
+      indexInfo.add("lastModified", new Date(Long.parseLong(s)));
+    }
     return indexInfo;
   }
   //////////////////////// SolrInfoMBeans methods //////////////////////

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/servlet/cache/HttpCacheHeaderUtil.java Mon Jan 30 17:56:40 2012
@@ -29,6 +29,7 @@ import javax.servlet.http.HttpServletRes
 
 import org.apache.lucene.index.IndexReader;
 
+import org.apache.solr.core.IndexDeletionPolicyWrapper;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrConfig;
 import org.apache.solr.core.SolrConfig.HttpCachingConfig.LastModFrom;
@@ -157,7 +158,7 @@ public final class HttpCacheHeaderUtil {
       // assume default, change if needed (getOpenTime() should be fast)
       lastMod =
         LastModFrom.DIRLASTMOD == lastModFrom
-        ? IndexReader.lastModified(searcher.getIndexReader().directory())
+        ? IndexDeletionPolicyWrapper.getCommitTimestamp(searcher.getIndexReader().getIndexCommit())
         : searcher.getOpenTime();
     } catch (IOException e) {
       // we're pretty freaking screwed if this happens

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java Mon Jan 30 17:56:40 2012
@@ -22,6 +22,8 @@ package org.apache.solr.update;
 
 import java.io.IOException;
 import java.net.URL;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicLong;
@@ -387,7 +389,9 @@ public class DirectUpdateHandler2 extend
         }
 
         // SolrCore.verbose("writer.commit() start writer=",writer);
-        writer.commit();
+        final Map<String,String> commitData = new HashMap<String,String>();
+        commitData.put(SolrIndexWriter.COMMIT_TIME_MSEC_KEY, String.valueOf(System.currentTimeMillis()));
+        writer.commit(commitData);
         // SolrCore.verbose("writer.commit() end");
         numDocsPending.set(0);
         callPostCommitCallbacks();

Modified: lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java Mon Jan 30 17:56:40 2012
@@ -50,6 +50,11 @@ public class SolrIndexWriter extends Ind
   public static final AtomicLong numOpens = new AtomicLong();
   public static final AtomicLong numCloses = new AtomicLong();
 
+
+  /** Stored into each Lucene commit to record the
+   *  System.currentTimeMillis() when commit was called. */
+  public static final String COMMIT_TIME_MSEC_KEY = "commitTimeMSec";
+
   String name;
   private DirectoryFactory directoryFactory;
 

Modified: lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/ChaosMonkey.java Mon Jan 30 17:56:40 2012
@@ -19,7 +19,6 @@ package org.apache.solr.cloud;
 
 import java.net.BindException;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -297,13 +296,7 @@ public class ChaosMonkey {
     JettySolrRunner jetty;
     if (chance <= 5 && aggressivelyKillLeaders) {
       // if killLeader, really aggressively go after leaders
-      Collection<CloudJettyRunner> leaders = shardToLeaderJetty.values();
-      List<CloudJettyRunner> leadersList = new ArrayList<CloudJettyRunner>(leaders.size());
-     
-      leadersList.addAll(leaders);
-
-      int index = random.nextInt(leadersList.size());
-      jetty = leadersList.get(index).jetty;
+      jetty = shardToLeaderJetty.get(slice).jetty;
     } else {
       // get random shard
       List<CloudJettyRunner> jetties = shardToJetty.get(slice);

Modified: lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java?rev=1237850&r1=1237849&r2=1237850&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java (original)
+++ lucene/dev/branches/lucene3661/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudTest.java Mon Jan 30 17:56:40 2012
@@ -59,6 +59,7 @@ import org.junit.Ignore;
  * what we test now - the default update chain
  * 
  */
+@Ignore
 public class FullSolrCloudTest extends AbstractDistributedZkTestCase {
   
   private static final String SHARD2 = "shard2";