You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/02/28 18:12:09 UTC

svn commit: r1573003 [2/3] - in /lucene/dev/branches/lucene5376: ./ dev-tools/ dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ lucene/analysis/common/src/test/org/...

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Fri Feb 28 17:12:07 2014
@@ -19,26 +19,27 @@ package org.apache.lucene.search.suggest
 
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.lucene.search.suggest.InputIterator;
 import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.Sort.SortInfo;
-import org.apache.lucene.search.suggest.Sort;
 import org.apache.lucene.search.suggest.fst.FSTCompletion.Completion;
 import org.apache.lucene.search.suggest.tst.TSTLookup;
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ByteArrayDataOutput;
 import org.apache.lucene.store.DataInput;
 import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
-import org.apache.lucene.util.*;
 import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.NoOutputs;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.OfflineSorter;
+import org.apache.lucene.util.OfflineSorter.SortInfo;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.UnicodeUtil;
 
 /**
  * An adapter from {@link Lookup} API to {@link FSTCompletion}.
@@ -150,12 +151,12 @@ public class FSTCompletionLookup extends
       throw new IllegalArgumentException("this suggester doesn't support payloads");
     }
     File tempInput = File.createTempFile(
-        FSTCompletionLookup.class.getSimpleName(), ".input", Sort.defaultTempDir());
+        FSTCompletionLookup.class.getSimpleName(), ".input", OfflineSorter.defaultTempDir());
     File tempSorted = File.createTempFile(
-        FSTCompletionLookup.class.getSimpleName(), ".sorted", Sort.defaultTempDir());
+        FSTCompletionLookup.class.getSimpleName(), ".sorted", OfflineSorter.defaultTempDir());
 
-    Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
-    Sort.ByteSequencesReader reader = null;
+    OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
+    OfflineSorter.ByteSequencesReader reader = null;
     ExternalRefSorter sorter = null;
 
     // Push floats up front before sequences to sort them. For now, assume they are non-negative.
@@ -180,13 +181,13 @@ public class FSTCompletionLookup extends
 
       // We don't know the distribution of scores and we need to bucket them, so we'll sort
       // and divide into equal buckets.
-      SortInfo info = new Sort().sort(tempInput, tempSorted);
+      SortInfo info = new OfflineSorter().sort(tempInput, tempSorted);
       tempInput.delete();
       FSTCompletionBuilder builder = new FSTCompletionBuilder(
-          buckets, sorter = new ExternalRefSorter(new Sort()), sharedTailLength);
+          buckets, sorter = new ExternalRefSorter(new OfflineSorter()), sharedTailLength);
 
       final int inputLines = info.lines;
-      reader = new Sort.ByteSequencesReader(tempSorted);
+      reader = new OfflineSorter.ByteSequencesReader(tempSorted);
       long line = 0;
       int previousBucket = 0;
       int previousScore = 0;

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Fri Feb 28 17:12:07 2014
@@ -25,7 +25,6 @@ import java.util.List;
 
 import org.apache.lucene.search.suggest.InputIterator;
 import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter;
 import org.apache.lucene.search.suggest.SortedInputIterator;
 import org.apache.lucene.store.ByteArrayDataInput;
 import org.apache.lucene.store.ByteArrayDataOutput;
@@ -43,6 +42,7 @@ import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.PositiveIntOutputs;
 import org.apache.lucene.util.fst.Util.MinResult;
 import org.apache.lucene.util.fst.Util;
+import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
 
 /**
  * Suggester based on a weighted FST: it first traverses the prefix, 

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java Fri Feb 28 17:12:07 2014
@@ -94,6 +94,7 @@ public class JaspellTernarySearchTrie {
       relatives[PARENT] = parent;
     }
 
+    /** Return an approximate memory usage for this node and its sub-nodes. */
     public long sizeInBytes() {
       long mem = RamUsageEstimator.shallowSizeOf(this) + RamUsageEstimator.shallowSizeOf(relatives);
       for (TSTNode node : relatives) {

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java Fri Feb 28 17:12:07 2014
@@ -83,7 +83,7 @@ public class DocumentValueSourceDictiona
     writer.close();
     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME,  new DoubleConstValueSource(10), PAYLOAD_FIELD_NAME);
-    InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+    InputIterator inputIterator = dictionary.getEntryIterator();
 
     assertNull(inputIterator.next());
     assertEquals(inputIterator.weight(), 0);
@@ -109,7 +109,7 @@ public class DocumentValueSourceDictiona
     IndexReader ir = DirectoryReader.open(dir);
     ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2), new LongFieldSource(WEIGHT_FIELD_NAME_3)};
     Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd), PAYLOAD_FIELD_NAME);
-    InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+    InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());
@@ -141,7 +141,7 @@ public class DocumentValueSourceDictiona
     IndexReader ir = DirectoryReader.open(dir);
     ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2), new LongFieldSource(WEIGHT_FIELD_NAME_3)};
     Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME,  new SumFloatFunction(toAdd));
-    InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+    InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());
@@ -195,7 +195,7 @@ public class DocumentValueSourceDictiona
     ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2)};
 
     Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME,  new SumFloatFunction(toAdd), PAYLOAD_FIELD_NAME);
-    InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+    InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());
@@ -226,7 +226,7 @@ public class DocumentValueSourceDictiona
 
     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new DoubleConstValueSource(10), PAYLOAD_FIELD_NAME);
-    InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+    InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Fri Feb 28 17:12:07 2014
@@ -21,6 +21,10 @@ import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -32,9 +36,9 @@ import org.apache.lucene.analysis.core.S
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.search.suggest.Lookup.LookupResult;
 import org.apache.lucene.search.suggest.Input;
 import org.apache.lucene.search.suggest.InputArrayIterator;
+import org.apache.lucene.search.suggest.Lookup.LookupResult;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -433,4 +437,458 @@ public class AnalyzingInfixSuggesterTest
     assertEquals("a bob for <b>a</b>pples", results.get(0).key);
     suggester.close();
   }
+
+  public void testEmptyAtStart() throws Exception {
+    File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+    Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+    AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
+        @Override
+        protected Directory getDirectory(File path) {
+          return newDirectory();
+        }
+      };
+    suggester.build(new InputArrayIterator(new Input[0]));
+    suggester.add(new BytesRef("a penny saved is a penny earned"), 10, new BytesRef("foobaz"));
+    suggester.add(new BytesRef("lend me your ear"), 8, new BytesRef("foobar"));
+    suggester.refresh();
+    List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+    assertEquals(2, results.size());
+    assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).key);
+    assertEquals(10, results.get(0).value);
+    assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+    assertEquals("lend me your <b>ear</b>", results.get(1).key);
+    assertEquals(8, results.get(1).value);
+    assertEquals(new BytesRef("foobar"), results.get(1).payload);
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("ear ", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("lend me your <b>ear</b>", results.get(0).key);
+    assertEquals(8, results.get(0).value);
+    assertEquals(new BytesRef("foobar"), results.get(0).payload);
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("pen", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("a <b>pen</b>ny saved is a <b>pen</b>ny earned", results.get(0).key);
+    assertEquals(10, results.get(0).value);
+    assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("p", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("a <b>p</b>enny saved is a <b>p</b>enny earned", results.get(0).key);
+    assertEquals(10, results.get(0).value);
+    assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+    suggester.close();
+  }
+
+  public void testBothExactAndPrefix() throws Exception {
+    File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+    Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+    AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
+        @Override
+        protected Directory getDirectory(File path) {
+          return newDirectory();
+        }
+      };
+    suggester.build(new InputArrayIterator(new Input[0]));
+    suggester.add(new BytesRef("the pen is pretty"), 10, new BytesRef("foobaz"));
+    suggester.refresh();
+
+    List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("pen p", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("the <b>pen</b> is <b>p</b>retty", results.get(0).key);
+    assertEquals(10, results.get(0).value);
+    assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+    suggester.close();
+  }
+
+  private static String randomText() {
+    int numWords = TestUtil.nextInt(random(), 1, 4);
+      
+    StringBuilder b = new StringBuilder();
+    for(int i=0;i<numWords;i++) {
+      if (i > 0) {
+        b.append(' ');
+      }
+      b.append(TestUtil.randomSimpleString(random(), 1, 10));
+    }
+
+    return b.toString();
+  }
+
+  private static class Update {
+    long weight;
+    int index;
+  }
+
+  private static class LookupThread extends Thread {
+    private final AnalyzingInfixSuggester suggester;
+    private volatile boolean stop;
+
+    public LookupThread(AnalyzingInfixSuggester suggester) {
+      this.suggester = suggester;
+    }
+
+    public void finish() throws InterruptedException {
+      stop = true;
+      this.join();
+    }
+
+    @Override
+    public void run() {
+      while (stop == false) {
+        String query = randomText();
+        int topN = TestUtil.nextInt(random(), 1, 100);
+        boolean allTermsRequired = random().nextBoolean();
+        boolean doHilite = random().nextBoolean();
+        // We don't verify the results; just doing
+        // simultaneous lookups while adding/updating to
+        // see if there are any thread hazards:
+        try {
+          suggester.lookup(TestUtil.stringToCharSequence(query, random()),
+                           topN, allTermsRequired, doHilite);
+        } catch (IOException ioe) {
+          throw new RuntimeException(ioe);
+        }
+      }
+    }
+  }
+
+  public void testRandomNRT() throws Exception {
+    final File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+    Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+    int minPrefixChars = random().nextInt(7);
+    if (VERBOSE) {
+      System.out.println("  minPrefixChars=" + minPrefixChars);
+    }
+
+    AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixChars) {
+        @Override
+        protected Directory getDirectory(File path) {
+          return newFSDirectory(path);
+        }
+      };
+
+    // Initial suggester built with nothing:
+    suggester.build(new InputArrayIterator(new Input[0]));
+
+    LookupThread lookupThread = new LookupThread(suggester);
+    lookupThread.start();
+
+    int iters = atLeast(1000);
+    int visibleUpto = 0;
+
+    Set<Long> usedWeights = new HashSet<Long>();
+    Set<String> usedKeys = new HashSet<String>();
+
+    List<Input> inputs = new ArrayList<Input>();
+    List<Update> pendingUpdates = new ArrayList<Update>();
+
+    for(int iter=0;iter<iters;iter++) {
+      String text;
+      while (true) {
+        text = randomText();
+        if (usedKeys.contains(text) == false) {
+          usedKeys.add(text);
+          break;
+        }
+      }
+
+      // Carefully pick a weight we never used, to sidestep
+      // tie-break problems:
+      long weight;
+      while (true) {
+        weight = random().nextInt(10*iters);
+        if (usedWeights.contains(weight) == false) {
+          usedWeights.add(weight);
+          break;
+        }
+      }
+
+      if (inputs.size() > 0 && random().nextInt(4) == 1) {
+        // Update an existing suggestion
+        Update update = new Update();
+        update.index = random().nextInt(inputs.size());
+        update.weight = weight;
+        Input input = inputs.get(update.index);
+        pendingUpdates.add(update);
+        if (VERBOSE) {
+          System.out.println("TEST: iter=" + iter + " update input=" + input.term.utf8ToString() + "/" + weight);
+        }
+        suggester.update(input.term, weight, input.term);
+        
+      } else {
+        // Add a new suggestion
+        inputs.add(new Input(text, weight, new BytesRef(text)));
+        if (VERBOSE) {
+          System.out.println("TEST: iter=" + iter + " add input=" + text + "/" + weight);
+        }
+        BytesRef br = new BytesRef(text);
+        suggester.add(br, weight, br);
+      }
+
+      if (random().nextInt(15) == 7) {
+        if (VERBOSE) {
+          System.out.println("TEST: now refresh suggester");
+        }
+        suggester.refresh();
+        visibleUpto = inputs.size();
+        for(Update update : pendingUpdates) {
+          Input oldInput = inputs.get(update.index);
+          Input newInput = new Input(oldInput.term, update.weight, oldInput.payload);
+          inputs.set(update.index, newInput);
+        }
+        pendingUpdates.clear();
+      }
+      
+      if (random().nextInt(50) == 7) {
+        if (VERBOSE) {
+          System.out.println("TEST: now close/reopen suggester");
+        }
+        lookupThread.finish();
+        suggester.close();
+        suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixChars) {
+            @Override
+            protected Directory getDirectory(File path) {
+              return newFSDirectory(path);
+            }
+          };
+        lookupThread = new LookupThread(suggester);
+        lookupThread.start();
+
+        visibleUpto = inputs.size();
+        for(Update update : pendingUpdates) {
+          Input oldInput = inputs.get(update.index);
+          Input newInput = new Input(oldInput.term, update.weight, oldInput.payload);
+          inputs.set(update.index, newInput);
+        }
+        pendingUpdates.clear();
+      }
+
+      if (visibleUpto > 0) {
+        String query = randomText();
+        boolean lastPrefix = random().nextInt(5) != 1;
+        if (lastPrefix == false) {
+          query += " ";
+        }
+
+        String[] queryTerms = query.split("\\s");
+        boolean allTermsRequired = random().nextInt(10) == 7;
+        boolean doHilite = random().nextBoolean();
+
+        if (VERBOSE) {
+          System.out.println("TEST: lookup \"" + query + "\" allTermsRequired=" + allTermsRequired + " doHilite=" + doHilite);
+        }
+
+        // Stupid slow but hopefully correct matching:
+        List<Input> expected = new ArrayList<Input>();
+        for(int i=0;i<visibleUpto;i++) {
+          Input input = inputs.get(i);
+          String[] inputTerms = input.term.utf8ToString().split("\\s");
+          boolean match = false;
+          for(int j=0;j<queryTerms.length;j++) {
+            if (j < queryTerms.length-1 || lastPrefix == false) {
+              // Exact match
+              for(int k=0;k<inputTerms.length;k++) {
+                if (inputTerms[k].equals(queryTerms[j])) {
+                  match = true;
+                  break;
+                }
+              }
+            } else {
+              // Prefix match
+              for(int k=0;k<inputTerms.length;k++) {
+                if (inputTerms[k].startsWith(queryTerms[j])) {
+                  match = true;
+                  break;
+                }
+              }
+            }
+            if (match) {
+              if (allTermsRequired == false) {
+                // At least one query term does match:
+                break;
+              }
+              match = false;
+            } else if (allTermsRequired) {
+              // At least one query term does not match:
+              break;
+            }
+          }
+
+          if (match) {
+            if (doHilite) {
+              expected.add(new Input(hilite(lastPrefix, inputTerms, queryTerms), input.v, input.term));
+            } else {
+              expected.add(input);
+            }
+          }
+        }
+
+        Collections.sort(expected,
+                         new Comparator<Input>() {
+                           @Override
+                           public int compare(Input a, Input b) {
+                             if (a.v > b.v) {
+                               return -1;
+                             } else if (a.v < b.v) {
+                               return 1;
+                             } else {
+                               return 0;
+                             }
+                           }
+                         });
+
+        if (expected.isEmpty() == false) {
+
+          int topN = TestUtil.nextInt(random(), 1, expected.size());
+
+          List<LookupResult> actual = suggester.lookup(TestUtil.stringToCharSequence(query, random()), topN, allTermsRequired, doHilite);
+
+          int expectedCount = Math.min(topN, expected.size());
+
+          if (VERBOSE) {
+            System.out.println("  expected:");
+            for(int i=0;i<expectedCount;i++) {
+              Input x = expected.get(i);
+              System.out.println("    " + x.term.utf8ToString() + "/" + x.v);
+            }
+            System.out.println("  actual:");
+            for(LookupResult result : actual) {
+              System.out.println("    " + result);
+            }
+          }
+
+          assertEquals(expectedCount, actual.size());
+          for(int i=0;i<expectedCount;i++) {
+            assertEquals(expected.get(i).term.utf8ToString(), actual.get(i).key.toString());
+            assertEquals(expected.get(i).v, actual.get(i).value);
+            assertEquals(expected.get(i).payload, actual.get(i).payload);
+          }
+        } else {
+          if (VERBOSE) {
+            System.out.println("  no expected matches");
+          }
+        }
+      }
+    }
+
+    lookupThread.finish();
+    suggester.close();
+  }
+
+  private static String hilite(boolean lastPrefix, String[] inputTerms, String[] queryTerms) {
+    // Stupid slow but hopefully correct highlighter:
+    //System.out.println("hilite: lastPrefix=" + lastPrefix + " inputTerms=" + Arrays.toString(inputTerms) + " queryTerms=" + Arrays.toString(queryTerms));
+    StringBuilder b = new StringBuilder();
+    for(int i=0;i<inputTerms.length;i++) {
+      if (i > 0) {
+        b.append(' ');
+      }
+      String inputTerm = inputTerms[i];
+      //System.out.println("  inputTerm=" + inputTerm);
+      boolean matched = false;
+      for(int j=0;j<queryTerms.length;j++) {
+        String queryTerm = queryTerms[j];
+        //System.out.println("    queryTerm=" + queryTerm);
+        if (j < queryTerms.length-1 || lastPrefix == false) {
+          //System.out.println("      check exact");
+          if (inputTerm.equals(queryTerm)) {
+            b.append("<b>");
+            b.append(inputTerm);
+            b.append("</b>");
+            matched = true;
+            break;
+          }
+        } else if (inputTerm.startsWith(queryTerm)) {
+          b.append("<b>");
+          b.append(queryTerm);
+          b.append("</b>");
+          b.append(inputTerm.substring(queryTerm.length(), inputTerm.length()));
+          matched = true;
+          break;
+        }
+      }
+
+      if (matched == false) {
+        b.append(inputTerm);
+      }
+    }
+
+    return b.toString();
+  }
+
+  public void testBasicNRT() throws Exception {
+    Input keys[] = new Input[] {
+      new Input("lend me your ear", 8, new BytesRef("foobar")),
+    };
+
+    File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+
+    Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+    AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
+        @Override
+        protected Directory getDirectory(File path) {
+          return newDirectory();
+        }
+      };
+    suggester.build(new InputArrayIterator(keys));
+
+    List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("lend me your <b>ear</b>", results.get(0).key);
+    assertEquals(8, results.get(0).value);
+    assertEquals(new BytesRef("foobar"), results.get(0).payload);
+
+    // Add a new suggestion:
+    suggester.add(new BytesRef("a penny saved is a penny earned"), 10, new BytesRef("foobaz"));
+
+    // Must refresh to see any newly added suggestions:
+    suggester.refresh();
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+    assertEquals(2, results.size());
+    assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).key);
+    assertEquals(10, results.get(0).value);
+    assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+    assertEquals("lend me your <b>ear</b>", results.get(1).key);
+    assertEquals(8, results.get(1).value);
+    assertEquals(new BytesRef("foobar"), results.get(1).payload);
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("ear ", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("lend me your <b>ear</b>", results.get(0).key);
+    assertEquals(8, results.get(0).value);
+    assertEquals(new BytesRef("foobar"), results.get(0).payload);
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("pen", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("a <b>pen</b>ny saved is a <b>pen</b>ny earned", results.get(0).key);
+    assertEquals(10, results.get(0).value);
+    assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("p", random()), 10, true, true);
+    assertEquals(1, results.size());
+    assertEquals("a <b>p</b>enny saved is a <b>p</b>enny earned", results.get(0).key);
+    assertEquals(10, results.get(0).value);
+    assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+    // Change the weight:
+    suggester.update(new BytesRef("lend me your ear"), 12, new BytesRef("foobox"));
+
+    // Must refresh to see any newly added suggestions:
+    suggester.refresh();
+
+    results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+    assertEquals(2, results.size());
+    assertEquals("lend me your <b>ear</b>", results.get(0).key);
+    assertEquals(12, results.get(0).value);
+    assertEquals(new BytesRef("foobox"), results.get(0).payload);
+    assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(1).key);
+    assertEquals(10, results.get(1).value);
+    assertEquals(new BytesRef("foobaz"), results.get(1).payload);
+    suggester.close();
+  }
 }

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java Fri Feb 28 17:12:07 2014
@@ -225,7 +225,7 @@ public class BlendedInfixSuggesterTest e
     suggester.close();
   }
 
-  private static long getInResults(BlendedInfixSuggester suggester, String prefix, BytesRef payload, int num) {
+  private static long getInResults(BlendedInfixSuggester suggester, String prefix, BytesRef payload, int num) throws IOException {
 
     List<Lookup.LookupResult> responses = suggester.lookup(prefix, num, true, false);
 

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java Fri Feb 28 17:12:07 2014
@@ -18,16 +18,16 @@ package org.apache.lucene.search.suggest
  */
 
 import org.apache.lucene.search.suggest.InMemorySorter;
-import org.apache.lucene.search.suggest.Sort;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefIterator;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.OfflineSorter;
 import org.junit.Test;
 
 public class BytesRefSortersTest extends LuceneTestCase {
   @Test
   public void testExternalRefSorter() throws Exception {
-    ExternalRefSorter s = new ExternalRefSorter(new Sort());
+    ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter());
     check(s);
     s.close();
   }

Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java Fri Feb 28 17:12:07 2014
@@ -17,10 +17,15 @@ package org.apache.lucene.search.suggest
  * limitations under the License.
  */
 
-import java.io.*;
 
-import org.apache.lucene.search.suggest.Sort;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.OfflineSorter;
 
 /**
  * Try to build a suggester from a large data set. The input is a simple text
@@ -33,7 +38,7 @@ public class LargeInputFST {
     int buckets = 20;
     int shareMaxTail = 10;
 
-    ExternalRefSorter sorter = new ExternalRefSorter(new Sort());
+    ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
     FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);
 
     BufferedReader reader = new BufferedReader(

Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Fri Feb 28 17:12:07 2014
@@ -209,7 +209,10 @@ public abstract class LuceneTestCase ext
   @Inherited
   @Retention(RetentionPolicy.RUNTIME)
   @TestGroup(enabled = false, sysProperty = SYSPROP_BADAPPLES)
-  public @interface BadApple {}
+  public @interface BadApple {
+    /** Point to JIRA entry. */
+    public String bugUrl();
+  }
 
   /**
    * Annotation for test classes that should avoid certain codec types

Modified: lucene/dev/branches/lucene5376/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/CHANGES.txt?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5376/solr/CHANGES.txt Fri Feb 28 17:12:07 2014
@@ -82,8 +82,12 @@ Detailed Change List
 
 New Features
 ----------------------
+
 * SOLR-5130: Implement addReplica Collections API (Noble Paul)
 
+* SOLR-5183: JSON updates now support nested child documents using a 
+  "_childDocument_" object key.  (Varun Thacker, hossman)
+
 Bug Fixes
 ----------------------
 
@@ -93,6 +97,12 @@ Bug Fixes
 * SOLR-5423: CSV output doesn't include function field
   (Arun Kumar, hossman, Steve Rowe)
 
+* SOLR-5777: Fix ordering of field values in JSON updates where
+  field name key is repeated (hossman)
+
+* SOLR-5782: The full MapReduceIndexer help text does not display when using
+  --help.  (Mark Miller, Wolfgang Hoschek)
+
 Optimizations
 ----------------------
 * SOLR-1880: Distributed Search skips GET_FIELDS stage if EXECUTE_QUERY

Modified: lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java Fri Feb 28 17:12:07 2014
@@ -27,8 +27,6 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
 import java.io.Writer;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -183,11 +181,7 @@ public class MapReduceIndexerTool extend
         .action(new HelpArgumentAction() {
           @Override
           public void run(ArgumentParser parser, Argument arg, Map<String, Object> attrs, String flag, Object value) throws ArgumentParserException {
-            try {
-              parser.printHelp(new PrintWriter(new OutputStreamWriter(System.out, "UTF-8")));
-            } catch (UnsupportedEncodingException e) {
-              throw new RuntimeException("Won't Happen for UTF-8");
-            }  
+            parser.printHelp();
             System.out.println();
             System.out.print(ToolRunnerHelpFormatter.getGenericCommandUsage());
             //ToolRunner.printGenericCommandUsage(System.out);
@@ -207,31 +201,6 @@ public class MapReduceIndexerTool extend
               "  --shards 1 \\\n" + 
               "  hdfs:///user/$USER/test-documents/sample-statuses-20120906-141433.avro\n" +
               "\n" +
-              "# (Re)index all files that match all of the following conditions:\n" +
-              "# 1) File is contained in dir tree hdfs:///user/$USER/solrloadtest/twitter/tweets\n" +
-              "# 2) file name matches the glob pattern 'sample-statuses*.gz'\n" +
-              "# 3) file was last modified less than 100000 minutes ago\n" +
-              "# 4) file size is between 1 MB and 1 GB\n" +
-              "# Also include extra library jar file containing JSON tweet Java parser:\n" +
-              "hadoop jar target/solr-map-reduce-*.jar " + "com.cloudera.cdk.morphline.hadoop.find.HdfsFindTool" + " \\\n" + 
-              "  -find hdfs:///user/$USER/solrloadtest/twitter/tweets \\\n" + 
-              "  -type f \\\n" + 
-              "  -name 'sample-statuses*.gz' \\\n" + 
-              "  -mmin -1000000 \\\n" + 
-              "  -size -100000000c \\\n" + 
-              "  -size +1000000c \\\n" + 
-              "| sudo -u hdfs hadoop \\\n" + 
-              "  --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" + 
-              "  jar target/solr-map-reduce-*.jar \\\n" +
-              "  -D 'mapred.child.java.opts=-Xmx500m' \\\n" + 
-//            "  -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" + 
-              "  --log4j src/test/resources/log4j.properties \\\n" + 
-              "  --morphline-file ../search-core/src/test/resources/test-morphlines/tutorialReadJsonTestTweets.conf \\\n" + 
-              "  --solr-home-dir src/test/resources/solr/minimr \\\n" + 
-              "  --output-dir hdfs://c2202.mycompany.com/user/$USER/test \\\n" + 
-              "  --shards 100 \\\n" + 
-              "  --input-list -\n" +
-              "\n" +
               "# Go live by merging resulting index shards into a live Solr cluster\n" +
               "# (explicitly specify Solr URLs - for a SolrCloud cluster see next example):\n" +
               "sudo -u hdfs hadoop \\\n" + 

Modified: lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java Fri Feb 28 17:12:07 2014
@@ -69,7 +69,6 @@ import org.apache.solr.common.util.Named
 import org.apache.solr.hadoop.hack.MiniMRClientCluster;
 import org.apache.solr.hadoop.hack.MiniMRClientClusterFactory;
 import org.apache.solr.morphlines.solr.AbstractSolrMorphlineTestBase;
-import org.apache.solr.util.ExternalPaths;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -89,7 +88,7 @@ import com.carrotsearch.randomizedtestin
 @ThreadLeakZombies(Consequence.CONTINUE)
 @ThreadLeakScope(Scope.NONE)
 @SuppressCodecs({"Lucene3x", "Lucene40"})
-@SuppressSSL
+@SuppressSSL // SSL does not work with this test for currently unknown reasons
 @Slow
 public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase {
   

Modified: lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java Fri Feb 28 17:12:07 2014
@@ -20,12 +20,15 @@ import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Calendar;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.TimeZone;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.io.FileUtils;
@@ -130,7 +133,9 @@ public class AbstractSolrMorphlineTestBa
       String[] files, 
       Map<String,Integer> expectedRecords, 
       Map<String, Map<String, Object>> expectedRecordContents) throws Exception {
-    
+
+    assumeTrue("This test has issues with this locale: https://issues.apache.org/jira/browse/SOLR-5778", 
+        "GregorianCalendar".equals(Calendar.getInstance(TimeZone.getDefault(), Locale.getDefault()).getClass().getSimpleName()));
     deleteAllDocuments();
     int numDocs = 0;    
     for (int i = 0; i < 1; i++) {

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java Fri Feb 28 17:12:07 2014
@@ -305,8 +305,7 @@ public class Overseer {
           makeMap(
           ZkStateReader.CORE_NAME_PROP, message.getStr(ZkStateReader.CORE_NAME_PROP),
           ZkStateReader.BASE_URL_PROP,message.getStr(ZkStateReader.BASE_URL_PROP),
-          ZkStateReader.STATE_PROP,message.getStr(ZkStateReader.STATE_PROP),
-          ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName));
+          ZkStateReader.STATE_PROP,message.getStr(ZkStateReader.STATE_PROP)));
       sl.getReplicasMap().put(coreNodeName, replica);
       return clusterState;
     }

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Fri Feb 28 17:12:07 2014
@@ -74,6 +74,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -1624,6 +1625,7 @@ public class OverseerCollectionProcessor
         throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully createcollection: " + message.getStr("name"));
 
       log.info("going to create cores replicas shardNames {} , repFactor : {}", shardNames, repFactor);
+      Map<String ,ShardRequest> coresToCreate = new LinkedHashMap<String, ShardRequest>();
       for (int i = 1; i <= shardNames.size(); i++) {
         String sliceName = shardNames.get(i-1);
         for (int j = 1; j <= repFactor; j++) {
@@ -1662,13 +1664,22 @@ public class OverseerCollectionProcessor
           ShardRequest sreq = new ShardRequest();
           params.set("qt", adminPath);
           sreq.purpose = 1;
-          String replica = zkStateReader.getBaseUrlForNodeName(nodeName);
-          sreq.shards = new String[] {replica};
+          sreq.shards = new String[] {baseUrl};
           sreq.actualShards = sreq.shards;
           sreq.params = params;
 
-          shardHandler.submit(sreq, replica, sreq.params);
+          if(isLegacyCloud) shardHandler.submit(sreq, sreq.shards[0], sreq.params);
+          else coresToCreate.put(coreName, sreq);
+        }
+      }
 
+      if(!isLegacyCloud) {
+        //wait for all replica entries to be created
+        Map<String, Replica> replicas = lookupReplicas(collectionName, coresToCreate.keySet());
+        for (Map.Entry<String, ShardRequest> e : coresToCreate.entrySet()) {
+          ShardRequest sreq = e.getValue();
+          sreq.params.set(CoreAdminParams.CORE_NODE_NAME, replicas.get(e.getKey()).getName());
+          shardHandler.submit(sreq, sreq.shards[0], sreq.params);
         }
       }
 
@@ -1690,6 +1701,37 @@ public class OverseerCollectionProcessor
     }
   }
 
+  private Map<String, Replica> lookupReplicas(String collectionName, Collection<String> coreNames) throws InterruptedException {
+    Map<String, Replica> result = new HashMap<String, Replica>();
+    long endTime = System.currentTimeMillis() +3000;
+    for(;;) {
+      DocCollection coll = zkStateReader.getClusterState().getCollection(collectionName);
+      for (String  coreName : coreNames) {
+        if(result.containsKey(coreName)) continue;
+        for (Slice slice : coll.getSlices()) {
+          for (Replica replica : slice.getReplicas()) {
+            if(coreName.equals(replica.getStr(ZkStateReader.CORE_NAME_PROP))) {
+              result.put(coreName,replica);
+              break;
+            }
+          }
+        }
+      }
+
+      if(result.size() == coreNames.size()) {
+        return result;
+      }
+      if( System.currentTimeMillis() > endTime) {
+        //time up . throw exception and go out
+        throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to create replica entries in ZK");
+      }
+
+      Thread.sleep(100);
+    }
+
+  }
+
+
   private void addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
     String collection = message.getStr(COLLECTION_PROP);
     String node = message.getStr("node");
@@ -1733,6 +1775,7 @@ public class OverseerCollectionProcessor
       }
       coreName = collection + "_" + shard + "_replica" + replicaNum;
     }
+    ModifiableSolrParams params = new ModifiableSolrParams();
 
     if(!Overseer.isLegacy(zkStateReader.getClusterProps())){
       ZkNodeProps props = new ZkNodeProps(
@@ -1743,6 +1786,7 @@ public class OverseerCollectionProcessor
           ZkStateReader.STATE_PROP, ZkStateReader.DOWN,
           ZkStateReader.BASE_URL_PROP,zkStateReader.getBaseUrlForNodeName(node));
       Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(props));
+      params.set(CoreAdminParams.CORE_NODE_NAME, lookupReplicas(collection, Collections.singletonList(coreName)).get(coreName).getName());
     }
 
 
@@ -1751,7 +1795,6 @@ public class OverseerCollectionProcessor
     String dataDir = message.getStr(CoreAdminParams.DATA_DIR);
     String instanceDir = message.getStr(CoreAdminParams.INSTANCE_DIR);
 
-    ModifiableSolrParams params = new ModifiableSolrParams();
     params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString());
     params.set(CoreAdminParams.NAME, coreName);
     params.set(COLL_CONF, configName);

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java Fri Feb 28 17:12:07 2014
@@ -1262,7 +1262,7 @@ public final class ZkController {
             String msgNodeName = getNodeName();
             String msgCore = descriptor.getName();
 
-            if (nodeName.equals(msgNodeName) && core.equals(msgCore)) {
+            if (msgNodeName.equals(nodeName) && core.equals(msgCore)) {
               descriptor.getCloudDescriptor()
                   .setCoreNodeName(replica.getName());
               return;
@@ -1354,10 +1354,11 @@ public final class ZkController {
 
     String coreNodeName = getCoreNodeName(cd);
 
-    checkStateInZk(cd);
     // before becoming available, make sure we are not live and active
     // this also gets us our assigned shard id if it was not specified
     try {
+      checkStateInZk(cd);
+
       CloudDescriptor cloudDesc = cd.getCloudDescriptor();
 
 
@@ -1385,28 +1386,26 @@ public final class ZkController {
 
   }
 
-  private void checkStateInZk(CoreDescriptor cd) {
-    if(!Overseer.isLegacy(zkStateReader.getClusterProps())){
-      DocCollection coll = zkStateReader.getClusterState().getCollection(cd.getCollectionName());
+  private void checkStateInZk(CoreDescriptor cd) throws InterruptedException {
+    if (!Overseer.isLegacy(zkStateReader.getClusterProps())) {
       CloudDescriptor cloudDesc = cd.getCloudDescriptor();
-      if(cloudDesc.getShardId() == null) throw new RuntimeException("No shard id for :"+ cd.toString());
-      Slice slice = coll.getSlice(cloudDesc.getShardId());
-      if(slice == null) throw new RuntimeException("Invalid slice : "+cloudDesc.getShardId());
-      Replica replica = null;
-      if(cloudDesc.getCoreNodeName() !=null){
-        replica = slice.getReplica(cloudDesc.getCoreNodeName());
-      } else {
-        for (Replica r : slice.getReplicas()) {
-          if(cd.getName().equals(r.get(ZkStateReader.CORE_NAME_PROP)) && getBaseUrl().equals(r.get(ZkStateReader.BASE_URL_PROP))){
-            replica = r;
-            break;
-          }
+      String coreNodeName = cloudDesc.getCoreNodeName();
+      assert coreNodeName != null;
+      if (cloudDesc.getShardId() == null) throw new SolrException(ErrorCode.SERVER_ERROR ,"No shard id for :" + cd);
+      long endTime = System.currentTimeMillis()+3000;
+      String errMessage= null;
+      for (; System.currentTimeMillis()<endTime; ) {
+        Thread.sleep(100);
+        errMessage = null;
+        Slice slice = zkStateReader.getClusterState().getSlice(cd.getCollectionName(), cloudDesc.getShardId());
+        if (slice == null) {
+          errMessage = "Invalid slice : " + cloudDesc.getShardId();
+          continue;
         }
+        if (slice.getReplica(coreNodeName) != null) return;
       }
-      if(replica == null){
-        throw new RuntimeException(" No such replica in clusterstate "+cd.toString());
-      }
-
+      if(errMessage == null)  errMessage = " no_such_replica in clusterstate ,replicaName :  " + coreNodeName;
+      throw new SolrException(ErrorCode.SERVER_ERROR,errMessage + "state : "+ zkStateReader.getClusterState().getCollection(cd.getCollectionName()));
     }
   }
 

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Fri Feb 28 17:12:07 2014
@@ -39,6 +39,7 @@ import org.apache.lucene.search.MatchAll
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.cloud.Overseer;
 import org.apache.solr.cloud.SyncStrategy;
 import org.apache.solr.cloud.ZkController;
 import org.apache.solr.common.SolrException;
@@ -489,6 +490,7 @@ public class CoreAdminHandler extends Re
   protected void handleCreateAction(SolrQueryRequest req, SolrQueryResponse rsp) throws SolrException {
 
     SolrParams params = req.getParams();
+    log.info("core create command {}", params);
     CoreDescriptor dcore = buildCoreDescriptor(params, coreContainer);
 
     if (coreContainer.getAllCoreNames().contains(dcore.getName())) {
@@ -500,6 +502,13 @@ public class CoreAdminHandler extends Re
     // TODO this should be moved into CoreContainer, really...
     try {
       if (coreContainer.getZkController() != null) {
+        if(!Overseer.isLegacy(coreContainer.getZkController() .getZkStateReader().getClusterProps())){
+          if(dcore.getCloudDescriptor().getCoreNodeName() ==null) {
+            throw new SolrException(ErrorCode.SERVER_ERROR,
+                "non legacy mode coreNodeName missing "+ params);
+
+          }
+        }
         coreContainer.preRegisterInZk(dcore);
       }
 

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java Fri Feb 28 17:12:07 2014
@@ -78,7 +78,7 @@ public class SuggestComponent extends Se
   protected NamedList initParams;
   
   /**
-   * Key is the dictionary name used in SolrConfig, value is the corrosponding {@link SolrSuggester}
+   * Key is the dictionary name used in SolrConfig, value is the corresponding {@link SolrSuggester}
    */
   protected Map<String, SolrSuggester> suggesters = new ConcurrentHashMap<String, SolrSuggester>();
   

Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java Fri Feb 28 17:12:07 2014
@@ -53,7 +53,8 @@ import org.slf4j.LoggerFactory;
  */
 public class JsonLoader extends ContentStreamLoader {
   final static Logger log = LoggerFactory.getLogger( JsonLoader.class );
-  
+  private static final String CHILD_DOC_KEY = "_childDocuments_";
+
   @Override
   public String getDefaultWT() {
     return "json";
@@ -409,35 +410,40 @@ public class JsonLoader extends ContentS
             +" at ["+parser.getPosition()+"]" );
       }
     }
-  
-  
+
+
     private SolrInputDocument parseDoc(int ev) throws IOException {
       assert ev == JSONParser.OBJECT_START;
-  
+
       SolrInputDocument sdoc = new SolrInputDocument();
       for (;;) {
-        SolrInputField sif = parseField();
-        if (sif == null) return sdoc;
-        SolrInputField prev = sdoc.put(sif.getName(), sif);
-        if (prev != null) {
-          // blech - repeated keys
-          sif.addValue(prev.getValue(), prev.getBoost());
+        ev = parser.nextEvent();
+        if (ev == JSONParser.OBJECT_END) {
+          return sdoc;
         }
+        String fieldName = parser.getString();
+
+        if(fieldName.equals(JsonLoader.CHILD_DOC_KEY)) {
+          ev = parser.nextEvent();
+          assertEvent(ev, JSONParser.ARRAY_START);
+          while( (ev = parser.nextEvent()) != JSONParser.ARRAY_END ) {
+            assertEvent(ev, JSONParser.OBJECT_START);
+
+            sdoc.addChildDocument(parseDoc(ev));
+          }
+        } else {
+          SolrInputField sif = new SolrInputField(fieldName);
+          parseFieldValue(sif);
+          // pulling out the pieces may seem weird, but it's because
+          // SolrInputDocument.addField will do the right thing
+          // if the doc already has another value for this field
+          // (ie: repeating fieldname keys)
+          sdoc.addField(sif.getName(), sif.getValue(), sif.getBoost());
+        }
+
       }
     }
-  
-    private SolrInputField parseField()  throws IOException {
-      int ev = parser.nextEvent();
-      if (ev == JSONParser.OBJECT_END) {
-        return null;
-      }
-  
-      String fieldName = parser.getString();
-      SolrInputField sif = new SolrInputField(fieldName);
-      parseFieldValue(sif);
-      return sif;
-    }
-  
+
     private void parseFieldValue(SolrInputField sif) throws IOException {
       int ev = parser.nextEvent();
       if (ev == JSONParser.OBJECT_START) {
@@ -550,4 +556,4 @@ public class JsonLoader extends ContentS
     }
   }
 
-}
\ No newline at end of file
+}

Modified: lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml Fri Feb 28 17:12:07 2014
@@ -126,6 +126,7 @@
       <str name="dictionaryImpl">FileDictionaryFactory</str>
       <str name="buildOnCommit">false</str>
       <str name="sourceLocation">blendedInfixSuggest.txt</str>
+      <str name="indexPath">blendedInfixDir1</str>
 
       <!-- Suggester properties -->
       <str name="blenderType">linear</str>
@@ -138,6 +139,7 @@
       <str name="dictionaryImpl">FileDictionaryFactory</str>
       <str name="buildOnCommit">false</str>
       <str name="sourceLocation">blendedInfixSuggest.txt</str>
+      <str name="indexPath">blendedInfixDir2</str>
 
       <!-- Suggester properties -->
       <str name="blenderType">reciprocal</str>

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java Fri Feb 28 17:12:07 2014
@@ -554,7 +554,7 @@ public class CursorPagingTest extends So
     final Collection<String> allFieldNames = getAllSortFieldNames();
 
     final int initialDocs = TestUtil.nextInt(random(), 100, 200);
-    final int totalDocs = atLeast(5000);
+    final int totalDocs = atLeast(500);
 
     // start with a smallish number of documents, and test that we can do a full walk using a 
     // sort on *every* field in the schema...
@@ -585,7 +585,7 @@ public class CursorPagingTest extends So
     }
     assertU(commit());
 
-    final int numRandomSorts = atLeast(5);
+    final int numRandomSorts = atLeast(3);
     for (int i = 0; i < numRandomSorts; i++) {
       final String sort = buildRandomSort(allFieldNames);
       final String rows = "" + TestUtil.nextInt(random(), 63, 113);
@@ -950,7 +950,7 @@ public class CursorPagingTest extends So
     Collections.shuffle(shuffledNames, random());
 
     final StringBuilder result = new StringBuilder();
-    final int numClauses = atLeast(2);
+    final int numClauses = TestUtil.nextInt(random(), 2, 5);
 
     for (int i = 0; i < numClauses; i++) {
       String field = shuffledNames.get(i);

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java Fri Feb 28 17:12:07 2014
@@ -20,6 +20,7 @@ package org.apache.solr.analytics.expres
 import com.google.common.collect.ObjectArrays;
 
 import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.solr.SolrTestCaseJ4;
 import org.apache.solr.analytics.AbstractAnalyticsStatsTest;
@@ -35,6 +36,7 @@ import java.util.ArrayList;
 import java.util.Scanner;
 
 @SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Appending", "Asserting"})
+@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5302")
 public class ExpressionTest extends AbstractAnalyticsStatsTest {
   private static final String fileName = "/analytics/requestFiles/expressions.txt";
 

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Fri Feb 28 17:12:07 2014
@@ -50,9 +50,7 @@ import org.apache.solr.util.AbstractSolr
  * This test simply does a bunch of basic things in solrcloud mode and asserts things
  * work as expected.
  */
-// disable SSL until this test is hardened more to deal with the 
-// consequences of how slow it can make some things
-@SuppressSSL
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
 public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
   private static final String SHARD2 = "shard2";
   private static final String SHARD1 = "shard1";

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Fri Feb 28 17:12:07 2014
@@ -37,8 +37,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.lucene.util.LuceneTestCase.BadApple;
 import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
 import org.apache.solr.JSONTestUtil;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrRequest;
@@ -71,11 +71,13 @@ import org.apache.solr.util.DefaultSolrT
 import org.junit.Before;
 import org.junit.BeforeClass;
 
+
 /**
  * This test simply does a bunch of basic things in solrcloud mode and asserts things
  * work as expected.
  */
-@Slow @BadApple
+@Slow 
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
 public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
   
   private static final String DEFAULT_COLLECTION = "collection1";

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Fri Feb 28 17:12:07 2014
@@ -32,7 +32,8 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 
-@Slow @BadApple
+@Slow
+@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5735")
 public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
   
   private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Fri Feb 28 17:12:07 2014
@@ -630,7 +630,16 @@ public class CollectionsAPIDistributedZk
   }
 
   private void testCollectionsAPI() throws Exception {
- 
+
+    boolean disableLegacy = random().nextBoolean();
+    CloudSolrServer client1 = null;
+
+    if(disableLegacy) {
+      log.info("legacyCloud=false");
+      client1 = createCloudClient(null);
+      setClusterProp(client1, ZkStateReader.LEGACY_CLOUD, "false");
+    }
+
     // TODO: fragile - because we dont pass collection.confName, it will only
     // find a default if a conf set with a name matching the collection name is found, or 
     // if there is only one conf set. That and the fact that other tests run first in this
@@ -907,6 +916,10 @@ public class CollectionsAPIDistributedZk
     checkForCollection(collectionInfos.keySet().iterator().next(), collectionInfos.entrySet().iterator().next().getValue(), createNodeList);
     
     checkNoTwoShardsUseTheSameIndexDir();
+    if(disableLegacy) {
+      setClusterProp(client1, ZkStateReader.LEGACY_CLOUD, null);
+      client1.shutdown();
+    }
   }
   
   private void testCollectionsAPIAddRemoveStress() throws Exception {

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java Fri Feb 28 17:12:07 2014
@@ -518,7 +518,7 @@ public class DistribCursorPagingTest ext
     final Collection<String> allFieldNames = getAllSortFieldNames();
 
     final int numInitialDocs = TestUtil.nextInt(random(), 100, 200);
-    final int totalDocs = atLeast(5000);
+    final int totalDocs = atLeast(500);
 
     // start with a smallish number of documents, and test that we can do a full walk using a 
     // sort on *every* field in the schema...
@@ -575,7 +575,7 @@ public class DistribCursorPagingTest ext
     }
     commit();
 
-    final int numRandomSorts = atLeast(5);
+    final int numRandomSorts = atLeast(3);
     for (int i = 0; i < numRandomSorts; i++) {
       final String sort = CursorPagingTest.buildRandomSort(allFieldNames);
       final String rows = "" + TestUtil.nextInt(random(), 63, 113);

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Fri Feb 28 17:12:07 2014
@@ -49,8 +49,7 @@ import org.junit.BeforeClass;
  * Super basic testing, no shard restarting or anything.
  */
 @Slow
-@SuppressSSL     // No SSL for now - it can be too slow
-@BadApple
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
 public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase {
   
   @BeforeClass

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java Fri Feb 28 17:12:07 2014
@@ -46,7 +46,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.BeforeClass;
 @LuceneTestCase.Slow
-@SuppressSSL     // SSL does not work with this feature for some reason
+@SuppressSSL     // Currently unknown why SSL does not work
 public class OverseerRolesTest  extends AbstractFullDistribZkTestBase{
   private CloudSolrServer client;
   

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Fri Feb 28 17:12:07 2014
@@ -47,8 +47,8 @@ import org.apache.solr.handler.component
 import org.apache.solr.util.DefaultSolrThreadFactory;
 import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.apache.zookeeper.data.Stat;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -87,11 +87,12 @@ public class OverseerTest extends SolrTe
     }
 
     private void deleteNode(final String path) {
+      
       try {
-        Stat stat = zkClient.exists(path, null, true);
-        if (stat != null) {
-          zkClient.delete(path, stat.getVersion(), true);
-        }
+        zkClient.delete(path, -1, true);
+      } catch (NoNodeException e) {
+        // fine
+        log.warn("cancelElection did not find election node to remove");
       } catch (KeeperException e) {
         fail("Unexpected KeeperException!" + e);
       } catch (InterruptedException e) {
@@ -134,8 +135,7 @@ public class OverseerTest extends SolrTe
       }
       
       for (int i = 0; i < 120; i++) {
-        String shardId = getShardId("http://" + nodeName
-            + "/solr/", coreName);
+        String shardId = getShardId("http://" + nodeName + "/solr/", coreName);
         if (shardId != null) {
           try {
             zkClient.makePath("/collections/" + collection + "/leader_elect/"
@@ -150,6 +150,7 @@ public class OverseerTest extends SolrTe
           ShardLeaderElectionContextBase ctx = new ShardLeaderElectionContextBase(
               elector, shardId, collection, nodeName + "_" + coreName, props,
               zkStateReader);
+          elector.setup(ctx);
           elector.joinElection(ctx, false);
           return shardId;
         }
@@ -545,46 +546,52 @@ public class OverseerTest extends SolrTe
       
       reader = new ZkStateReader(controllerClient);
       reader.createClusterStateWatchersAndUpdate();
-
-      mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
+      
+      mockController = new MockZKController(server.getZkAddress(), "node1",
+          "collection1");
       
       overseerClient = electNewOverseer(server.getZkAddress());
-
+      
       Thread.sleep(1000);
-      mockController.publishState("core1", "core_node1", ZkStateReader.RECOVERING, 1);
-
+      mockController.publishState("core1", "core_node1",
+          ZkStateReader.RECOVERING, 1);
+      
       waitForCollections(reader, "collection1");
       verifyStatus(reader, ZkStateReader.RECOVERING);
-
+      
       int version = getClusterStateVersion(controllerClient);
       
-      mockController.publishState("core1", "core_node1", ZkStateReader.ACTIVE, 1);
+      mockController.publishState("core1", "core_node1", ZkStateReader.ACTIVE,
+          1);
+      
+      while (version == getClusterStateVersion(controllerClient))
+        ;
       
-      while(version == getClusterStateVersion(controllerClient));
-
       verifyStatus(reader, ZkStateReader.ACTIVE);
       version = getClusterStateVersion(controllerClient);
       overseerClient.close();
-      Thread.sleep(1000); //wait for overseer to get killed
-
-      mockController.publishState("core1",  "core_node1", ZkStateReader.RECOVERING, 1);
+      Thread.sleep(1000); // wait for overseer to get killed
+      
+      mockController.publishState("core1", "core_node1",
+          ZkStateReader.RECOVERING, 1);
       version = getClusterStateVersion(controllerClient);
       
       overseerClient = electNewOverseer(server.getZkAddress());
-
-      while(version == getClusterStateVersion(controllerClient));
-
+      
+      while (version == getClusterStateVersion(controllerClient));
+      
       verifyStatus(reader, ZkStateReader.RECOVERING);
       
-      assertEquals("Live nodes count does not match", 1, reader.getClusterState()
-          .getLiveNodes().size());
+      assertEquals("Live nodes count does not match", 1, reader
+          .getClusterState().getLiveNodes().size());
       assertEquals("Shard count does not match", 1, reader.getClusterState()
           .getSlice("collection1", "shard1").getReplicasMap().size());
       version = getClusterStateVersion(controllerClient);
-      mockController.publishState("core1", "core_node1", null,1);
-      while(version == getClusterStateVersion(controllerClient));
+      mockController.publishState("core1", "core_node1", null, 1);
+      while (version == getClusterStateVersion(controllerClient));
       Thread.sleep(500);
-      assertFalse("collection1 should be gone after publishing the null state", reader.getClusterState().getCollections().contains("collection1"));
+      assertFalse("collection1 should be gone after publishing the null state",
+          reader.getClusterState().getCollections().contains("collection1"));
     } finally {
       
       close(mockController);
@@ -898,18 +905,21 @@ public class OverseerTest extends SolrTe
   }
 
 
-  private SolrZkClient electNewOverseer(String address) throws InterruptedException,
- TimeoutException, IOException,
+  private SolrZkClient electNewOverseer(String address)
+      throws InterruptedException, TimeoutException, IOException,
       KeeperException, ParserConfigurationException, SAXException {
     SolrZkClient zkClient = new SolrZkClient(address, TIMEOUT);
     ZkStateReader reader = new ZkStateReader(zkClient);
     readers.add(reader);
     LeaderElector overseerElector = new LeaderElector(zkClient);
-    // TODO: close Overseer
+    if (overseers.size() > 0) {
+      overseers.get(overseers.size() -1).close();
+    }
     Overseer overseer = new Overseer(
         new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", reader);
     overseers.add(overseer);
-    ElectionContext ec = new OverseerElectionContext(zkClient, overseer, address.replaceAll("/", "_"));
+    ElectionContext ec = new OverseerElectionContext(zkClient, overseer,
+        address.replaceAll("/", "_"));
     overseerElector.setup(ec);
     overseerElector.joinElection(ec, false);
     return zkClient;

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java Fri Feb 28 17:12:07 2014
@@ -24,7 +24,7 @@ import java.util.concurrent.SynchronousQ
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
 import org.apache.lucene.util.LuceneTestCase.Slow;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServer;
@@ -45,7 +45,8 @@ import org.junit.BeforeClass;
  * This test simply does a bunch of basic things in solrcloud mode and asserts things
  * work as expected.
  */
-@Slow @BadApple
+@Slow
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
 public class UnloadDistributedZkTest extends BasicDistributedZkTest {
   
   @BeforeClass

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java Fri Feb 28 17:12:07 2014
@@ -47,7 +47,6 @@ import org.junit.Test;
 /**
  * Incorporate the open/close stress tests into unit tests.
  */
-@BadApple
 public class OpenCloseCoreStressTest extends SolrTestCaseJ4 {
 
   private final Object locker = new Object();

Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java Fri Feb 28 17:12:07 2014
@@ -209,6 +209,54 @@ public class JsonLoaderTest extends Solr
     req.close();
   }
 
+  public void testFieldValueOrdering() throws Exception {
+    final String pre = "{'add':[{'id':'1',";
+    final String post = "},{'id':'2'}]}";
+
+    // list
+    checkFieldValueOrdering((pre+ "'f':[45,67,89]" +post)
+                            .replace('\'', '"'),
+                            1.0F);
+    // dup fieldname keys
+    checkFieldValueOrdering((pre+ "'f':45,'f':67,'f':89" +post)
+                            .replace('\'', '"'),
+                            1.0F);
+    // extended w/boost
+    checkFieldValueOrdering((pre+ "'f':{'boost':4.0,'value':[45,67,89]}" +post)
+                            .replace('\'', '"'),
+                            4.0F);
+    // dup keys extended w/ multiplicitive boost
+    checkFieldValueOrdering((pre+ 
+                             "'f':{'boost':2.0,'value':[45,67]}," +
+                             "'f':{'boost':2.0,'value':89}" 
+                             +post)
+                            .replace('\'', '"'),
+                            4.0F);
+
+  }
+  private void checkFieldValueOrdering(String rawJson, float fBoost) throws Exception {
+    SolrQueryRequest req = req();
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+    JsonLoader loader = new JsonLoader();
+    loader.load(req, rsp, new ContentStreamBase.StringStream(rawJson), p);
+    assertEquals( 2, p.addCommands.size() );
+
+    SolrInputDocument d = p.addCommands.get(0).solrDoc;
+    assertEquals(2, d.getFieldNames().size());
+    assertEquals("1", d.getFieldValue("id"));
+    assertEquals(new Object[] {45L, 67L, 89L} , d.getFieldValues("f").toArray());
+    assertEquals(0.0F, fBoost, d.getField("f").getBoost());
+
+    d = p.addCommands.get(1).solrDoc;
+    assertEquals(1, d.getFieldNames().size());
+    assertEquals("2", d.getFieldValue("id"));
+
+    req.close();
+  }
+
+
+
   public void testExtendedFieldValues() throws Exception {
     String str = "[{'id':'1', 'val_s':{'add':'foo'}}]".replace('\'', '"');
     SolrQueryRequest req = req();
@@ -499,5 +547,161 @@ public class JsonLoaderTest extends Solr
     req.close();
   }
 
+  @Test
+  public void testSimpleChildDocs() throws Exception {
+    String str = "{\n" +
+        "    \"add\": {\n" +
+        "        \"doc\": {\n" +
+        "            \"id\": \"1\",\n" +
+        "            \"_childDocuments_\": [\n" +
+        "                {\n" +
+        "                    \"id\": \"2\"\n" +
+        "                },\n" +
+        "                {\n" +
+        "                    \"id\": \"3\",\n" +
+        "                    \"foo_i\": [666,777]\n" +
+        "                }\n" +
+        "            ]\n" +
+        "        }\n" +
+        "    }\n" +
+        "}";
+    checkTwoChildDocs(str);
+  }
+
+  @Test
+  public void testDupKeysChildDocs() throws Exception {
+    String str = "{\n" +
+        "    \"add\": {\n" +
+        "        \"doc\": {\n" +
+        "            \"_childDocuments_\": [\n" +
+        "                {\n" +
+        "                    \"id\": \"2\"\n" +
+        "                }\n" +
+        "            ],\n" +
+        "            \"id\": \"1\",\n" +
+        "            \"_childDocuments_\": [\n" +
+        "                {\n" +
+        "                    \"id\": \"3\",\n" +
+        "                    \"foo_i\": 666,\n" +
+        "                    \"foo_i\": 777\n" +
+        "                }\n" +
+        "            ]\n" +
+        "        }\n" +
+        "    }\n" +
+        "}";
+    checkTwoChildDocs(str);
+  }
+
+  private void checkTwoChildDocs(String rawJsonStr) throws Exception {
+    SolrQueryRequest req = req("commit","true");
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+    JsonLoader loader = new JsonLoader();
+    loader.load(req, rsp, new ContentStreamBase.StringStream(rawJsonStr), p);
+
+    assertEquals( 1, p.addCommands.size() );
+
+    AddUpdateCommand add = p.addCommands.get(0);
+    SolrInputDocument d = add.solrDoc;
+    SolrInputField f = d.getField( "id" );
+    assertEquals("1", f.getValue());
+
+    SolrInputDocument cd = d.getChildDocuments().get(0);
+    SolrInputField cf = cd.getField( "id" );
+    assertEquals("2", cf.getValue());
+
+    cd = d.getChildDocuments().get(1);
+    cf = cd.getField( "id" );
+    assertEquals("3", cf.getValue());
+    cf = cd.getField( "foo_i" );
+    assertEquals(2, cf.getValueCount());
+
+    assertEquals(new Object[] {666L,777L}, cf.getValues().toArray());
+
+    req.close();
+  }
+
+  @Test
+  public void testEmptyChildDocs() throws Exception {
+    String str = "{\n" +
+        "    \"add\": {\n" +
+        "        \"doc\": {\n" +
+        "            \"id\": \"1\",\n" +
+        "            \"_childDocuments_\": []\n" +
+        "        }\n" +
+        "    }\n" +
+        "}";
+    SolrQueryRequest req = req("commit","true");
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+    JsonLoader loader = new JsonLoader();
+    loader.load(req, rsp, new ContentStreamBase.StringStream(str), p);
+
+    assertEquals( 1, p.addCommands.size() );
+
+    AddUpdateCommand add = p.addCommands.get(0);
+    SolrInputDocument d = add.solrDoc;
+    SolrInputField f = d.getField( "id" );
+    assertEquals("1", f.getValue());
+    List<SolrInputDocument> cd = d.getChildDocuments();
+    assertNull(cd);
+
+    req.close();
+  }
+
+  @Test
+  public void testGrandChildDocs() throws Exception {
+    String str = "{\n" +
+        "    \"add\": {\n" +
+        "        \"doc\": {\n" +
+        "            \"id\": \"1\",\n" +
+        "            \"_childDocuments_\": [\n" +
+        "                {\n" +
+        "                    \"id\": \"2\",\n" +
+        "                    \"_childDocuments_\": [\n" +
+        "                        {\n" +
+        "                           \"id\": \"4\",\n" +
+        "                           \"foo_s\": \"Baz\"\n" +
+        "                        }\n" +
+        "                    ],\n" +
+        "                    \"foo_s\": \"Yaz\"\n" +
+        "                },\n" +
+        "                {\n" +
+        "                    \"id\": \"3\",\n" +
+        "                    \"foo_s\": \"Bar\"\n" +
+        "                }\n" +
+        "            ]\n" +
+        "        }\n" +
+        "    }\n" +
+        "}";
+
+    SolrQueryRequest req = req("commit","true");
+    SolrQueryResponse rsp = new SolrQueryResponse();
+    BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+    JsonLoader loader = new JsonLoader();
+    loader.load(req, rsp, new ContentStreamBase.StringStream(str), p);
+
+    assertEquals( 1, p.addCommands.size() );
+
+    AddUpdateCommand add = p.addCommands.get(0);
+    SolrInputDocument one = add.solrDoc;
+    assertEquals("1", one.getFieldValue("id"));
+
+    SolrInputDocument two = one.getChildDocuments().get(0);
+    assertEquals("2", two.getFieldValue("id"));
+    assertEquals("Yaz", two.getFieldValue("foo_s"));
+
+    SolrInputDocument four = two.getChildDocuments().get(0);
+    assertEquals("4", four.getFieldValue("id"));
+    assertEquals("Baz", four.getFieldValue("foo_s"));
+
+    SolrInputDocument three = one.getChildDocuments().get(1);
+    assertEquals("3", three.getFieldValue("id"));
+    assertEquals("Bar", three.getFieldValue("foo_s"));
+
+    req.close();
+
+  }
+
 
-}
\ No newline at end of file
+}