You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/02/28 18:12:09 UTC
svn commit: r1573003 [2/3] - in /lucene/dev/branches/lucene5376: ./
dev-tools/ dev-tools/scripts/ lucene/ lucene/analysis/
lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/analysis/common/src/test/org/...
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Fri Feb 28 17:12:07 2014
@@ -19,26 +19,27 @@ package org.apache.lucene.search.suggest
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.Sort.SortInfo;
-import org.apache.lucene.search.suggest.Sort;
import org.apache.lucene.search.suggest.fst.FSTCompletion.Completion;
import org.apache.lucene.search.suggest.tst.TSTLookup;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
-import org.apache.lucene.util.*;
import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.NoOutputs;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.OfflineSorter;
+import org.apache.lucene.util.OfflineSorter.SortInfo;
+import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.UnicodeUtil;
/**
* An adapter from {@link Lookup} API to {@link FSTCompletion}.
@@ -150,12 +151,12 @@ public class FSTCompletionLookup extends
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
File tempInput = File.createTempFile(
- FSTCompletionLookup.class.getSimpleName(), ".input", Sort.defaultTempDir());
+ FSTCompletionLookup.class.getSimpleName(), ".input", OfflineSorter.defaultTempDir());
File tempSorted = File.createTempFile(
- FSTCompletionLookup.class.getSimpleName(), ".sorted", Sort.defaultTempDir());
+ FSTCompletionLookup.class.getSimpleName(), ".sorted", OfflineSorter.defaultTempDir());
- Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput);
- Sort.ByteSequencesReader reader = null;
+ OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
+ OfflineSorter.ByteSequencesReader reader = null;
ExternalRefSorter sorter = null;
// Push floats up front before sequences to sort them. For now, assume they are non-negative.
@@ -180,13 +181,13 @@ public class FSTCompletionLookup extends
// We don't know the distribution of scores and we need to bucket them, so we'll sort
// and divide into equal buckets.
- SortInfo info = new Sort().sort(tempInput, tempSorted);
+ SortInfo info = new OfflineSorter().sort(tempInput, tempSorted);
tempInput.delete();
FSTCompletionBuilder builder = new FSTCompletionBuilder(
- buckets, sorter = new ExternalRefSorter(new Sort()), sharedTailLength);
+ buckets, sorter = new ExternalRefSorter(new OfflineSorter()), sharedTailLength);
final int inputLines = info.lines;
- reader = new Sort.ByteSequencesReader(tempSorted);
+ reader = new OfflineSorter.ByteSequencesReader(tempSorted);
long line = 0;
int previousBucket = 0;
int previousScore = 0;
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Fri Feb 28 17:12:07 2014
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
-import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter;
import org.apache.lucene.search.suggest.SortedInputIterator;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
@@ -43,6 +42,7 @@ import org.apache.lucene.util.fst.FST;
import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.Util.MinResult;
import org.apache.lucene.util.fst.Util;
+import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
/**
* Suggester based on a weighted FST: it first traverses the prefix,
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellTernarySearchTrie.java Fri Feb 28 17:12:07 2014
@@ -94,6 +94,7 @@ public class JaspellTernarySearchTrie {
relatives[PARENT] = parent;
}
+ /** Return an approximate memory usage for this node and its sub-nodes. */
public long sizeInBytes() {
long mem = RamUsageEstimator.shallowSizeOf(this) + RamUsageEstimator.shallowSizeOf(relatives);
for (TSTNode node : relatives) {
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java Fri Feb 28 17:12:07 2014
@@ -83,7 +83,7 @@ public class DocumentValueSourceDictiona
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new DoubleConstValueSource(10), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
assertNull(inputIterator.next());
assertEquals(inputIterator.weight(), 0);
@@ -109,7 +109,7 @@ public class DocumentValueSourceDictiona
IndexReader ir = DirectoryReader.open(dir);
ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2), new LongFieldSource(WEIGHT_FIELD_NAME_3)};
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -141,7 +141,7 @@ public class DocumentValueSourceDictiona
IndexReader ir = DirectoryReader.open(dir);
ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2), new LongFieldSource(WEIGHT_FIELD_NAME_3)};
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd));
- InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -195,7 +195,7 @@ public class DocumentValueSourceDictiona
ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2)};
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -226,7 +226,7 @@ public class DocumentValueSourceDictiona
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new DoubleConstValueSource(10), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Fri Feb 28 17:12:07 2014
@@ -21,6 +21,10 @@ import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -32,9 +36,9 @@ import org.apache.lucene.analysis.core.S
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.util.CharArraySet;
-import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.Input;
import org.apache.lucene.search.suggest.InputArrayIterator;
+import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
@@ -433,4 +437,458 @@ public class AnalyzingInfixSuggesterTest
assertEquals("a bob for <b>a</b>pples", results.get(0).key);
suggester.close();
}
+
+ public void testEmptyAtStart() throws Exception {
+ File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
+ @Override
+ protected Directory getDirectory(File path) {
+ return newDirectory();
+ }
+ };
+ suggester.build(new InputArrayIterator(new Input[0]));
+ suggester.add(new BytesRef("a penny saved is a penny earned"), 10, new BytesRef("foobaz"));
+ suggester.add(new BytesRef("lend me your ear"), 8, new BytesRef("foobar"));
+ suggester.refresh();
+ List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+ assertEquals(2, results.size());
+ assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).key);
+ assertEquals(10, results.get(0).value);
+ assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+ assertEquals("lend me your <b>ear</b>", results.get(1).key);
+ assertEquals(8, results.get(1).value);
+ assertEquals(new BytesRef("foobar"), results.get(1).payload);
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear ", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("lend me your <b>ear</b>", results.get(0).key);
+ assertEquals(8, results.get(0).value);
+ assertEquals(new BytesRef("foobar"), results.get(0).payload);
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("pen", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("a <b>pen</b>ny saved is a <b>pen</b>ny earned", results.get(0).key);
+ assertEquals(10, results.get(0).value);
+ assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("p", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("a <b>p</b>enny saved is a <b>p</b>enny earned", results.get(0).key);
+ assertEquals(10, results.get(0).value);
+ assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+ suggester.close();
+ }
+
+ public void testBothExactAndPrefix() throws Exception {
+ File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
+ @Override
+ protected Directory getDirectory(File path) {
+ return newDirectory();
+ }
+ };
+ suggester.build(new InputArrayIterator(new Input[0]));
+ suggester.add(new BytesRef("the pen is pretty"), 10, new BytesRef("foobaz"));
+ suggester.refresh();
+
+ List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("pen p", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("the <b>pen</b> is <b>p</b>retty", results.get(0).key);
+ assertEquals(10, results.get(0).value);
+ assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+ suggester.close();
+ }
+
+ private static String randomText() {
+ int numWords = TestUtil.nextInt(random(), 1, 4);
+
+ StringBuilder b = new StringBuilder();
+ for(int i=0;i<numWords;i++) {
+ if (i > 0) {
+ b.append(' ');
+ }
+ b.append(TestUtil.randomSimpleString(random(), 1, 10));
+ }
+
+ return b.toString();
+ }
+
+ private static class Update {
+ long weight;
+ int index;
+ }
+
+ private static class LookupThread extends Thread {
+ private final AnalyzingInfixSuggester suggester;
+ private volatile boolean stop;
+
+ public LookupThread(AnalyzingInfixSuggester suggester) {
+ this.suggester = suggester;
+ }
+
+ public void finish() throws InterruptedException {
+ stop = true;
+ this.join();
+ }
+
+ @Override
+ public void run() {
+ while (stop == false) {
+ String query = randomText();
+ int topN = TestUtil.nextInt(random(), 1, 100);
+ boolean allTermsRequired = random().nextBoolean();
+ boolean doHilite = random().nextBoolean();
+ // We don't verify the results; just doing
+ // simultaneous lookups while adding/updating to
+ // see if there are any thread hazards:
+ try {
+ suggester.lookup(TestUtil.stringToCharSequence(query, random()),
+ topN, allTermsRequired, doHilite);
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ }
+ }
+
+ public void testRandomNRT() throws Exception {
+ final File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ int minPrefixChars = random().nextInt(7);
+ if (VERBOSE) {
+ System.out.println(" minPrefixChars=" + minPrefixChars);
+ }
+
+ AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixChars) {
+ @Override
+ protected Directory getDirectory(File path) {
+ return newFSDirectory(path);
+ }
+ };
+
+ // Initial suggester built with nothing:
+ suggester.build(new InputArrayIterator(new Input[0]));
+
+ LookupThread lookupThread = new LookupThread(suggester);
+ lookupThread.start();
+
+ int iters = atLeast(1000);
+ int visibleUpto = 0;
+
+ Set<Long> usedWeights = new HashSet<Long>();
+ Set<String> usedKeys = new HashSet<String>();
+
+ List<Input> inputs = new ArrayList<Input>();
+ List<Update> pendingUpdates = new ArrayList<Update>();
+
+ for(int iter=0;iter<iters;iter++) {
+ String text;
+ while (true) {
+ text = randomText();
+ if (usedKeys.contains(text) == false) {
+ usedKeys.add(text);
+ break;
+ }
+ }
+
+ // Carefully pick a weight we never used, to sidestep
+ // tie-break problems:
+ long weight;
+ while (true) {
+ weight = random().nextInt(10*iters);
+ if (usedWeights.contains(weight) == false) {
+ usedWeights.add(weight);
+ break;
+ }
+ }
+
+ if (inputs.size() > 0 && random().nextInt(4) == 1) {
+ // Update an existing suggestion
+ Update update = new Update();
+ update.index = random().nextInt(inputs.size());
+ update.weight = weight;
+ Input input = inputs.get(update.index);
+ pendingUpdates.add(update);
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + iter + " update input=" + input.term.utf8ToString() + "/" + weight);
+ }
+ suggester.update(input.term, weight, input.term);
+
+ } else {
+ // Add a new suggestion
+ inputs.add(new Input(text, weight, new BytesRef(text)));
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + iter + " add input=" + text + "/" + weight);
+ }
+ BytesRef br = new BytesRef(text);
+ suggester.add(br, weight, br);
+ }
+
+ if (random().nextInt(15) == 7) {
+ if (VERBOSE) {
+ System.out.println("TEST: now refresh suggester");
+ }
+ suggester.refresh();
+ visibleUpto = inputs.size();
+ for(Update update : pendingUpdates) {
+ Input oldInput = inputs.get(update.index);
+ Input newInput = new Input(oldInput.term, update.weight, oldInput.payload);
+ inputs.set(update.index, newInput);
+ }
+ pendingUpdates.clear();
+ }
+
+ if (random().nextInt(50) == 7) {
+ if (VERBOSE) {
+ System.out.println("TEST: now close/reopen suggester");
+ }
+ lookupThread.finish();
+ suggester.close();
+ suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, minPrefixChars) {
+ @Override
+ protected Directory getDirectory(File path) {
+ return newFSDirectory(path);
+ }
+ };
+ lookupThread = new LookupThread(suggester);
+ lookupThread.start();
+
+ visibleUpto = inputs.size();
+ for(Update update : pendingUpdates) {
+ Input oldInput = inputs.get(update.index);
+ Input newInput = new Input(oldInput.term, update.weight, oldInput.payload);
+ inputs.set(update.index, newInput);
+ }
+ pendingUpdates.clear();
+ }
+
+ if (visibleUpto > 0) {
+ String query = randomText();
+ boolean lastPrefix = random().nextInt(5) != 1;
+ if (lastPrefix == false) {
+ query += " ";
+ }
+
+ String[] queryTerms = query.split("\\s");
+ boolean allTermsRequired = random().nextInt(10) == 7;
+ boolean doHilite = random().nextBoolean();
+
+ if (VERBOSE) {
+ System.out.println("TEST: lookup \"" + query + "\" allTermsRequired=" + allTermsRequired + " doHilite=" + doHilite);
+ }
+
+ // Stupid slow but hopefully correct matching:
+ List<Input> expected = new ArrayList<Input>();
+ for(int i=0;i<visibleUpto;i++) {
+ Input input = inputs.get(i);
+ String[] inputTerms = input.term.utf8ToString().split("\\s");
+ boolean match = false;
+ for(int j=0;j<queryTerms.length;j++) {
+ if (j < queryTerms.length-1 || lastPrefix == false) {
+ // Exact match
+ for(int k=0;k<inputTerms.length;k++) {
+ if (inputTerms[k].equals(queryTerms[j])) {
+ match = true;
+ break;
+ }
+ }
+ } else {
+ // Prefix match
+ for(int k=0;k<inputTerms.length;k++) {
+ if (inputTerms[k].startsWith(queryTerms[j])) {
+ match = true;
+ break;
+ }
+ }
+ }
+ if (match) {
+ if (allTermsRequired == false) {
+ // At least one query term does match:
+ break;
+ }
+ match = false;
+ } else if (allTermsRequired) {
+ // At least one query term does not match:
+ break;
+ }
+ }
+
+ if (match) {
+ if (doHilite) {
+ expected.add(new Input(hilite(lastPrefix, inputTerms, queryTerms), input.v, input.term));
+ } else {
+ expected.add(input);
+ }
+ }
+ }
+
+ Collections.sort(expected,
+ new Comparator<Input>() {
+ @Override
+ public int compare(Input a, Input b) {
+ if (a.v > b.v) {
+ return -1;
+ } else if (a.v < b.v) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+ });
+
+ if (expected.isEmpty() == false) {
+
+ int topN = TestUtil.nextInt(random(), 1, expected.size());
+
+ List<LookupResult> actual = suggester.lookup(TestUtil.stringToCharSequence(query, random()), topN, allTermsRequired, doHilite);
+
+ int expectedCount = Math.min(topN, expected.size());
+
+ if (VERBOSE) {
+ System.out.println(" expected:");
+ for(int i=0;i<expectedCount;i++) {
+ Input x = expected.get(i);
+ System.out.println(" " + x.term.utf8ToString() + "/" + x.v);
+ }
+ System.out.println(" actual:");
+ for(LookupResult result : actual) {
+ System.out.println(" " + result);
+ }
+ }
+
+ assertEquals(expectedCount, actual.size());
+ for(int i=0;i<expectedCount;i++) {
+ assertEquals(expected.get(i).term.utf8ToString(), actual.get(i).key.toString());
+ assertEquals(expected.get(i).v, actual.get(i).value);
+ assertEquals(expected.get(i).payload, actual.get(i).payload);
+ }
+ } else {
+ if (VERBOSE) {
+ System.out.println(" no expected matches");
+ }
+ }
+ }
+ }
+
+ lookupThread.finish();
+ suggester.close();
+ }
+
+ private static String hilite(boolean lastPrefix, String[] inputTerms, String[] queryTerms) {
+ // Stupid slow but hopefully correct highlighter:
+ //System.out.println("hilite: lastPrefix=" + lastPrefix + " inputTerms=" + Arrays.toString(inputTerms) + " queryTerms=" + Arrays.toString(queryTerms));
+ StringBuilder b = new StringBuilder();
+ for(int i=0;i<inputTerms.length;i++) {
+ if (i > 0) {
+ b.append(' ');
+ }
+ String inputTerm = inputTerms[i];
+ //System.out.println(" inputTerm=" + inputTerm);
+ boolean matched = false;
+ for(int j=0;j<queryTerms.length;j++) {
+ String queryTerm = queryTerms[j];
+ //System.out.println(" queryTerm=" + queryTerm);
+ if (j < queryTerms.length-1 || lastPrefix == false) {
+ //System.out.println(" check exact");
+ if (inputTerm.equals(queryTerm)) {
+ b.append("<b>");
+ b.append(inputTerm);
+ b.append("</b>");
+ matched = true;
+ break;
+ }
+ } else if (inputTerm.startsWith(queryTerm)) {
+ b.append("<b>");
+ b.append(queryTerm);
+ b.append("</b>");
+ b.append(inputTerm.substring(queryTerm.length(), inputTerm.length()));
+ matched = true;
+ break;
+ }
+ }
+
+ if (matched == false) {
+ b.append(inputTerm);
+ }
+ }
+
+ return b.toString();
+ }
+
+ public void testBasicNRT() throws Exception {
+ Input keys[] = new Input[] {
+ new Input("lend me your ear", 8, new BytesRef("foobar")),
+ };
+
+ File tempDir = TestUtil.getTempDir("AnalyzingInfixSuggesterTest");
+
+ Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
+ AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
+ @Override
+ protected Directory getDirectory(File path) {
+ return newDirectory();
+ }
+ };
+ suggester.build(new InputArrayIterator(keys));
+
+ List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("lend me your <b>ear</b>", results.get(0).key);
+ assertEquals(8, results.get(0).value);
+ assertEquals(new BytesRef("foobar"), results.get(0).payload);
+
+ // Add a new suggestion:
+ suggester.add(new BytesRef("a penny saved is a penny earned"), 10, new BytesRef("foobaz"));
+
+ // Must refresh to see any newly added suggestions:
+ suggester.refresh();
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+ assertEquals(2, results.size());
+ assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).key);
+ assertEquals(10, results.get(0).value);
+ assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+ assertEquals("lend me your <b>ear</b>", results.get(1).key);
+ assertEquals(8, results.get(1).value);
+ assertEquals(new BytesRef("foobar"), results.get(1).payload);
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear ", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("lend me your <b>ear</b>", results.get(0).key);
+ assertEquals(8, results.get(0).value);
+ assertEquals(new BytesRef("foobar"), results.get(0).payload);
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("pen", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("a <b>pen</b>ny saved is a <b>pen</b>ny earned", results.get(0).key);
+ assertEquals(10, results.get(0).value);
+ assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("p", random()), 10, true, true);
+ assertEquals(1, results.size());
+ assertEquals("a <b>p</b>enny saved is a <b>p</b>enny earned", results.get(0).key);
+ assertEquals(10, results.get(0).value);
+ assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+
+ // Change the weight:
+ suggester.update(new BytesRef("lend me your ear"), 12, new BytesRef("foobox"));
+
+ // Must refresh to see any newly added suggestions:
+ suggester.refresh();
+
+ results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
+ assertEquals(2, results.size());
+ assertEquals("lend me your <b>ear</b>", results.get(0).key);
+ assertEquals(12, results.get(0).value);
+ assertEquals(new BytesRef("foobox"), results.get(0).payload);
+ assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(1).key);
+ assertEquals(10, results.get(1).value);
+ assertEquals(new BytesRef("foobaz"), results.get(1).payload);
+ suggester.close();
+ }
}
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggesterTest.java Fri Feb 28 17:12:07 2014
@@ -225,7 +225,7 @@ public class BlendedInfixSuggesterTest e
suggester.close();
}
- private static long getInResults(BlendedInfixSuggester suggester, String prefix, BytesRef payload, int num) {
+ private static long getInResults(BlendedInfixSuggester suggester, String prefix, BytesRef payload, int num) throws IOException {
List<Lookup.LookupResult> responses = suggester.lookup(prefix, num, true, false);
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/BytesRefSortersTest.java Fri Feb 28 17:12:07 2014
@@ -18,16 +18,16 @@ package org.apache.lucene.search.suggest
*/
import org.apache.lucene.search.suggest.InMemorySorter;
-import org.apache.lucene.search.suggest.Sort;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.OfflineSorter;
import org.junit.Test;
public class BytesRefSortersTest extends LuceneTestCase {
@Test
public void testExternalRefSorter() throws Exception {
- ExternalRefSorter s = new ExternalRefSorter(new Sort());
+ ExternalRefSorter s = new ExternalRefSorter(new OfflineSorter());
check(s);
s.close();
}
Modified: lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java (original)
+++ lucene/dev/branches/lucene5376/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/LargeInputFST.java Fri Feb 28 17:12:07 2014
@@ -17,10 +17,15 @@ package org.apache.lucene.search.suggest
* limitations under the License.
*/
-import java.io.*;
-import org.apache.lucene.search.suggest.Sort;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.OfflineSorter;
/**
* Try to build a suggester from a large data set. The input is a simple text
@@ -33,7 +38,7 @@ public class LargeInputFST {
int buckets = 20;
int shareMaxTail = 10;
- ExternalRefSorter sorter = new ExternalRefSorter(new Sort());
+ ExternalRefSorter sorter = new ExternalRefSorter(new OfflineSorter());
FSTCompletionBuilder builder = new FSTCompletionBuilder(buckets, sorter, shareMaxTail);
BufferedReader reader = new BufferedReader(
Modified: lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene5376/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Fri Feb 28 17:12:07 2014
@@ -209,7 +209,10 @@ public abstract class LuceneTestCase ext
@Inherited
@Retention(RetentionPolicy.RUNTIME)
@TestGroup(enabled = false, sysProperty = SYSPROP_BADAPPLES)
- public @interface BadApple {}
+ public @interface BadApple {
+ /** Point to JIRA entry. */
+ public String bugUrl();
+ }
/**
* Annotation for test classes that should avoid certain codec types
Modified: lucene/dev/branches/lucene5376/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/CHANGES.txt?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5376/solr/CHANGES.txt Fri Feb 28 17:12:07 2014
@@ -82,8 +82,12 @@ Detailed Change List
New Features
----------------------
+
* SOLR-5130: Implement addReplica Collections API (Noble Paul)
+* SOLR-5183: JSON updates now support nested child documents using a
+ "_childDocument_" object key. (Varun Thacker, hossman)
+
Bug Fixes
----------------------
@@ -93,6 +97,12 @@ Bug Fixes
* SOLR-5423: CSV output doesn't include function field
(Arun Kumar, hossman, Steve Rowe)
+* SOLR-5777: Fix ordering of field values in JSON updates where
+ field name key is repeated (hossman)
+
+* SOLR-5782: The full MapReduceIndexer help text does not display when using
+ --help. (Mark Miller, Wolfgang Hoschek)
+
Optimizations
----------------------
* SOLR-1880: Distributed Search skips GET_FIELDS stage if EXECUTE_QUERY
Modified: lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/MapReduceIndexerTool.java Fri Feb 28 17:12:07 2014
@@ -27,8 +27,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.URISyntaxException;
import java.net.URL;
@@ -183,11 +181,7 @@ public class MapReduceIndexerTool extend
.action(new HelpArgumentAction() {
@Override
public void run(ArgumentParser parser, Argument arg, Map<String, Object> attrs, String flag, Object value) throws ArgumentParserException {
- try {
- parser.printHelp(new PrintWriter(new OutputStreamWriter(System.out, "UTF-8")));
- } catch (UnsupportedEncodingException e) {
- throw new RuntimeException("Won't Happen for UTF-8");
- }
+ parser.printHelp();
System.out.println();
System.out.print(ToolRunnerHelpFormatter.getGenericCommandUsage());
//ToolRunner.printGenericCommandUsage(System.out);
@@ -207,31 +201,6 @@ public class MapReduceIndexerTool extend
" --shards 1 \\\n" +
" hdfs:///user/$USER/test-documents/sample-statuses-20120906-141433.avro\n" +
"\n" +
- "# (Re)index all files that match all of the following conditions:\n" +
- "# 1) File is contained in dir tree hdfs:///user/$USER/solrloadtest/twitter/tweets\n" +
- "# 2) file name matches the glob pattern 'sample-statuses*.gz'\n" +
- "# 3) file was last modified less than 100000 minutes ago\n" +
- "# 4) file size is between 1 MB and 1 GB\n" +
- "# Also include extra library jar file containing JSON tweet Java parser:\n" +
- "hadoop jar target/solr-map-reduce-*.jar " + "com.cloudera.cdk.morphline.hadoop.find.HdfsFindTool" + " \\\n" +
- " -find hdfs:///user/$USER/solrloadtest/twitter/tweets \\\n" +
- " -type f \\\n" +
- " -name 'sample-statuses*.gz' \\\n" +
- " -mmin -1000000 \\\n" +
- " -size -100000000c \\\n" +
- " -size +1000000c \\\n" +
- "| sudo -u hdfs hadoop \\\n" +
- " --config /etc/hadoop/conf.cloudera.mapreduce1 \\\n" +
- " jar target/solr-map-reduce-*.jar \\\n" +
- " -D 'mapred.child.java.opts=-Xmx500m' \\\n" +
-// " -D 'mapreduce.child.java.opts=-Xmx500m' \\\n" +
- " --log4j src/test/resources/log4j.properties \\\n" +
- " --morphline-file ../search-core/src/test/resources/test-morphlines/tutorialReadJsonTestTweets.conf \\\n" +
- " --solr-home-dir src/test/resources/solr/minimr \\\n" +
- " --output-dir hdfs://c2202.mycompany.com/user/$USER/test \\\n" +
- " --shards 100 \\\n" +
- " --input-list -\n" +
- "\n" +
"# Go live by merging resulting index shards into a live Solr cluster\n" +
"# (explicitly specify Solr URLs - for a SolrCloud cluster see next example):\n" +
"sudo -u hdfs hadoop \\\n" +
Modified: lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/map-reduce/src/test/org/apache/solr/hadoop/MorphlineGoLiveMiniMRTest.java Fri Feb 28 17:12:07 2014
@@ -69,7 +69,6 @@ import org.apache.solr.common.util.Named
import org.apache.solr.hadoop.hack.MiniMRClientCluster;
import org.apache.solr.hadoop.hack.MiniMRClientClusterFactory;
import org.apache.solr.morphlines.solr.AbstractSolrMorphlineTestBase;
-import org.apache.solr.util.ExternalPaths;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -89,7 +88,7 @@ import com.carrotsearch.randomizedtestin
@ThreadLeakZombies(Consequence.CONTINUE)
@ThreadLeakScope(Scope.NONE)
@SuppressCodecs({"Lucene3x", "Lucene40"})
-@SuppressSSL
+@SuppressSSL // SSL does not work with this test for currently unknown reasons
@Slow
public class MorphlineGoLiveMiniMRTest extends AbstractFullDistribZkTestBase {
Modified: lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java (original)
+++ lucene/dev/branches/lucene5376/solr/contrib/morphlines-core/src/test/org/apache/solr/morphlines/solr/AbstractSolrMorphlineTestBase.java Fri Feb 28 17:12:07 2014
@@ -20,12 +20,15 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Calendar;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TimeZone;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
@@ -130,7 +133,9 @@ public class AbstractSolrMorphlineTestBa
String[] files,
Map<String,Integer> expectedRecords,
Map<String, Map<String, Object>> expectedRecordContents) throws Exception {
-
+
+ assumeTrue("This test has issues with this locale: https://issues.apache.org/jira/browse/SOLR-5778",
+ "GregorianCalendar".equals(Calendar.getInstance(TimeZone.getDefault(), Locale.getDefault()).getClass().getSimpleName()));
deleteAllDocuments();
int numDocs = 0;
for (int i = 0; i < 1; i++) {
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/Overseer.java Fri Feb 28 17:12:07 2014
@@ -305,8 +305,7 @@ public class Overseer {
makeMap(
ZkStateReader.CORE_NAME_PROP, message.getStr(ZkStateReader.CORE_NAME_PROP),
ZkStateReader.BASE_URL_PROP,message.getStr(ZkStateReader.BASE_URL_PROP),
- ZkStateReader.STATE_PROP,message.getStr(ZkStateReader.STATE_PROP),
- ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName));
+ ZkStateReader.STATE_PROP,message.getStr(ZkStateReader.STATE_PROP)));
sl.getReplicasMap().put(coreNodeName, replica);
return clusterState;
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Fri Feb 28 17:12:07 2014
@@ -74,6 +74,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -1624,6 +1625,7 @@ public class OverseerCollectionProcessor
throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully createcollection: " + message.getStr("name"));
log.info("going to create cores replicas shardNames {} , repFactor : {}", shardNames, repFactor);
+ Map<String ,ShardRequest> coresToCreate = new LinkedHashMap<String, ShardRequest>();
for (int i = 1; i <= shardNames.size(); i++) {
String sliceName = shardNames.get(i-1);
for (int j = 1; j <= repFactor; j++) {
@@ -1662,13 +1664,22 @@ public class OverseerCollectionProcessor
ShardRequest sreq = new ShardRequest();
params.set("qt", adminPath);
sreq.purpose = 1;
- String replica = zkStateReader.getBaseUrlForNodeName(nodeName);
- sreq.shards = new String[] {replica};
+ sreq.shards = new String[] {baseUrl};
sreq.actualShards = sreq.shards;
sreq.params = params;
- shardHandler.submit(sreq, replica, sreq.params);
+ if(isLegacyCloud) shardHandler.submit(sreq, sreq.shards[0], sreq.params);
+ else coresToCreate.put(coreName, sreq);
+ }
+ }
+ if(!isLegacyCloud) {
+ //wait for all replica entries to be created
+ Map<String, Replica> replicas = lookupReplicas(collectionName, coresToCreate.keySet());
+ for (Map.Entry<String, ShardRequest> e : coresToCreate.entrySet()) {
+ ShardRequest sreq = e.getValue();
+ sreq.params.set(CoreAdminParams.CORE_NODE_NAME, replicas.get(e.getKey()).getName());
+ shardHandler.submit(sreq, sreq.shards[0], sreq.params);
}
}
@@ -1690,6 +1701,37 @@ public class OverseerCollectionProcessor
}
}
+ private Map<String, Replica> lookupReplicas(String collectionName, Collection<String> coreNames) throws InterruptedException {
+ Map<String, Replica> result = new HashMap<String, Replica>();
+ long endTime = System.currentTimeMillis() +3000;
+ for(;;) {
+ DocCollection coll = zkStateReader.getClusterState().getCollection(collectionName);
+ for (String coreName : coreNames) {
+ if(result.containsKey(coreName)) continue;
+ for (Slice slice : coll.getSlices()) {
+ for (Replica replica : slice.getReplicas()) {
+ if(coreName.equals(replica.getStr(ZkStateReader.CORE_NAME_PROP))) {
+ result.put(coreName,replica);
+ break;
+ }
+ }
+ }
+ }
+
+ if(result.size() == coreNames.size()) {
+ return result;
+ }
+ if( System.currentTimeMillis() > endTime) {
+ //time up . throw exception and go out
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to create replica entries in ZK");
+ }
+
+ Thread.sleep(100);
+ }
+
+ }
+
+
private void addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
String collection = message.getStr(COLLECTION_PROP);
String node = message.getStr("node");
@@ -1733,6 +1775,7 @@ public class OverseerCollectionProcessor
}
coreName = collection + "_" + shard + "_replica" + replicaNum;
}
+ ModifiableSolrParams params = new ModifiableSolrParams();
if(!Overseer.isLegacy(zkStateReader.getClusterProps())){
ZkNodeProps props = new ZkNodeProps(
@@ -1743,6 +1786,7 @@ public class OverseerCollectionProcessor
ZkStateReader.STATE_PROP, ZkStateReader.DOWN,
ZkStateReader.BASE_URL_PROP,zkStateReader.getBaseUrlForNodeName(node));
Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(props));
+ params.set(CoreAdminParams.CORE_NODE_NAME, lookupReplicas(collection, Collections.singletonList(coreName)).get(coreName).getName());
}
@@ -1751,7 +1795,6 @@ public class OverseerCollectionProcessor
String dataDir = message.getStr(CoreAdminParams.DATA_DIR);
String instanceDir = message.getStr(CoreAdminParams.INSTANCE_DIR);
- ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString());
params.set(CoreAdminParams.NAME, coreName);
params.set(COLL_CONF, configName);
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/cloud/ZkController.java Fri Feb 28 17:12:07 2014
@@ -1262,7 +1262,7 @@ public final class ZkController {
String msgNodeName = getNodeName();
String msgCore = descriptor.getName();
- if (nodeName.equals(msgNodeName) && core.equals(msgCore)) {
+ if (msgNodeName.equals(nodeName) && core.equals(msgCore)) {
descriptor.getCloudDescriptor()
.setCoreNodeName(replica.getName());
return;
@@ -1354,10 +1354,11 @@ public final class ZkController {
String coreNodeName = getCoreNodeName(cd);
- checkStateInZk(cd);
// before becoming available, make sure we are not live and active
// this also gets us our assigned shard id if it was not specified
try {
+ checkStateInZk(cd);
+
CloudDescriptor cloudDesc = cd.getCloudDescriptor();
@@ -1385,28 +1386,26 @@ public final class ZkController {
}
- private void checkStateInZk(CoreDescriptor cd) {
- if(!Overseer.isLegacy(zkStateReader.getClusterProps())){
- DocCollection coll = zkStateReader.getClusterState().getCollection(cd.getCollectionName());
+ private void checkStateInZk(CoreDescriptor cd) throws InterruptedException {
+ if (!Overseer.isLegacy(zkStateReader.getClusterProps())) {
CloudDescriptor cloudDesc = cd.getCloudDescriptor();
- if(cloudDesc.getShardId() == null) throw new RuntimeException("No shard id for :"+ cd.toString());
- Slice slice = coll.getSlice(cloudDesc.getShardId());
- if(slice == null) throw new RuntimeException("Invalid slice : "+cloudDesc.getShardId());
- Replica replica = null;
- if(cloudDesc.getCoreNodeName() !=null){
- replica = slice.getReplica(cloudDesc.getCoreNodeName());
- } else {
- for (Replica r : slice.getReplicas()) {
- if(cd.getName().equals(r.get(ZkStateReader.CORE_NAME_PROP)) && getBaseUrl().equals(r.get(ZkStateReader.BASE_URL_PROP))){
- replica = r;
- break;
- }
+ String coreNodeName = cloudDesc.getCoreNodeName();
+ assert coreNodeName != null;
+ if (cloudDesc.getShardId() == null) throw new SolrException(ErrorCode.SERVER_ERROR ,"No shard id for :" + cd);
+ long endTime = System.currentTimeMillis()+3000;
+ String errMessage= null;
+ for (; System.currentTimeMillis()<endTime; ) {
+ Thread.sleep(100);
+ errMessage = null;
+ Slice slice = zkStateReader.getClusterState().getSlice(cd.getCollectionName(), cloudDesc.getShardId());
+ if (slice == null) {
+ errMessage = "Invalid slice : " + cloudDesc.getShardId();
+ continue;
}
+ if (slice.getReplica(coreNodeName) != null) return;
}
- if(replica == null){
- throw new RuntimeException(" No such replica in clusterstate "+cd.toString());
- }
-
+ if(errMessage == null) errMessage = " no_such_replica in clusterstate ,replicaName : " + coreNodeName;
+ throw new SolrException(ErrorCode.SERVER_ERROR,errMessage + "state : "+ zkStateReader.getClusterState().getCollection(cd.getCollectionName()));
}
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Fri Feb 28 17:12:07 2014
@@ -39,6 +39,7 @@ import org.apache.lucene.search.MatchAll
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.SyncStrategy;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
@@ -489,6 +490,7 @@ public class CoreAdminHandler extends Re
protected void handleCreateAction(SolrQueryRequest req, SolrQueryResponse rsp) throws SolrException {
SolrParams params = req.getParams();
+ log.info("core create command {}", params);
CoreDescriptor dcore = buildCoreDescriptor(params, coreContainer);
if (coreContainer.getAllCoreNames().contains(dcore.getName())) {
@@ -500,6 +502,13 @@ public class CoreAdminHandler extends Re
// TODO this should be moved into CoreContainer, really...
try {
if (coreContainer.getZkController() != null) {
+ if(!Overseer.isLegacy(coreContainer.getZkController() .getZkStateReader().getClusterProps())){
+ if(dcore.getCloudDescriptor().getCoreNodeName() ==null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "non legacy mode coreNodeName missing "+ params);
+
+ }
+ }
coreContainer.preRegisterInZk(dcore);
}
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/component/SuggestComponent.java Fri Feb 28 17:12:07 2014
@@ -78,7 +78,7 @@ public class SuggestComponent extends Se
protected NamedList initParams;
/**
- * Key is the dictionary name used in SolrConfig, value is the corrosponding {@link SolrSuggester}
+ * Key is the dictionary name used in SolrConfig, value is the corresponding {@link SolrSuggester}
*/
protected Map<String, SolrSuggester> suggesters = new ConcurrentHashMap<String, SolrSuggester>();
Modified: lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java Fri Feb 28 17:12:07 2014
@@ -53,7 +53,8 @@ import org.slf4j.LoggerFactory;
*/
public class JsonLoader extends ContentStreamLoader {
final static Logger log = LoggerFactory.getLogger( JsonLoader.class );
-
+ private static final String CHILD_DOC_KEY = "_childDocuments_";
+
@Override
public String getDefaultWT() {
return "json";
@@ -409,35 +410,40 @@ public class JsonLoader extends ContentS
+" at ["+parser.getPosition()+"]" );
}
}
-
-
+
+
private SolrInputDocument parseDoc(int ev) throws IOException {
assert ev == JSONParser.OBJECT_START;
-
+
SolrInputDocument sdoc = new SolrInputDocument();
for (;;) {
- SolrInputField sif = parseField();
- if (sif == null) return sdoc;
- SolrInputField prev = sdoc.put(sif.getName(), sif);
- if (prev != null) {
- // blech - repeated keys
- sif.addValue(prev.getValue(), prev.getBoost());
+ ev = parser.nextEvent();
+ if (ev == JSONParser.OBJECT_END) {
+ return sdoc;
}
+ String fieldName = parser.getString();
+
+ if(fieldName.equals(JsonLoader.CHILD_DOC_KEY)) {
+ ev = parser.nextEvent();
+ assertEvent(ev, JSONParser.ARRAY_START);
+ while( (ev = parser.nextEvent()) != JSONParser.ARRAY_END ) {
+ assertEvent(ev, JSONParser.OBJECT_START);
+
+ sdoc.addChildDocument(parseDoc(ev));
+ }
+ } else {
+ SolrInputField sif = new SolrInputField(fieldName);
+ parseFieldValue(sif);
+ // pulling out the pieces may seem weird, but it's because
+ // SolrInputDocument.addField will do the right thing
+ // if the doc already has another value for this field
+ // (ie: repeating fieldname keys)
+ sdoc.addField(sif.getName(), sif.getValue(), sif.getBoost());
+ }
+
}
}
-
- private SolrInputField parseField() throws IOException {
- int ev = parser.nextEvent();
- if (ev == JSONParser.OBJECT_END) {
- return null;
- }
-
- String fieldName = parser.getString();
- SolrInputField sif = new SolrInputField(fieldName);
- parseFieldValue(sif);
- return sif;
- }
-
+
private void parseFieldValue(SolrInputField sif) throws IOException {
int ev = parser.nextEvent();
if (ev == JSONParser.OBJECT_START) {
@@ -550,4 +556,4 @@ public class JsonLoader extends ContentS
}
}
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml Fri Feb 28 17:12:07 2014
@@ -126,6 +126,7 @@
<str name="dictionaryImpl">FileDictionaryFactory</str>
<str name="buildOnCommit">false</str>
<str name="sourceLocation">blendedInfixSuggest.txt</str>
+ <str name="indexPath">blendedInfixDir1</str>
<!-- Suggester properties -->
<str name="blenderType">linear</str>
@@ -138,6 +139,7 @@
<str name="dictionaryImpl">FileDictionaryFactory</str>
<str name="buildOnCommit">false</str>
<str name="sourceLocation">blendedInfixSuggest.txt</str>
+ <str name="indexPath">blendedInfixDir2</str>
<!-- Suggester properties -->
<str name="blenderType">reciprocal</str>
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/CursorPagingTest.java Fri Feb 28 17:12:07 2014
@@ -554,7 +554,7 @@ public class CursorPagingTest extends So
final Collection<String> allFieldNames = getAllSortFieldNames();
final int initialDocs = TestUtil.nextInt(random(), 100, 200);
- final int totalDocs = atLeast(5000);
+ final int totalDocs = atLeast(500);
// start with a smallish number of documents, and test that we can do a full walk using a
// sort on *every* field in the schema...
@@ -585,7 +585,7 @@ public class CursorPagingTest extends So
}
assertU(commit());
- final int numRandomSorts = atLeast(5);
+ final int numRandomSorts = atLeast(3);
for (int i = 0; i < numRandomSorts; i++) {
final String sort = buildRandomSort(allFieldNames);
final String rows = "" + TestUtil.nextInt(random(), 63, 113);
@@ -950,7 +950,7 @@ public class CursorPagingTest extends So
Collections.shuffle(shuffledNames, random());
final StringBuilder result = new StringBuilder();
- final int numClauses = atLeast(2);
+ final int numClauses = TestUtil.nextInt(random(), 2, 5);
for (int i = 0; i < numClauses; i++) {
String field = shuffledNames.get(i);
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/analytics/expression/ExpressionTest.java Fri Feb 28 17:12:07 2014
@@ -20,6 +20,7 @@ package org.apache.solr.analytics.expres
import com.google.common.collect.ObjectArrays;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.analytics.AbstractAnalyticsStatsTest;
@@ -35,6 +36,7 @@ import java.util.ArrayList;
import java.util.Scanner;
@SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Appending", "Asserting"})
+@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5302")
public class ExpressionTest extends AbstractAnalyticsStatsTest {
private static final String fileName = "/analytics/requestFiles/expressions.txt";
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java Fri Feb 28 17:12:07 2014
@@ -50,9 +50,7 @@ import org.apache.solr.util.AbstractSolr
* This test simply does a bunch of basic things in solrcloud mode and asserts things
* work as expected.
*/
-// disable SSL until this test is hardened more to deal with the
-// consequences of how slow it can make some things
-@SuppressSSL
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
private static final String SHARD2 = "shard2";
private static final String SHARD1 = "shard1";
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java Fri Feb 28 17:12:07 2014
@@ -37,8 +37,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang.StringUtils;
-import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
@@ -71,11 +71,13 @@ import org.apache.solr.util.DefaultSolrT
import org.junit.Before;
import org.junit.BeforeClass;
+
/**
* This test simply does a bunch of basic things in solrcloud mode and asserts things
* work as expected.
*/
-@Slow @BadApple
+@Slow
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
private static final String DEFAULT_COLLECTION = "collection1";
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Fri Feb 28 17:12:07 2014
@@ -32,7 +32,8 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
-@Slow @BadApple
+@Slow
+@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5735")
public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
private static final Integer RUN_LENGTH = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1"));
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/CollectionsAPIDistributedZkTest.java Fri Feb 28 17:12:07 2014
@@ -630,7 +630,16 @@ public class CollectionsAPIDistributedZk
}
private void testCollectionsAPI() throws Exception {
-
+
+ boolean disableLegacy = random().nextBoolean();
+ CloudSolrServer client1 = null;
+
+ if(disableLegacy) {
+ log.info("legacyCloud=false");
+ client1 = createCloudClient(null);
+ setClusterProp(client1, ZkStateReader.LEGACY_CLOUD, "false");
+ }
+
// TODO: fragile - because we dont pass collection.confName, it will only
// find a default if a conf set with a name matching the collection name is found, or
// if there is only one conf set. That and the fact that other tests run first in this
@@ -907,6 +916,10 @@ public class CollectionsAPIDistributedZk
checkForCollection(collectionInfos.keySet().iterator().next(), collectionInfos.entrySet().iterator().next().getValue(), createNodeList);
checkNoTwoShardsUseTheSameIndexDir();
+ if(disableLegacy) {
+ setClusterProp(client1, ZkStateReader.LEGACY_CLOUD, null);
+ client1.shutdown();
+ }
}
private void testCollectionsAPIAddRemoveStress() throws Exception {
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/DistribCursorPagingTest.java Fri Feb 28 17:12:07 2014
@@ -518,7 +518,7 @@ public class DistribCursorPagingTest ext
final Collection<String> allFieldNames = getAllSortFieldNames();
final int numInitialDocs = TestUtil.nextInt(random(), 100, 200);
- final int totalDocs = atLeast(5000);
+ final int totalDocs = atLeast(500);
// start with a smallish number of documents, and test that we can do a full walk using a
// sort on *every* field in the schema...
@@ -575,7 +575,7 @@ public class DistribCursorPagingTest ext
}
commit();
- final int numRandomSorts = atLeast(5);
+ final int numRandomSorts = atLeast(3);
for (int i = 0; i < numRandomSorts; i++) {
final String sort = CursorPagingTest.buildRandomSort(allFieldNames);
final String rows = "" + TestUtil.nextInt(random(), 63, 113);
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Fri Feb 28 17:12:07 2014
@@ -49,8 +49,7 @@ import org.junit.BeforeClass;
* Super basic testing, no shard restarting or anything.
*/
@Slow
-@SuppressSSL // No SSL for now - it can be too slow
-@BadApple
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase {
@BeforeClass
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java Fri Feb 28 17:12:07 2014
@@ -46,7 +46,7 @@ import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
@LuceneTestCase.Slow
-@SuppressSSL // SSL does not work with this feature for some reason
+@SuppressSSL // Currently unknown why SSL does not work
public class OverseerRolesTest extends AbstractFullDistribZkTestBase{
private CloudSolrServer client;
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Fri Feb 28 17:12:07 2014
@@ -47,8 +47,8 @@ import org.apache.solr.handler.component
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
import org.apache.zookeeper.KeeperException.NodeExistsException;
-import org.apache.zookeeper.data.Stat;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -87,11 +87,12 @@ public class OverseerTest extends SolrTe
}
private void deleteNode(final String path) {
+
try {
- Stat stat = zkClient.exists(path, null, true);
- if (stat != null) {
- zkClient.delete(path, stat.getVersion(), true);
- }
+ zkClient.delete(path, -1, true);
+ } catch (NoNodeException e) {
+ // fine
+ log.warn("cancelElection did not find election node to remove");
} catch (KeeperException e) {
fail("Unexpected KeeperException!" + e);
} catch (InterruptedException e) {
@@ -134,8 +135,7 @@ public class OverseerTest extends SolrTe
}
for (int i = 0; i < 120; i++) {
- String shardId = getShardId("http://" + nodeName
- + "/solr/", coreName);
+ String shardId = getShardId("http://" + nodeName + "/solr/", coreName);
if (shardId != null) {
try {
zkClient.makePath("/collections/" + collection + "/leader_elect/"
@@ -150,6 +150,7 @@ public class OverseerTest extends SolrTe
ShardLeaderElectionContextBase ctx = new ShardLeaderElectionContextBase(
elector, shardId, collection, nodeName + "_" + coreName, props,
zkStateReader);
+ elector.setup(ctx);
elector.joinElection(ctx, false);
return shardId;
}
@@ -545,46 +546,52 @@ public class OverseerTest extends SolrTe
reader = new ZkStateReader(controllerClient);
reader.createClusterStateWatchersAndUpdate();
-
- mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
+
+ mockController = new MockZKController(server.getZkAddress(), "node1",
+ "collection1");
overseerClient = electNewOverseer(server.getZkAddress());
-
+
Thread.sleep(1000);
- mockController.publishState("core1", "core_node1", ZkStateReader.RECOVERING, 1);
-
+ mockController.publishState("core1", "core_node1",
+ ZkStateReader.RECOVERING, 1);
+
waitForCollections(reader, "collection1");
verifyStatus(reader, ZkStateReader.RECOVERING);
-
+
int version = getClusterStateVersion(controllerClient);
- mockController.publishState("core1", "core_node1", ZkStateReader.ACTIVE, 1);
+ mockController.publishState("core1", "core_node1", ZkStateReader.ACTIVE,
+ 1);
+
+ while (version == getClusterStateVersion(controllerClient))
+ ;
- while(version == getClusterStateVersion(controllerClient));
-
verifyStatus(reader, ZkStateReader.ACTIVE);
version = getClusterStateVersion(controllerClient);
overseerClient.close();
- Thread.sleep(1000); //wait for overseer to get killed
-
- mockController.publishState("core1", "core_node1", ZkStateReader.RECOVERING, 1);
+ Thread.sleep(1000); // wait for overseer to get killed
+
+ mockController.publishState("core1", "core_node1",
+ ZkStateReader.RECOVERING, 1);
version = getClusterStateVersion(controllerClient);
overseerClient = electNewOverseer(server.getZkAddress());
-
- while(version == getClusterStateVersion(controllerClient));
-
+
+ while (version == getClusterStateVersion(controllerClient));
+
verifyStatus(reader, ZkStateReader.RECOVERING);
- assertEquals("Live nodes count does not match", 1, reader.getClusterState()
- .getLiveNodes().size());
+ assertEquals("Live nodes count does not match", 1, reader
+ .getClusterState().getLiveNodes().size());
assertEquals("Shard count does not match", 1, reader.getClusterState()
.getSlice("collection1", "shard1").getReplicasMap().size());
version = getClusterStateVersion(controllerClient);
- mockController.publishState("core1", "core_node1", null,1);
- while(version == getClusterStateVersion(controllerClient));
+ mockController.publishState("core1", "core_node1", null, 1);
+ while (version == getClusterStateVersion(controllerClient));
Thread.sleep(500);
- assertFalse("collection1 should be gone after publishing the null state", reader.getClusterState().getCollections().contains("collection1"));
+ assertFalse("collection1 should be gone after publishing the null state",
+ reader.getClusterState().getCollections().contains("collection1"));
} finally {
close(mockController);
@@ -898,18 +905,21 @@ public class OverseerTest extends SolrTe
}
- private SolrZkClient electNewOverseer(String address) throws InterruptedException,
- TimeoutException, IOException,
+ private SolrZkClient electNewOverseer(String address)
+ throws InterruptedException, TimeoutException, IOException,
KeeperException, ParserConfigurationException, SAXException {
SolrZkClient zkClient = new SolrZkClient(address, TIMEOUT);
ZkStateReader reader = new ZkStateReader(zkClient);
readers.add(reader);
LeaderElector overseerElector = new LeaderElector(zkClient);
- // TODO: close Overseer
+ if (overseers.size() > 0) {
+ overseers.get(overseers.size() -1).close();
+ }
Overseer overseer = new Overseer(
new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", reader);
overseers.add(overseer);
- ElectionContext ec = new OverseerElectionContext(zkClient, overseer, address.replaceAll("/", "_"));
+ ElectionContext ec = new OverseerElectionContext(zkClient, overseer,
+ address.replaceAll("/", "_"));
overseerElector.setup(ec);
overseerElector.joinElection(ec, false);
return zkClient;
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java Fri Feb 28 17:12:07 2014
@@ -24,7 +24,7 @@ import java.util.concurrent.SynchronousQ
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import org.apache.lucene.util.LuceneTestCase.BadApple;
+import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
@@ -45,7 +45,8 @@ import org.junit.BeforeClass;
* This test simply does a bunch of basic things in solrcloud mode and asserts things
* work as expected.
*/
-@Slow @BadApple
+@Slow
+@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class UnloadDistributedZkTest extends BasicDistributedZkTest {
@BeforeClass
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/core/OpenCloseCoreStressTest.java Fri Feb 28 17:12:07 2014
@@ -47,7 +47,6 @@ import org.junit.Test;
/**
* Incorporate the open/close stress tests into unit tests.
*/
-@BadApple
public class OpenCloseCoreStressTest extends SolrTestCaseJ4 {
private final Object locker = new Object();
Modified: lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java?rev=1573003&r1=1573002&r2=1573003&view=diff
==============================================================================
--- lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java (original)
+++ lucene/dev/branches/lucene5376/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java Fri Feb 28 17:12:07 2014
@@ -209,6 +209,54 @@ public class JsonLoaderTest extends Solr
req.close();
}
+ public void testFieldValueOrdering() throws Exception {
+ final String pre = "{'add':[{'id':'1',";
+ final String post = "},{'id':'2'}]}";
+
+ // list
+ checkFieldValueOrdering((pre+ "'f':[45,67,89]" +post)
+ .replace('\'', '"'),
+ 1.0F);
+ // dup fieldname keys
+ checkFieldValueOrdering((pre+ "'f':45,'f':67,'f':89" +post)
+ .replace('\'', '"'),
+ 1.0F);
+ // extended w/boost
+ checkFieldValueOrdering((pre+ "'f':{'boost':4.0,'value':[45,67,89]}" +post)
+ .replace('\'', '"'),
+ 4.0F);
+ // dup keys extended w/ multiplicitive boost
+ checkFieldValueOrdering((pre+
+ "'f':{'boost':2.0,'value':[45,67]}," +
+ "'f':{'boost':2.0,'value':89}"
+ +post)
+ .replace('\'', '"'),
+ 4.0F);
+
+ }
+ private void checkFieldValueOrdering(String rawJson, float fBoost) throws Exception {
+ SolrQueryRequest req = req();
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+ JsonLoader loader = new JsonLoader();
+ loader.load(req, rsp, new ContentStreamBase.StringStream(rawJson), p);
+ assertEquals( 2, p.addCommands.size() );
+
+ SolrInputDocument d = p.addCommands.get(0).solrDoc;
+ assertEquals(2, d.getFieldNames().size());
+ assertEquals("1", d.getFieldValue("id"));
+ assertEquals(new Object[] {45L, 67L, 89L} , d.getFieldValues("f").toArray());
+ assertEquals(0.0F, fBoost, d.getField("f").getBoost());
+
+ d = p.addCommands.get(1).solrDoc;
+ assertEquals(1, d.getFieldNames().size());
+ assertEquals("2", d.getFieldValue("id"));
+
+ req.close();
+ }
+
+
+
public void testExtendedFieldValues() throws Exception {
String str = "[{'id':'1', 'val_s':{'add':'foo'}}]".replace('\'', '"');
SolrQueryRequest req = req();
@@ -499,5 +547,161 @@ public class JsonLoaderTest extends Solr
req.close();
}
+ @Test
+ public void testSimpleChildDocs() throws Exception {
+ String str = "{\n" +
+ " \"add\": {\n" +
+ " \"doc\": {\n" +
+ " \"id\": \"1\",\n" +
+ " \"_childDocuments_\": [\n" +
+ " {\n" +
+ " \"id\": \"2\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"id\": \"3\",\n" +
+ " \"foo_i\": [666,777]\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ checkTwoChildDocs(str);
+ }
+
+ @Test
+ public void testDupKeysChildDocs() throws Exception {
+ String str = "{\n" +
+ " \"add\": {\n" +
+ " \"doc\": {\n" +
+ " \"_childDocuments_\": [\n" +
+ " {\n" +
+ " \"id\": \"2\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"id\": \"1\",\n" +
+ " \"_childDocuments_\": [\n" +
+ " {\n" +
+ " \"id\": \"3\",\n" +
+ " \"foo_i\": 666,\n" +
+ " \"foo_i\": 777\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ checkTwoChildDocs(str);
+ }
+
+ private void checkTwoChildDocs(String rawJsonStr) throws Exception {
+ SolrQueryRequest req = req("commit","true");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+ JsonLoader loader = new JsonLoader();
+ loader.load(req, rsp, new ContentStreamBase.StringStream(rawJsonStr), p);
+
+ assertEquals( 1, p.addCommands.size() );
+
+ AddUpdateCommand add = p.addCommands.get(0);
+ SolrInputDocument d = add.solrDoc;
+ SolrInputField f = d.getField( "id" );
+ assertEquals("1", f.getValue());
+
+ SolrInputDocument cd = d.getChildDocuments().get(0);
+ SolrInputField cf = cd.getField( "id" );
+ assertEquals("2", cf.getValue());
+
+ cd = d.getChildDocuments().get(1);
+ cf = cd.getField( "id" );
+ assertEquals("3", cf.getValue());
+ cf = cd.getField( "foo_i" );
+ assertEquals(2, cf.getValueCount());
+
+ assertEquals(new Object[] {666L,777L}, cf.getValues().toArray());
+
+ req.close();
+ }
+
+ @Test
+ public void testEmptyChildDocs() throws Exception {
+ String str = "{\n" +
+ " \"add\": {\n" +
+ " \"doc\": {\n" +
+ " \"id\": \"1\",\n" +
+ " \"_childDocuments_\": []\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+ SolrQueryRequest req = req("commit","true");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+ JsonLoader loader = new JsonLoader();
+ loader.load(req, rsp, new ContentStreamBase.StringStream(str), p);
+
+ assertEquals( 1, p.addCommands.size() );
+
+ AddUpdateCommand add = p.addCommands.get(0);
+ SolrInputDocument d = add.solrDoc;
+ SolrInputField f = d.getField( "id" );
+ assertEquals("1", f.getValue());
+ List<SolrInputDocument> cd = d.getChildDocuments();
+ assertNull(cd);
+
+ req.close();
+ }
+
+ @Test
+ public void testGrandChildDocs() throws Exception {
+ String str = "{\n" +
+ " \"add\": {\n" +
+ " \"doc\": {\n" +
+ " \"id\": \"1\",\n" +
+ " \"_childDocuments_\": [\n" +
+ " {\n" +
+ " \"id\": \"2\",\n" +
+ " \"_childDocuments_\": [\n" +
+ " {\n" +
+ " \"id\": \"4\",\n" +
+ " \"foo_s\": \"Baz\"\n" +
+ " }\n" +
+ " ],\n" +
+ " \"foo_s\": \"Yaz\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"id\": \"3\",\n" +
+ " \"foo_s\": \"Bar\"\n" +
+ " }\n" +
+ " ]\n" +
+ " }\n" +
+ " }\n" +
+ "}";
+
+ SolrQueryRequest req = req("commit","true");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ BufferingRequestProcessor p = new BufferingRequestProcessor(null);
+ JsonLoader loader = new JsonLoader();
+ loader.load(req, rsp, new ContentStreamBase.StringStream(str), p);
+
+ assertEquals( 1, p.addCommands.size() );
+
+ AddUpdateCommand add = p.addCommands.get(0);
+ SolrInputDocument one = add.solrDoc;
+ assertEquals("1", one.getFieldValue("id"));
+
+ SolrInputDocument two = one.getChildDocuments().get(0);
+ assertEquals("2", two.getFieldValue("id"));
+ assertEquals("Yaz", two.getFieldValue("foo_s"));
+
+ SolrInputDocument four = two.getChildDocuments().get(0);
+ assertEquals("4", four.getFieldValue("id"));
+ assertEquals("Baz", four.getFieldValue("foo_s"));
+
+ SolrInputDocument three = one.getChildDocuments().get(1);
+ assertEquals("3", three.getFieldValue("id"));
+ assertEquals("Bar", three.getFieldValue("foo_s"));
+
+ req.close();
+
+ }
+
-}
\ No newline at end of file
+}