You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/20 17:27:23 UTC
svn commit: r1458924 [10/13] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/eclipse/dot.settings/ dev-tools/maven/
dev-tools/maven/solr/solrj/src/java/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/ lucene/analysis/c...
Modified: lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Wed Mar 20 16:27:13 2013
@@ -53,6 +53,8 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.search.suggest.Lookup.LookupResult;
import org.apache.lucene.search.suggest.TermFreq;
import org.apache.lucene.search.suggest.TermFreqArrayIterator;
+import org.apache.lucene.search.suggest.TermFreqPayload;
+import org.apache.lucene.search.suggest.TermFreqPayloadArrayIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -103,6 +105,56 @@ public class AnalyzingSuggesterTest exte
assertEquals(6, results.get(2).value, 0.01F);
}
+ public void testKeywordWithPayloads() throws Exception {
+ TermFreqPayload keys[] = new TermFreqPayload[] {
+ new TermFreqPayload("foo", 50, new BytesRef("hello")),
+ new TermFreqPayload("bar", 10, new BytesRef("goodbye")),
+ new TermFreqPayload("barbar", 12, new BytesRef("thank you")),
+ new TermFreqPayload("barbara", 6, new BytesRef("for all the fish"))
+ };
+
+ AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false));
+ suggester.build(new TermFreqPayloadArrayIterator(keys));
+
+ // top N of 2, but only foo is available
+ List<LookupResult> results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2);
+ assertEquals(1, results.size());
+ assertEquals("foo", results.get(0).key.toString());
+ assertEquals(50, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("hello"), results.get(0).payload);
+
+ // top N of 1 for 'bar': we return this even though
+ // barbar is higher because exactFirst is enabled:
+ results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random()), false, 1);
+ assertEquals(1, results.size());
+ assertEquals("bar", results.get(0).key.toString());
+ assertEquals(10, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("goodbye"), results.get(0).payload);
+
+ // top N Of 2 for 'b'
+ results = suggester.lookup(_TestUtil.stringToCharSequence("b", random()), false, 2);
+ assertEquals(2, results.size());
+ assertEquals("barbar", results.get(0).key.toString());
+ assertEquals(12, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("thank you"), results.get(0).payload);
+ assertEquals("bar", results.get(1).key.toString());
+ assertEquals(10, results.get(1).value, 0.01F);
+ assertEquals(new BytesRef("goodbye"), results.get(1).payload);
+
+ // top N of 3 for 'ba'
+ results = suggester.lookup(_TestUtil.stringToCharSequence("ba", random()), false, 3);
+ assertEquals(3, results.size());
+ assertEquals("barbar", results.get(0).key.toString());
+ assertEquals(12, results.get(0).value, 0.01F);
+ assertEquals(new BytesRef("thank you"), results.get(0).payload);
+ assertEquals("bar", results.get(1).key.toString());
+ assertEquals(10, results.get(1).value, 0.01F);
+ assertEquals(new BytesRef("goodbye"), results.get(1).payload);
+ assertEquals("barbara", results.get(2).key.toString());
+ assertEquals(6, results.get(2).value, 0.01F);
+ assertEquals(new BytesRef("for all the fish"), results.get(2).payload);
+ }
+
// TODO: more tests
/**
* basic "standardanalyzer" test with stopword removal
@@ -435,11 +487,13 @@ public class AnalyzingSuggesterTest exte
public final String surfaceForm;
public final String analyzedForm;
public final long weight;
+ public final BytesRef payload;
- public TermFreq2(String surfaceForm, String analyzedForm, long weight) {
+ public TermFreq2(String surfaceForm, String analyzedForm, long weight, BytesRef payload) {
this.surfaceForm = surfaceForm;
this.analyzedForm = analyzedForm;
this.weight = weight;
+ this.payload = payload;
}
@Override
@@ -549,7 +603,15 @@ public class AnalyzingSuggesterTest exte
final TreeSet<String> allPrefixes = new TreeSet<String>();
final Set<String> seen = new HashSet<String>();
- TermFreq[] keys = new TermFreq[numQueries];
+ boolean doPayloads = random().nextBoolean();
+
+ TermFreq[] keys = null;
+ TermFreqPayload[] payloadKeys = null;
+ if (doPayloads) {
+ payloadKeys = new TermFreqPayload[numQueries];
+ } else {
+ keys = new TermFreq[numQueries];
+ }
boolean preserveSep = random().nextBoolean();
@@ -567,6 +629,7 @@ public class AnalyzingSuggesterTest exte
while(true) {
key = "";
analyzedKey = "";
+ boolean lastRemoved = false;
for(int token=0;token < numTokens;token++) {
String s;
while (true) {
@@ -582,10 +645,12 @@ public class AnalyzingSuggesterTest exte
}
key += s;
if (s.length() == 1 && isStopChar(s.charAt(0), numStopChars)) {
+ lastRemoved = true;
if (preserveSep && preserveHoles) {
analyzedKey += SEP;
}
} else {
+ lastRemoved = false;
analyzedKey += s;
}
break;
@@ -595,6 +660,10 @@ public class AnalyzingSuggesterTest exte
analyzedKey = analyzedKey.replaceAll("(^|" + SEP + ")" + SEP + "$", "");
+ if (preserveSep && lastRemoved) {
+ analyzedKey += SEP;
+ }
+
// Don't add same surface form more than once:
if (!seen.contains(key)) {
seen.add(key);
@@ -607,9 +676,18 @@ public class AnalyzingSuggesterTest exte
}
// we can probably do Integer.MAX_VALUE here, but why worry.
int weight = random().nextInt(1<<24);
- keys[i] = new TermFreq(key, weight);
+ BytesRef payload;
+ if (doPayloads) {
+ byte[] bytes = new byte[random().nextInt(10)];
+ random().nextBytes(bytes);
+ payload = new BytesRef(bytes);
+ payloadKeys[i] = new TermFreqPayload(key, weight, payload);
+ } else {
+ keys[i] = new TermFreq(key, weight);
+ payload = null;
+ }
- slowCompletor.add(new TermFreq2(key, analyzedKey, weight));
+ slowCompletor.add(new TermFreq2(key, analyzedKey, weight, payload));
}
if (VERBOSE) {
@@ -625,7 +703,11 @@ public class AnalyzingSuggesterTest exte
Analyzer a = new MockTokenEatingAnalyzer(numStopChars, preserveHoles);
AnalyzingSuggester suggester = new AnalyzingSuggester(a, a,
preserveSep ? AnalyzingSuggester.PRESERVE_SEP : 0, 256, -1);
- suggester.build(new TermFreqArrayIterator(keys));
+ if (doPayloads) {
+ suggester.build(new TermFreqPayloadArrayIterator(payloadKeys));
+ } else {
+ suggester.build(new TermFreqArrayIterator(keys));
+ }
for (String prefix : allPrefixes) {
@@ -642,6 +724,7 @@ public class AnalyzingSuggesterTest exte
// "Analyze" the key:
String[] tokens = prefix.split(" ");
StringBuilder builder = new StringBuilder();
+ boolean lastRemoved = false;
for(int i=0;i<tokens.length;i++) {
String token = tokens[i];
if (preserveSep && builder.length() > 0 && !builder.toString().endsWith(""+SEP)) {
@@ -652,8 +735,10 @@ public class AnalyzingSuggesterTest exte
if (preserveSep && preserveHoles) {
builder.append(SEP);
}
+ lastRemoved = true;
} else {
builder.append(token);
+ lastRemoved = false;
}
}
@@ -676,6 +761,10 @@ public class AnalyzingSuggesterTest exte
continue;
}
+ if (preserveSep && (prefix.endsWith(" ") || lastRemoved)) {
+ analyzedKey += SEP;
+ }
+
if (VERBOSE) {
System.out.println(" analyzed: " + analyzedKey);
}
@@ -725,6 +814,9 @@ public class AnalyzingSuggesterTest exte
//System.out.println(" check hit " + hit);
assertEquals(matches.get(hit).surfaceForm.toString(), r.get(hit).key.toString());
assertEquals(matches.get(hit).weight, r.get(hit).value, 0f);
+ if (doPayloads) {
+ assertEquals(matches.get(hit).payload, r.get(hit).payload);
+ }
}
}
}
@@ -1060,4 +1152,15 @@ public class AnalyzingSuggesterTest exte
}));
assertEquals("[a a/7, a c/6, a b/5]", suggester.lookup("a", false, 3).toString());
}
+
+ public void testEndingSpace() throws Exception {
+ Analyzer a = new MockAnalyzer(random());
+ AnalyzingSuggester suggester = new AnalyzingSuggester(a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1);
+ suggester.build(new TermFreqArrayIterator(new TermFreq[] {
+ new TermFreq("i love lucy", 7),
+ new TermFreq("isla de muerta", 8),
+ }));
+ assertEquals("[isla de muerta/8, i love lucy/7]", suggester.lookup("i", false, 3).toString());
+ assertEquals("[i love lucy/7]", suggester.lookup("i ", false, 3).toString());
+ }
}
Modified: lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java Wed Mar 20 16:27:13 2013
@@ -594,6 +594,7 @@ public class FuzzySuggesterTest extends
while(true) {
key = "";
analyzedKey = "";
+ boolean lastRemoved = false;
for(int token=0;token < numTokens;token++) {
String s;
while (true) {
@@ -612,8 +613,10 @@ public class FuzzySuggesterTest extends
if (preserveSep && preserveHoles) {
analyzedKey += '\u0000';
}
+ lastRemoved = true;
} else {
analyzedKey += s;
+ lastRemoved = false;
}
break;
}
@@ -622,6 +625,10 @@ public class FuzzySuggesterTest extends
analyzedKey = analyzedKey.replaceAll("(^| )\u0000$", "");
+ if (preserveSep && lastRemoved) {
+ analyzedKey += " ";
+ }
+
// Don't add same surface form more than once:
if (!seen.contains(key)) {
seen.add(key);
@@ -669,6 +676,7 @@ public class FuzzySuggesterTest extends
// "Analyze" the key:
String[] tokens = prefix.split(" ");
StringBuilder builder = new StringBuilder();
+ boolean lastRemoved = false;
for(int i=0;i<tokens.length;i++) {
String token = tokens[i];
if (preserveSep && builder.length() > 0 && !builder.toString().endsWith(" ")) {
@@ -679,8 +687,10 @@ public class FuzzySuggesterTest extends
if (preserveSep && preserveHoles) {
builder.append("\u0000");
}
+ lastRemoved = true;
} else {
builder.append(token);
+ lastRemoved = false;
}
}
@@ -704,6 +714,10 @@ public class FuzzySuggesterTest extends
continue;
}
+ if (preserveSep && (prefix.endsWith(" ") || lastRemoved)) {
+ analyzedKey += " ";
+ }
+
if (VERBOSE) {
System.out.println(" analyzed: " + analyzedKey);
}
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java Wed Mar 20 16:27:13 2013
@@ -100,12 +100,21 @@ public class MockTokenizer extends Token
public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
this(input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
}
-
/** Calls {@link #MockTokenizer(Reader, CharacterRunAutomaton, boolean) MockTokenizer(Reader, WHITESPACE, true)} */
public MockTokenizer(Reader input) {
this(input, WHITESPACE, true);
}
-
+
+ public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(factory, input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
+ }
+
+ /** Calls {@link #MockTokenizer(org.apache.lucene.util.AttributeSource.AttributeFactory,Reader,CharacterRunAutomaton,boolean)
+ * MockTokenizer(AttributeFactory, Reader, WHITESPACE, true)} */
+ public MockTokenizer(AttributeFactory factory, Reader input) {
+ this(input, WHITESPACE, true);
+ }
+
@Override
public final boolean incrementToken() throws IOException {
assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT)
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java Wed Mar 20 16:27:13 2013
@@ -433,6 +433,11 @@ public final class RAMOnlyPostingsFormat
public int docID() {
return current.docID;
}
+
+ @Override
+ public long cost() {
+ return ramTerm.docs.size();
+ }
}
private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum {
@@ -505,6 +510,11 @@ public final class RAMOnlyPostingsFormat
return null;
}
}
+
+ @Override
+ public long cost() {
+ return ramTerm.docs.size();
+ }
}
// Holds all indexes created, keyed by the ID assigned in fieldsConsumer
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Wed Mar 20 16:27:13 2013
@@ -45,6 +45,7 @@ import org.apache.lucene.document.Stored
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldCache;
@@ -702,6 +703,77 @@ public abstract class BaseDocValuesForma
directory.close();
}
+ public void testSortedTermsEnum() throws IOException {
+ Directory directory = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriterConfig iwconfig = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+
+ Document doc = new Document();
+ doc.add(new SortedDocValuesField("field", new BytesRef("hello")));
+ iwriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new SortedDocValuesField("field", new BytesRef("world")));
+ iwriter.addDocument(doc);
+
+ doc = new Document();
+ doc.add(new SortedDocValuesField("field", new BytesRef("beer")));
+ iwriter.addDocument(doc);
+ iwriter.forceMerge(1);
+
+ DirectoryReader ireader = iwriter.getReader();
+ iwriter.close();
+
+ SortedDocValues dv = getOnlySegmentReader(ireader).getSortedDocValues("field");
+ assertEquals(3, dv.getValueCount());
+
+ TermsEnum termsEnum = dv.termsEnum();
+
+ // next()
+ assertEquals("beer", termsEnum.next().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ assertEquals("hello", termsEnum.next().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ assertEquals("world", termsEnum.next().utf8ToString());
+ assertEquals(2, termsEnum.ord());
+
+ // seekCeil()
+ assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(new BytesRef("ha!")));
+ assertEquals("hello", termsEnum.term().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef("beer")));
+ assertEquals("beer", termsEnum.term().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("zzz")));
+
+ // seekExact()
+ assertTrue(termsEnum.seekExact(new BytesRef("beer"), true));
+ assertEquals("beer", termsEnum.term().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ assertTrue(termsEnum.seekExact(new BytesRef("hello"), true));
+ assertEquals(Codec.getDefault().toString(), "hello", termsEnum.term().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ assertTrue(termsEnum.seekExact(new BytesRef("world"), true));
+ assertEquals("world", termsEnum.term().utf8ToString());
+ assertEquals(2, termsEnum.ord());
+ assertFalse(termsEnum.seekExact(new BytesRef("bogus"), true));
+
+ // seek(ord)
+ termsEnum.seekExact(0);
+ assertEquals("beer", termsEnum.term().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ termsEnum.seekExact(1);
+ assertEquals("hello", termsEnum.term().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ termsEnum.seekExact(2);
+ assertEquals("world", termsEnum.term().utf8ToString());
+ assertEquals(2, termsEnum.ord());
+ ireader.close();
+ directory.close();
+ }
+
public void testEmptySortedBytes() throws IOException {
Analyzer analyzer = new MockAnalyzer(random());
@@ -1658,6 +1730,71 @@ public abstract class BaseDocValuesForma
directory.close();
}
+ public void testSortedSetTermsEnum() throws IOException {
+ assumeTrue("Codec does not support SORTED_SET", defaultCodecSupportsSortedSet());
+ Directory directory = newDirectory();
+ Analyzer analyzer = new MockAnalyzer(random());
+ IndexWriterConfig iwconfig = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ iwconfig.setMergePolicy(newLogMergePolicy());
+ RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
+
+ Document doc = new Document();
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("world")));
+ doc.add(new SortedSetDocValuesField("field", new BytesRef("beer")));
+ iwriter.addDocument(doc);
+
+ DirectoryReader ireader = iwriter.getReader();
+ iwriter.close();
+
+ SortedSetDocValues dv = getOnlySegmentReader(ireader).getSortedSetDocValues("field");
+ assertEquals(3, dv.getValueCount());
+
+ TermsEnum termsEnum = dv.termsEnum();
+
+ // next()
+ assertEquals("beer", termsEnum.next().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ assertEquals("hello", termsEnum.next().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ assertEquals("world", termsEnum.next().utf8ToString());
+ assertEquals(2, termsEnum.ord());
+
+ // seekCeil()
+ assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(new BytesRef("ha!")));
+ assertEquals("hello", termsEnum.term().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef("beer")));
+ assertEquals("beer", termsEnum.term().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef("zzz")));
+
+ // seekExact()
+ assertTrue(termsEnum.seekExact(new BytesRef("beer"), true));
+ assertEquals("beer", termsEnum.term().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ assertTrue(termsEnum.seekExact(new BytesRef("hello"), true));
+ assertEquals("hello", termsEnum.term().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ assertTrue(termsEnum.seekExact(new BytesRef("world"), true));
+ assertEquals("world", termsEnum.term().utf8ToString());
+ assertEquals(2, termsEnum.ord());
+ assertFalse(termsEnum.seekExact(new BytesRef("bogus"), true));
+
+ // seek(ord)
+ termsEnum.seekExact(0);
+ assertEquals("beer", termsEnum.term().utf8ToString());
+ assertEquals(0, termsEnum.ord());
+ termsEnum.seekExact(1);
+ assertEquals("hello", termsEnum.term().utf8ToString());
+ assertEquals(1, termsEnum.ord());
+ termsEnum.seekExact(2);
+ assertEquals("world", termsEnum.term().utf8ToString());
+ assertEquals(2, termsEnum.ord());
+ ireader.close();
+ directory.close();
+ }
+
private void doTestSortedSetVsStoredFields(int minLength, int maxLength) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Wed Mar 20 16:27:13 2013
@@ -280,6 +280,11 @@ public abstract class BasePostingsFormat
}
return docID;
}
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
}
private static class FieldAndTerm {
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/index/DocHelper.java Wed Mar 20 16:27:13 2013
@@ -33,6 +33,7 @@ import org.apache.lucene.document.Stored
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
@@ -287,7 +288,7 @@ class DocHelper {
*/
public static SegmentInfoPerCommit writeDoc(Random random, Directory dir, Analyzer analyzer, Similarity similarity, Document doc) throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( /* LuceneTestCase.newIndexWriterConfig(random, */
- TEST_VERSION_CURRENT, analyzer).setSimilarity(similarity));
+ TEST_VERSION_CURRENT, analyzer).setSimilarity(similarity == null ? IndexSearcher.getDefaultSimilarity() : similarity));
//writer.setUseCompoundFile(false);
writer.addDocument(doc);
writer.commit();
Modified: lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/lucene4258/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java Wed Mar 20 16:27:13 2013
@@ -655,6 +655,7 @@ public abstract class LuceneTestCase ext
* Return <code>args</code> as a {@link Set} instance. The order of elements is not
* preserved in iterators.
*/
+ @SafeVarargs @SuppressWarnings("varargs")
public static <T> Set<T> asSet(T... args) {
return new HashSet<T>(Arrays.asList(args));
}
Modified: lucene/dev/branches/lucene4258/lucene/tools/clover/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/tools/clover/README.txt?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/tools/clover/README.txt (original)
+++ lucene/dev/branches/lucene4258/lucene/tools/clover/README.txt Wed Mar 20 16:27:13 2013
@@ -2,6 +2,26 @@
DOCUMENTATION ABOUT ATLASSIAN CLOVER LICENSE
################################################
+From: Nicholas Muldoon <nm...@atlassian.com>
+To: Ted Yu <yu...@gmail.com>
+Cc: Enis Soztutar <en...@hortonworks.com>, Todd Lipcon <to...@cloudera.com>
+Content-Type: multipart/alternative; boundary=047d7b10cff34cee0f04c376df70
+X-Gm-Message-State: ALoCoQmzowRKrtL3txnON+W+U2vn7gwwWSn/U5dvchuTV0Nn3xaMoAojvbPal5TBkC6foBnuHPWU
+
+--047d7b10cff34cee0f04c376df70
+Content-Type: text/plain; charset=ISO-8859-1
+
+Hi Ted,
+
+Please find a renewed license below - Clover 3 compatible. Should you have
+any difficulty please let me know.
+
+[license]
+
+Thanks Ted, have a great day, thanks for your support,
+Nicholas
+
+
On Fri, Dec 18, 2009 at 1:33 AM, Nicholas Muldoon <nm...@atlassian.com> wrote:
---------------------------------------------------------------------------------
Modified: lucene/dev/branches/lucene4258/lucene/tools/clover/clover.license
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/tools/clover/clover.license?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/tools/clover/clover.license (original)
+++ lucene/dev/branches/lucene4258/lucene/tools/clover/clover.license Wed Mar 20 16:27:13 2013
@@ -1,7 +1,5 @@
-AAABLQ0ODAoPeNptkF1rwjAUhu/zKwK7rjRVhwiBSdsLQVtZdVeDkaXHGkzTkA+n/36xtbiLXeQmh
-+d533NeKq/wyjeYzHCSLOfxkiQ4zfY4iQlBGVhuhHaiUzSV3QXM5xKXGhSuOm84oNQAu08z5oDek
-SheRGSO0k45xl3BWqBbcQa85ZypWoK1iPeiSeHbbzDl8WDBWBqRkcm3TEjaBqjtmTemGT/BpDMNy
-i9M+j6QHpm0MLoCJi5AnfHPLym7H6j1udEGjuIKlgbDZJAhKTgoCx8h+i5LUAhVDhRTHPKrFub23
-IgkEVmM2s0A7m8a+uXKXV58VeXhPc1RaRqmhB36rYagKi9oeBGZvs4W0yl68OuMbtbZP4M/4V5J0
-QoHNdqFW5+YhbFT/Oj0C1FClngwLgIVAJJTCumwsuBLa8eB0JcPVMcTcRiKAhUAgS9VbXCNOqGDY
-ooFNN9744FAK2s=X02f7
\ No newline at end of file
+RMnmmVojSVTAxfPXiUgnIWdgpmMJbqADeQonukSmgbVIhd
+mi2K<6pTN6gi825uWHxIERlJ2KvVSx3LOJtfGCJvZnLyCp
+qOPqVTPpPqNRmNomonNUXqmqqrrpQMNpNPuVuutwVsxspP
+MmonmqmUUnpqmspoummmmmUUnpqmspoummmmmUU1mXZebU
+Unmmmm
Modified: lucene/dev/branches/lucene4258/lucene/tools/javadoc/ecj.javadocs.prefs
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/tools/javadoc/ecj.javadocs.prefs?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/tools/javadoc/ecj.javadocs.prefs (original)
+++ lucene/dev/branches/lucene4258/lucene/tools/javadoc/ecj.javadocs.prefs Wed Mar 20 16:27:13 2013
@@ -1,7 +1,7 @@
#Sun Sep 23 20:55:03 EDT 2012
eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
+org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.doc.comment.support=enabled
org.eclipse.jdt.core.compiler.problem.annotationSuperInterface=ignore
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
@@ -88,4 +88,4 @@ org.eclipse.jdt.core.compiler.problem.un
org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=ignore
org.eclipse.jdt.core.compiler.problem.unusedWarningToken=ignore
org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=ignore
-org.eclipse.jdt.core.compiler.source=1.6
+org.eclipse.jdt.core.compiler.source=1.7
Modified: lucene/dev/branches/lucene4258/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/CHANGES.txt?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene4258/solr/CHANGES.txt Wed Mar 20 16:27:13 2013
@@ -12,7 +12,7 @@ See http://lucene.apache.org/solr for mo
Getting Started
---------------
-You need a Java 1.6 VM or later installed.
+You need a Java 1.7 VM or later installed.
In this release, there is an example Solr server including a bundled
servlet container in the directory named "example".
See the tutorial at http://lucene.apache.org/solr/tutorial.html
@@ -59,14 +59,169 @@ Detailed Change List
New Features
----------------------
+* SOLR-4196 (and others). Solr.xml is being deprecated in favor of a simple
+ properties file. In the absence of a <solr_home>/solr.xml but the presence of
+ <solr_home>/solr.properties, two things will happen
+ 1> The attributes that, in the solr.xml file, were in the <solr> and <cores> tags will
+ be read from the solr.properties
+ 2> The <solr_home> will be walked and any cores fond will be discovered, which will
+ be inferred by the presence of a "core.properties" file which will contain the
+ data formerly in the individual <core> tags. The implication here is that there will
+ be no individual core information in the solr.properties file.
+ See the discussion on the wiki page titled "Core Discovery (4.3 and beyond)" for
+ the formats of both solr.properties and the individual core.properties files
+ (Erick Erickson)
+
Bug Fixes
----------------------
+* SOLR-4543: setting shardHandlerFactory in solr.xml/solr.properties does not work.
+ (Ryan Ernst, Robert Muir via Erick Erickson)
+
+Optimizations
+----------------------
+
+Other Changes
+----------------------
+
+* SOLR-4317: SolrTestCaseJ4: Can't avoid "collection1" convention (Tricia Jenkins, via Erick Erickson)
+
+* SOLR-4571: SolrZkClient#setData should return Stat object. (Mark Miller)
+
+* SOLR-4603: CachingDirectoryFactory should use an IdentityHashMap for
+ byDirectoryCache. (Mark Miller)
+
+* SOLR-4544: Refactor HttpShardHandlerFactory so load-balancing logic can be customized.
+ (Ryan Ernst via Robert Muir)
+
+* SOLR-4607: Use noggit 0.5 release jar rather than a forked copy. (Yonik Seeley, Robert Muir)
+
+================== 4.2.1 ==================
+
+Versions of Major Components
+---------------------
+Apache Tika 1.3
+Carrot2 3.6.2
+Velocity 1.7 and Velocity Tools 2.0
+Apache UIMA 2.3.1
+Apache ZooKeeper 3.4.5
+
+Detailed Change List
+----------------------
+
+Bug Fixes
+----------------------
+
+* SOLR-4567: copyField source glob matching explicit field(s) stopped working
+ in Solr 4.2. (Alexandre Rafalovitch, Steve Rowe)
+
+* SOLR-4475: Fix various places that still assume File based paths even when
+ not using a file based DirectoryFactory. (Mark Miller)
+
+* SOLR-4551: CachingDirectoryFactory needs to create CacheEntry's with the
+ fullpath not path. (Mark Miller)
+
+* SOLR-4555: When forceNew is used with CachingDirectoryFactory#get, the old
+ CachValue should give up it's path as it will be used by a new Directory
+ instance. (Mark Miller)
+
+* SOLR-4578: CoreAdminHandler#handleCreateAction gets a SolrCore and does not
+ close it in SolrCloud mode when a core with the same name already exists.
+ (Mark Miller)
+
+* SOLR-4574: The Collections API will silently return success on an unknown
+ ACTION parameter. (Mark Miller)
+
+* SOLR-4576: Collections API validation errors should cause an exception on
+ clients and otherwise act as validation errors with the Core Admin API.
+ (Mark Miller)
+
+* SOLR-4577: The collections API should return responses (success or failure)
+ for each node it attempts to work with. (Mark Miller)
+
+* SOLR-4568: The lastPublished state check before becoming a leader is not
+ working correctly. (Mark Miller)
+
+* SOLR-4570: Even if an explicit shard id is used, ZkController#preRegister
+ should still wait to see the shard id in it's current ClusterState.
+ (Mark Miller)
+
+* SOLR-4585: The Collections API validates numShards with < 0 but should use
+ <= 0. (Mark Miller)
+
+* SOLR-4592: DefaultSolrCoreState#doRecovery needs to check the CoreContainer
+ shutdown flag inside the recoveryLock sync block. (Mark Miller)
+
+* SOLR-4595: CachingDirectoryFactory#close can throw a concurrent
+ modification exception. (Mark Miller)
+
+* SOLR-4573: Accessing Admin UI files in SolrCloud mode logs warnings.
+ (Mark Miller, Phil John)
+
+* SOLR-4594: StandardDirectoryFactory#remove accesses byDirectoryCache
+ without a lock. (Mark Miller)
+
+* SOLR-4597: CachingDirectoryFactory#remove should not attempt to empty/remove
+ the index right away but flag for removal after close. (Mark Miller)
+
+* SOLR-4598: The Core Admin unload command's option 'deleteDataDir', should use
+ the DirectoryFactory API to remove the data dir. (Mark Miller)
+
+* SOLR-4599: CachingDirectoryFactory calls close(Directory) on forceNew if the
+ Directory has a refCnt of 0, but it should call closeDirectory(CacheValue).
+ (Mark Miller)
+
+* SOLR-4602: ZkController#unregister should cancel it's election participation
+ before asking the Overseer to delete the SolrCore information. (Mark Miller)
+
+* SOLR-4601: A Collection that is only partially created and then deleted will
+ leave pre allocated shard information in ZooKeeper. (Mark Miller)
+
+* SOLR-4604: UpdateLog#init is over called on SolrCore#reload. (Mark Miller)
+
+* SOLR-4605: Rollback does not work correctly. (Mark S, Mark Miller)
+
+* SOLR-4609: The Collections API should only send the reload command to ACTIVE
+ cores. (Mark Miller)
+
+* SOLR-4297: Atomic update request containing null=true sets all subsequent
+ fields to null (Ben Pennell, Rob, shalin)
+
+* SOLR-4371: Admin UI - Analysis Screen shows empty result (steffkes)
+
+* SOLR-4318: NPE encountered with querying with wildcards on a field that uses
+ the DefaultAnalyzer (i.e. no analysis chain defined). (Erick Erickson)
+
+* SOLR-4361: DataImportHandler would throw UnsupportedOperationException if
+ handler-level parameters were specified containing periods in the name
+ (James Dyer)
+
+* SOLR-4538: Date Math expressions were being truncated to 32 characters
+ when used in field:value queries in the lucene QParser. (hossman, yonik)
+
+* SOLR-4617: SolrCore#reload needs to pass the deletion policy to the next
+ SolrCore through it's constructor rather than setting a field after.
+ (Mark Miller)
+
+* SOLR-4589: Fixed CPU spikes and poor performance in lazy field loading
+ of multivalued fields. (hossman)
+
+Optimizations
+----------------------
+
+* SOLR-4569: waitForReplicasToComeUp should bail right away if it doesn't see the
+ expected slice in the clusterstate rather than waiting. (Mark Miller)
+
+* SOLR-4311: Admin UI - Optimize Caching Behaviour (steffkes)
+
Other Changes
----------------------
* SOLR-4537: Clean up schema information REST API. (Steve Rowe)
+* SOLR-4596: DistributedQueue should ensure its full path exists in the constructor.
+ (Mark Miller)
+
================== 4.2.0 ==================
Versions of Major Components
@@ -911,7 +1066,7 @@ Other Changes
* SOLR-4208: ExtendedDismaxQParserPlugin has been refactored to make
subclassing easier. (Tomás Fernández Löbbe, hossman)
-
+
* SOLR-3735: Relocate the example mime-to-extension mapping, and
upgrade Velocity Engine to 1.7 (ehatcher)
Modified: lucene/dev/branches/lucene4258/solr/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/README.txt?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/README.txt (original)
+++ lucene/dev/branches/lucene4258/solr/README.txt Wed Mar 20 16:27:13 2013
@@ -62,10 +62,10 @@ docs/index.html
Instructions for Building Apache Solr from Source
-------------------------------------------------
-1. Download the Java SE 6 JDK (Java Development Kit) or later from http://java.sun.com/
+1. Download the Java SE 7 JDK (Java Development Kit) or later from http://java.sun.com/
You will need the JDK installed, and the $JAVA_HOME/bin (Windows: %JAVA_HOME%\bin)
folder included on your command path. To test this, issue a "java -version" command
- from your shell (command prompt) and verify that the Java version is 1.6 or later.
+ from your shell (command prompt) and verify that the Java version is 1.7 or later.
2. Download the Apache Ant binary distribution (1.8.2+) from
http://ant.apache.org/ You will need Ant installed and the $ANT_HOME/bin (Windows:
Modified: lucene/dev/branches/lucene4258/solr/SYSTEM_REQUIREMENTS.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/SYSTEM_REQUIREMENTS.txt?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/SYSTEM_REQUIREMENTS.txt (original)
+++ lucene/dev/branches/lucene4258/solr/SYSTEM_REQUIREMENTS.txt Wed Mar 20 16:27:13 2013
@@ -1,6 +1,6 @@
# System Requirements
-Apache Solr runs of Java 6 or greater. When using Java 7, be sure to
+Apache Solr runs of Java 7 or greater. When using Java 7, be sure to
install at least Update 1! With all Java versions it is strongly
recommended to not use experimental `-XX` JVM options. It is also
recommended to always use the latest update version of your Java VM,
Modified: lucene/dev/branches/lucene4258/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/build.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/build.xml (original)
+++ lucene/dev/branches/lucene4258/solr/build.xml Wed Mar 20 16:27:13 2013
@@ -70,7 +70,7 @@
<java jar="${example}/start.jar" fork="true" dir="${example}" maxmemory="${example.heap.size}">
<jvmarg line="${example.jvm.line}"/>
<sysproperty key="solr.solr.home" file="${example.solr.home}"/>
- <sysproperty key="solr.data.dir" file="${example.data.dir}"/>
+ <sysproperty key="solr.data.dir" value="${example.data.dir}"/>
<sysproperty key="jetty.port" value="${example.jetty.port}"/>
</java>
</target>
@@ -409,7 +409,7 @@
<svn-export-source source.dir=".."/>
<!-- Exclude javadoc package-list files under licenses incompatible with the ASL -->
- <delete dir="${svn.export.dir}/lucene/tools/javadoc/java6"/>
+ <delete dir="${svn.export.dir}/lucene/tools/javadoc/java7"/>
<!-- Exclude clover license files incompatible with the ASL -->
<delete dir="${svn.export.dir}/lucene/tools/clover"/>
Modified: lucene/dev/branches/lucene4258/solr/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/common-build.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/common-build.xml (original)
+++ lucene/dev/branches/lucene4258/solr/common-build.xml Wed Mar 20 16:27:13 2013
@@ -29,9 +29,9 @@
<property name="fullnamever" value="${fullname}-${version}"/>
<property name="final.name" value="${fullnamever}"/>
- <!-- solr uses 1.6 -->
- <property name="javac.source" value="1.6"/>
- <property name="javac.target" value="1.6"/>
+ <!-- solr uses 1.7 -->
+ <property name="javac.source" value="1.7"/>
+ <property name="javac.target" value="1.7"/>
<property name="javac.args" value=""/>
<property name="dest" location="${common-solr.dir}/build" />
Modified: lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/clustering/src/test-files/clustering/solr/collection1/conf/schema.xml Wed Mar 20 16:27:13 2013
@@ -243,7 +243,7 @@
See the Java Regular Expression documentation for more
infomation on pattern and replacement string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory"
pattern="([^a-z])" replacement="" replace="all"
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Wed Mar 20 16:27:13 2013
@@ -130,7 +130,7 @@ public class DocBuilder {
indexerNamespace.put(LAST_INDEX_TIME, EPOCH);
}
indexerNamespace.put(INDEX_START_TIME, dataImporter.getIndexStartTime());
- indexerNamespace.put("request", reqParams.getRawParams());
+ indexerNamespace.put("request", new HashMap<String,Object>(reqParams.getRawParams()));
for (Entity entity : dataImporter.getConfig().getEntities()) {
String key = entity.getName() + "." + SolrWriter.LAST_INDEX_KEY;
Object lastIndex = persistedProperties.get(key);
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/VariableResolver.java Wed Mar 20 16:27:13 2013
@@ -91,8 +91,9 @@ public class VariableResolver {
Object r = null;
if (name != null) {
String[] nameParts = DOT_PATTERN.split(name);
- Map<String,Object> currentLevel = currentLevelMap(nameParts,
+ CurrentLevel cr = currentLevelMap(nameParts,
rootNamespace, false);
+ Map<String,Object> currentLevel = cr.map;
r = currentLevel.get(nameParts[nameParts.length - 1]);
if (r == null && name.startsWith(FUNCTIONS_NAMESPACE)
&& name.length() > FUNCTIONS_NAMESPACE.length()) {
@@ -103,6 +104,16 @@ public class VariableResolver {
return resolveEvaluator(FUNCTIONS_NAMESPACE_SHORT, name);
}
if (r == null) {
+ StringBuilder sb = new StringBuilder();
+ for(int i=cr.level ; i<nameParts.length ; i++) {
+ if(sb.length()>0) {
+ sb.append(".");
+ }
+ sb.append(nameParts[i]);
+ }
+ r = cr.map.get(sb.toString());
+ }
+ if (r == null) {
r = System.getProperty(name);
}
}
@@ -181,28 +192,41 @@ public class VariableResolver {
if (name != null) {
String[] nameParts = DOT_PATTERN.split(name);
Map<String,Object> nameResolveLevel = currentLevelMap(nameParts,
- rootNamespace, false);
+ rootNamespace, false).map;
nameResolveLevel.put(nameParts[nameParts.length - 1], newMap);
} else {
for (Map.Entry<String,Object> entry : newMap.entrySet()) {
String[] keyParts = DOT_PATTERN.split(entry.getKey());
Map<String,Object> currentLevel = rootNamespace;
- currentLevel = currentLevelMap(keyParts, currentLevel, false);
+ currentLevel = currentLevelMap(keyParts, currentLevel, false).map;
currentLevel.put(keyParts[keyParts.length - 1], entry.getValue());
}
}
}
}
- private Map<String,Object> currentLevelMap(String[] keyParts,
+ class CurrentLevel {
+ final Map<String,Object> map;
+ final int level;
+ CurrentLevel(int level, Map<String,Object> map) {
+ this.level = level;
+ this.map = map;
+ }
+ }
+
+ private CurrentLevel currentLevelMap(String[] keyParts,
Map<String,Object> currentLevel, boolean includeLastLevel) {
int j = includeLastLevel ? keyParts.length : keyParts.length - 1;
for (int i = 0; i < j; i++) {
Object o = currentLevel.get(keyParts[i]);
if (o == null) {
- Map<String,Object> nextLevel = new HashMap<String,Object>();
- currentLevel.put(keyParts[i], nextLevel);
- currentLevel = nextLevel;
+ if(i == j-1) {
+ Map<String,Object> nextLevel = new HashMap<String,Object>();
+ currentLevel.put(keyParts[i], nextLevel);
+ currentLevel = nextLevel;
+ } else {
+ return new CurrentLevel(i, currentLevel);
+ }
} else if (o instanceof Map<?,?>) {
@SuppressWarnings("unchecked")
Map<String,Object> nextLevel = (Map<String,Object>) o;
@@ -212,7 +236,7 @@ public class VariableResolver {
"Non-leaf nodes should be of type java.util.Map");
}
}
- return currentLevel;
+ return new CurrentLevel(j-1, currentLevel);
}
public void removeNamespace(String name) {
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solr_id-schema.xml Wed Mar 20 16:27:13 2013
@@ -224,7 +224,7 @@
See the Java Regular Expression documentation for more
infomation on pattern and replacement string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory"
pattern="([^a-z])" replacement="" replace="all"
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test-files/dih/solr/collection1/conf/dataimport-solrconfig.xml Wed Mar 20 16:27:13 2013
@@ -239,6 +239,9 @@
</requestHandler>
<requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
+ <lst name="defaults">
+ <str name="dots.in.hsqldb.driver">org.hsqldb.jdbcDriver</str>
+ </lst>
</requestHandler>
<!--
Modified: lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestVariableResolverEndToEnd.java Wed Mar 20 16:27:13 2013
@@ -62,7 +62,7 @@ public class TestVariableResolverEndToEn
String thirdLocaleParam = random().nextBoolean() ? "" : (", '" + Locale.getDefault() + "'");
StringBuilder sb = new StringBuilder();
sb.append("<dataConfig> \n");
- sb.append("<dataSource name=\"hsqldb\" driver=\"org.hsqldb.jdbcDriver\" url=\"jdbc:hsqldb:mem:.\" /> \n");
+ sb.append("<dataSource name=\"hsqldb\" driver=\"${dataimporter.request.dots.in.hsqldb.driver}\" url=\"jdbc:hsqldb:mem:.\" /> \n");
sb.append("<document name=\"TestEvaluators\"> \n");
sb.append("<entity name=\"FIRST\" processor=\"SqlEntityProcessor\" dataSource=\"hsqldb\" ");
sb.append(" query=\"" +
Modified: lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/solr/collection1/conf/schema.xml Wed Mar 20 16:27:13 2013
@@ -406,7 +406,7 @@
documentation for more information on pattern and replacement
string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory" pattern="([^a-z])"
replacement="" replace="all" />
Modified: lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml (original)
+++ lucene/dev/branches/lucene4258/solr/contrib/uima/src/test-files/uima/uima-tokenizers-schema.xml Wed Mar 20 16:27:13 2013
@@ -402,7 +402,7 @@
documentation for more information on pattern and replacement
string syntax.
- http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
+ http://docs.oracle.com/javase/7/docs/api/java/util/regex/package-summary.html
-->
<filter class="solr.PatternReplaceFilterFactory" pattern="([^a-z])"
replacement="" replace="all" />
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java Wed Mar 20 16:27:13 2013
@@ -17,15 +17,21 @@
package org.apache.solr.analysis;
import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
import org.apache.solr.common.SolrException;
import org.apache.solr.schema.DateField;
import static org.apache.solr.schema.TrieField.TrieTypes;
import java.io.IOException;
import java.io.Reader;
+import java.util.Iterator;
/**
* Tokenizer for trie fields. It uses NumericTokenStream to create multiple trie encoded string per number.
@@ -49,8 +55,8 @@ public class TrieTokenizerFactory extend
}
@Override
- public TrieTokenizer create(Reader input) {
- return new TrieTokenizer(input, type, TrieTokenizer.getNumericTokenStream(precisionStep));
+ public TrieTokenizer create(AttributeFactory factory, Reader input) {
+ return new TrieTokenizer(input, type, TrieTokenizer.getNumericTokenStream(factory, precisionStep));
}
}
@@ -59,36 +65,47 @@ final class TrieTokenizer extends Tokeni
protected final TrieTypes type;
protected final NumericTokenStream ts;
+ // NumericTokenStream does not support CharTermAttribute so keep it local
+ private final CharTermAttribute termAtt = new CharTermAttributeImpl();
protected final OffsetAttribute ofsAtt = addAttribute(OffsetAttribute.class);
protected int startOfs, endOfs;
protected boolean hasValue;
- protected final char[] buf = new char[32];
- static NumericTokenStream getNumericTokenStream(int precisionStep) {
- return new NumericTokenStream(precisionStep);
+ static NumericTokenStream getNumericTokenStream(AttributeFactory factory, int precisionStep) {
+ return new NumericTokenStream(factory, precisionStep);
}
- public TrieTokenizer(Reader input, TrieTypes type, NumericTokenStream ts) {
- // must share the attribute source with the NumericTokenStream we delegate to
- super(ts, input);
+ public TrieTokenizer(Reader input, TrieTypes type, final NumericTokenStream ts) {
+ // Häckidy-Hick-Hack: must share the attributes with the NumericTokenStream we delegate to, so we create a fake factory:
+ super(new AttributeFactory() {
+ @Override
+ public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
+ return (AttributeImpl) ts.addAttribute(attClass);
+ }
+ }, input);
+ // add all attributes:
+ for (Iterator<Class<? extends Attribute>> it = ts.getAttributeClassesIterator(); it.hasNext();) {
+ addAttribute(it.next());
+ }
this.type = type;
this.ts = ts;
+ // dates tend to be longer, especially when math is involved
+ termAtt.resizeBuffer( type == TrieTypes.DATE ? 128 : 32 );
}
@Override
public void reset() {
try {
int upto = 0;
- while (upto < buf.length) {
- final int length = input.read(buf, upto, buf.length - upto);
+ char[] buf = termAtt.buffer();
+ while (true) {
+ final int length = input.read(buf, upto, buf.length-upto);
if (length == -1) break;
upto += length;
+ if (upto == buf.length)
+ buf = termAtt.resizeBuffer(1+buf.length);
}
- // skip remaining data if buffer was too short:
- if (upto == buf.length) {
- input.skip(Long.MAX_VALUE);
- }
-
+ termAtt.setLength(upto);
this.startOfs = correctOffset(0);
this.endOfs = correctOffset(upto);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/CloudDescriptor.java Wed Mar 20 16:27:13 2013
@@ -17,6 +17,7 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.SolrParams;
public class CloudDescriptor {
@@ -28,7 +29,7 @@ public class CloudDescriptor {
private String nodeName = null;
volatile boolean isLeader = false;
- volatile String lastPublished;
+ volatile String lastPublished = ZkStateReader.ACTIVE;
public String getLastPublished() {
return lastPublished;
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/DistributedQueue.java Wed Mar 20 16:27:13 2013
@@ -22,7 +22,10 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.TreeMap;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
@@ -53,6 +56,16 @@ public class DistributedQueue {
public DistributedQueue(SolrZkClient zookeeper, String dir, List<ACL> acl) {
this.dir = dir;
+ ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(30);
+ try {
+ cmdExecutor.ensureExists(dir, zookeeper);
+ } catch (KeeperException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(ErrorCode.SERVER_ERROR, e);
+ }
+
if (acl != null) {
this.acl = acl;
}
@@ -82,7 +95,7 @@ public class DistributedQueue {
try {
// Check format
if (!childName.regionMatches(0, prefix, 0, prefix.length())) {
- LOG.warn("Found child node with improper name: " + childName);
+ LOG.debug("Found child node with improper name: " + childName);
continue;
}
String suffix = childName.substring(prefix.length());
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java Wed Mar 20 16:27:13 2013
@@ -132,6 +132,9 @@ final class ShardLeaderElectionContext e
syncStrategy.close();
}
+ /*
+ * weAreReplacement: has someone else been the leader already?
+ */
@Override
void runLeaderProcess(boolean weAreReplacement) throws KeeperException,
InterruptedException, IOException {
@@ -302,7 +305,7 @@ final class ShardLeaderElectionContext e
found = zkClient.getChildren(shardsElectZkPath, null, true).size();
} catch (KeeperException e) {
SolrException.log(log,
- "Errir checking for the number of election participants", e);
+ "Error checking for the number of election participants", e);
}
// on startup and after connection timeout, wait for all known shards
@@ -321,6 +324,11 @@ final class ShardLeaderElectionContext e
log.info("Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later");
return;
}
+ } else {
+ log.warn("Shard not found: " + shardId + " for collection " + collection);
+
+ return;
+
}
Thread.sleep(500);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/Overseer.java Wed Mar 20 16:27:13 2013
@@ -459,6 +459,7 @@ public class Overseer {
}
Map<String, Slice> newSlices = new LinkedHashMap<String, Slice>();
+ boolean lastSlice = false;
for (Slice slice : coll.getSlices()) {
Replica replica = slice.getReplica(cnn);
if (replica != null) {
@@ -469,6 +470,7 @@ public class Overseer {
// if there are no replicas left for the slice remove it
if (newReplicas.size() == 0) {
slice = null;
+ lastSlice = true;
} else {
slice = new Slice(slice.getName(), newReplicas, slice.getProperties());
}
@@ -478,6 +480,15 @@ public class Overseer {
newSlices.put(slice.getName(), slice);
}
}
+
+ if (lastSlice) {
+ // remove all empty pre allocated slices
+ for (Slice slice : coll.getSlices()) {
+ if (slice.getReplicas().size() == 0) {
+ newSlices.remove(slice.getName());
+ }
+ }
+ }
// if there are no slices left in the collection, remove it?
if (newSlices.size() == 0) {
@@ -581,12 +592,26 @@ public class Overseer {
public void close() {
isClosed = true;
if (updaterThread != null) {
- updaterThread.close();
- updaterThread.interrupt();
+ try {
+ updaterThread.close();
+ updaterThread.interrupt();
+ } catch (Throwable t) {
+ log.error("Error closing updaterThread", t);
+ }
}
if (ccThread != null) {
- ccThread.close();
- ccThread.interrupt();
+ try {
+ ccThread.close();
+ ccThread.interrupt();
+ } catch (Throwable t) {
+ log.error("Error closing ccThread", t);
+ }
+ }
+
+ try {
+ reader.close();
+ } catch (Throwable t) {
+ log.error("Error closing zkStateReader", t);
}
}
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Wed Mar 20 16:27:13 2013
@@ -41,6 +41,7 @@ import org.apache.solr.common.params.Cor
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardRequest;
@@ -156,22 +157,22 @@ public class OverseerCollectionProcessor
NamedList results = new NamedList();
try {
if (CREATECOLLECTION.equals(operation)) {
- createCollection(zkStateReader.getClusterState(), message);
+ createCollection(zkStateReader.getClusterState(), message, results);
} else if (DELETECOLLECTION.equals(operation)) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.UNLOAD.toString());
params.set(CoreAdminParams.DELETE_INSTANCE_DIR, true);
- collectionCmd(zkStateReader.getClusterState(), message, params);
+ collectionCmd(zkStateReader.getClusterState(), message, params, results, null);
} else if (RELOADCOLLECTION.equals(operation)) {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.RELOAD.toString());
- collectionCmd(zkStateReader.getClusterState(), message, params);
+ collectionCmd(zkStateReader.getClusterState(), message, params, results, ZkStateReader.ACTIVE);
} else if (CREATEALIAS.equals(operation)) {
createAlias(zkStateReader.getAliases(), message);
} else if (DELETEALIAS.equals(operation)) {
deleteAlias(zkStateReader.getAliases(), message);
} else {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Unknow the operation:"
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ operation);
}
int failed = 0;
@@ -194,6 +195,10 @@ public class OverseerCollectionProcessor
SolrException.log(log, "Collection " + operation + " of " + operation
+ " failed", ex);
results.add("Operation " + operation + " caused exception:", ex);
+ SimpleOrderedMap nl = new SimpleOrderedMap();
+ nl.add("msg", ex.getMessage());
+ nl.add("rspCode", ex instanceof SolrException ? ((SolrException)ex).code() : -1);
+ results.add("exception", nl);
} finally {
return new OverseerSolrResponse(results);
}
@@ -300,11 +305,10 @@ public class OverseerCollectionProcessor
}
- private boolean createCollection(ClusterState clusterState, ZkNodeProps message) {
+ private void createCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) {
String collectionName = message.getStr("name");
if (clusterState.getCollections().contains(collectionName)) {
- SolrException.log(log, "collection already exists: " + collectionName);
- return false;
+ throw new SolrException(ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName);
}
try {
@@ -312,19 +316,22 @@ public class OverseerCollectionProcessor
// if it does not, find best nodes to create more cores
int repFactor = msgStrToInt(message, REPLICATION_FACTOR, 1);
- int numSlices = msgStrToInt(message, NUM_SLICES, 0);
+ Integer numSlices = msgStrToInt(message, NUM_SLICES, null);
+
+ if (numSlices == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName);
+ }
+
int maxShardsPerNode = msgStrToInt(message, MAX_SHARDS_PER_NODE, 1);
String createNodeSetStr;
List<String> createNodeList = ((createNodeSetStr = message.getStr(CREATE_NODE_SET)) == null)?null:StrUtils.splitSmart(createNodeSetStr, ",", true);
if (repFactor <= 0) {
- SolrException.log(log, REPLICATION_FACTOR + " must be > 0");
- return false;
+ throw new SolrException(ErrorCode.BAD_REQUEST, NUM_SLICES + " is a required paramater");
}
- if (numSlices < 0) {
- SolrException.log(log, NUM_SLICES + " must be > 0");
- return false;
+ if (numSlices <= 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, NUM_SLICES + " must be > 0");
}
String configName = message.getStr("collection.configName");
@@ -343,9 +350,8 @@ public class OverseerCollectionProcessor
Collections.shuffle(nodeList);
if (nodeList.size() <= 0) {
- log.error("Cannot create collection " + collectionName
- + ". No live Solr-instaces" + ((createNodeList != null)?" among Solr-instances specified in " + CREATE_NODE_SET:""));
- return false;
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName
+ + ". No live Solr-instances" + ((createNodeList != null)?" among Solr-instances specified in " + CREATE_NODE_SET + ":" + createNodeSetStr:""));
}
if (repFactor > nodeList.size()) {
@@ -355,7 +361,7 @@ public class OverseerCollectionProcessor
+ repFactor
+ " on collection "
+ collectionName
- + " is higher than or equal to the number of Solr instances currently live ("
+ + " is higher than or equal to the number of Solr instances currently live or part of your " + CREATE_NODE_SET + "("
+ nodeList.size()
+ "). Its unusual to run two replica of the same slice on the same Solr-instance.");
}
@@ -363,7 +369,7 @@ public class OverseerCollectionProcessor
int maxShardsAllowedToCreate = maxShardsPerNode * nodeList.size();
int requestedShardsToCreate = numSlices * repFactor;
if (maxShardsAllowedToCreate < requestedShardsToCreate) {
- log.error("Cannot create collection " + collectionName + ". Value of "
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of "
+ MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode
+ ", and the number of live nodes is " + nodeList.size()
+ ". This allows a maximum of " + maxShardsAllowedToCreate
@@ -371,7 +377,6 @@ public class OverseerCollectionProcessor
+ " and value of " + REPLICATION_FACTOR + " is " + repFactor
+ ". This requires " + requestedShardsToCreate
+ " shards to be created (higher than the allowed number)");
- return false;
}
for (int i = 1; i <= numSlices; i++) {
@@ -394,6 +399,7 @@ public class OverseerCollectionProcessor
params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices);
ShardRequest sreq = new ShardRequest();
+ sreq.nodeName = nodeName;
params.set("qt", adminPath);
sreq.purpose = 1;
String replica = zkStateReader.getZkClient()
@@ -408,36 +414,25 @@ public class OverseerCollectionProcessor
}
}
- int failed = 0;
ShardResponse srsp;
do {
srsp = shardHandler.takeCompletedOrError();
if (srsp != null) {
- Throwable e = srsp.getException();
- if (e != null) {
- // should we retry?
- // TODO: we should return errors to the client
- // TODO: what if one fails and others succeed?
- failed++;
- log.error("Error talking to shard: " + srsp.getShard(), e);
- }
+ processResponse(results, srsp);
}
} while (srsp != null);
-
- // if all calls succeeded, return true
- if (failed > 0) {
- return false;
- }
- log.info("Successfully created all shards for collection "
+
+ log.info("Finished create command on all shards for collection: "
+ collectionName);
- return true;
+
+ } catch (SolrException ex) {
+ throw ex;
} catch (Exception ex) {
- // Expecting that the necessary logging has already been performed
- return false;
+ throw new SolrException(ErrorCode.SERVER_ERROR, null, ex);
}
}
- private boolean collectionCmd(ClusterState clusterState, ZkNodeProps message, ModifiableSolrParams params) {
+ private void collectionCmd(ClusterState clusterState, ZkNodeProps message, ModifiableSolrParams params, NamedList results, String stateMatcher) {
log.info("Executing Collection Cmd : " + params);
String collectionName = message.getStr("name");
@@ -454,7 +449,7 @@ public class OverseerCollectionProcessor
Set<Map.Entry<String,Replica>> shardEntries = shards.entrySet();
for (Map.Entry<String,Replica> shardEntry : shardEntries) {
final ZkNodeProps node = shardEntry.getValue();
- if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
+ if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP)) && (stateMatcher != null ? node.getStr(ZkStateReader.STATE_PROP).equals(stateMatcher) : true)) {
// For thread safety, only simple clone the ModifiableSolrParams
ModifiableSolrParams cloneParams = new ModifiableSolrParams();
cloneParams.add(params);
@@ -463,7 +458,7 @@ public class OverseerCollectionProcessor
String replica = node.getStr(ZkStateReader.BASE_URL_PROP);
ShardRequest sreq = new ShardRequest();
-
+ sreq.nodeName = node.getStr(ZkStateReader.NODE_NAME_PROP);
// yes, they must use same admin handler path everywhere...
cloneParams.set("qt", adminPath);
sreq.purpose = 1;
@@ -479,28 +474,39 @@ public class OverseerCollectionProcessor
}
}
- int failed = 0;
ShardResponse srsp;
do {
srsp = shardHandler.takeCompletedOrError();
if (srsp != null) {
- Throwable e = srsp.getException();
- if (e != null) {
- // should we retry?
- // TODO: we should return errors to the client
- // TODO: what if one fails and others succeed?
- failed++;
- log.error("Error talking to shard: " + srsp.getShard(), e);
- }
+ processResponse(results, srsp);
}
} while (srsp != null);
-
- // if all calls succeeded, return true
- if (failed > 0) {
- return false;
+ }
+
+ private void processResponse(NamedList results, ShardResponse srsp) {
+ Throwable e = srsp.getException();
+ if (e != null) {
+ log.error("Error from shard: " + srsp.getShard(), e);
+
+ SimpleOrderedMap failure = (SimpleOrderedMap) results.get("failure");
+ if (failure == null) {
+ failure = new SimpleOrderedMap();
+ results.add("failure", failure);
+ }
+
+ failure.add(srsp.getNodeName(), e.getClass().getName() + ":" + e.getMessage());
+
+ } else {
+
+ SimpleOrderedMap success = (SimpleOrderedMap) results.get("success");
+ if (success == null) {
+ success = new SimpleOrderedMap();
+ results.add("success", success);
+ }
+
+ success.add(srsp.getNodeName(), srsp.getSolrResponse().getResponse());
}
- return true;
}
private int msgStrToInt(ZkNodeProps message, String key, Integer def)
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java Wed Mar 20 16:27:13 2013
@@ -190,7 +190,7 @@ public class ZkCLI {
if (isXml) {
cfg = new ConfigSolrXmlBackCompat(loader, null, is, null, false);
} else {
- cfg = new SolrProperties(null, is, null);
+ cfg = new SolrProperties(null, loader, is, null);
}
} finally {
IOUtils.closeQuietly(is);
Modified: lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1458924&r1=1458923&r2=1458924&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene4258/solr/core/src/java/org/apache/solr/cloud/ZkController.java Wed Mar 20 16:27:13 2013
@@ -349,6 +349,12 @@ public final class ZkController {
}
try {
+ zkStateReader.close();
+ } catch(Throwable t) {
+ log.error("Error closing zkStateReader", t);
+ }
+
+ try {
zkClient.close();;
} catch(Throwable t) {
log.error("Error closing zkClient", t);
@@ -830,20 +836,20 @@ public final class ZkController {
.getCoreUrl();
// now wait until our currently cloud state contains the latest leader
- String clusterStateLeader = zkStateReader.getLeaderUrl(collection,
+ String clusterStateLeaderUrl = zkStateReader.getLeaderUrl(collection,
shardId, timeoutms * 2); // since we found it in zk, we are willing to
// wait a while to find it in state
int tries = 0;
- while (!leaderUrl.equals(clusterStateLeader)) {
+ while (!leaderUrl.equals(clusterStateLeaderUrl)) {
if (tries == 60) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"There is conflicting information about the leader of shard: "
+ cloudDesc.getShardId() + " our state says:"
- + clusterStateLeader + " but zookeeper says:" + leaderUrl);
+ + clusterStateLeaderUrl + " but zookeeper says:" + leaderUrl);
}
Thread.sleep(1000);
tries++;
- clusterStateLeader = zkStateReader.getLeaderUrl(collection, shardId,
+ clusterStateLeaderUrl = zkStateReader.getLeaderUrl(collection, shardId,
timeoutms);
leaderUrl = getLeaderProps(collection, cloudDesc.getShardId(), timeoutms)
.getCoreUrl();
@@ -852,7 +858,7 @@ public final class ZkController {
} catch (Exception e) {
log.error("Error getting leader from zk", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Error getting leader from zk", e);
+ "Error getting leader from zk for shard " + shardId, e);
}
return leaderUrl;
}
@@ -993,7 +999,9 @@ public final class ZkController {
: null,
ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName != null ? coreNodeName
: null);
- cd.getCloudDescriptor().lastPublished = state;
+ if (updateLastState) {
+ cd.getCloudDescriptor().lastPublished = state;
+ }
overseerJobQueue.offer(ZkStateReader.toJSON(m));
}
@@ -1013,17 +1021,17 @@ public final class ZkController {
public void unregister(String coreName, CoreDescriptor cd)
throws InterruptedException, KeeperException {
- ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
- "deletecore", ZkStateReader.CORE_NAME_PROP, coreName,
- ZkStateReader.NODE_NAME_PROP, getNodeName(),
- ZkStateReader.COLLECTION_PROP, cd.getCloudDescriptor().getCollectionName());
- overseerJobQueue.offer(ZkStateReader.toJSON(m));
-
final String zkNodeName = getCoreNodeName(cd);
ElectionContext context = electionContexts.remove(zkNodeName);
if (context != null) {
context.cancelElection();
}
+
+ ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
+ "deletecore", ZkStateReader.CORE_NAME_PROP, coreName,
+ ZkStateReader.NODE_NAME_PROP, getNodeName(),
+ ZkStateReader.COLLECTION_PROP, cd.getCloudDescriptor().getCollectionName());
+ overseerJobQueue.offer(ZkStateReader.toJSON(m));
}
public void createCollection(String collection) throws KeeperException,
@@ -1192,11 +1200,10 @@ public final class ZkController {
}
private String doGetShardIdProcess(String coreName, CoreDescriptor descriptor) {
- final String shardZkNodeName = getCoreNodeName(descriptor);
+ final String coreNodeName = getCoreNodeName(descriptor);
int retryCount = 320;
while (retryCount-- > 0) {
- final String shardId = zkStateReader.getClusterState().getShardId(
- shardZkNodeName);
+ final String shardId = zkStateReader.getClusterState().getShardId(coreNodeName);
if (shardId != null) {
return shardId;
}
@@ -1208,7 +1215,7 @@ public final class ZkController {
}
throw new SolrException(ErrorCode.SERVER_ERROR,
- "Could not get shard_id for core: " + coreName + " coreNodeName:" + shardZkNodeName);
+ "Could not get shard_id for core: " + coreName + " coreNodeName:" + coreNodeName);
}
public static void uploadToZK(SolrZkClient zkClient, File dir, String zkPath) throws IOException, KeeperException, InterruptedException {
@@ -1266,12 +1273,21 @@ public final class ZkController {
public void preRegister(CoreDescriptor cd) throws KeeperException, InterruptedException {
// before becoming available, make sure we are not live and active
// this also gets us our assigned shard id if it was not specified
- publish(cd, ZkStateReader.DOWN);
- String shardZkNodeName = getCoreNodeName(cd);
- if (cd.getCloudDescriptor().getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getClusterState(), shardZkNodeName)) {
+ publish(cd, ZkStateReader.DOWN, false);
+ String coreNodeName = getCoreNodeName(cd);
+
+ // make sure the node name is set on the descriptor
+ if (cd.getCloudDescriptor().getCoreNodeName() == null) {
+ cd.getCloudDescriptor().setCoreNodeName(coreNodeName);
+ }
+
+ if (cd.getCloudDescriptor().getShardId() == null && needsToBeAssignedShardId(cd, zkStateReader.getClusterState(), coreNodeName)) {
String shardId;
shardId = doGetShardIdProcess(cd.getName(), cd);
cd.getCloudDescriptor().setShardId(shardId);
+ } else {
+ // still wait till we see us in local state
+ doGetShardIdProcess(cd.getName(), cd);
}
}