You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2015/04/01 03:10:13 UTC
svn commit: r1670533 [3/4] - in /lucene/dev/branches/lucene6271: ./
dev-tools/ lucene/ lucene/analysis/ lucene/analysis/common/
lucene/backward-codecs/ lucene/benchmark/ lucene/classification/
lucene/codecs/ lucene/core/ lucene/core/src/java/org/apache...
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java Wed Apr 1 01:10:11 2015
@@ -106,7 +106,7 @@ public class TestNearSpansOrdered extend
}
public String s(Spans span) {
- return s(span.doc(), span.start(), span.end());
+ return s(span.docID(), span.startPosition(), span.endPosition());
}
public String s(int doc, int start, int end) {
return "s(" + doc + "," + start + "," + end +")";
@@ -114,12 +114,10 @@ public class TestNearSpansOrdered extend
public void testNearSpansNext() throws Exception {
SpanNearQuery q = makeQuery();
- Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
- assertEquals(true, span.next());
- assertEquals(s(0,0,3), s(span));
- assertEquals(true, span.next());
- assertEquals(s(1,0,4), s(span));
- assertEquals(false, span.next());
+ Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q);
+ TestSpans.tstNextSpans(span,0,0,3);
+ TestSpans.tstNextSpans(span,1,0,4);
+ TestSpans.tstEndSpans(span);
}
/**
@@ -127,51 +125,58 @@ public class TestNearSpansOrdered extend
* same as next -- it's only applicable in this case since we know doc
* does not contain more than one span
*/
- public void testNearSpansSkipToLikeNext() throws Exception {
+ public void testNearSpansAdvanceLikeNext() throws Exception {
SpanNearQuery q = makeQuery();
- Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
- assertEquals(true, span.skipTo(0));
+ Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q);
+ assertEquals(0, span.advance(0));
+ assertEquals(0, span.nextStartPosition());
assertEquals(s(0,0,3), s(span));
- assertEquals(true, span.skipTo(1));
+ assertEquals(1, span.advance(1));
+ assertEquals(0, span.nextStartPosition());
assertEquals(s(1,0,4), s(span));
- assertEquals(false, span.skipTo(2));
+ assertEquals(Spans.NO_MORE_DOCS, span.advance(2));
}
- public void testNearSpansNextThenSkipTo() throws Exception {
+ public void testNearSpansNextThenAdvance() throws Exception {
SpanNearQuery q = makeQuery();
- Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
- assertEquals(true, span.next());
+ Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q);
+ assertNotSame(Spans.NO_MORE_DOCS, span.nextDoc());
+ assertEquals(0, span.nextStartPosition());
assertEquals(s(0,0,3), s(span));
- assertEquals(true, span.skipTo(1));
+ assertNotSame(Spans.NO_MORE_DOCS, span.advance(1));
+ assertEquals(0, span.nextStartPosition());
assertEquals(s(1,0,4), s(span));
- assertEquals(false, span.next());
+ assertEquals(Spans.NO_MORE_DOCS, span.nextDoc());
}
- public void testNearSpansNextThenSkipPast() throws Exception {
+ public void testNearSpansNextThenAdvancePast() throws Exception {
SpanNearQuery q = makeQuery();
- Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
- assertEquals(true, span.next());
+ Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q);
+ assertNotSame(Spans.NO_MORE_DOCS, span.nextDoc());
+ assertEquals(0, span.nextStartPosition());
assertEquals(s(0,0,3), s(span));
- assertEquals(false, span.skipTo(2));
+ assertEquals(Spans.NO_MORE_DOCS, span.advance(2));
}
- public void testNearSpansSkipPast() throws Exception {
+ public void testNearSpansAdvancePast() throws Exception {
SpanNearQuery q = makeQuery();
- Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
- assertEquals(false, span.skipTo(2));
+ Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q);
+ assertEquals(Spans.NO_MORE_DOCS, span.advance(2));
}
- public void testNearSpansSkipTo0() throws Exception {
+ public void testNearSpansAdvanceTo0() throws Exception {
SpanNearQuery q = makeQuery();
- Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
- assertEquals(true, span.skipTo(0));
+ Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q);
+ assertEquals(0, span.advance(0));
+ assertEquals(0, span.nextStartPosition());
assertEquals(s(0,0,3), s(span));
}
- public void testNearSpansSkipTo1() throws Exception {
+ public void testNearSpansAdvanceTo1() throws Exception {
SpanNearQuery q = makeQuery();
- Spans span = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), q);
- assertEquals(true, span.skipTo(1));
+ Spans span = MultiSpansWrapper.wrap(searcher.getIndexReader(), q);
+ assertEquals(1, span.advance(1));
+ assertEquals(0, span.nextStartPosition());
assertEquals(s(1,0,4), s(span));
}
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java Wed Apr 1 01:10:11 2015
@@ -67,12 +67,12 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq;
Spans spans;
stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "seventy"));
- spans = MultiSpansWrapper.wrap(indexReader.getContext(), stq);
+ spans = MultiSpansWrapper.wrap(indexReader, stq);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 100, 1, 1, 1);
stq = new SpanTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "seventy"));
- spans = MultiSpansWrapper.wrap(indexReader.getContext(), stq);
+ spans = MultiSpansWrapper.wrap(indexReader, stq);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 100, 0, 0, 0);
}
@@ -83,7 +83,7 @@ public class TestPayloadSpans extends Lu
SpanFirstQuery sfq;
match = new SpanTermQuery(new Term(PayloadHelper.FIELD, "one"));
sfq = new SpanFirstQuery(match, 2);
- Spans spans = MultiSpansWrapper.wrap(indexReader.getContext(), sfq);
+ Spans spans = MultiSpansWrapper.wrap(indexReader, sfq);
checkSpans(spans, 109, 1, 1, 1);
//Test more complicated subclause
SpanQuery[] clauses = new SpanQuery[2];
@@ -91,11 +91,11 @@ public class TestPayloadSpans extends Lu
clauses[1] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "hundred"));
match = new SpanNearQuery(clauses, 0, true);
sfq = new SpanFirstQuery(match, 2);
- checkSpans(MultiSpansWrapper.wrap(indexReader.getContext(), sfq), 100, 2, 1, 1);
+ checkSpans(MultiSpansWrapper.wrap(indexReader, sfq), 100, 2, 1, 1);
match = new SpanNearQuery(clauses, 0, false);
sfq = new SpanFirstQuery(match, 2);
- checkSpans(MultiSpansWrapper.wrap(indexReader.getContext(), sfq), 100, 2, 1, 1);
+ checkSpans(MultiSpansWrapper.wrap(indexReader, sfq), 100, 2, 1, 1);
}
@@ -119,7 +119,7 @@ public class TestPayloadSpans extends Lu
writer.close();
- checkSpans(MultiSpansWrapper.wrap(reader.getContext(), snq), 1,new int[]{2});
+ checkSpans(MultiSpansWrapper.wrap(reader, snq), 1,new int[]{2});
reader.close();
directory.close();
}
@@ -129,10 +129,8 @@ public class TestPayloadSpans extends Lu
Spans spans;
IndexSearcher searcher = getSearcher();
stq = new SpanTermQuery(new Term(PayloadHelper.FIELD, "mark"));
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), stq);
- assertTrue("spans is null and it shouldn't be", spans != null);
- checkSpans(spans, 0, null);
-
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), stq);
+ assertNull(spans);
SpanQuery[] clauses = new SpanQuery[3];
clauses[0] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "rr"));
@@ -140,7 +138,7 @@ public class TestPayloadSpans extends Lu
clauses[2] = new SpanTermQuery(new Term(PayloadHelper.FIELD, "xx"));
SpanNearQuery spanNearQuery = new SpanNearQuery(clauses, 12, false);
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), spanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 2, new int[]{3,3});
@@ -151,7 +149,7 @@ public class TestPayloadSpans extends Lu
spanNearQuery = new SpanNearQuery(clauses, 6, true);
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), spanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), spanNearQuery);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 1, new int[]{3});
@@ -174,7 +172,7 @@ public class TestPayloadSpans extends Lu
// yy within 6 of xx within 6 of rr
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), nestedSpanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 2, new int[]{3,3});
closeIndexReader.close();
@@ -205,7 +203,7 @@ public class TestPayloadSpans extends Lu
clauses3[1] = snq;
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), nestedSpanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 1, new int[]{3});
@@ -243,7 +241,7 @@ public class TestPayloadSpans extends Lu
SpanNearQuery nestedSpanNearQuery = new SpanNearQuery(clauses3, 6, false);
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), nestedSpanNearQuery);
+ spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), nestedSpanNearQuery);
assertTrue("spans is null and it shouldn't be", spans != null);
checkSpans(spans, 2, new int[]{8, 8});
closeIndexReader.close();
@@ -267,16 +265,18 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 1, true);
- Spans spans = MultiSpansWrapper.wrap(is.getTopReaderContext(), snq);
+ Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
TopDocs topDocs = is.search(snq, 1);
Set<String> payloadSet = new HashSet<>();
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- while (spans.next()) {
- Collection<byte[]> payloads = spans.getPayload();
-
- for (final byte [] payload : payloads) {
- payloadSet.add(new String(payload, StandardCharsets.UTF_8));
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ Collection<byte[]> payloads = spans.getPayload();
+
+ for (final byte [] payload : payloads) {
+ payloadSet.add(new String(payload, StandardCharsets.UTF_8));
+ }
}
}
}
@@ -303,15 +303,18 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
- Spans spans = MultiSpansWrapper.wrap(is.getTopReaderContext(), snq);
+ Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
TopDocs topDocs = is.search(snq, 1);
Set<String> payloadSet = new HashSet<>();
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- while (spans.next()) {
- Collection<byte[]> payloads = spans.getPayload();
- for (final byte[] payload : payloads) {
- payloadSet.add(new String(payload, StandardCharsets.UTF_8));
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ Collection<byte[]> payloads = spans.getPayload();
+
+ for (final byte [] payload : payloads) {
+ payloadSet.add(new String(payload, StandardCharsets.UTF_8));
+ }
}
}
}
@@ -338,16 +341,18 @@ public class TestPayloadSpans extends Lu
SpanTermQuery stq2 = new SpanTermQuery(new Term("content", "k"));
SpanQuery[] sqs = { stq1, stq2 };
SpanNearQuery snq = new SpanNearQuery(sqs, 0, true);
- Spans spans = MultiSpansWrapper.wrap(is.getTopReaderContext(), snq);
+ Spans spans = MultiSpansWrapper.wrap(is.getIndexReader(), snq);
TopDocs topDocs = is.search(snq, 1);
Set<String> payloadSet = new HashSet<>();
for (int i = 0; i < topDocs.scoreDocs.length; i++) {
- while (spans.next()) {
- Collection<byte[]> payloads = spans.getPayload();
-
- for (final byte [] payload : payloads) {
- payloadSet.add(new String(payload, StandardCharsets.UTF_8));
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ Collection<byte[]> payloads = spans.getPayload();
+
+ for (final byte [] payload : payloads) {
+ payloadSet.add(new String(payload, StandardCharsets.UTF_8));
+ }
}
}
}
@@ -395,31 +400,22 @@ public class TestPayloadSpans extends Lu
//each position match should have a span associated with it, since there is just one underlying term query, there should
//only be one entry in the span
int seen = 0;
- while (spans.next() == true)
- {
- //if we expect payloads, then isPayloadAvailable should be true
- if (expectedNumPayloads > 0) {
- assertTrue("isPayloadAvailable is not returning the correct value: " + spans.isPayloadAvailable()
- + " and it should be: " + (expectedNumPayloads > 0),
- spans.isPayloadAvailable() == true);
- } else {
- assertTrue("isPayloadAvailable should be false", spans.isPayloadAvailable() == false);
- }
- //See payload helper, for the PayloadHelper.FIELD field, there is a single byte payload at every token
- if (spans.isPayloadAvailable()) {
- Collection<byte[]> payload = spans.getPayload();
- assertTrue("payload Size: " + payload.size() + " is not: " + expectedNumPayloads, payload.size() == expectedNumPayloads);
- for (final byte [] thePayload : payload) {
- assertTrue("payload[0] Size: " + thePayload.length + " is not: " + expectedPayloadLength,
- thePayload.length == expectedPayloadLength);
- assertTrue(thePayload[0] + " does not equal: " + expectedFirstByte, thePayload[0] == expectedFirstByte);
-
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ assertEquals("isPayloadAvailable should return true/false as payloads are expected", expectedNumPayloads > 0, spans.isPayloadAvailable());
+ //See payload helper, for the PayloadHelper.FIELD field, there is a single byte payload at every token
+ if (spans.isPayloadAvailable()) {
+ Collection<byte[]> payload = spans.getPayload();
+ assertEquals("payload size", expectedNumPayloads, payload.size());
+ for (final byte [] thePayload : payload) {
+ assertEquals("payload length", expectedPayloadLength, thePayload.length);
+ assertEquals("payload first byte", expectedFirstByte, thePayload[0]);
+ }
}
-
+ seen++;
}
- seen++;
}
- assertTrue(seen + " does not equal: " + expectedNumSpans, seen == expectedNumSpans);
+ assertEquals("expectedNumSpans", expectedNumSpans, seen);
}
private IndexSearcher getSearcher() throws Exception {
@@ -446,27 +442,28 @@ public class TestPayloadSpans extends Lu
private void checkSpans(Spans spans, int numSpans, int[] numPayloads) throws IOException {
int cnt = 0;
- while (spans.next() == true) {
- if(VERBOSE)
- System.out.println("\nSpans Dump --");
- if (spans.isPayloadAvailable()) {
- Collection<byte[]> payload = spans.getPayload();
- if(VERBOSE) {
- System.out.println("payloads for span:" + payload.size());
- for (final byte [] bytes : payload) {
- System.out.println("doc:" + spans.doc() + " s:" + spans.start() + " e:" + spans.end() + " "
- + new String(bytes, StandardCharsets.UTF_8));
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ if(VERBOSE)
+ System.out.println("\nSpans Dump --");
+ if (spans.isPayloadAvailable()) {
+ Collection<byte[]> payload = spans.getPayload();
+ if(VERBOSE) {
+ System.out.println("payloads for span:" + payload.size());
+ for (final byte [] bytes : payload) {
+ System.out.println("doc:" + spans.docID() + " s:" + spans.startPosition() + " e:" + spans.endPosition() + " "
+ + new String(bytes, StandardCharsets.UTF_8));
+ }
}
+ assertEquals("payload size", numPayloads[cnt], payload.size());
+ } else { // no payload available
+ assertFalse("Expected spans:" + numPayloads[cnt] + " found: 0", numPayloads.length > 0 && numPayloads[cnt] > 0 );
}
-
- assertEquals(numPayloads[cnt],payload.size());
- } else {
- assertFalse("Expected spans:" + numPayloads[cnt] + " found: 0",numPayloads.length > 0 && numPayloads[cnt] > 0 );
+ cnt++;
}
- cnt++;
}
- assertEquals(numSpans, cnt);
+ assertEquals("expected numSpans", numSpans, cnt);
}
final class PayloadAnalyzer extends Analyzer {
Modified: lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/lucene6271/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java Wed Apr 1 01:10:11 2015
@@ -22,7 +22,6 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.IndexWriter;
@@ -201,117 +200,55 @@ public class TestSpans extends LuceneTes
makeSpanTermQuery("t3") },
slop,
ordered);
- Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
+ Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), snq);
- assertTrue("first range", spans.next());
- assertEquals("first doc", 11, spans.doc());
- assertEquals("first start", 0, spans.start());
- assertEquals("first end", 4, spans.end());
-
- assertTrue("second range", spans.next());
- assertEquals("second doc", 11, spans.doc());
- assertEquals("second start", 2, spans.start());
- assertEquals("second end", 6, spans.end());
+ assertEquals("first doc", 11, spans.nextDoc());
+ assertEquals("first start", 0, spans.nextStartPosition());
+ assertEquals("first end", 4, spans.endPosition());
- assertFalse("third range", spans.next());
- }
+ assertEquals("second start", 2, spans.nextStartPosition());
+ assertEquals("second end", 6, spans.endPosition());
+ tstEndSpans(spans);
+ }
public void testSpanNearUnOrdered() throws Exception {
-
//See http://www.gossamer-threads.com/lists/lucene/java-dev/52270 for discussion about this test
- SpanNearQuery snq;
- snq = new SpanNearQuery(
+ SpanNearQuery senq;
+ senq = new SpanNearQuery(
new SpanQuery[] {
makeSpanTermQuery("u1"),
makeSpanTermQuery("u2") },
0,
false);
- Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 4, spans.doc());
- assertEquals("start", 1, spans.start());
- assertEquals("end", 3, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 5, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 8, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 9, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 10, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
- assertTrue("Has next and it shouldn't: " + spans.doc(), spans.next() == false);
+ Spans spans = MultiSpansWrapper.wrap(reader, senq);
+ tstNextSpans(spans, 4, 1, 3);
+ tstNextSpans(spans, 5, 2, 4);
+ tstNextSpans(spans, 8, 2, 4);
+ tstNextSpans(spans, 9, 0, 2);
+ tstNextSpans(spans, 10, 0, 2);
+ tstEndSpans(spans);
SpanNearQuery u1u2 = new SpanNearQuery(new SpanQuery[]{makeSpanTermQuery("u1"),
makeSpanTermQuery("u2")}, 0, false);
- snq = new SpanNearQuery(
+ senq = new SpanNearQuery(
new SpanQuery[] {
u1u2,
makeSpanTermQuery("u2")
},
1,
false);
- spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 4, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 3, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- //unordered spans can be subsets
- assertEquals("doc", 4, spans.doc());
- assertEquals("start", 1, spans.start());
- assertEquals("end", 3, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 5, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 5, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 8, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 4, spans.end());
-
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 8, spans.doc());
- assertEquals("start", 2, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 9, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 9, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 4, spans.end());
-
- assertTrue("Does not have next and it should", spans.next());
- assertEquals("doc", 10, spans.doc());
- assertEquals("start", 0, spans.start());
- assertEquals("end", 2, spans.end());
-
- assertTrue("Has next and it shouldn't", spans.next() == false);
+ spans = MultiSpansWrapper.wrap(reader, senq);
+ tstNextSpans(spans, 4, 0, 3);
+ tstNextSpans(spans, 4, 1, 3); // unordered spans can be subsets
+ tstNextSpans(spans, 5, 0, 4);
+ tstNextSpans(spans, 5, 2, 4);
+ tstNextSpans(spans, 8, 0, 4);
+ tstNextSpans(spans, 8, 2, 4);
+ tstNextSpans(spans, 9, 0, 2);
+ tstNextSpans(spans, 9, 0, 4);
+ tstNextSpans(spans, 10, 0, 2);
+ tstEndSpans(spans);
}
@@ -321,21 +258,40 @@ public class TestSpans extends LuceneTes
for (int i = 0; i < terms.length; i++) {
sqa[i] = makeSpanTermQuery(terms[i]);
}
- return MultiSpansWrapper.wrap(searcher.getTopReaderContext(), new SpanOrQuery(sqa));
+ return MultiSpansWrapper.wrap(searcher.getIndexReader(), new SpanOrQuery(sqa));
}
- private void tstNextSpans(Spans spans, int doc, int start, int end)
- throws Exception {
- assertTrue("next", spans.next());
- assertEquals("doc", doc, spans.doc());
- assertEquals("start", start, spans.start());
- assertEquals("end", end, spans.end());
+ public static void tstNextSpans(Spans spans, int doc, int start, int end) throws IOException {
+ if (spans.docID() >= doc) {
+ assertEquals("docId", doc, spans.docID());
+ } else { // nextDoc needed before testing start/end
+ if (spans.docID() >= 0) {
+ assertEquals("nextStartPosition of previous doc", Spans.NO_MORE_POSITIONS, spans.nextStartPosition());
+ assertEquals("endPosition of previous doc", Spans.NO_MORE_POSITIONS, spans.endPosition());
+ }
+ assertEquals("nextDoc", doc, spans.nextDoc());
+ if (doc != Spans.NO_MORE_DOCS) {
+ assertEquals("first startPosition", -1, spans.startPosition());
+ assertEquals("first endPosition", -1, spans.endPosition());
+ }
+ }
+ if (doc != Spans.NO_MORE_DOCS) {
+ assertEquals("nextStartPosition", start, spans.nextStartPosition());
+ assertEquals("startPosition", start, spans.startPosition());
+ assertEquals("endPosition", end, spans.endPosition());
+ }
+ }
+
+ public static void tstEndSpans(Spans spans) throws Exception {
+ if (spans != null) { // null Spans is empty
+ tstNextSpans(spans, Spans.NO_MORE_DOCS, -2, -2); // start and end positions will be ignored
+ }
}
public void testSpanOrEmpty() throws Exception {
Spans spans = orSpans(new String[0]);
- assertFalse("empty next", spans.next());
-
+ tstEndSpans(spans);
+
SpanOrQuery a = new SpanOrQuery();
SpanOrQuery b = new SpanOrQuery();
assertTrue("empty should equal", a.equals(b));
@@ -344,24 +300,7 @@ public class TestSpans extends LuceneTes
public void testSpanOrSingle() throws Exception {
Spans spans = orSpans(new String[] {"w5"});
tstNextSpans(spans, 0, 4, 5);
- assertFalse("final next", spans.next());
- }
-
- public void testSpanOrMovesForward() throws Exception {
- Spans spans = orSpans(new String[] {"w1", "xx"});
-
- spans.next();
- int doc = spans.doc();
- assertEquals(0, doc);
-
- spans.skipTo(0);
- doc = spans.doc();
-
- // LUCENE-1583:
- // according to Spans, a skipTo to the same doc or less
- // should still call next() on the underlying Spans
- assertEquals(1, doc);
-
+ tstEndSpans(spans);
}
public void testSpanOrDouble() throws Exception {
@@ -370,17 +309,15 @@ public class TestSpans extends LuceneTes
tstNextSpans(spans, 2, 3, 4);
tstNextSpans(spans, 3, 4, 5);
tstNextSpans(spans, 7, 3, 4);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
- public void testSpanOrDoubleSkip() throws Exception {
+ public void testSpanOrDoubleAdvance() throws Exception {
Spans spans = orSpans(new String[] {"w5", "yy"});
- assertTrue("initial skipTo", spans.skipTo(3));
- assertEquals("doc", 3, spans.doc());
- assertEquals("start", 4, spans.start());
- assertEquals("end", 5, spans.end());
+ assertEquals("initial advance", 3, spans.advance(3));
+ tstNextSpans(spans, 3, 4, 5);
tstNextSpans(spans, 7, 3, 4);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
public void testSpanOrUnused() throws Exception {
@@ -389,7 +326,7 @@ public class TestSpans extends LuceneTes
tstNextSpans(spans, 2, 3, 4);
tstNextSpans(spans, 3, 4, 5);
tstNextSpans(spans, 7, 3, 4);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
public void testSpanOrTripleSameDoc() throws Exception {
@@ -400,7 +337,7 @@ public class TestSpans extends LuceneTes
tstNextSpans(spans, 11, 3, 4);
tstNextSpans(spans, 11, 4, 5);
tstNextSpans(spans, 11, 5, 6);
- assertFalse("final next", spans.next());
+ tstEndSpans(spans);
}
public void testSpanScorerZeroSloppyFreq() throws Exception {
@@ -439,8 +376,8 @@ public class TestSpans extends LuceneTes
assertEquals("first doc number", spanScorer.docID() + ctx.docBase, 11);
float score = spanScorer.score();
assertTrue("first doc score should be zero, " + score, score == 0.0f);
- } else {
- assertTrue("no second doc", spanScorer.nextDoc() == DocIdSetIterator.NO_MORE_DOCS);
+ } else {
+ assertTrue("no second doc", spanScorer == null || spanScorer.nextDoc() == DocIdSetIterator.NO_MORE_DOCS);
}
}
}
@@ -542,11 +479,15 @@ public class TestSpans extends LuceneTes
SpanTermQuery iq = new SpanTermQuery(new Term(field, include));
SpanTermQuery eq = new SpanTermQuery(new Term(field, exclude));
SpanNotQuery snq = new SpanNotQuery(iq, eq, pre, post);
- Spans spans = MultiSpansWrapper.wrap(searcher.getTopReaderContext(), snq);
+ Spans spans = MultiSpansWrapper.wrap(searcher.getIndexReader(), snq);
int i = 0;
- while (spans.next()){
- i++;
+ if (spans != null) {
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS){
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ i++;
+ }
+ }
}
return i;
}
Modified: lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java (original)
+++ lucene/dev/branches/lucene6271/lucene/facet/src/java/org/apache/lucene/facet/RandomSamplingFacetsCollector.java Wed Apr 1 01:10:11 2015
@@ -215,7 +215,7 @@ public class RandomSamplingFacetsCollect
return new MatchingDocs(docs.context, new BitDocIdSet(sampleDocs), docs.totalHits, null);
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
Modified: lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/lucene6271/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Wed Apr 1 01:10:11 2015
@@ -308,10 +308,11 @@ public class WeightedSpanTermExtractor {
final Spans spans = q.getSpans(context, acceptDocs, termContexts);
// collect span positions
- while (spans.next()) {
- spanPositions.add(new PositionSpan(spans.start(), spans.end() - 1));
+ while (spans.nextDoc() != Spans.NO_MORE_DOCS) {
+ while (spans.nextStartPosition() != Spans.NO_MORE_POSITIONS) {
+ spanPositions.add(new PositionSpan(spans.startPosition(), spans.endPosition() - 1));
+ }
}
-
}
if (spanPositions.size() == 0) {
Modified: lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java (original)
+++ lucene/dev/branches/lucene6271/lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestMultiTermHighlighting.java Wed Apr 1 01:10:11 2015
@@ -681,7 +681,7 @@ public class TestMultiTermHighlighting e
}
};
SpanQuery childQuery = new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term("body", "te*")));
- Query query = new SpanNearQuery(new SpanQuery[] { childQuery }, 0, true);
+ Query query = new SpanNearQuery(new SpanQuery[] { childQuery, childQuery }, 0, false);
TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.totalHits);
String snippets[] = highlighter.highlight("body", query, searcher, topDocs);
Modified: lucene/dev/branches/lucene6271/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java (original)
+++ lucene/dev/branches/lucene6271/lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java Wed Apr 1 01:10:11 2015
@@ -895,7 +895,7 @@ public class DocTermOrds implements Acco
try {
return getOrdTermsEnum(reader);
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
}
Modified: lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java (original)
+++ lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java Wed Apr 1 01:10:11 2015
@@ -145,7 +145,7 @@ public class DocumentValueSourceDictiona
try {
currentWeightValues = weightsValueSource.getValues(new HashMap<String, Object>(), leaves.get(currentLeafIndex));
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
return currentWeightValues.longVal(docId - starts[subIndex]);
Modified: lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java (original)
+++ lucene/dev/branches/lucene6271/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java Wed Apr 1 01:10:11 2015
@@ -113,7 +113,7 @@ public class FileDictionary implements D
try {
return new FileIterator();
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
Modified: lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (original)
+++ lucene/dev/branches/lucene6271/lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java Wed Apr 1 01:10:11 2015
@@ -2681,7 +2681,7 @@ public abstract class BaseDocValuesForma
}
}
} catch (Throwable e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
};
Modified: lucene/dev/branches/lucene6271/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/CHANGES.txt?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/CHANGES.txt (original)
+++ lucene/dev/branches/lucene6271/solr/CHANGES.txt Wed Apr 1 01:10:11 2015
@@ -58,6 +58,9 @@ Other Changes
* SOLR-6954: Deprecated SolrClient.shutdown() method removed (Alan Woodward)
+================== 5.2.0 ==================
+(No Changes)
+
================== 5.1.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
@@ -91,15 +94,6 @@ Upgrading from Solr 5.0
contain some LGPL-only code. Until that's resolved by Tika, you can download the
.jar yourself and place it under contrib/extraction/lib.
-* A twitter engineer discovered a JVM bug that causes GC pause problems. The
- workaround for those problems makes certain functionality impossible, such as
- running the jstat program on your Solr instance. That workaround has been
- implemented in the bin/solr start scripts. If you need the missing java
- functionality, delete the "-XX:+PerfDisableSharedMem" parameter from
- bin/solr.in.sh or bin/solr.in.cmd.
-
- http://www.evanjones.ca/jvm-mmap-pause.html
-
Detailed Change List
----------------------
@@ -350,11 +344,8 @@ Bug Fixes
* SOLR-7309: Make bin/solr, bin/post work when Solr installation directory contains spaces
(Ramkumar Aiyengar, Martijn Koster)
-* SOLR-7319: Workaround for the "Four Month Bug" GC pause problem discovered
- by a twitter software engineer. This causes GC pauses when JVM statistics
- are left enabled and there is heavy MMAP write activity.
- http://www.evanjones.ca/jvm-mmap-pause.html
- (Shawn Heisey)
+* SOLR-6924: The config API forcefully refreshes all replicas in the collection to ensure all are
+ updated (Noble Paul)
Optimizations
----------------------
@@ -368,6 +359,9 @@ Optimizations
* SOLR-7239: improved performance of min & max in StatsComponent, as well as situations
where local params disable all stats (hossman)
+ * SOLR-7324: IndexFetcher does not need to call isIndexStale if full copy is already needed
+ (Stephan Lagraulet via Varun Thacker)
+
Other Changes
----------------------
Modified: lucene/dev/branches/lucene6271/solr/bin/solr.in.cmd
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/bin/solr.in.cmd?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/bin/solr.in.cmd (original)
+++ lucene/dev/branches/lucene6271/solr/bin/solr.in.cmd Wed Apr 1 01:10:11 2015
@@ -40,7 +40,6 @@ set GC_TUNE=-XX:NewRatio=3 ^
-XX:+UseCMSInitiatingOccupancyOnly ^
-XX:CMSInitiatingOccupancyFraction=50 ^
-XX:CMSMaxAbortablePrecleanTime=6000 ^
- -XX:+PerfDisableSharedMem ^
-XX:+CMSParallelRemarkEnabled ^
-XX:+ParallelRefProcEnabled
Modified: lucene/dev/branches/lucene6271/solr/bin/solr.in.sh
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/bin/solr.in.sh?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/bin/solr.in.sh (original)
+++ lucene/dev/branches/lucene6271/solr/bin/solr.in.sh Wed Apr 1 01:10:11 2015
@@ -37,7 +37,6 @@ GC_TUNE="-XX:NewRatio=3 \
-XX:PretenureSizeThreshold=64m \
-XX:+UseCMSInitiatingOccupancyOnly \
-XX:CMSInitiatingOccupancyFraction=50 \
--XX:+PerfDisableSharedMem \
-XX:CMSMaxAbortablePrecleanTime=6000 \
-XX:+CMSParallelRemarkEnabled \
-XX:+ParallelRefProcEnabled"
Modified: lucene/dev/branches/lucene6271/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java (original)
+++ lucene/dev/branches/lucene6271/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/RegexRulesPasswordProvider.java Wed Apr 1 01:10:11 2015
@@ -103,7 +103,7 @@ public class RegexRulesPasswordProvider
}
is.close();
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
return rules;
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/cloud/ZkController.java Wed Apr 1 01:10:11 2015
@@ -2175,9 +2175,10 @@ public final class ZkController {
*
* @return true on success
*/
- public static boolean persistConfigResourceToZooKeeper(ZkSolrResourceLoader zkLoader, int znodeVersion,
+ public static int persistConfigResourceToZooKeeper(ZkSolrResourceLoader zkLoader, int znodeVersion,
String resourceName, byte[] content,
boolean createIfNotExists) {
+ int latestVersion = znodeVersion;
final ZkController zkController = zkLoader.getZkController();
final SolrZkClient zkClient = zkController.getZkClient();
final String resourceLocation = zkLoader.getConfigSetZkPath() + "/" + resourceName;
@@ -2185,17 +2186,19 @@ public final class ZkController {
try {
try {
zkClient.setData(resourceLocation, content, znodeVersion, true);
+ latestVersion = znodeVersion + 1;// if the set succeeded , it should have incremented the version by one always
log.info("Persisted config data to node {} ", resourceLocation);
touchConfDir(zkLoader);
} catch (NoNodeException e) {
if (createIfNotExists) {
try {
zkClient.create(resourceLocation, content, CreateMode.PERSISTENT, true);
+ latestVersion = 0;//just created so version must be zero
touchConfDir(zkLoader);
} catch (KeeperException.NodeExistsException nee) {
try {
Stat stat = zkClient.exists(resourceLocation, null, true);
- log.info("failed to set data version in zk is {0} and expected version is {1} ", stat.getVersion(), znodeVersion);
+ log.info("failed to set data version in zk is {} and expected version is {} ", stat.getVersion(), znodeVersion);
} catch (Exception e1) {
log.warn("could not get stat");
}
@@ -2227,7 +2230,7 @@ public final class ZkController {
log.error(msg, e);
throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
}
- return true;
+ return latestVersion;
}
public static void touchConfDir(ZkSolrResourceLoader zkLoader) {
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/ConfigOverlay.java Wed Apr 1 01:10:11 2015
@@ -187,14 +187,14 @@ public class ConfigOverlay implements Ma
public static final String RESOURCE_NAME = "configoverlay.json";
- private static final Long STR_ATTR = 0L;
+ /*private static final Long STR_ATTR = 0L;
private static final Long STR_NODE = 1L;
private static final Long BOOL_ATTR = 10L;
private static final Long BOOL_NODE = 11L;
private static final Long INT_ATTR = 20L;
private static final Long INT_NODE = 21L;
private static final Long FLOAT_ATTR = 30L;
- private static final Long FLOAT_NODE = 31L;
+ private static final Long FLOAT_NODE = 31L;*/
private static Map editable_prop_map;
//The path maps to the xml xpath and value of 1 means it is a tag with a string value and value
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/RequestParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/RequestParams.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/RequestParams.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/RequestParams.java Wed Apr 1 01:10:11 2015
@@ -148,6 +148,7 @@ public class RequestParams implements Ma
ZkSolrResourceLoader resourceLoader = (ZkSolrResourceLoader) loader;
try {
Stat stat = resourceLoader.getZkController().getZkClient().exists(resourceLoader.getConfigSetZkPath() + "/" + RequestParams.RESOURCE, null, true);
+ log.debug("latest version of {} in ZK is : {}", resourceLoader.getConfigSetZkPath() + "/" + RequestParams.RESOURCE, stat == null ? "": stat.getVersion());
if (stat == null) {
requestParams = new RequestParams(Collections.EMPTY_MAP, -1);
} else if (requestParams == null || stat.getVersion() > requestParams.getZnodeVersion()) {
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrConfig.java Wed Apr 1 01:10:11 2015
@@ -77,6 +77,7 @@ import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
import static org.apache.solr.core.SolrConfig.PluginOpts.LAZY;
import static org.apache.solr.core.SolrConfig.PluginOpts.MULTI_OK;
import static org.apache.solr.core.SolrConfig.PluginOpts.NOOP;
@@ -819,7 +820,7 @@ public class SolrConfig extends Config i
@Override
public Map<String, Object> toMap() {
LinkedHashMap result = new LinkedHashMap();
- if (getZnodeVersion() > -1) result.put("znodeVersion", getZnodeVersion());
+ if (getZnodeVersion() > -1) result.put(ZNODEVER, getZnodeVersion());
result.put("luceneMatchVersion", luceneMatchVersion);
result.put("updateHandler", getUpdateHandlerInfo().toMap());
Map m = new LinkedHashMap();
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrCore.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/core/SolrCore.java Wed Apr 1 01:10:11 2015
@@ -66,6 +66,7 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.cloud.CloudDescriptor;
@@ -89,6 +90,7 @@ import org.apache.solr.handler.RequestHa
import org.apache.solr.handler.admin.ShowFileRequestHandler;
import org.apache.solr.handler.component.HighlightComponent;
import org.apache.solr.handler.component.SearchComponent;
+import org.apache.solr.logging.MDCUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.BinaryResponseWriter;
@@ -132,7 +134,6 @@ import org.apache.solr.update.processor.
import org.apache.solr.update.processor.UpdateRequestProcessorChain;
import org.apache.solr.update.processor.UpdateRequestProcessorChain.ProcessorInfo;
import org.apache.solr.update.processor.UpdateRequestProcessorFactory;
-import org.apache.solr.util.ConcurrentLRUCache;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.PropertiesInputStream;
import org.apache.solr.util.RefCounted;
@@ -691,6 +692,8 @@ public final class SolrCore implements S
public SolrCore(String name, String dataDir, SolrConfig config, IndexSchema schema, CoreDescriptor cd, UpdateHandler updateHandler, IndexDeletionPolicyWrapper delPolicy, SolrCore prev) {
coreDescriptor = cd;
this.setName( name );
+ MDCUtils.setCore(name); // show the core name in the error logs
+
resourceLoader = config.getResourceLoader();
this.solrConfig = config;
@@ -2077,13 +2080,13 @@ public final class SolrCore implements S
HashMap<String, QueryResponseWriter> m= new HashMap<>();
m.put("xml", new XMLResponseWriter());
m.put("standard", m.get("xml"));
- m.put("json", new JSONResponseWriter());
+ m.put(CommonParams.JSON, new JSONResponseWriter());
m.put("python", new PythonResponseWriter());
m.put("php", new PHPResponseWriter());
m.put("phps", new PHPSerializedResponseWriter());
m.put("ruby", new RubyResponseWriter());
m.put("raw", new RawResponseWriter());
- m.put("javabin", new BinaryResponseWriter());
+ m.put(CommonParams.JAVABIN, new BinaryResponseWriter());
m.put("csv", new CSVResponseWriter());
m.put("xsort", new SortingResponseWriter());
m.put("schema.xml", new SchemaXmlResponseWriter());
@@ -2463,12 +2466,12 @@ public final class SolrCore implements S
zkSolrResourceLoader.getZkController().registerConfListenerForCore(
zkSolrResourceLoader.getConfigSetZkPath(),
this,
- getListener(this, zkSolrResourceLoader));
+ getConfListener(this, zkSolrResourceLoader));
}
- private static Runnable getListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
+ public static Runnable getConfListener(SolrCore core, ZkSolrResourceLoader zkSolrResourceLoader) {
final String coreName = core.getName();
final CoreContainer cc = core.getCoreDescriptor().getCoreContainer();
final String overlayPath = zkSolrResourceLoader.getConfigSetZkPath() + "/" + ConfigOverlay.RESOURCE_NAME;
@@ -2506,9 +2509,7 @@ public final class SolrCore implements S
cc.reload(coreName);
return;
}
- //some files in conf directoy has changed other than schema.xml,
- // solrconfig.xml. so fire event listeners
-
+ //some files in conf directory may have other than managedschema, overlay, params
try (SolrCore core = cc.solrCores.getCoreFromAnyList(coreName, true)) {
if (core == null || core.isClosed()) return;
for (Runnable listener : core.confListeners) {
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java Wed Apr 1 01:10:11 2015
@@ -362,8 +362,10 @@ public class IndexFetcher {
indexDir = core.getDirectoryFactory().get(indexDirPath, DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
try {
-
- if (isIndexStale(indexDir)) {
+
+ //We will compare all the index files from the master vs the index files on disk to see if there is a mismatch
+ //in the metadata. If there is a mismatch for the same index file then we download the entire index again.
+ if (!isFullCopyNeeded && isIndexStale(indexDir)) {
isFullCopyNeeded = true;
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java Wed Apr 1 01:10:11 2015
@@ -21,6 +21,7 @@ package org.apache.solr.handler;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -29,35 +30,63 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import com.google.common.collect.ImmutableSet;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.cloud.ZkCLI;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.ConfigOverlay;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.core.ImplicitPlugins;
import org.apache.solr.core.RequestParams;
import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
+import org.apache.solr.response.BinaryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaManager;
import org.apache.solr.util.CommandOperation;
+import org.apache.solr.util.DefaultSolrThreadFactory;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonList;
+import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
import static org.apache.solr.common.params.CoreAdminParams.NAME;
import static org.apache.solr.common.util.StrUtils.formatString;
import static org.apache.solr.core.ConfigOverlay.NOT_EDITABLE;
+import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_CLASS;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME_IN_OVERLAY;
@@ -67,6 +96,7 @@ public class SolrConfigHandler extends R
public static final Logger log = LoggerFactory.getLogger(SolrConfigHandler.class);
public static final boolean configEditing_disabled = Boolean.getBoolean("disable.configEdit");
private static final Map<String, SolrConfig.SolrPluginInfo> namedPlugins;
+ private Lock reloadLock = new ReentrantLock(true);
static {
Map<String, SolrConfig.SolrPluginInfo> map = new HashMap<>();
@@ -99,7 +129,7 @@ public class SolrConfigHandler extends R
}
- private static class Command {
+ private class Command {
private final SolrQueryRequest req;
private final SolrQueryResponse resp;
private final String method;
@@ -122,6 +152,7 @@ public class SolrConfigHandler extends R
private void handleGET() {
if (parts.size() == 1) {
+ //this is the whole config. sent out the whole payload
resp.add("config", getConfigDetails());
} else {
if (ConfigOverlay.NAME.equals(parts.get(1))) {
@@ -131,9 +162,9 @@ public class SolrConfigHandler extends R
RequestParams params = req.getCore().getSolrConfig().getRequestParams();
MapSolrParams p = params.getParams(parts.get(2));
Map m = new LinkedHashMap<>();
- m.put(ConfigOverlay.ZNODEVER, params.getZnodeVersion());
+ m.put(ZNODEVER, params.getZnodeVersion());
if (p != null) {
- m.put(RequestParams.NAME, ZkNodeProps.makeMap(parts.get(2), p.getMap()));
+ m.put(RequestParams.NAME, makeMap(parts.get(2), p.getMap()));
}
resp.add(SolrQueryResponse.NAME, m);
} else {
@@ -141,8 +172,53 @@ public class SolrConfigHandler extends R
}
} else {
- Map<String, Object> m = getConfigDetails();
- resp.add("config", ZkNodeProps.makeMap(parts.get(1), m.get(parts.get(1))));
+ if (ZNODEVER.equals(parts.get(1))) {
+ resp.add(ZNODEVER, ZkNodeProps.makeMap(
+ ConfigOverlay.NAME, req.getCore().getSolrConfig().getOverlay().getZnodeVersion(),
+ RequestParams.NAME, req.getCore().getSolrConfig().getRequestParams().getZnodeVersion()));
+ boolean checkStale = false;
+ int expectedVersion = req.getParams().getInt(ConfigOverlay.NAME, -1);
+ int actualVersion = req.getCore().getSolrConfig().getOverlay().getZnodeVersion();
+ if (expectedVersion > actualVersion) {
+ log.info("expecting overlay version {} but my version is {}", expectedVersion, actualVersion);
+ checkStale = true;
+ } else if (expectedVersion != -1) {
+ log.info("I already have the expected version {} of config", expectedVersion);
+ }
+ expectedVersion = req.getParams().getInt(RequestParams.NAME, -1);
+ actualVersion = req.getCore().getSolrConfig().getRequestParams().getZnodeVersion();
+ if (expectedVersion > actualVersion) {
+ log.info("expecting params version {} but my version is {}", expectedVersion, actualVersion);
+ checkStale = true;
+ } else if (expectedVersion != -1) {
+ log.info("I already have the expected version {} of params", expectedVersion);
+ }
+ if (checkStale && req.getCore().getResourceLoader() instanceof ZkSolrResourceLoader) {
+ new Thread(SolrConfigHandler.class.getSimpleName() + "-refreshconf") {
+ @Override
+ public void run() {
+ if (!reloadLock.tryLock()) {
+ log.info("Another reload is in progress . Not doing anything");
+ return;
+ }
+ try {
+ log.info("Trying to update my configs");
+ SolrCore.getConfListener(req.getCore(), (ZkSolrResourceLoader) req.getCore().getResourceLoader()).run();
+ } catch (Exception e) {
+ log.error("Unable to refresh conf ", e);
+ } finally {
+ reloadLock.unlock();
+ }
+ }
+ }.start();
+ } else {
+ log.info("checkStale {} , resourceloader {}", checkStale, req.getCore().getResourceLoader().getClass().getName());
+ }
+
+ } else {
+ Map<String, Object> m = getConfigDetails();
+ resp.add("config", makeMap(parts.get(1), m.get(parts.get(1))));
+ }
}
}
}
@@ -277,8 +353,15 @@ public class SolrConfigHandler extends R
if (ops.isEmpty()) {
ZkController.touchConfDir(zkLoader);
} else {
- ZkController.persistConfigResourceToZooKeeper(zkLoader, params.getZnodeVersion(),
- RequestParams.RESOURCE, params.toByteArray(), true);
+ log.info("persisting params version : {}", params.toMap());
+ int latestVersion = ZkController.persistConfigResourceToZooKeeper(zkLoader,
+ params.getZnodeVersion(),
+ RequestParams.RESOURCE,
+ params.toByteArray(), true);
+ waitForAllReplicasState(req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
+ req.getCore().getCoreDescriptor().getCoreContainer().getZkController(),
+ RequestParams.NAME,
+ latestVersion, 30);
}
} else {
@@ -326,17 +409,20 @@ public class SolrConfigHandler extends R
}
List errs = CommandOperation.captureErrors(ops);
if (!errs.isEmpty()) {
- log.info("Failed to run commands errors are {}", StrUtils.join(errs, ','));
+ log.info("Failed to run commands. errors are {}", StrUtils.join(errs, ','));
resp.add(CommandOperation.ERR_MSGS, errs);
return;
}
SolrResourceLoader loader = req.getCore().getResourceLoader();
if (loader instanceof ZkSolrResourceLoader) {
- ZkController.persistConfigResourceToZooKeeper((ZkSolrResourceLoader) loader, overlay.getZnodeVersion(),
+ int latestVersion = ZkController.persistConfigResourceToZooKeeper((ZkSolrResourceLoader) loader, overlay.getZnodeVersion(),
ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true);
-
- log.info("Executed config commands successfully and persited to ZK {}", ops);
+ log.info("Executed config commands successfully and persisted to ZK {}", ops);
+ waitForAllReplicasState(req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName(),
+ req.getCore().getCoreDescriptor().getCoreContainer().getZkController(),
+ ConfigOverlay.NAME,
+ latestVersion, 30);
} else {
SolrResourceLoader.persistConfLocally(loader, ConfigOverlay.RESOURCE_NAME, overlay.toByteArray());
req.getCore().getCoreDescriptor().getCoreContainer().reload(req.getCore().getName());
@@ -519,7 +605,7 @@ public class SolrConfigHandler extends R
private static Set<String> subPaths = new HashSet<>(Arrays.asList("/overlay", "/params",
- "/query", "/jmx", "/requestDispatcher"));
+ "/query", "/jmx", "/requestDispatcher", "/znodeVersion"));
static {
for (SolrConfig.SolrPluginInfo solrPluginInfo : SolrConfig.plugins)
@@ -556,4 +642,170 @@ public class SolrConfigHandler extends R
public static final String CREATE = "create";
private static Set<String> cmdPrefixes = ImmutableSet.of(CREATE, UPDATE, "delete", "add");
+ /**
+ * Block up to a specified maximum time until we see agreement on the schema
+ * version in ZooKeeper across all replicas for a collection.
+ */
+ private static void waitForAllReplicasState(String collection,
+ ZkController zkController,
+ String prop,
+ int expectedVersion,
+ int maxWaitSecs) {
+ long startMs = System.currentTimeMillis();
+ // get a list of active replica cores to query for the schema zk version (skipping this core of course)
+ List<PerReplicaCallable> concurrentTasks = new ArrayList<>();
+
+ for (String coreUrl : getActiveReplicaCoreUrls(zkController, collection)) {
+ PerReplicaCallable e = new PerReplicaCallable(coreUrl, prop, expectedVersion, maxWaitSecs);
+ concurrentTasks.add(e);
+ }
+ if (concurrentTasks.isEmpty()) return; // nothing to wait for ...
+
+ log.info(formatString("Waiting up to {0} secs for {1} replicas to set the property {2} to be of version {3} for collection {4}",
+ maxWaitSecs, concurrentTasks.size(), prop, expectedVersion, collection));
+
+ // use an executor service to invoke schema zk version requests in parallel with a max wait time
+ int poolSize = Math.min(concurrentTasks.size(), 10);
+ ExecutorService parallelExecutor =
+ Executors.newFixedThreadPool(poolSize, new DefaultSolrThreadFactory("solrHandlerExecutor"));
+ try {
+ List<Future<Boolean>> results =
+ parallelExecutor.invokeAll(concurrentTasks, maxWaitSecs, TimeUnit.SECONDS);
+
+ // determine whether all replicas have the update
+ List<String> failedList = null; // lazily init'd
+ for (int f = 0; f < results.size(); f++) {
+ Boolean success = false;
+ Future<Boolean> next = results.get(f);
+ if (next.isDone() && !next.isCancelled()) {
+ // looks to have finished, but need to check if it succeeded
+ try {
+ success = next.get();
+ } catch (ExecutionException e) {
+ // shouldn't happen since we checked isCancelled
+ }
+ }
+
+ if (!success) {
+ String coreUrl = concurrentTasks.get(f).coreUrl;
+ log.warn("Core " + coreUrl + "could not get the expected version " + expectedVersion);
+ if (failedList == null) failedList = new ArrayList<>();
+ failedList.add(coreUrl);
+ }
+ }
+
+ // if any tasks haven't completed within the specified timeout, it's an error
+ if (failedList != null)
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ formatString("{0} out of {1} the property {2} to be of version {3} within {4} seconds! Failed cores: {5}",
+ failedList.size(), concurrentTasks.size() + 1, prop, expectedVersion, maxWaitSecs, failedList));
+
+ } catch (InterruptedException ie) {
+ log.warn(formatString(
+ "Core was interrupted . trying to set the property {1} to version {2} to propagate to {3} replicas for collection {4}",
+ prop, expectedVersion, concurrentTasks.size(), collection));
+ Thread.currentThread().interrupt();
+ } finally {
+ if (!parallelExecutor.isShutdown())
+ parallelExecutor.shutdownNow();
+ }
+
+ long diffMs = (System.currentTimeMillis() - startMs);
+ log.info(formatString(
+ "Took {0} secs to set the property {1} to be of version {2} for collection {3}",
+ Math.round(diffMs / 1000d), prop, expectedVersion, collection));
+ }
+
+ public static List<String> getActiveReplicaCoreUrls(ZkController zkController,
+ String collection) {
+ List<String> activeReplicaCoreUrls = new ArrayList<>();
+ ClusterState clusterState = zkController.getZkStateReader().getClusterState();
+ Set<String> liveNodes = clusterState.getLiveNodes();
+ Collection<Slice> activeSlices = clusterState.getActiveSlices(collection);
+ if (activeSlices != null && activeSlices.size() > 0) {
+ for (Slice next : activeSlices) {
+ Map<String, Replica> replicasMap = next.getReplicasMap();
+ if (replicasMap != null) {
+ for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
+ Replica replica = entry.getValue();
+ if (ZkStateReader.ACTIVE.equals(replica.getStr(ZkStateReader.STATE_PROP)) &&
+ liveNodes.contains(replica.getNodeName())) {
+ activeReplicaCoreUrls.add(replica.getCoreUrl());
+ }
+ }
+ }
+ }
+ }
+ return activeReplicaCoreUrls;
+ }
+
+ private static class PerReplicaCallable extends SolrRequest implements Callable<Boolean> {
+ String coreUrl;
+ String prop;
+ int expectedZkVersion;
+ Number remoteVersion = null;
+ int maxWait;
+
+ PerReplicaCallable(String coreUrl, String prop, int expectedZkVersion, int maxWait) {
+ super(METHOD.GET, "/config/" + ZNODEVER);
+ this.coreUrl = coreUrl;
+ this.expectedZkVersion = expectedZkVersion;
+ this.prop = prop;
+ this.maxWait = maxWait;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ return new ModifiableSolrParams()
+ .set(prop, expectedZkVersion)
+ .set(CommonParams.WT, CommonParams.JAVABIN);
+ }
+
+ @Override
+ public Boolean call() throws Exception {
+ long startTime = System.currentTimeMillis();
+ int attempts = 0;
+ try (HttpSolrClient solr = new HttpSolrClient(coreUrl)) {
+ // eventually, this loop will get killed by the ExecutorService's timeout
+ while (true) {
+ try {
+ long timeElapsed = (System.currentTimeMillis() - startTime) / 1000;
+ if (timeElapsed >= maxWait) {
+ return false;
+ }
+ log.info("Time elapsed : {} secs, maxWait {}", timeElapsed, maxWait);
+ Thread.sleep(100);
+ NamedList<Object> resp = solr.httpUriRequest(this).future.get();
+ if (resp != null) {
+ Map m = (Map) resp.get(ZNODEVER);
+ if (m != null) {
+ remoteVersion = (Number) m.get(prop);
+ if (remoteVersion != null && remoteVersion.intValue() >= expectedZkVersion) break;
+ }
+ }
+
+ attempts++;
+ log.info(formatString("Could not get expectedVersion {0} from {1} for prop {2} after {3} attempts", expectedZkVersion, coreUrl, prop, attempts));
+ } catch (Exception e) {
+ if (e instanceof InterruptedException) {
+ break; // stop looping
+ } else {
+ log.warn("Failed to get /schema/zkversion from " + coreUrl + " due to: " + e);
+ }
+ }
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public Collection<ContentStream> getContentStreams() throws IOException {
+ return null;
+ }
+
+ @Override
+ protected SolrResponse createResponse(SolrClient client) {
+ return null;
+ }
+ }
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/admin/LoggingHandler.java Wed Apr 1 01:10:11 2015
@@ -121,7 +121,7 @@ public class LoggingHandler extends Requ
SimpleOrderedMap<Object> info = new SimpleOrderedMap<>();
if(time>0) {
info.add("since", time);
- info.add("found", found);
+ info.add("found", found.get());
}
else {
info.add("levels", watcher.getAllLevels()); // show for the first request
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Wed Apr 1 01:10:11 2015
@@ -199,9 +199,7 @@ public class HttpShardHandler extends Sh
params.remove(CommonParams.WT); // use default (currently javabin)
params.remove(CommonParams.VERSION);
- // SolrRequest req = new QueryRequest(SolrRequest.METHOD.POST, "/select");
- // use generic request to avoid extra processing of queries
- QueryRequest req = new QueryRequest(params);
+ QueryRequest req = makeQueryRequest(sreq, params, shard);
req.setMethod(SolrRequest.METHOD.POST);
// no need to set the response parser as binary is the default
@@ -239,12 +237,29 @@ public class HttpShardHandler extends Sh
ssr.elapsedTime = TimeUnit.MILLISECONDS.convert(System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- return srsp;
+ return transfomResponse(sreq, srsp, shard);
}
};
pending.add( completionService.submit(task) );
}
+
+ /**
+ * Subclasses could modify the request based on the shard
+ */
+ protected QueryRequest makeQueryRequest(final ShardRequest sreq, ModifiableSolrParams params, String shard)
+ {
+ // use generic request to avoid extra processing of queries
+ return new QueryRequest(params);
+ }
+
+ /**
+ * Subclasses could modify the Response based on the the shard
+ */
+ protected ShardResponse transfomResponse(final ShardRequest sreq, ShardResponse rsp, String shard)
+ {
+ return rsp;
+ }
/** returns a ShardResponse of the last response correlated with a ShardRequest. This won't
* return early if it runs into an error.
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/logging/log4j/Log4jWatcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/logging/log4j/Log4jWatcher.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/logging/log4j/Log4jWatcher.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/logging/log4j/Log4jWatcher.java Wed Apr 1 01:10:11 2015
@@ -16,26 +16,27 @@
*/
package org.apache.solr.logging.log4j;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
-import com.google.common.base.Throwables;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
import org.apache.log4j.spi.ThrowableInformation;
import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.logging.CircularList;
import org.apache.solr.logging.ListenerConfig;
import org.apache.solr.logging.LogWatcher;
import org.apache.solr.logging.LoggerInfo;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import com.google.common.base.Throwables;
public class Log4jWatcher extends LogWatcher<LoggingEvent> {
@@ -157,6 +158,12 @@ public class Log4jWatcher extends LogWat
if(t!=null) {
doc.setField("trace", Throwables.getStackTraceAsString(t.getThrowable()));
}
+
+ // Will be null if not present
+ doc.setField("core", event.getMDC(ZkStateReader.CORE_NAME_PROP));
+ doc.setField("collection", event.getMDC(ZkStateReader.COLLECTION_PROP));
+ doc.setField("replica", event.getMDC(ZkStateReader.REPLICA_PROP));
+ doc.setField("shard", event.getMDC(ZkStateReader.SHARD_ID_PROP));
return doc;
}
}
\ No newline at end of file
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/search/ExtendedDismaxQParser.java Wed Apr 1 01:10:11 2015
@@ -1494,7 +1494,7 @@ public class ExtendedDismaxQParser exten
try {
queryFields = DisMaxQParser.parseQueryFields(req.getSchema(), solrParams); // req.getSearcher() here causes searcher refcount imbalance
} catch (SyntaxError e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
// Phrase slop array
int pslop[] = new int[4];
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/DocumentExpressionDictionaryFactory.java Wed Apr 1 01:10:11 2015
@@ -99,7 +99,7 @@ public class DocumentExpressionDictionar
try {
expression = JavascriptCompiler.compile(weightExpression);
} catch (ParseException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
SimpleBindings bindings = new SimpleBindings();
for (SortField sortField : sortFields) {
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/FileDictionaryFactory.java Wed Apr 1 01:10:11 2015
@@ -55,7 +55,7 @@ public class FileDictionaryFactory exten
return new FileDictionary(new InputStreamReader(
core.getResourceLoader().openResource(sourceLocation), StandardCharsets.UTF_8), fieldDelimiter);
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/spelling/suggest/fst/AnalyzingInfixLookupFactory.java Wed Apr 1 01:10:11 2015
@@ -135,7 +135,7 @@ public class AnalyzingInfixLookupFactory
}
};
} catch (IOException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/util/SimplePostTool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/util/SimplePostTool.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/util/SimplePostTool.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/java/org/apache/solr/util/SimplePostTool.java Wed Apr 1 01:10:11 2015
@@ -518,7 +518,7 @@ public class SimplePostTool {
Thread.sleep(delay * 1000);
filesPosted++;
} catch (InterruptedException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
return filesPosted;
@@ -610,7 +610,7 @@ public class SimplePostTool {
} catch (IOException e) {
warn("Caught exception when trying to open connection to "+u+": "+e.getMessage());
} catch (InterruptedException e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
}
if(!subStack.isEmpty()) {
@@ -1209,7 +1209,7 @@ public class SimplePostTool {
} catch (IOException e) {
warn("IOException opening URL "+url+": "+e.getMessage());
} catch (Exception e) {
- throw new RuntimeException();
+ throw new RuntimeException(e);
}
return l;
}
Modified: lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/core/TestSolrConfigHandler.java Wed Apr 1 01:10:11 2015
@@ -21,7 +21,6 @@ package org.apache.solr.core;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
-import java.text.MessageFormat;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
Modified: lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java (original)
+++ lucene/dev/branches/lucene6271/solr/core/src/test/org/apache/solr/handler/TestReqParamsAPI.java Wed Apr 1 01:10:11 2015
@@ -41,7 +41,6 @@ import static org.apache.solr.handler.Te
* limitations under the License.
*/
-@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-6924")
public class TestReqParamsAPI extends AbstractFullDistribZkTestBase {
static final Logger log = LoggerFactory.getLogger(TestSolrConfigHandlerCloud.class);
private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
Modified: lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java (original)
+++ lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/cloud/Replica.java Wed Apr 1 01:10:11 2015
@@ -21,6 +21,8 @@ import org.noggit.JSONUtil;
import java.util.Map;
+import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
public class Replica extends ZkNodeProps {
private final String name;
@@ -35,6 +37,9 @@ public class Replica extends ZkNodeProps
public String getName() {
return name;
}
+ public String getCoreUrl() {
+ return ZkCoreNodeProps.getCoreUrl(getStr(BASE_URL_PROP), getStr(CORE_NAME_PROP));
+ }
/** The name of the node this replica resides on */
public String getNodeName() {
Modified: lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java?rev=1670533&r1=1670532&r2=1670533&view=diff
==============================================================================
--- lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java (original)
+++ lucene/dev/branches/lucene6271/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java Wed Apr 1 01:10:11 2015
@@ -224,5 +224,9 @@ public interface CommonParams {
* When querying a node, prefer local node's cores for distributed queries.
*/
public static final String PREFER_LOCAL_SHARDS = "preferLocalShards";
+
+ public static final String JAVABIN = "javabin";
+
+ public static final String JSON = "json";
}