You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/08/10 09:13:41 UTC
[01/31] lucene-solr:jira/http2: LUCENE-8445: Tighten condition when
two planes are identical to prevent constructing bogus tiles when building
GeoPolygons
Repository: lucene-solr
Updated Branches:
refs/heads/jira/http2 51b2a1be3 -> 9f554f6de
LUCENE-8445: Tighten condition when two planes are identical to prevent constructing bogus tiles when building GeoPolygons
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/70a4939a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/70a4939a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/70a4939a
Branch: refs/heads/jira/http2
Commit: 70a4939ab4a640554aebd1aaa68935605d9af112
Parents: 3b15be3
Author: Ignacio Vera <iv...@apache.org>
Authored: Mon Aug 6 09:34:11 2018 +0200
Committer: Ignacio Vera <iv...@apache.org>
Committed: Mon Aug 6 09:34:11 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../org/apache/lucene/spatial3d/geom/Plane.java | 2 +-
.../lucene/spatial3d/geom/GeoPolygonTest.java | 23 +++++++++++++++++++-
3 files changed, 26 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/70a4939a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7e261bc..14e2b02 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -176,6 +176,9 @@ API Changes:
Bug Fixes:
+* LUCENE-8445: Tighten condition when two planes are identical to prevent constructing
+ bogus tiles when building GeoPolygons. (Ignacio Vera)
+
* LUCENE-8380: UTF8TaxonomyWriterCache inconsistency. (Ruslan Torobaev, Dawid Weiss)
* LUCENE-8164: IndexWriter silently accepts broken payload. This has been fixed
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/70a4939a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
index 1f9339c..038dda0 100755
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/Plane.java
@@ -2361,7 +2361,7 @@ public class Plane extends Vector {
//System.out.println("cross product magnitude = "+(cross1 * cross1 + cross2 * cross2 + cross3 * cross3));
// Should be MINIMUM_RESOLUTION_SQUARED, but that gives us planes that are *almost* parallel, and those are problematic too,
// so we have a tighter constraint on parallelism in this method.
- if (cross1 * cross1 + cross2 * cross2 + cross3 * cross3 >= MINIMUM_RESOLUTION) {
+ if (cross1 * cross1 + cross2 * cross2 + cross3 * cross3 >= 5 * MINIMUM_RESOLUTION) {
return false;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/70a4939a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index d301de1..c8cbc1b 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -1849,5 +1849,26 @@ shape:
assertTrue(largePolygon.isWithin(thePoint) == smallPolygon.isWithin(thePoint));
}
-
+
+ @Test
+ public void testLUCENE8445() {
+ //POLYGON((32.18017946378854 -17.397683785381247,49.51018758330871 -9.870219317504647,58.77903721991479 33.90553510354402,2.640604559432277 9.363173880050821,3.1673235739886286E-10 8.853669066894417E-11,0.0 -5.7E-322,4.820339742500488E-5 5.99784517213369E-7,32.18017946378854 -17.397683785381247))
+ final List<GeoPoint> points = new ArrayList<>();
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(-17.397683785381247), Geo3DUtil.fromDegrees(32.18017946378854)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(-9.870219317504647), Geo3DUtil.fromDegrees(49.51018758330871)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(33.90553510354402), Geo3DUtil.fromDegrees(58.77903721991479)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(9.363173880050821), Geo3DUtil.fromDegrees(2.640604559432277)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(8.853669066894417E-11), Geo3DUtil.fromDegrees(3.1673235739886286E-10)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(-5.7E-322), Geo3DUtil.fromDegrees(0.0)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(5.99784517213369E-7), Geo3DUtil.fromDegrees(4.820339742500488E-5)));
+
+ final GeoPolygonFactory.PolygonDescription description = new GeoPolygonFactory.PolygonDescription(points);
+ final GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, description);
+ final GeoPolygon largePolygon = GeoPolygonFactory.makeLargeGeoPolygon(PlanetModel.SPHERE, Collections.singletonList(description));
+
+ //POINT(179.99999999999983 -5.021400461974724E-11)
+ final GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(-5.021400461974724E-11), Geo3DUtil.fromDegrees(179.99999999999983));
+ assertTrue(polygon.isWithin(point) == largePolygon.isWithin(point));
+ }
+
}
[12/31] lucene-solr:jira/http2: SOLR-12485: Solr-Update XML format
now accepts child documents under a field thus providing a semantic
relationship. (like allowed for JSON). Improved XML.java a bit Closes #430
Posted by da...@apache.org.
SOLR-12485: Solr-Update XML format now accepts child documents under a field
thus providing a semantic relationship. (like allowed for JSON).
Improved XML.java a bit
Closes #430
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/38bf976c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/38bf976c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/38bf976c
Branch: refs/heads/jira/http2
Commit: 38bf976cd4b9e324c21664bd7ae3d554df803705
Parents: c3887b3
Author: Moshe <mo...@mail.com>
Authored: Tue Aug 7 13:52:11 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Aug 7 13:52:11 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../apache/solr/handler/loader/XMLLoader.java | 18 ++-
.../apache/solr/update/AddBlockUpdateTest.java | 148 +++++++++++++++++++
.../solr/client/solrj/util/ClientUtils.java | 20 ++-
.../java/org/apache/solr/common/util/XML.java | 98 +++---------
.../org/apache/solr/util/BaseTestHarness.java | 23 +--
6 files changed, 215 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 94960bb..3ede98b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -150,6 +150,9 @@ New Features
* SOLR-12592: support #EQUAL function, range operator, decimal and percentage in cores in autoscaling policies (noble)
+* SOLR-12485: Uploading docs in XML now supports child documents as field values, thus providing a label to the
+ relationship instead of the current "anonymous" relationship. (Moshe Bla, David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
index a07aff2..724a40c 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
@@ -404,6 +404,7 @@ public class XMLLoader extends ContentStreamLoader {
StringBuilder text = new StringBuilder();
String name = null;
boolean isNull = false;
+ boolean isLabeledChildDoc = false;
String update = null;
Collection<SolrInputDocument> subDocs = null;
Map<String, Map<String, Object>> updateMap = null;
@@ -453,7 +454,13 @@ public class XMLLoader extends ContentStreamLoader {
}
break;
}
- doc.addField(name, v);
+ if(!isLabeledChildDoc){
+ // only add data if this is not a childDoc, since it was added already
+ doc.addField(name, v);
+ } else {
+ // reset so next field is not treated as child doc
+ isLabeledChildDoc = false;
+ }
// field is over
name = null;
}
@@ -463,6 +470,15 @@ public class XMLLoader extends ContentStreamLoader {
text.setLength(0);
String localName = parser.getLocalName();
if ("doc".equals(localName)) {
+ if(name != null) {
+ // flag to prevent spaces after doc from being added
+ isLabeledChildDoc = true;
+ if(!doc.containsKey(name)) {
+ doc.setField(name, Lists.newArrayList());
+ }
+ doc.addField(name, readDoc(parser));
+ break;
+ }
if (subDocs == null)
subDocs = Lists.newArrayList();
subDocs.add(readDoc(parser));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
index a302d58..50c0c39 100644
--- a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
+++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
@@ -502,6 +502,154 @@ public class AddBlockUpdateTest extends SolrTestCaseJ4 {
}
@Test
+ public void testXMLMultiLevelLabeledChildren() throws IOException, XMLStreamException {
+ String xml_doc1 =
+ "<doc >" +
+ " <field name=\"id\">1</field>" +
+ " <field name=\"empty_s\"></field>" +
+ " <field name=\"parent_s\">X</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >2</field>" +
+ " <field name=\"child_s\">y</field>" +
+ " </doc>" +
+ " <doc> " +
+ " <field name=\"id\" >3</field>" +
+ " <field name=\"child_s\">z</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ String xml_doc2 =
+ "<doc >" +
+ " <field name=\"id\">4</field>" +
+ " <field name=\"parent_s\">A</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >5</field>" +
+ " <field name=\"child_s\">b</field>" +
+ " <field name=\"grandChild\">" +
+ " <doc> " +
+ " <field name=\"id\" >7</field>" +
+ " <field name=\"child_s\">d</field>" +
+ " </doc>" +
+ " </field>" +
+ " </doc>" +
+ " </field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >6</field>" +
+ " <field name=\"child_s\">c</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ XMLStreamReader parser =
+ inputFactory.createXMLStreamReader(new StringReader(xml_doc1));
+ parser.next(); // read the START document...
+ //null for the processor is all right here
+ XMLLoader loader = new XMLLoader();
+ SolrInputDocument document1 = loader.readDoc(parser);
+
+ XMLStreamReader parser2 =
+ inputFactory.createXMLStreamReader(new StringReader(xml_doc2));
+ parser2.next(); // read the START document...
+ //null for the processor is all right here
+ //XMLLoader loader = new XMLLoader();
+ SolrInputDocument document2 = loader.readDoc(parser2);
+
+ assertFalse(document1.hasChildDocuments());
+ assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test",
+ sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString());
+
+ assertFalse(document2.hasChildDocuments());
+ assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test",
+ sdocs(sdoc("id", "5", "child_s", "b", "grandChild", Collections.singleton(sdoc("id", "7", "child_s", "d"))),
+ sdoc("id", "6", "child_s", "c"))).toString());
+ }
+
+ @Test
+ public void testXMLLabeledChildren() throws IOException, XMLStreamException {
+ UpdateRequest req = new UpdateRequest();
+
+ List<SolrInputDocument> docs = new ArrayList<>();
+
+ String xml_doc1 =
+ "<doc >" +
+ " <field name=\"id\">1</field>" +
+ " <field name=\"empty_s\"></field>" +
+ " <field name=\"parent_s\">X</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >2</field>" +
+ " <field name=\"child_s\">y</field>" +
+ " </doc>"+
+ " <doc> " +
+ " <field name=\"id\" >3</field>" +
+ " <field name=\"child_s\">z</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ String xml_doc2 =
+ "<doc >" +
+ " <field name=\"id\">4</field>" +
+ " <field name=\"parent_s\">A</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >5</field>" +
+ " <field name=\"child_s\">b</field>" +
+ " </doc>"+
+ " </field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >6</field>" +
+ " <field name=\"child_s\">c</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ XMLStreamReader parser =
+ inputFactory.createXMLStreamReader( new StringReader( xml_doc1 ) );
+ parser.next(); // read the START document...
+ //null for the processor is all right here
+ XMLLoader loader = new XMLLoader();
+ SolrInputDocument document1 = loader.readDoc( parser );
+
+ XMLStreamReader parser2 =
+ inputFactory.createXMLStreamReader( new StringReader( xml_doc2 ) );
+ parser2.next(); // read the START document...
+ //null for the processor is all right here
+ //XMLLoader loader = new XMLLoader();
+ SolrInputDocument document2 = loader.readDoc( parser2 );
+
+ assertFalse(document1.hasChildDocuments());
+ assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test",
+ sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString());
+
+ assertFalse(document2.hasChildDocuments());
+ assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test",
+ sdocs(sdoc("id", "5", "child_s", "b"), sdoc("id", "6", "child_s", "c"))).toString());
+
+ docs.add(document1);
+ docs.add(document2);
+
+ Collections.shuffle(docs, random());
+ req.add(docs);
+
+ RequestWriter requestWriter = new RequestWriter();
+ OutputStream os = new ByteArrayOutputStream();
+ requestWriter.write(req, os);
+ assertBlockU(os.toString());
+ assertU(commit());
+
+ final SolrIndexSearcher searcher = getSearcher();
+ assertSingleParentOf(searcher, one("yz"), "X");
+ assertSingleParentOf(searcher, one("bc"), "A");
+
+ }
+
+ @Test
public void testJavaBinCodecNestedRelation() throws IOException {
SolrInputDocument topDocument = new SolrInputDocument();
topDocument.addField("parent_f1", "v1");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
index 54986db..26a188d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
@@ -72,7 +72,9 @@ public class ClientUtils
for( Object v : field ) {
String update = null;
- if (v instanceof Map) {
+ if(v instanceof SolrInputDocument) {
+ writeVal(writer, name, v , null);
+ } else if (v instanceof Map) {
// currently only supports a single value
for (Entry<Object,Object> entry : ((Map<Object,Object>)v).entrySet()) {
update = entry.getKey().toString();
@@ -112,20 +114,28 @@ public class ClientUtils
v = Base64.byteArrayToBase64(bytes.array(), bytes.position(),bytes.limit() - bytes.position());
}
+ XML.Writable valWriter = null;
+ if(v instanceof SolrInputDocument) {
+ final SolrInputDocument solrDoc = (SolrInputDocument) v;
+ valWriter = (writer1) -> writeXML(solrDoc, writer1);
+ } else if(v != null) {
+ final Object val = v;
+ valWriter = (writer1) -> XML.escapeCharData(val.toString(), writer1);
+ }
+
if (update == null) {
if (v != null) {
- XML.writeXML(writer, "field", v.toString(), "name", name );
+ XML.writeXML(writer, "field", valWriter, "name", name);
}
} else {
if (v == null) {
- XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true);
+ XML.writeXML(writer, "field", (XML.Writable) null, "name", name, "update", update, "null", true);
} else {
- XML.writeXML(writer, "field", v.toString(), "name", name, "update", update);
+ XML.writeXML(writer, "field", valWriter, "name", name, "update", update);
}
}
}
-
public static String toXML( SolrInputDocument doc )
{
StringWriter str = new StringWriter();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/solrj/src/java/org/apache/solr/common/util/XML.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java
index c6e5205..9d1b8a8 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java
@@ -16,9 +16,10 @@
*/
package org.apache.solr.common.util;
-import java.io.Writer;
import java.io.IOException;
+import java.io.Writer;
import java.util.Map;
+import java.util.stream.Stream;
/**
*
@@ -37,9 +38,7 @@ public class XML {
private static final String[] attribute_escapes=
{"#0;","#1;","#2;","#3;","#4;","#5;","#6;","#7;","#8;",null,null,"#11;","#12;",null,"#14;","#15;","#16;","#17;","#18;","#19;","#20;","#21;","#22;","#23;","#24;","#25;","#26;","#27;","#28;","#29;","#30;","#31;",null,null,""",null,null,null,"&",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"<"};
-
-
- /*****************************************
+ /*
#Simple python script used to generate the escape table above. -YCS
#
#use individual char arrays or one big char array for better efficiency
@@ -65,13 +64,9 @@ public class XML {
result += val + ','
print result
- ****************************************/
+ */
-/*********
- *
- * @throws IOException If there is a low-level I/O error.
- */
public static void escapeCharData(String str, Writer out) throws IOException {
escape(str, out, chardata_escapes);
}
@@ -84,53 +79,28 @@ public class XML {
escape(chars, start, length, out, attribute_escapes);
}
+ /** does NOT escape character data in val; it must already be valid XML. Attributes are always escaped. */
+ public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ writeXML(out, tag, (writer1) -> writer1.write(val), attrs);
+ }
- public final static void writeXML(Writer out, String tag, String val) throws IOException {
- out.write('<');
- out.write(tag);
- if (val == null) {
- out.write('/');
- out.write('>');
- } else {
- out.write('>');
- escapeCharData(val,out);
- out.write('<');
- out.write('/');
- out.write(tag);
- out.write('>');
- }
+ /** escapes character data in val and attributes */
+ public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ final Writable writable = val != null ? (writer1) -> XML.escapeCharData(val, writer1) : null;
+ writeXML(out, tag, writable, attrs);
}
- /** does NOT escape character data in val, must already be valid XML */
- public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException {
- out.write('<');
- out.write(tag);
- for (int i=0; i<attrs.length; i++) {
- out.write(' ');
- out.write(attrs[i++].toString());
- out.write('=');
- out.write('"');
- out.write(attrs[i].toString());
- out.write('"');
- }
- if (val == null) {
- out.write('/');
- out.write('>');
- } else {
- out.write('>');
- out.write(val);
- out.write('<');
- out.write('/');
- out.write(tag);
- out.write('>');
- }
+ /** escapes character data in val and attributes */
+ public static void writeXML(Writer out, String tag, String val, Map<String, String> attrs) throws IOException {
+ writeXML(out, tag, val, attrs.entrySet().stream().flatMap((entry) -> Stream.of(entry.getKey(), entry.getValue())).toArray());
}
- /** escapes character data in val */
- public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ /** @lucene.internal */
+ public final static void writeXML(Writer out, String tag, Writable valWritable, Object... attrs) throws IOException {
out.write('<');
out.write(tag);
- for (int i=0; i<attrs.length; i++) {
+ final int attrsLen = attrs == null ? 0 : attrs.length;
+ for (int i = 0; i< attrsLen; i++) {
out.write(' ');
out.write(attrs[i++].toString());
out.write('=');
@@ -138,12 +108,12 @@ public class XML {
escapeAttributeValue(attrs[i].toString(), out);
out.write('"');
}
- if (val == null) {
+ if (valWritable == null) {
out.write('/');
out.write('>');
} else {
out.write('>');
- escapeCharData(val,out);
+ valWritable.write(out);
out.write('<');
out.write('/');
out.write(tag);
@@ -151,29 +121,9 @@ public class XML {
}
}
- /** escapes character data in val */
- public static void writeXML(Writer out, String tag, String val, Map<String, String> attrs) throws IOException {
- out.write('<');
- out.write(tag);
- for (Map.Entry<String, String> entry : attrs.entrySet()) {
- out.write(' ');
- out.write(entry.getKey());
- out.write('=');
- out.write('"');
- escapeAttributeValue(entry.getValue(), out);
- out.write('"');
- }
- if (val == null) {
- out.write('/');
- out.write('>');
- } else {
- out.write('>');
- escapeCharData(val,out);
- out.write('<');
- out.write('/');
- out.write(tag);
- out.write('>');
- }
+ @FunctionalInterface
+ public interface Writable {
+ void write(Writer w) throws IOException;
}
private static void escape(char [] chars, int offset, int length, Writer out, String [] escapes) throws IOException{
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
index a84d6d1..ad1d38e 100644
--- a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
@@ -15,11 +15,6 @@
* limitations under the License.
*/
package org.apache.solr.util;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.util.XML;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
import javax.xml.namespace.QName;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -27,13 +22,17 @@ import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
-
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.XML;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
abstract public class BaseTestHarness {
private static final ThreadLocal<DocumentBuilder> builderTL = new ThreadLocal<>();
private static final ThreadLocal<XPath> xpathTL = new ThreadLocal<>();
@@ -200,15 +199,9 @@ abstract public class BaseTestHarness {
public static String simpleTag(String tag, String... args) {
try {
- StringWriter r = new StringWriter();
-
- // this is annoying
- if (null == args || 0 == args.length) {
- XML.writeXML(r, tag, null);
- } else {
- XML.writeXML(r, tag, null, (Object[])args);
- }
- return r.getBuffer().toString();
+ StringWriter writer = new StringWriter();
+ XML.writeXML(writer, tag, (String) null, (Object[])args);
+ return writer.getBuffer().toString();
} catch (IOException e) {
throw new RuntimeException
("this should never happen with a StringWriter", e);
[07/31] lucene-solr:jira/http2: SOLR-11770: NPE in tvrh if no field
is specified and document doesn't contain any fields with term vectors,
forgot to add CHANGES.txt
Posted by da...@apache.org.
SOLR-11770: NPE in tvrh if no field is specified and document doesn't contain any fields with term vectors, forgot to add CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/08ca54f1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/08ca54f1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/08ca54f1
Branch: refs/heads/jira/http2
Commit: 08ca54f1f89eb98f6a3238f0f91c1f8383c12119
Parents: 96e8392
Author: Erick Erickson <Er...@gmail.com>
Authored: Mon Aug 6 20:05:52 2018 -0700
Committer: Erick Erickson <Er...@gmail.com>
Committed: Mon Aug 6 20:05:52 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/08ca54f1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e21b931..ac8ff3c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -211,7 +211,6 @@ Bug Fixes
with empty values. All documents with empty values ( string , numeric ) will be processed by worker=0
This would fix the NPE when using the search stream with partitionKeys. (Varun Thacker)
-
* SOLR-11770: NPE in tvrh if no field is specified and document doesn't contain any fields with term vectors
Optimizations
[30/31] lucene-solr:jira/http2: Resolved merge conflict caused by
SOLR-11881
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
index 6241188,0000000..5a10ae6
mode 100644,000000..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/Http2SolrClient.java
@@@ -1,802 -1,0 +1,804 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.ConnectException;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpStatus;
+import org.apache.http.entity.ContentType;
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.V2RequestSupport;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.RequestWriter;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.request.V2Request;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.ObjectReleaseTracker;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.HttpClientTransport;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.api.Result;
+import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP;
+import org.eclipse.jetty.client.util.BufferingResponseListener;
+import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.FormContentProvider;
+import org.eclipse.jetty.client.util.InputStreamContentProvider;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
+import org.eclipse.jetty.client.util.MultiPartContentProvider;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.http.HttpField;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.http2.client.HTTP2Client;
+import org.eclipse.jetty.http2.client.http.HttpClientTransportOverHTTP2;
+import org.eclipse.jetty.util.Fields;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.eclipse.jetty.util.thread.QueuedThreadPool;
+
+import static org.apache.solr.common.util.Utils.getObjectByPath;
+
+// TODO: error handling, small Http2SolrClient features, basic auth, security, ssl, apiV2 ...
+/**
+ * @lucene.experimental
+ */
+public class Http2SolrClient extends SolrClient {
+ private static volatile SslContextFactory defaultSslContextFactory = getDefaultSslContextFactory();
+ private static final int MAX_OUTSTANDING_REQUESTS = 1000;
+ private static final String AGENT = "Solr[" + Http2SolrClient.class.getName() + "] 2.0";
+ private static final String UTF_8 = StandardCharsets.UTF_8.name();
+ private static final String DEFAULT_PATH = "/select";
+ private static final List<String> errPath = Arrays.asList("metadata", "error-class");
+
+ private HttpClient httpClient;
+ private volatile Set<String> queryParams = Collections.emptySet();
+ private Phaser phaser = new Phaser(1);
+ private final Semaphore available;
+ private int idleTimeout;
+
+ private ResponseParser parser = new BinaryResponseParser();
+ private volatile RequestWriter requestWriter = new BinaryRequestWriter();
+
+ private Request.QueuedListener requestQueuedListener = new Request.QueuedListener() {
+
+ @Override
+ public void onQueued(Request request) {
+ phaser.register();
+ try {
+ available.acquire();
+ } catch (InterruptedException e) {
+
+ }
+ }
+ };
+
+ private Request.BeginListener beginListener = req -> {
+
+ };
+
+ private Response.CompleteListener requestCompleteListener = new Response.CompleteListener() {
+
+ @Override
+ public void onComplete(Result arg0) {
+ phaser.arriveAndDeregister();
+ available.release();
+ }
+ };
+
+ /**
+ * The URL of the Solr server.
+ */
+ private String serverBaseUrl;
+ private boolean closeClient;
+
+ protected Http2SolrClient(String serverBaseUrl, Builder builder) {
+ // TODO: what about shared instances?
+ available = new Semaphore(MAX_OUTSTANDING_REQUESTS, false);
+
- if (!serverBaseUrl.equals("/") && serverBaseUrl.endsWith("/")) {
- serverBaseUrl = serverBaseUrl.substring(0, serverBaseUrl.length() - 1);
- }
++ if (serverBaseUrl != null) {
++ if (!serverBaseUrl.equals("/") && serverBaseUrl.endsWith("/")) {
++ serverBaseUrl = serverBaseUrl.substring(0, serverBaseUrl.length() - 1);
++ }
+
- if (serverBaseUrl.startsWith("//")) {
- serverBaseUrl = serverBaseUrl.substring(1, serverBaseUrl.length());
++ if (serverBaseUrl.startsWith("//")) {
++ serverBaseUrl = serverBaseUrl.substring(1, serverBaseUrl.length());
++ }
++ this.serverBaseUrl = serverBaseUrl;
+ }
+
+ if (builder.idleTimeout != null) idleTimeout = builder.idleTimeout;
+ else idleTimeout = HttpClientUtil.DEFAULT_SO_TIMEOUT;
+
- this.serverBaseUrl = serverBaseUrl;
+ if (builder.httpClient == null) {
+ httpClient = createHttpClient(builder);
+ closeClient = true;
+ } else {
+ httpClient = builder.httpClient;
+ }
+ if (builder.beginListener != null) this.beginListener = builder.beginListener;
+ if (!httpClient.isStarted()) {
+ try {
+ httpClient.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ assert ObjectReleaseTracker.track(this);
+ }
+
+ private HttpClient createHttpClient(Builder builder) {
+
+ HttpClient httpClient;
+
+ QueuedThreadPool httpClientExecutor = new QueuedThreadPool(100, 4);
+ httpClientExecutor.setDaemon(true);
+
+ HttpClientTransport transport;
+ if (builder.useHttp1_1 || builder.sslContextFactory != null) {
+ transport = new HttpClientTransportOverHTTP(2);
+ httpClient = new HttpClient(transport, builder.sslContextFactory);
+ } else {
+ HTTP2Client http2client = new HTTP2Client();
+ transport = new HttpClientTransportOverHTTP2(http2client);
+ httpClient = new HttpClient(transport, builder.sslContextFactory);
+ }
+ httpClient.setExecutor(httpClientExecutor);
+ httpClient.setStrictEventOrdering(false);
+ httpClient.setConnectBlocking(true);
+ httpClient.setFollowRedirects(false);
+ httpClient.setMaxConnectionsPerDestination(4);
+ httpClient.setMaxRequestsQueuedPerDestination(MAX_OUTSTANDING_REQUESTS * 4); // comfortably above max outstanding // requests
+ httpClient.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, AGENT));
+
+ if (builder.idleTimeout != null) httpClient.setIdleTimeout(builder.idleTimeout);
+ if (builder.connectionTimeout != null) httpClient.setConnectTimeout(builder.connectionTimeout);
+ return httpClient;
+ }
+
+
+ public HttpClient getHttpClient() {
+ return httpClient;
+ }
+
+ public void close() {
+ // we wait for async requests, so far devs don't want to give sugar for this
+ phaser.arriveAndAwaitAdvance();
+ phaser.arriveAndDeregister();
+ if (closeClient) {
+ close(httpClient);
+ }
+
+ assert ObjectReleaseTracker.release(this);
+ }
+
+ public static void close(HttpClient httpClient) {
+ try {
+ // TODO: stop time?
+ httpClient.setStopTimeout(1000);
+ httpClient.stop();
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public Http2ClientResponse request(SolrRequest solrRequest, String collection, OnComplete onComplete)
+ throws SolrServerException, IOException {
+ return request(solrRequest, collection, onComplete, false);
+ }
+
+ private boolean isV2ApiRequest(final SolrRequest request) {
+ return request instanceof V2Request || request.getPath().contains("/____v2");
+ }
+
+ private Http2ClientResponse request(SolrRequest solrRequest,
+ String collection,
+ OnComplete onComplete,
+ boolean returnStream) throws IOException, SolrServerException {
+ Request req = makeRequest(solrRequest, collection);
+ try {
+ if (onComplete != null) {
+ req.onRequestBegin(beginListener).onRequestQueued(requestQueuedListener)
+ .onComplete(requestCompleteListener).send(new BufferingResponseListener() {
+
+ @Override
+ public void onComplete(Result result) {
+ if (result.isFailed()) {
+ onComplete.onFailure(result.getFailure());
+ return;
+ }
+
+ // TODO: should we stream this?
+ try (InputStream ris = getContentAsInputStream()) {
+ NamedList<Object> rsp;
+ try {
+ rsp = processErrorsAndResponse(result.getResponse(),
+ parser, ris, getEncoding(), isV2ApiRequest(solrRequest));
+ onComplete.onSuccess(rsp);
+ } catch (Exception e) {
+ onComplete.onFailure(e);
+ }
+ } catch (IOException e1) {
+ onComplete.onFailure(e1);
+ }
+ }
+ });
+ return null;
+ } else {
+ Http2ClientResponse arsp = new Http2ClientResponse();
+ if (returnStream) {
+ InputStreamResponseListener listener = new InputStreamResponseListener();
+ req.onRequestBegin(beginListener).send(listener);
+ // Wait for the response headers to arrive
+ listener.get(idleTimeout, TimeUnit.SECONDS);
+ // TODO: process response
+ arsp.stream = listener.getInputStream();
+ } else {
+ ContentResponse response = req.onRequestBegin(beginListener).send();
+ ByteArrayInputStream is = new ByteArrayInputStream(response.getContent());
+ arsp.response = processErrorsAndResponse(response, parser,
+ is, response.getEncoding(), isV2ApiRequest(solrRequest));
+ }
+ return arsp;
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(e);
+ } catch (TimeoutException e) {
+ throw new SolrServerException(
+ "Timeout occured while waiting response from server at: "
+ + getBaseURL(), e);
+ } catch (ExecutionException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof ConnectException) {
+ throw new SolrServerException("Server refused connection at: "
+ + getBaseURL(), cause);
+ }
+ if (cause instanceof SolrServerException) {
+ throw (SolrServerException) cause;
+ } else if (cause instanceof IOException) {
+ throw new SolrServerException(
+ "IOException occured when talking to server at: " + getBaseURL(), cause);
+ }
+ throw new SolrServerException(cause.getMessage(), cause);
+ }
+ }
+
+ private Request makeRequest(SolrRequest solrRequest, String collection)
+ throws SolrServerException, IOException {
+ if (solrRequest.getBasePath() == null && serverBaseUrl == null)
+ throw new IllegalArgumentException("Destination node is not provided!");
+
+ if (solrRequest instanceof V2RequestSupport) {
+ solrRequest = ((V2RequestSupport) solrRequest).getV2Request();
+ }
+ SolrParams params = solrRequest.getParams();
+ RequestWriter.ContentWriter contentWriter = requestWriter.getContentWriter(solrRequest);
+ Collection<ContentStream> streams = contentWriter == null ? requestWriter.getContentStreams(solrRequest) : null;
+ String path = requestWriter.getPath(solrRequest);
+ if (path == null || !path.startsWith("/")) {
+ path = DEFAULT_PATH;
+ }
+
+ ResponseParser parser = solrRequest.getResponseParser();
+ if (parser == null) {
+ parser = this.parser;
+ }
+
+ // The parser 'wt=' and 'version=' params are used instead of the original
+ // params
+ ModifiableSolrParams wparams = new ModifiableSolrParams(params);
+ if (parser != null) {
+ wparams.set(CommonParams.WT, parser.getWriterType());
+ wparams.set(CommonParams.VERSION, parser.getVersion());
+ }
+
+ //TODO add invariantParams support
+
+ String basePath = solrRequest.getBasePath() == null ? serverBaseUrl : solrRequest.getBasePath();
+ if (collection != null)
+ basePath += "/" + collection;
+
+ if (solrRequest instanceof V2Request) {
+ if (System.getProperty("solr.v2RealPath") == null) {
+ basePath = serverBaseUrl.replace("/solr", "/api");
+ } else {
+ basePath = serverBaseUrl + "/____v2";
+ }
+ }
+
+ if (SolrRequest.METHOD.GET == solrRequest.getMethod()) {
+ if (streams != null || contentWriter != null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "GET can't send streams!");
+ }
+
+ return httpClient.newRequest(basePath + path + wparams.toQueryString()).method(HttpMethod.GET);
+ }
+
+ if (SolrRequest.METHOD.DELETE == solrRequest.getMethod()) {
+ return httpClient.newRequest(basePath + path + wparams.toQueryString()).method(HttpMethod.DELETE);
+ }
+
+ if (SolrRequest.METHOD.POST == solrRequest.getMethod() || SolrRequest.METHOD.PUT == solrRequest.getMethod()) {
+
+ String url = basePath + path;
+ boolean hasNullStreamName = false;
+ if (streams != null) {
+ hasNullStreamName = streams.stream().anyMatch(cs -> cs.getName() == null);
+ }
+
+ boolean isMultipart = streams != null && streams.size() > 1 && !hasNullStreamName;
+
+ HttpMethod method = SolrRequest.METHOD.POST == solrRequest.getMethod() ? HttpMethod.POST : HttpMethod.PUT;
+
+ if (contentWriter != null) {
+ Request req = httpClient
+ .newRequest(url + wparams.toQueryString())
+ .method(method);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ contentWriter.write(baos);
+
+ //TODO reduce memory usage
+ return req.content(new BytesContentProvider(contentWriter.getContentType(), baos.toByteArray()));
+ } else if (streams == null || isMultipart) {
+ // send server list and request list as query string params
+ ModifiableSolrParams queryParams = calculateQueryParams(this.queryParams, wparams);
+ queryParams.add(calculateQueryParams(solrRequest.getQueryParams(), wparams));
+ Request req = httpClient
+ .newRequest(url + queryParams.toQueryString())
+ .method(method);
+ return fillContentStream(req, streams, wparams, isMultipart);
+ } else {
+ // It is has one stream, it is the post body, put the params in the URL
+ ContentStream contentStream = streams.iterator().next();
+ return httpClient
+ .newRequest(url + wparams.toQueryString())
+ .method(method)
+ .content(new InputStreamContentProvider(contentStream.getStream()), contentStream.getContentType());
+ }
+ }
+
+ throw new SolrServerException("Unsupported method: " + solrRequest.getMethod());
+ }
+
+ private Request fillContentStream(Request req, Collection<ContentStream> streams,
+ ModifiableSolrParams wparams,
+ boolean isMultipart) throws IOException {
+ if (isMultipart) {
+ // multipart/form-data
+ MultiPartContentProvider content = new MultiPartContentProvider();
+ Iterator<String> iter = wparams.getParameterNamesIterator();
+ while (iter.hasNext()) {
+ String key = iter.next();
+ String[] vals = wparams.getParams(key);
+ if (vals != null) {
+ for (String val : vals) {
+ content.addFieldPart(key, new StringContentProvider(val), null);
+ }
+ }
+ }
+ if (streams != null) {
+ for (ContentStream contentStream : streams) {
+ String name = contentStream.getName();
+ if (name == null) {
+ name = "";
+ }
+ content.addFieldPart(name, new InputStreamContentProvider(contentStream.getStream()), null);
+ }
+ }
+ } else {
+ // application/x-www-form-urlencoded
+ Fields fields = new Fields();
+ Iterator<String> iter = wparams.getParameterNamesIterator();
+ while (iter.hasNext()) {
+ String key = iter.next();
+ String[] vals = wparams.getParams(key);
+ if (vals != null) {
+ for (String val : vals) {
+ fields.add(key, val);
+ }
+ }
+ }
+ req.content(new FormContentProvider(fields, StandardCharsets.UTF_8));
+ }
+
+ return req;
+ }
+
+ private NamedList<Object> processErrorsAndResponse(Response response,
+ final ResponseParser processor,
+ InputStream is,
+ String encoding,
+ final boolean isV2Api)
+ throws SolrServerException {
+ // handle some http level checks before trying to parse the response
+ int httpStatus = response.getStatus();
+
+ String contentType;
+ contentType = response.getHeaders().get("content-type");
+ if (contentType == null) contentType = "";
+
+ switch (httpStatus) {
+ case HttpStatus.SC_OK:
+ case HttpStatus.SC_BAD_REQUEST:
+ case HttpStatus.SC_CONFLICT: // 409
+ break;
+ case HttpStatus.SC_MOVED_PERMANENTLY:
+ case HttpStatus.SC_MOVED_TEMPORARILY:
+ if (!httpClient.isFollowRedirects()) {
+ throw new SolrServerException("Server at " + getBaseURL()
+ + " sent back a redirect (" + httpStatus + ").");
+ }
+ break;
+ default:
+ if (processor == null || "".equals(contentType)) {
+ throw new RemoteSolrException(serverBaseUrl, httpStatus, "non ok status: " + httpStatus
+ + ", message:" + response.getReason(),
+ null);
+ }
+ }
+
+ String procCt = processor.getContentType();
+ if (procCt != null) {
+ String procMimeType = ContentType.parse(procCt).getMimeType().trim().toLowerCase(Locale.ROOT);
+ String mimeType = ContentType.parse(contentType).getMimeType().trim().toLowerCase(Locale.ROOT);
+ if (!procMimeType.equals(mimeType)) {
+ // unexpected mime type
+ String msg = "Expected mime type " + procMimeType + " but got " + mimeType + ".";
+ try {
+ msg = msg + " " + IOUtils.toString(is, encoding);
+ } catch (IOException e) {
+ throw new RemoteSolrException(serverBaseUrl, httpStatus, "Could not parse response with encoding " + encoding, e);
+ }
+ throw new RemoteSolrException(serverBaseUrl, httpStatus, msg, null);
+ }
+ }
+
+ NamedList<Object> rsp;
+ try {
+ rsp = parser.processResponse(is, encoding);
+ } catch (Exception e) {
+ throw new RemoteSolrException(serverBaseUrl, httpStatus, e.getMessage(), e);
+ }
+
+ Object error = rsp == null ? null : rsp.get("error");
+ if (error != null && (String.valueOf(getObjectByPath(error, true, errPath)).endsWith("ExceptionWithErrObject"))) {
+ throw RemoteExecutionException.create(serverBaseUrl, rsp);
+ }
+ if (httpStatus != HttpStatus.SC_OK && !isV2Api) {
+ NamedList<String> metadata = null;
+ String reason = null;
+ try {
+ NamedList err = (NamedList) rsp.get("error");
+ if (err != null) {
+ reason = (String) err.get("msg");
+ if (reason == null) {
+ reason = (String) err.get("trace");
+ }
+ metadata = (NamedList<String>) err.get("metadata");
+ }
+ } catch (Exception ex) {
+ }
+ if (reason == null) {
+ StringBuilder msg = new StringBuilder();
+ msg.append(response.getReason())
+ .append("\n\n")
+ .append("request: ")
+ .append(response.getRequest().getMethod());
+ try {
+ reason = java.net.URLDecoder.decode(msg.toString(), UTF_8);
+ } catch (UnsupportedEncodingException e) {
+ }
+ }
+ RemoteSolrException rss = new RemoteSolrException(serverBaseUrl, httpStatus, reason, null);
+ if (metadata != null) rss.setMetadata(metadata);
+ throw rss;
+ }
+ return rsp;
+ }
+
+ @Override
+ public NamedList<Object> request(SolrRequest request, String collection) throws SolrServerException, IOException {
+ return request(request, collection, null).response;
+ }
+
+ public void setRequestWriter(RequestWriter requestWriter) {
+ this.requestWriter = requestWriter;
+ }
+
+ // sample new async method
+ public void add(String collection, SolrInputDocument doc, int commitWithinMs, OnComplete<UpdateResponse> onComplete)
+ throws SolrServerException, IOException {
+
+ UpdateRequest req = new UpdateRequest();
+ req.add(doc);
+ req.setCommitWithin(commitWithinMs);
+ request(req, collection, onComplete);
+ }
+
+ public String get(String url) throws InterruptedException, ExecutionException, TimeoutException {
+ ContentResponse response = httpClient.GET(url);
+ return response.getContentAsString();
+ }
+
+ // sample new async method
+ public void query(String collection, SolrParams params, OnComplete<QueryResponse> onComplete)
+ throws SolrServerException, IOException {
+ QueryRequest queryRequest = new QueryRequest(params);
+ request(queryRequest, collection, onComplete);
+ }
+
+ public InputStream queryAndStreamResponse(String collection, SolrParams params)
+ throws SolrServerException, IOException {
+ QueryRequest queryRequest = new QueryRequest(params);
+ Http2ClientResponse resp = request(queryRequest, collection, null, true);
+ assert resp.stream != null;
+ return resp.stream;
+ }
+
+ public void commit(String collection, boolean softCommit, boolean waitSearcher, OnComplete<UpdateResponse> onComplete)
+ throws SolrServerException, IOException {
+ UpdateRequest req = new UpdateRequest();
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ params.set(UpdateParams.COMMIT, "true");
+ params.set(UpdateParams.SOFT_COMMIT, String.valueOf(softCommit));
+
+ params.set(UpdateParams.WAIT_SEARCHER, String.valueOf(waitSearcher));
+ req.setParams(params);
+
+ request(req, collection, onComplete);
+ }
+
+ public interface OnComplete<T> {
+ void onSuccess(T result);
+
+ void onFailure(Throwable e);
+ }
+
+ public void setFollowRedirects(boolean follow) {
+ httpClient.setFollowRedirects(follow);
+ }
+
+ public String getBaseURL() {
+ return serverBaseUrl;
+ }
+
+ public static class Builder {
+
+ private HttpClient httpClient;
+ private SslContextFactory sslContextFactory;
+ private Integer idleTimeout;
+ private Integer connectionTimeout;
+ private boolean useHttp1_1 = false;
+ protected String baseSolrUrl;
+ private Request.BeginListener beginListener = request -> {};
+
+ public Builder() {
+ this.sslContextFactory = defaultSslContextFactory;
+ }
+
+ public Builder(String baseSolrUrl) {
+ this.baseSolrUrl = baseSolrUrl;
+ this.sslContextFactory = defaultSslContextFactory;
+ }
+
+ public Http2SolrClient build() {
+ return new Http2SolrClient(baseSolrUrl, this);
+ }
+
+ public Builder withHttpClient(HttpClient httpClient) {
+ this.httpClient = httpClient;
+ return this;
+ }
+
+ public Builder withSslContextFactory(SslContextFactory factory) {
+ this.sslContextFactory = factory;
+ return this;
+ }
+
+ public Builder idleTimeout(int idleConnectionTimeout) {
+ this.idleTimeout = idleConnectionTimeout;
+ return this;
+ }
+
+ public Builder useHttp1_1(boolean useHttp1_1) {
+ this.useHttp1_1 = useHttp1_1;
+ return this;
+ }
+
+ public Builder connectionTimeout(int connectionTimeOut) {
+ this.connectionTimeout = connectionTimeOut;
+ return this;
+ }
+
+ public Builder withListener(Request.BeginListener beginListener) {
+ this.beginListener = beginListener;
+ return this;
+ }
+ }
+
+ /**
+ * Subclass of SolrException that allows us to capture an arbitrary HTTP status code that may have been returned by
+ * the remote server or a proxy along the way.
+ */
+ public static class RemoteSolrException extends SolrException {
+ /**
+ * @param remoteHost the host the error was received from
+ * @param code Arbitrary HTTP status code
+ * @param msg Exception Message
+ * @param th Throwable to wrap with this Exception
+ */
+ public RemoteSolrException(String remoteHost, int code, String msg, Throwable th) {
+ super(code, "Error from server at " + remoteHost + ": " + msg, th);
+ }
+ }
+
+ /**
+ * This should be thrown when a server has an error in executing the request and it sends a proper payload back to the
+ * client
+ */
+ public static class RemoteExecutionException extends RemoteSolrException {
+ private NamedList meta;
+
+ public RemoteExecutionException(String remoteHost, int code, String msg, NamedList meta) {
+ super(remoteHost, code, msg, null);
+ this.meta = meta;
+ }
+
+ public static RemoteExecutionException create(String host, NamedList errResponse) {
+ Object errObj = errResponse.get("error");
+ if (errObj != null) {
+ Number code = (Number) getObjectByPath(errObj, true, Collections.singletonList("code"));
+ String msg = (String) getObjectByPath(errObj, true, Collections.singletonList("msg"));
+ return new RemoteExecutionException(host, code == null ? ErrorCode.UNKNOWN.code : code.intValue(),
+ msg == null ? "Unknown Error" : msg, errResponse);
+
+ } else {
+ throw new RuntimeException("No error");
+ }
+ }
+
+ public NamedList getMetaData() {
+ return meta;
+ }
+ }
+
+ protected static class Http2ClientResponse {
+ NamedList response;
+ InputStream stream;
+ }
+
+ public Set<String> getQueryParams() {
+ return queryParams;
+ }
+
+ /**
+ * Expert Method
+ *
+ * @param queryParams set of param keys to only send via the query string
+ * Note that the param will be sent as a query string if the key is part
+ * of this Set or the SolrRequest's query params.
+ * @see org.apache.solr.client.solrj.SolrRequest#getQueryParams
+ */
+ public void setQueryParams(Set<String> queryParams) {
+ this.queryParams = queryParams;
+ }
+
+ private ModifiableSolrParams calculateQueryParams(Set<String> queryParamNames,
+ ModifiableSolrParams wparams) {
+ ModifiableSolrParams queryModParams = new ModifiableSolrParams();
+ if (queryParamNames != null) {
+ for (String param : queryParamNames) {
+ String[] value = wparams.getParams(param);
+ if (value != null) {
+ for (String v : value) {
+ queryModParams.add(param, v);
+ }
+ wparams.remove(param);
+ }
+ }
+ }
+ return queryModParams;
+ }
+
+ public ResponseParser getParser() {
+ return parser;
+ }
+
+ public void setParser(ResponseParser processor) {
+ parser = processor;
+ }
+
+ // public for testing, only used by tests
+ public static void setSslContextFactory(SslContextFactory sslContextFactory) {
+ Http2SolrClient.defaultSslContextFactory = sslContextFactory;
+ }
+
+ // public for testing, only used by tests
+ public static void resetSslContextFactory() {
+ Http2SolrClient.defaultSslContextFactory = getDefaultSslContextFactory();
+ }
+
+ private static SslContextFactory getDefaultSslContextFactory() {
+ SslContextFactory sslContextFactory = new SslContextFactory(false);
+
+ if (null != System.getProperty("javax.net.ssl.keyStore")) {
+ sslContextFactory.setKeyStorePath
+ (System.getProperty("javax.net.ssl.keyStore"));
+ }
+ if (null != System.getProperty("javax.net.ssl.keyStorePassword")) {
+ sslContextFactory.setKeyStorePassword
+ (System.getProperty("javax.net.ssl.keyStorePassword"));
+ }
+ if (null != System.getProperty("javax.net.ssl.trustStore")) {
+ sslContextFactory.setTrustStorePath
+ (System.getProperty("javax.net.ssl.trustStore"));
+ }
+ if (null != System.getProperty("javax.net.ssl.trustStorePassword")) {
+ sslContextFactory.setTrustStorePassword
+ (System.getProperty("javax.net.ssl.trustStorePassword"));
+ }
+
+ String checkPeerNameStr = System.getProperty(HttpClientUtil.SYS_PROP_CHECK_PEER_NAME);
+ boolean sslCheckPeerName = true;
+ if (checkPeerNameStr == null && "false".equalsIgnoreCase(checkPeerNameStr)) {
+ sslCheckPeerName = false;
+ }
+
+ if (System.getProperty("tests.jettySsl.clientAuth") != null) {
+ sslCheckPeerName = sslCheckPeerName || Boolean.getBoolean("tests.jettySsl.clientAuth");
+ }
+
+ sslContextFactory.setNeedClientAuth(sslCheckPeerName);
+ return sslContextFactory;
+ }
+
+}
[15/31] lucene-solr:jira/http2: SOLR-8207: Collections with
underscores in name no longer cause a crash the Cloud->Nodes UI
Posted by da...@apache.org.
SOLR-8207: Collections with underscores in name no longer cause a crash the Cloud->Nodes UI
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6845bbff
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6845bbff
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6845bbff
Branch: refs/heads/jira/http2
Commit: 6845bbff55707ca219fd29f9c4d3d9e3c20175ec
Parents: 9306922
Author: Jan Høydahl <ja...@apache.org>
Authored: Wed Aug 8 15:25:43 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Wed Aug 8 15:25:43 2018 +0200
----------------------------------------------------------------------
solr/webapp/web/js/angular/controllers/cloud.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6845bbff/solr/webapp/web/js/angular/controllers/cloud.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js
index 8cbd4db..8c65037 100644
--- a/solr/webapp/web/js/angular/controllers/cloud.js
+++ b/solr/webapp/web/js/angular/controllers/cloud.js
@@ -407,13 +407,13 @@ var nodesSubController = function($scope, Collections, System, Metrics) {
if (cores) {
for (coreId in cores) {
var core = cores[coreId];
- var keyName = "solr.core." + core['core'].replace('_', '.').replace('_', '.');
+ var keyName = "solr.core." + core['core'].replace(/(.*?)_(shard\d+)_(replica_.\d+)/, '\$1.\$2.\$3');
var nodeMetric = m.metrics[keyName];
var size = nodeMetric['INDEX.sizeInBytes'];
size = (typeof size !== 'undefined') ? size : 0;
core['sizeInBytes'] = size;
core['size'] = bytesToSize(size);
- core['label'] = core['core'].replace('_shard', '_s').replace(/_replica_./, 'r');
+ core['label'] = core['core'].replace(/(.*?)_shard(\d+)_replica_.(\d+)/, '\$1_s\$2r\$3');
indexSizeTotal += size;
var numDocs = nodeMetric['SEARCHER.searcher.numDocs'];
numDocs = (typeof numDocs !== 'undefined') ? numDocs : 0;
[22/31] lucene-solr:jira/http2: SOLR-12616: Optimize Export writer
upto 4 sort fields to get better performance. This was removed in SOLR-11598
but brought back in the same version
Posted by da...@apache.org.
SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance. This was removed in SOLR-11598 but brought back in the same version
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e9f3a3ce
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e9f3a3ce
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e9f3a3ce
Branch: refs/heads/jira/http2
Commit: e9f3a3ce1d482bd90ba8aca6e8cb7fe6c86756eb
Parents: 7c4584b
Author: Varun Thacker <va...@apache.org>
Authored: Wed Aug 8 11:10:21 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Aug 8 13:28:00 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../solr/handler/export/DoubleValueSortDoc.java | 93 ++++++++++++++
.../solr/handler/export/ExportWriter.java | 13 +-
.../solr/handler/export/QuadValueSortDoc.java | 126 +++++++++++++++++++
.../solr/handler/export/SingleValueSortDoc.java | 82 ++++++++++++
.../org/apache/solr/handler/export/SortDoc.java | 7 +-
.../solr/handler/export/TripleValueSortDoc.java | 110 ++++++++++++++++
7 files changed, 431 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9c80773..9e6ece5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -253,6 +253,9 @@ Optimizations
* SOLR-11881: Retry update requests sent by leaders to it's followers (Varun Thacker, Mark Miller, Tomás Fernández Löbbe)
+* SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance.
+ This was removed in SOLR-11598 but brought back in the same version (Amrit Sarkar, Varun Thacker)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
new file mode 100644
index 0000000..30fff33
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class DoubleValueSortDoc extends SingleValueSortDoc {
+
+ protected SortValue value2;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((DoubleValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((DoubleValueSortDoc)sortDoc).value2);
+ }
+
+ public DoubleValueSortDoc(SortValue value1, SortValue value2) {
+ super(value1);
+ this.value2 = value2;
+ }
+
+ public SortDoc copy() {
+ return new DoubleValueSortDoc(value1.copy(), value2.copy());
+ }
+
+ public boolean lessThan(Object o) {
+ DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if (comp == 0) {
+ return value2.compareTo(sd.value2);
+ } else {
+ return comp;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 57ce600..9cacfcc 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -431,7 +431,18 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
}
}
-
+ //SingleValueSortDoc etc are specialized classes which don't have array lookups. On benchmarking large datasets
+ //This is faster than the using an array in SortDoc . So upto 4 sort fields we still want to keep specialized classes.
+ //SOLR-12616 has more details
+ if (sortValues.length == 1) {
+ return new SingleValueSortDoc(sortValues[0]);
+ } else if (sortValues.length == 2) {
+ return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
+ } else if (sortValues.length == 3) {
+ return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
+ } else if (sortValues.length == 4) {
+ return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
+ }
return new SortDoc(sortValues);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
new file mode 100644
index 0000000..c2dfcba
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class QuadValueSortDoc extends TripleValueSortDoc {
+
+ protected SortValue value4;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ value3.setNextReader(context);
+ value4.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ value3.reset();
+ value4.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ value3.setCurrentValue(docId);
+ value4.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((QuadValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((QuadValueSortDoc)sortDoc).value2);
+ value3.setCurrentValue(((QuadValueSortDoc)sortDoc).value3);
+ value4.setCurrentValue(((QuadValueSortDoc)sortDoc).value4);
+ }
+
+ public QuadValueSortDoc(SortValue value1, SortValue value2, SortValue value3, SortValue value4) {
+ super(value1, value2, value3);
+ this.value4 = value4;
+ }
+
+ public SortDoc copy() {
+ return new QuadValueSortDoc(value1.copy(), value2.copy(), value3.copy(), value4.copy());
+ }
+
+ public boolean lessThan(Object o) {
+
+ QuadValueSortDoc sd = (QuadValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value3.compareTo(sd.value3);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value4.compareTo(sd.value4);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ QuadValueSortDoc sd = (QuadValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == 0) {
+ comp = value2.compareTo(sd.value2);
+ if(comp == 0) {
+ comp = value3.compareTo(sd.value3);
+ if(comp == 0) {
+ return value4.compareTo(sd.value4);
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
new file mode 100644
index 0000000..88fc799
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class SingleValueSortDoc extends SortDoc {
+
+ protected SortValue value1;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ this.value1.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((SingleValueSortDoc)sortDoc).value1);
+ }
+
+ public SingleValueSortDoc(SortValue value1) {
+ super();
+ this.value1 = value1;
+ }
+
+ public SortDoc copy() {
+ return new SingleValueSortDoc(value1.copy());
+ }
+
+ public boolean lessThan(Object o) {
+ SingleValueSortDoc sd = (SingleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+
+ public int compareTo(Object o) {
+ SingleValueSortDoc sd = (SingleValueSortDoc)o;
+ return value1.compareTo(sd.value1);
+ }
+
+ public String toString() {
+ return docId+":"+value1.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
index 3cf478b..1302332 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -32,6 +32,8 @@ class SortDoc {
public SortDoc(SortValue[] sortValues) {
this.sortValues = sortValues;
}
+ public SortDoc() {
+ }
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
@@ -43,6 +45,7 @@ class SortDoc {
public void reset() {
this.docId = -1;
+ this.docBase = -1;
for (SortValue value : sortValues) {
value.reset();
}
@@ -82,9 +85,9 @@ class SortDoc {
SortValue[] sortValues1 = sd.sortValues;
for(int i=0; i<sortValues.length; i++) {
int comp = sortValues[i].compareTo(sortValues1[i]);
- if(comp < 0) {
+ if (comp < 0) {
return true;
- } if(comp > 0) {
+ } else if (comp > 0) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
new file mode 100644
index 0000000..0241ebc
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class TripleValueSortDoc extends DoubleValueSortDoc {
+
+ protected SortValue value3;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ value3.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ value3.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ value3.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((TripleValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((TripleValueSortDoc)sortDoc).value2);
+ value3.setCurrentValue(((TripleValueSortDoc)sortDoc).value3);
+ }
+
+ public TripleValueSortDoc(SortValue value1, SortValue value2, SortValue value3) {
+ super(value1, value2);
+ this.value3 = value3;
+ }
+
+ public SortDoc copy() {
+ return new TripleValueSortDoc(value1.copy(), value2.copy(), value3.copy());
+ }
+
+ public boolean lessThan(Object o) {
+
+ TripleValueSortDoc sd = (TripleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value3.compareTo(sd.value3);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ TripleValueSortDoc sd = (TripleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if (comp == 0) {
+ comp = value2.compareTo(sd.value2);
+ if (comp == 0) {
+ return value3.compareTo(sd.value3);
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ }
+}
[10/31] lucene-solr:jira/http2: SOLR-12629: The predict evaluator
should work with the polyfit function
Posted by da...@apache.org.
SOLR-12629: The predict evaluator should work with the polyfit function
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2ca5a6d1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2ca5a6d1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2ca5a6d1
Branch: refs/heads/jira/http2
Commit: 2ca5a6d12e48e5d58811d7991da028cb2d70e86f
Parents: 3898653
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue Aug 7 10:12:07 2018 -0400
Committer: Joel <jo...@alfresco.com>
Committed: Tue Aug 7 10:31:53 2018 -0400
----------------------------------------------------------------------
.../solr/client/solrj/io/eval/PolyFitEvaluator.java | 7 ++++++-
.../client/solrj/io/stream/MathExpressionTest.java | 15 ++++++++++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2ca5a6d1/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
index 5320f18..5292c94 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
@@ -96,6 +96,11 @@ public class PolyFitEvaluator extends RecursiveNumericEvaluator implements ManyV
list.add(yvalue);
}
- return list;
+
+ VectorFunction vec = new VectorFunction(pf, list);
+ vec.addToContext("x", x);
+ vec.addToContext("y", y);
+
+ return vec;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2ca5a6d1/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 3806b22..98a52a6 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -2856,7 +2856,8 @@ public class MathExpressionTest extends SolrCloudTestCase {
public void testPolyfit() throws Exception {
String cexpr = "let(echo=true," +
" a=array(0,1,2,3,4,5,6,7)," +
- " fit=polyfit(a, 1))";
+ " fit=polyfit(a, 1)," +
+ " predictions=predict(fit, a))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", cexpr);
paramsLoc.set("qt", "/stream");
@@ -2876,6 +2877,18 @@ public class MathExpressionTest extends SolrCloudTestCase {
assertTrue(out.get(5).intValue() == 5);
assertTrue(out.get(6).intValue() == 6);
assertTrue(out.get(7).intValue() == 7);
+
+
+ out = (List<Number>)tuples.get(0).get("predictions");
+ assertTrue(out.size() == 8);
+ assertTrue(out.get(0).intValue() == 0);
+ assertTrue(out.get(1).intValue() == 1);
+ assertTrue(out.get(2).intValue() == 2);
+ assertTrue(out.get(3).intValue() == 3);
+ assertTrue(out.get(4).intValue() == 4);
+ assertTrue(out.get(5).intValue() == 5);
+ assertTrue(out.get(6).intValue() == 6);
+ assertTrue(out.get(7).intValue() == 7);
}
@Test
[13/31] lucene-solr:jira/http2: LUCENE-8439: Disjunction max queries
can skip blocks to select the top documents when the total hit count is not
required
Posted by da...@apache.org.
LUCENE-8439: Disjunction max queries can skip blocks to select the top documents when the total hit count is not required
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ba9b18f3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ba9b18f3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ba9b18f3
Branch: refs/heads/jira/http2
Commit: ba9b18f36743dc9674478dce5bbf2da509ef41c0
Parents: 38bf976
Author: Jim Ferenczi <ji...@apache.org>
Authored: Wed Aug 8 12:34:42 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed Aug 8 12:34:42 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/search/BlockMaxDISI.java | 94 ++++++++++++++++++++
.../lucene/search/Boolean2ScorerSupplier.java | 2 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../lucene/search/DisjunctionMaxQuery.java | 2 +-
.../lucene/search/DisjunctionMaxScorer.java | 80 ++++++++++-------
.../apache/lucene/search/DisjunctionScorer.java | 22 ++++-
.../lucene/search/DisjunctionSumScorer.java | 4 +-
.../lucene/search/TestDisjunctionMaxQuery.java | 77 +++++++++++++++-
9 files changed, 244 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 865f5e2..f0360a9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -134,6 +134,9 @@ Optimizations
or phrase queries as sub queries, which know how to leverage this information
to run faster. (Adrien Grand)
+* LUCENE-8439: Disjunction max queries can skip blocks to select the top documents
+ when the total hit count is not required. (Jim Ferenczi, Adrien Grand)
+
======================= Lucene 7.5.0 =======================
API Changes:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java
new file mode 100644
index 0000000..804f96f
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+import java.io.IOException;
+
+/**
+ * {@link DocIdSetIterator} that skips non-competitive docs by checking
+ * the max score of the provided {@link Scorer} for the current block.
+ * Call {@link #setMinCompetitiveScore(float)} in order to give this iterator the ability
+ * to skip low-scoring documents.
+ * @lucene.internal
+ */
+public class BlockMaxDISI extends DocIdSetIterator {
+ protected final Scorer scorer;
+ private final DocIdSetIterator in;
+ private float minScore;
+ private float maxScore;
+ private int upTo = -1;
+
+ public BlockMaxDISI(DocIdSetIterator iterator, Scorer scorer) {
+ this.in = iterator;
+ this.scorer = scorer;
+ }
+
+ @Override
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(docID()+1);
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ int doc = advanceImpacts(target);
+ return in.advance(doc);
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ public void setMinCompetitiveScore(float minScore) {
+ this.minScore = minScore;
+ }
+
+ private void moveToNextBlock(int target) throws IOException {
+ upTo = scorer.advanceShallow(target);
+ maxScore = scorer.getMaxScore(upTo);
+ }
+
+ private int advanceImpacts(int target) throws IOException {
+ if (minScore == -1 || target == NO_MORE_DOCS) {
+ return target;
+ }
+
+ if (target > upTo) {
+ moveToNextBlock(target);
+ }
+
+ while (true) {
+ if (maxScore >= minScore) {
+ return target;
+ }
+
+ if (upTo == NO_MORE_DOCS) {
+ return NO_MORE_DOCS;
+ }
+
+ target = upTo + 1;
+
+ moveToNextBlock(target);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
index 3401215..bae7941 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
@@ -187,7 +187,7 @@ final class Boolean2ScorerSupplier extends ScorerSupplier {
} else if (scoreMode == ScoreMode.TOP_SCORES) {
return new WANDScorer(weight, optionalScorers);
} else {
- return new DisjunctionSumScorer(weight, optionalScorers, scoreMode.needsScores());
+ return new DisjunctionSumScorer(weight, optionalScorers, scoreMode);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
index 1fddd7c..5f20a6b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
@@ -309,7 +309,7 @@ final class BooleanWeight extends Weight {
} else {
Scorer prohibitedScorer = prohibited.size() == 1
? prohibited.get(0)
- : new DisjunctionSumScorer(this, prohibited, false);
+ : new DisjunctionSumScorer(this, prohibited, ScoreMode.COMPLETE_NO_SCORES);
if (prohibitedScorer.twoPhaseIterator() != null) {
// ReqExclBulkScorer can't deal efficiently with two-phased prohibited clauses
return null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
index 5b99c3c..3cdc0bc 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
@@ -148,7 +148,7 @@ public final class DisjunctionMaxQuery extends Query implements Iterable<Query>
// only one sub-scorer in this segment
return scorers.get(0);
} else {
- return new DisjunctionMaxScorer(this, tieBreakerMultiplier, scorers, scoreMode.needsScores());
+ return new DisjunctionMaxScorer(this, tieBreakerMultiplier, scorers, scoreMode);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
index 0434e0f..dd04d4c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
@@ -21,6 +21,8 @@ import java.util.List;
import org.apache.lucene.util.MathUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* The Scorer for DisjunctionMaxQuery. The union of all documents generated by the the subquery scorers
* is generated in document number order. The score for each document is the maximum of the scores computed
@@ -28,9 +30,9 @@ import org.apache.lucene.util.MathUtil;
* for the other subqueries that generate the document.
*/
final class DisjunctionMaxScorer extends DisjunctionScorer {
+ private final List<Scorer> subScorers;
/* Multiplier applied to non-maximum-scoring subqueries for a document as they are summed into the result. */
private final float tieBreakerMultiplier;
- private final float maxScore;
/**
* Creates a new instance of DisjunctionMaxScorer
@@ -43,40 +45,13 @@ final class DisjunctionMaxScorer extends DisjunctionScorer {
* @param subScorers
* The sub scorers this Scorer should iterate on
*/
- DisjunctionMaxScorer(Weight weight, float tieBreakerMultiplier, List<Scorer> subScorers, boolean needsScores) throws IOException {
- super(weight, subScorers, needsScores);
+ DisjunctionMaxScorer(Weight weight, float tieBreakerMultiplier, List<Scorer> subScorers, ScoreMode scoreMode) throws IOException {
+ super(weight, subScorers, scoreMode);
+ this.subScorers = subScorers;
this.tieBreakerMultiplier = tieBreakerMultiplier;
if (tieBreakerMultiplier < 0 || tieBreakerMultiplier > 1) {
throw new IllegalArgumentException("tieBreakerMultiplier must be in [0, 1]");
}
-
- if (needsScores == false) {
- this.maxScore = Float.MAX_VALUE;
- } else {
- float scoreMax = 0;
- double otherScoreSum = 0;
- for (Scorer scorer : subScorers) {
- scorer.advanceShallow(0);
- float subScore = scorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
- if (subScore >= scoreMax) {
- otherScoreSum += scoreMax;
- scoreMax = subScore;
- } else {
- otherScoreSum += subScore;
- }
- }
-
- if (tieBreakerMultiplier == 0) {
- this.maxScore = scoreMax;
- } else {
- // The error of sums depends on the order in which values are summed up. In
- // order to avoid this issue, we compute an upper bound of the value that
- // the sum may take. If the max relative error is b, then it means that two
- // sums are always within 2*b of each other.
- otherScoreSum *= (1 + 2 * MathUtil.sumRelativeErrorBound(subScorers.size() - 1));
- this.maxScore = (float) (scoreMax + otherScoreSum * tieBreakerMultiplier);
- }
- }
}
@Override
@@ -96,7 +71,48 @@ final class DisjunctionMaxScorer extends DisjunctionScorer {
}
@Override
+ public int advanceShallow(int target) throws IOException {
+ int upTo = NO_MORE_DOCS;
+ for (Scorer scorer : subScorers) {
+ if (scorer.docID() <= target) {
+ upTo = Math.min(scorer.advanceShallow(target), upTo);
+ } else if (scorer.docID() < NO_MORE_DOCS) {
+ upTo = Math.min(scorer.docID()-1, upTo);
+ }
+ }
+ return upTo;
+ }
+
+ @Override
public float getMaxScore(int upTo) throws IOException {
- return maxScore;
+ float scoreMax = 0;
+ double otherScoreSum = 0;
+ for (Scorer scorer : subScorers) {
+ if (scorer.docID() <= upTo) {
+ float subScore = scorer.getMaxScore(upTo);
+ if (subScore >= scoreMax) {
+ otherScoreSum += scoreMax;
+ scoreMax = subScore;
+ } else {
+ otherScoreSum += subScore;
+ }
+ }
+ }
+
+ if (tieBreakerMultiplier == 0) {
+ return scoreMax;
+ } else {
+ // The error of sums depends on the order in which values are summed up. In
+ // order to avoid this issue, we compute an upper bound of the value that
+ // the sum may take. If the max relative error is b, then it means that two
+ // sums are always within 2*b of each other.
+ otherScoreSum *= (1 + 2 * MathUtil.sumRelativeErrorBound(subScorers.size() - 1));
+ return (float) (scoreMax + otherScoreSum * tieBreakerMultiplier);
+ }
+ }
+
+ @Override
+ public void setMinCompetitiveScore(float minScore) {
+ getBlockMaxApprox().setMinCompetitiveScore(minScore);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
index 147b993..249521e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
@@ -32,10 +32,11 @@ abstract class DisjunctionScorer extends Scorer {
private final boolean needsScores;
private final DisiPriorityQueue subScorers;
- private final DisjunctionDISIApproximation approximation;
+ private final DocIdSetIterator approximation;
+ private final BlockMaxDISI blockMaxApprox;
private final TwoPhase twoPhase;
- protected DisjunctionScorer(Weight weight, List<Scorer> subScorers, boolean needsScores) {
+ protected DisjunctionScorer(Weight weight, List<Scorer> subScorers, ScoreMode scoreMode) throws IOException {
super(weight);
if (subScorers.size() <= 1) {
throw new IllegalArgumentException("There must be at least 2 subScorers");
@@ -45,8 +46,17 @@ abstract class DisjunctionScorer extends Scorer {
final DisiWrapper w = new DisiWrapper(scorer);
this.subScorers.add(w);
}
- this.needsScores = needsScores;
- this.approximation = new DisjunctionDISIApproximation(this.subScorers);
+ this.needsScores = scoreMode != ScoreMode.COMPLETE_NO_SCORES;
+ if (scoreMode == ScoreMode.TOP_SCORES) {
+ for (Scorer scorer : subScorers) {
+ scorer.advanceShallow(0);
+ }
+ this.blockMaxApprox = new BlockMaxDISI(new DisjunctionDISIApproximation(this.subScorers), this);
+ this.approximation = blockMaxApprox;
+ } else {
+ this.approximation = new DisjunctionDISIApproximation(this.subScorers);
+ this.blockMaxApprox = null;
+ }
boolean hasApproximation = false;
float sumMatchCost = 0;
@@ -167,6 +177,10 @@ abstract class DisjunctionScorer extends Scorer {
return subScorers.top().doc;
}
+ BlockMaxDISI getBlockMaxApprox() {
+ return blockMaxApprox;
+ }
+
DisiWrapper getSubMatches() throws IOException {
if (twoPhase == null) {
return subScorers.topList();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
index 06e8c4a..18835ed 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
@@ -28,8 +28,8 @@ final class DisjunctionSumScorer extends DisjunctionScorer {
* @param weight The weight to be used.
* @param subScorers Array of at least two subscorers.
*/
- DisjunctionSumScorer(Weight weight, List<Scorer> subScorers, boolean needsScores) throws IOException {
- super(weight, subScorers, needsScores);
+ DisjunctionSumScorer(Weight weight, List<Scorer> subScorers, ScoreMode scoreMode) throws IOException {
+ super(weight, subScorers, scoreMode);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
index 2d72fc1..7fa1ed3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
@@ -16,18 +16,22 @@
*/
package org.apache.lucene.search;
-
import java.io.IOException;
+import java.io.StringReader;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -522,6 +526,77 @@ public class TestDisjunctionMaxQuery extends LuceneTestCase {
assertEquals(bq.clauses().get(0), new BooleanClause(sub1, BooleanClause.Occur.SHOULD));
assertEquals(bq.clauses().get(1), new BooleanClause(sub2, BooleanClause.Occur.SHOULD));
}
+
+ public void testRandomTopDocs() throws Exception {
+ doTestRandomTopDocs(2, 0.05f, 0.05f);
+ doTestRandomTopDocs(2, 1.0f, 0.05f);
+ doTestRandomTopDocs(3, 1.0f, 0.5f, 0.05f);
+ doTestRandomTopDocs(4, 1.0f, 0.5f, 0.05f, 0f);
+ doTestRandomTopDocs(4, 1.0f, 0.5f, 0.05f, 0f);
+ }
+
+ private void doTestRandomTopDocs(int numFields, double... freqs) throws IOException {
+ assert numFields == freqs.length;
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+ IndexWriter w = new IndexWriter(dir, config);
+
+ int numDocs = atLeast(1000); // make sure some terms have skip data
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ for (int j = 0; j < numFields; j++) {
+ StringBuilder builder = new StringBuilder();
+ int numAs = random().nextDouble() < freqs[j] ? 0 : 1 + random().nextInt(5);
+ for (int k = 0; k < numAs; k++) {
+ if (builder.length() > 0) {
+ builder.append(' ');
+ }
+ builder.append('a');
+ }
+ if (random().nextBoolean()) {
+ doc.add(new StringField("field", "c", Field.Store.NO));
+ }
+ int numOthers = random().nextBoolean() ? 0 : 1 + random().nextInt(5);
+ for (int k = 0; k < numOthers; k++) {
+ if (builder.length() > 0) {
+ builder.append(' ');
+ }
+ builder.append(Integer.toString(random().nextInt()));
+ }
+ doc.add(new TextField(Integer.toString(j), new StringReader(builder.toString())));
+ }
+ w.addDocument(doc);
+ }
+ IndexReader reader = DirectoryReader.open(w);
+ w.close();
+ IndexSearcher searcher = newSearcher(reader);
+ for (int i = 0; i < 4; i++) {
+ List<Query> clauses = new ArrayList<>();
+ for (int j = 0; j < numFields; j++) {
+ if (i % 2 == 1) {
+ clauses.add(tq(Integer.toString(j), "a"));
+ } else {
+ float boost = random().nextBoolean() ? 0 : random().nextFloat();
+ if (boost > 0) {
+ clauses.add(tq(Integer.toString(j), "a", boost));
+ } else {
+ clauses.add(tq(Integer.toString(j), "a"));
+ }
+ }
+ }
+ float tieBreaker = random().nextFloat();
+ Query query = new DisjunctionMaxQuery(clauses, tieBreaker);
+ CheckHits.checkTopScores(random(), query, searcher);
+
+ query = new BooleanQuery.Builder()
+ .add(new DisjunctionMaxQuery(clauses, tieBreaker), BooleanClause.Occur.MUST)
+ .add(tq("field", "c"), BooleanClause.Occur.FILTER)
+ .build();
+ CheckHits.checkTopScores(random(), query, searcher);
+ }
+ reader.close();
+ dir.close();
+ }
/** macro */
protected Query tq(String f, String t) {
[05/31] lucene-solr:jira/http2: SOLR-12222: Improve
TestDistributedSearch search
Posted by da...@apache.org.
SOLR-12222: Improve TestDistributedSearch search
Consider the case when the distributed requests fails with timeAllowed before it reaches the shards
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/896fd0eb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/896fd0eb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/896fd0eb
Branch: refs/heads/jira/http2
Commit: 896fd0ebd55aaafa51222aa9200e16f2dbb093e4
Parents: db1e064
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Mon Aug 6 16:58:08 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Mon Aug 6 16:58:08 2018 -0700
----------------------------------------------------------------------
.../test/org/apache/solr/TestDistributedSearch.java | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/896fd0eb/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
index c9be70f..3ec1c6d 100644
--- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
+++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
@@ -1186,14 +1186,20 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
String s = shardsArr[i];
if (shard.contains(s)) {
found = true;
- // make sure that it responded if it's up
+ // make sure that it responded if it's up and the landing node didn't error before sending the request to the shard
if (upShards.contains(s)) {
// this is no longer true if there was a query timeout on an up shard
// assertTrue("Expected to find numFound in the up shard info",info.get("numFound") != null);
- assertTrue("Expected to find shardAddress in the up shard info: " + info.toString(), info.get("shardAddress") != null);
- }
- else {
- assertEquals("Expected to find the "+SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" header set if a shard is down",
+ boolean timeAllowedError = info.get("error") != null && info.get("error").toString().contains("Time allowed to handle this request");
+ if (timeAllowedError) {
+ assertEquals("Expected to find the " + SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY + " header set if a shard is down",
+ Boolean.TRUE, rsp.getHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+ assertTrue("Expected to find error in the down shard info: " + info.toString(), info.get("error") != null);
+ } else {
+ assertTrue("Expected timeAllowedError or to find shardAddress in the up shard info: " + info.toString(), info.get("shardAddress") != null);
+ }
+ } else {
+ assertEquals("Expected to find the " + SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY + " header set if a shard is down",
Boolean.TRUE, rsp.getHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
assertTrue("Expected to find error in the down shard info: " + info.toString(), info.get("error") != null);
}
[20/31] lucene-solr:jira/http2: SOLR-12636: Improve autoscaling
policy rules documentation
Posted by da...@apache.org.
SOLR-12636: Improve autoscaling policy rules documentation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/37e8fea2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/37e8fea2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/37e8fea2
Branch: refs/heads/jira/http2
Commit: 37e8fea22e532f9b9deb9ce040427cb8abc4a455
Parents: cbaedb4
Author: Steve Rowe <sa...@apache.org>
Authored: Wed Aug 8 14:23:08 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Wed Aug 8 14:23:08 2018 -0400
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
...olrcloud-autoscaling-policy-preferences.adoc | 167 +++++++++++++------
2 files changed, 117 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37e8fea2/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
index 2c4c433..2215c86 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
@@ -61,7 +61,7 @@ You can learn more about preferences in the <<solrcloud-autoscaling-policy-prefe
A cluster policy is a set of conditions that a node, shard, or collection must satisfy before it can be chosen as the target of a cluster management operation. These conditions are applied across the cluster regardless of the collection being managed. For example, the condition `{"cores":"<10", "node":"#ANY"}` means that any node must have less than 10 Solr cores in total, regardless of which collection they belong to.
-There are many metrics on which the condition can be based, e.g., system load average, heap usage, free disk space, etc. The full list of supported metrics can be found in the section describing <<solrcloud-autoscaling-policy-preferences.adoc#policy-attributes,Autoscaling Policy Attributes>>.
+There are many metrics on which the condition can be based, e.g., system load average, heap usage, free disk space, etc. The full list of supported metrics can be found in the section describing <<solrcloud-autoscaling-policy-preferences.adoc#policy-rule-attributes,Autoscaling Policy Rule Attributes>>.
When a node, shard, or collection does not satisfy the policy, we call it a *violation*. Solr ensures that cluster management operations minimize the number of violations. Cluster management operations are currently invoked manually. In the future, these cluster management operations may be invoked automatically in response to cluster events such as a node being added or lost.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37e8fea2/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
index 9641424..82af94c 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
@@ -20,9 +20,11 @@
The autoscaling policy and preferences are a set of rules and sorting preferences that help Solr select the target of cluster management operations so the overall load on the cluster remains balanced.
+Solr consults the configured policy and preferences when performing <<Commands That Use Autoscaling Policy and Preferences,Collections API commands>> in all contexts: manual, e.g. using `bin/solr`; semi-automatic, via the <<solrcloud-autoscaling-api.adoc#suggestions-api,Suggestions API>> or the Admin UI's <<suggestions-screen.adoc#suggestions-screen,Suggestions Screen>>; or fully automatic, via configured <<solrcloud-autoscaling-triggers.adoc#solrcloud-autoscaling-triggers,Triggers>>.
+
== Cluster Preferences Specification
-A preference is a hint to Solr on how to sort nodes based on their utilization. The default cluster preference is to sort by the total number of Solr cores (or replicas) hosted by a node. Therefore, by default, when selecting a node to add a replica, Solr can apply the preferences and choose the node with the least number of cores.
+A preference is a hint to Solr on how to sort nodes based on their utilization. The default cluster preference is to sort by the total number of Solr cores (or replicas) hosted by a node. Therefore, by default, when selecting a node to which to add a replica, Solr can apply the preferences and choose the node with the fewest cores.
More than one preference can be added to break ties. For example, we may choose to use free disk space to break ties if the number of cores on two nodes are the same. The node with the higher free disk space can be chosen as the target of the cluster operation.
@@ -86,16 +88,85 @@ In this example, we add a precision to the `freedisk` parameter so that nodes wi
A policy is a hard rule to be satisfied by each node. If a node does not satisfy the rule then it is called a *violation*. Solr ensures that the number of violations are minimized while invoking any cluster management operations.
-=== Policy Attributes
-A policy can have the following attributes:
+=== Policy Rule Structure
+
+==== Rule Types
+
+Policy rules can be either global or per-collection:
+
+* *Global rules* constrain the number of cores per node or node group. This type of rule applies to cores from all collections hosted on the specified node(s). As a result, <<Defining Collection-Specific Policies,collection-specific policies>>, which are associated with individual collections, may not contain global rules.
+* *Per-collection rules* constrain the number of replicas per node or node group.
+
+Global rules have three parts:
+
+* <<Node Selector>>
+* <<Core Count Constraint>>
+* <<Rule Strictness>>
+
+Per-collection rules have four parts:
+
+* <<Node Selector>>
+* <<Replica Selector and Rule Evaluation Context>>
+* <<Replica Count Constraint>>
+* <<Rule Strictness>>
+
+==== Node Selector
+
+Rule evaluation is restricted to node(s) matching the value of one of the following attributes: `node`, `port`, `ip_\*`, `sysprop.*`, or `diskType`. For replica/core count constraints other than `#EQUAL`, a condition specified in one of the following attributes may instead be used to select nodes: `freedisk`, `host`, `sysLoadAvg`, `heapUsage`, `nodeRole`, or `metrics.*`.
+
+Except for `node`, the attributes above cause selected nodes to be partitioned into node groups. A node group is referred to as a "bucket". Those attributes usable with the `#EQUAL` directive may define buckets either via the value `#EACH` or an array `["value1", ...]` (a subset of all possible values); in both cases, each node is placed in the bucket corresponding to the matching attribute value.
+
+The `node` attribute always places each selected node into its own bucket, regardless of the attribute value's form (`#ANY`, `node-name`, or `["node1-name", ...]`).
+
+Replica and core count constraints, described below, are evaluated against the total number in each bucket.
+
+==== Core Count Constraint
+
+The `cores` attribute value can be specified in one of the following forms:
+
+* the `#EQUAL` directive, which will cause cores to be distributed equally among the nodes specified via the rule's <<Node Selector>>.
+* a constraint on the core count on each <<Node Selector,selected node>>, specified as one of:
+** an integer value (e.g. `2`), a lower bound (e.g. `>0`), or an upper bound (e.g. `<3`)
+** a decimal value, interpreted as an acceptable range of core counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g. `1.6`: `1` or `2` is acceptable, and `2` is preferred)
+** a range of acceptable core counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g. `3-5`)
+** a percentage (e.g. `33%`), which is multiplied by the number of cores in the cluster at runtime. This value is then interpreted as described above for literal decimal values.
+
+==== Replica Selector and Rule Evaluation Context
+
+Rule evaluation can be restricted to replicas that meet any combination of the following conditions:
+
+* The replica is of a shard belonging to the collection specified in the `collection` attribute value. (Not usable with per-collection policies.)
+* The replica is of a shard specified in the `shard` attribute value.
+* The replica has the replica type specified in the `type` attribute value (`NRT`, `TLOG`, or `PULL`).
+
+If none of the above attributes is specified, then the rule is evaluated separately for each collection against all types of replicas of all shards.
+
+Specifying `#EACH` as the `shard` attribute value causes the rule to be evaluated separately for each shard of each collection.
+
+==== Replica Count Constraint
+
+The `replica` attribute value can be specified in one of the following forms:
+
+* `#ALL`: All <<Replica Selector and Rule Evaluation Context,selected replicas>> will be placed on the <<Node Selector,selected nodes>>.
+* `#EQUAL`: Distribute <<Replica Selector and Rule Evaluation Context,selected replicas>> evenly among all the <<Node Selector,selected nodes>>.
+* a constraint on the replica count on each <<Node Selector,selected node>>, specified as one of:
+** an integer value (e.g. `2`), a lower bound (e.g. `>0`), or an upper bound (e.g. `<3`)
+** a decimal value, interpreted as an acceptable range of replica counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g. `1.6`: `1` or `2` is acceptable, and `2` is preferred)
+** a range of acceptable replica counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g. `3-5`)
+** a percentage (e.g. `33%`), which is multiplied by the number of <<Replica Selector and Rule Evaluation Context,selected replicas>> at runtime. This value is then interpreted as described above for literal decimal values.
+
+==== Rule Strictness
+
+By default, the rule must be satisfied, and if it can't, then no action will be taken.
+
+If the `strict` attribute value is specified as `false`, Solr tries to satisfy the rule on a best effort basis, but if no node can satisfy the rule then any node may be chosen.
+
+=== Policy Rule Attributes
+
+A policy rule can have the following attributes:
`cores`::
-This is a special attribute that applies to the entire cluster. It can only be used along with the `node` attribute and no other. The value of this attribute can be
-* a positive integer . e.g : "`3`"
-* a number with a decimal value . e.g: "`1.66`" . This means both 1 and 2 are acceptable values but the system would prefer `2`
-* a number range. Such as `"3-5"` . This means `3,4,5` are acceptable values
-* a percentage value . e.g: `33%` . This is computed to a decimal value at runtime
-* `#EQUAL` : Divide the no:of cores equally among all the nodes or a subset of nodes
+This is a required attribute for <<Rule Types,global rules>>. It can only be used along with the `node` attribute and no other. See <<Core Count Constraint>> for possible attribute values.
`collection`::
The name of the collection to which the policy rule should apply. If omitted, the rule applies to all collections. This attribute is optional.
@@ -107,15 +178,7 @@ The name of the shard to which the policy rule should apply. If omitted, the rul
The type of the replica to which the policy rule should apply. If omitted, the rule is applied for all replica types of this collection/shard. The allowed values are `NRT`, `TLOG` and `PULL`
`replica`::
-This is a required attribute. The number of replicas that must exist to satisfy the rule. This must be one of
-
-* a positive integer . e.g : "`3`"
-* a number with a decimal value . e.g: "`1.66`" . This means both 1 and 2 are acceptable values but the system would prefer `2`
-* a number range. Such as `"3-5"` . This means `3,4,5` are acceptable values
-* `#ALL` : All replicas of a given collection or shard
-* a percentage value . e.g: `33%` . This is computed to a decimal value at runtime
-* `#EQUAL` : Divide the no:of replicas equally among all the nodes qualifying a certain property and place equal no:of them in each node
-
+This is a required attribute for <<Rule Types,per-collection rules>>. The number of replicas that must exist to satisfy the rule. See <<Replica Count Constraint>> for possible attribute values.
`strict`::
An optional boolean value. The default is `true`. If true, the rule must be satisfied. If false, Solr tries to satisfy the rule on a best effort basis but if no node can satisfy the rule then any node may be chosen.
@@ -123,7 +186,7 @@ An optional boolean value. The default is `true`. If true, the rule must be sati
One and only one of the following attributes can be specified in addition to the above attributes:
`node`::
-The name of the node to which the rule should apply. The default value is `#ANY` which means that any node in the cluster may satisfy the rule.
+The name of the node to which the rule should apply.
`port`::
The port of the node to which the rule should apply.
@@ -155,7 +218,7 @@ Any arbitrary metric. For example, `metrics:solr.node:CONTAINER.fs.totalSpace`.
`diskType`::
The type of disk drive being used for Solr's `coreRootDirectory`. The only two supported values are `rotational` and `ssd`. Refer to `coreRootDirectory` parameter in the <<format-of-solr-xml.adoc#solr-xml-parameters, Solr.xml Parameters>> section.
+
-It's value is fetched from the Metrics API with the key named `solr.node:CONTAINER.fs.coreRoot.spins`. The disk type is auto-detected by Lucene using various heuristics and it is not guaranteed to be correct across all platforms or operating systems. Refer to the <<taking-solr-to-production.adoc#dynamic-defaults-for-concurrentmergescheduler, Dynamic defaults for ConcurrentMergeScheduler>> section for more details.
+Its value is fetched from the Metrics API with the key named `solr.node:CONTAINER.fs.coreRoot.spins`. The disk type is auto-detected by Lucene using various heuristics and it is not guaranteed to be correct across all platforms or operating systems. Refer to the <<taking-solr-to-production.adoc#dynamic-defaults-for-concurrentmergescheduler, Dynamic defaults for ConcurrentMergeScheduler>> section for more details.
=== Policy Operators
@@ -164,9 +227,8 @@ Each attribute in the policy may specify one of the following operators along wi
* `<`: Less than
* `>`: Greater than
* `!`: Not
-* Range operator `(-)` : a value such as `"3-5"` means a value between 3 to 5 (inclusive). This is only supported in the following attributes
-** `replica`
-* array operator . e.g: `sysprop.zone = ["east", "west","apac"]`. This is equivalent to having multiple rules with each of these values. This can be used in the following attributes
+* Range operator `(-)`: a value such as `"3-5"` means a value between 3 to 5 (inclusive). This is only supported in the `replica` and `cores` attributes.
+* Array operator `[]`. e.g: `sysprop.zone = ["east", "west","apac"]`. This is equivalent to having multiple rules with each of these values. This can be used in the following attributes
** `sysprop.*`
** `port`
** `ip_*`
@@ -174,46 +236,47 @@ Each attribute in the policy may specify one of the following operators along wi
** `diskType`
* None means equal
-==== Special functions
+==== Special Functions
+
This supports values calculated at the time of execution.
* `%` : A certain percentage of the value. This is supported by the following attributes
** `replica`
+** `cores`
** `freedisk`
-* `#ALL` : This is applied to the `replica` attribute only. This means all replicas qualifying a certain clause
-* `#EQUAL`: This is applied to the `replica` attribute only. This means equal no:of replicas in each bucket.The buckets can be defined using an array operator (`[]`) or `#EACH` .The buckets can be defined on the following properties
-** `node`
+* `#ALL` : This is applied to the `replica` attribute only. This means all replicas that meet the rule condition.
+* `#EQUAL`: This is applied to the `replica` and `cores` attributes only. This means equal number of replicas/cores in each bucket. The buckets can be defined using an array operator (`[]`) or `#EACH`. The buckets can be defined using the following properties:
+** `node` <- <<Rule Types,global rules>>, i.e. with the `cores` attribute, may only specify this attribute
** `sysprop.*`
** `port`
** `diskType`
** `ip_*`
-****
-Some content here
-****
-
-
=== Examples of Policy Rules
==== Limit Replica Placement
+
Do not place more than one replica of the same shard on the same node:
[source,json]
{"replica": "<2", "shard": "#EACH", "node": "#ANY"}
==== Limit Cores per Node
+
Do not place more than 10 cores in any node. This rule can only be added to the cluster policy because it mentions the `cores` attribute that is only applicable cluster-wide.
[source,json]
{"cores": "<10", "node": "#ANY"}
==== Place Replicas Based on Port
+
Place exactly 1 replica of each shard of collection `xyz` on a node running on port `8983`
[source,json]
{"replica": 1, "shard": "#EACH", "collection": "xyz", "port": "8983"}
==== Place Replicas Based on a System Property
+
Place all replicas on a node with system property `availability_zone=us-east-1a`.
[source,json]
@@ -221,25 +284,28 @@ Place all replicas on a node with system property `availability_zone=us-east-1a`
===== Use Percentage
-====== example 1
-Place roughly a maximum of a 3rd of the replicas of a shard in a node. In the following example, the value of replica is computed in real time.
+====== Example 1
+
+Place roughly a maximum of a 3rd of the replicas of a shard in a node. In the following example, the value of `replica` is computed in real time:
+
[source,json]
{"replica": "33%", "shard": "#EACH", "node": "#ANY"}
-If the no:of of replicas in a shard is `2` , `33% of 2 = 0.66` . This means a node may have a maximum of `1` and a minimum of `0` replicas of each shard.
+If the number of replicas in a shard is `2`, `33% of 2 = 0.66`. This means a node may have a maximum of `1` and a minimum of `0` replicas of each shard.
-It is possible to get the same effect by hard coding the value of replica as follows
+It is possible to get the same effect by hard coding the value of `replica` as follows:
[source,json]
{"replica": 0.66, "shard": "#EACH", "node": "#ANY"}
-or using the range operator
+or using the range operator:
[source,json]
{"replica": "0-1", "shard": "#EACH", "node": "#ANY"}
-====== example 2
-Distribute replicas across datacenters east and west at a `1:2` ratio
+====== Example 2
+
+Distribute replicas across datacenters `east` and `west` at a `1:2` ratio:
[source,json]
{"replica": "33%", "shard": "#EACH", "sysprop.zone": "east"}
@@ -247,9 +313,7 @@ Distribute replicas across datacenters east and west at a `1:2` ratio
For the above rule to work, all nodes must the started with a system property called `"zone"`
-== example 3
-
-Distribute replicas equally in each zone
+==== Distribute Replicas Equally in Each Zone
[source,json]
{"replica": "#EQUAL", "shard": "#EACH", "sysprop.zone": ["east", "west"]}
@@ -259,15 +323,16 @@ or simply as follows
[source,json]
{"replica": "#EQUAL", "shard": "#EACH", "sysprop.zone": "#EACH"}
-
==== Place Replicas Based on Node Role
+
Do not place any replica on a node which has the overseer role. Note that the role is added by the `addRole` collection API. It is *not* automatically the node which is currently the overseer.
[source,json]
{"replica": 0, "nodeRole": "overseer"}
==== Place Replicas Based on Free Disk
-Place all replicas in nodes with freedisk more than 500GB. Here again, we have to write the rule in the negative sense.
+
+Place all replicas in nodes with freedisk more than 500GB.
[source,json]
{"replica": "#ALL", "freedisk": ">500"}
@@ -276,19 +341,19 @@ Keep all replicas in nodes with over `50%` freedisk
[source,json]
{"replica": "#ALL", "freedisk": ">50%"}
-
==== Try to Place Replicas Based on Free Disk
+
Place all replicas in nodes with freedisk more than 500GB when possible. Here we use the strict keyword to signal that this rule is to be honored on a best effort basis.
[source,json]
{"replica": "#ALL", "freedisk": ">500", "strict" : false}
-==== Try to Place all Replicas of type TLOG on Nodes with SSD Drives
+==== Try to Place All Replicas of Type TLOG on Nodes with SSD Drives
[source,json]
{ "replica": "#ALL","type" : "TLOG", "diskType" : "ssd" }
-==== Try to Place all Replicas of type PULL on Nodes with Rotational Disk Drives
+==== Try to Place All Replicas of Type PULL on Nodes with Rotational Disk Drives
[source,json]
{ "replica": "#ALL", "type" : "PULL" , "diskType" : "rotational"}
@@ -304,9 +369,11 @@ It is possible to override conditions specified in the cluster policy using coll
Also, if `maxShardsPerNode` is specified during the time of collection creation, then both `maxShardsPerNode` and the policy rules must be satisfied.
-Some attributes such as `cores` can only be used in the cluster policy. See the section above on policy attributes for details.
+Some attributes such as `cores` can only be used in the cluster policy. See the section <<Policy Rule Attributes>> for details.
+
+== Commands That Use Autoscaling Policy and Preferences
-The policy is used by these <<collections-api.adoc#collections-api,Collections API>> commands:
+The configured autoscaling policy and preferences are used by these <<collections-api.adoc#collections-api,Collections API>> commands:
* CREATE
* CREATESHARD
@@ -315,5 +382,3 @@ The policy is used by these <<collections-api.adoc#collections-api,Collections A
* SPLITSHARD
* UTILIZENODE
* MOVEREPLICA
-
-In the future, the policy and preferences will be used by the Autoscaling framework to automatically change the cluster in response to events such as a node being added or lost.
[24/31] lucene-solr:jira/http2: SOLR-12636: escape arrow glyph to fix
precommit
Posted by da...@apache.org.
SOLR-12636: escape arrow glyph to fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6c11284e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6c11284e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6c11284e
Branch: refs/heads/jira/http2
Commit: 6c11284e7c46bac110b1915719907d647d490091
Parents: cb1db48
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Aug 9 09:56:17 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Aug 9 09:56:17 2018 -0400
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-policy-preferences.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c11284e/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
index 82af94c..8714b38 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
@@ -246,7 +246,7 @@ This supports values calculated at the time of execution.
** `freedisk`
* `#ALL` : This is applied to the `replica` attribute only. This means all replicas that meet the rule condition.
* `#EQUAL`: This is applied to the `replica` and `cores` attributes only. This means equal number of replicas/cores in each bucket. The buckets can be defined using an array operator (`[]`) or `#EACH`. The buckets can be defined using the following properties:
-** `node` <- <<Rule Types,global rules>>, i.e. with the `cores` attribute, may only specify this attribute
+** `node` \<- <<Rule Types,global rules>>, i.e. with the `cores` attribute, may only specify this attribute
** `sysprop.*`
** `port`
** `diskType`
[06/31] lucene-solr:jira/http2: SOLR-11770: NPE in tvrh if no field
is specified and document doesn't contain any fields with term vectors
Posted by da...@apache.org.
SOLR-11770: NPE in tvrh if no field is specified and document doesn't contain any fields with term vectors
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/96e83929
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/96e83929
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/96e83929
Branch: refs/heads/jira/http2
Commit: 96e8392921792b9ec281c28f28f49b50f998ceed
Parents: 896fd0e
Author: Erick Erickson <Er...@gmail.com>
Authored: Mon Aug 6 20:04:59 2018 -0700
Committer: Erick Erickson <Er...@gmail.com>
Committed: Mon Aug 6 20:04:59 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../handler/component/TermVectorComponent.java | 89 ++---
.../solr/response/RetrieveFieldsOptimizer.java | 8 +-
.../transform/BaseEditorialTransformer.java | 2 +
.../test-files/solr/collection1/conf/schema.xml | 2 +-
.../component/QueryElevationComponentTest.java | 30 ++
.../component/TermVectorComponentTest.java | 340 +++++++++++--------
7 files changed, 280 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96e83929/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5fb1f88..e21b931 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -212,6 +212,8 @@ Bug Fixes
This would fix the NPE when using the search stream with partitionKeys. (Varun Thacker)
+* SOLR-11770: NPE in tvrh if no field is specified and document doesn't contain any fields with term vectors
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96e83929/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
index 4cb95c2..784dc81 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java
@@ -17,7 +17,6 @@
package org.apache.solr.handler.component;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -28,15 +27,16 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.PostingsEnum;
-import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
@@ -44,11 +44,14 @@ import org.apache.solr.common.params.TermVectorParams;
import org.apache.solr.common.util.Base64;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
+import org.apache.solr.response.DocsStreamer;
+import org.apache.solr.response.RetrieveFieldsOptimizer;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.util.SolrPluginUtils;
@@ -262,54 +265,49 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar
//Only load the id field to get the uniqueKey of that
//field
-
- final String finalUniqFieldName = uniqFieldName;
-
- final List<String> uniqValues = new ArrayList<>();
-
- // TODO: is this required to be single-valued? if so, we should STOP
- // once we find it...
- final StoredFieldVisitor getUniqValue = new StoredFieldVisitor() {
- @Override
- public void stringField(FieldInfo fieldInfo, byte[] bytes) {
- uniqValues.add(new String(bytes, StandardCharsets.UTF_8));
- }
-
- @Override
- public void intField(FieldInfo fieldInfo, int value) {
- uniqValues.add(Integer.toString(value));
- }
-
- @Override
- public void longField(FieldInfo fieldInfo, long value) {
- uniqValues.add(Long.toString(value));
- }
-
- @Override
- public Status needsField(FieldInfo fieldInfo) {
- return (fieldInfo.name.equals(finalUniqFieldName)) ? Status.YES : Status.NO;
- }
- };
+ SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
+ SolrReturnFields srf = new SolrReturnFields(uniqFieldName, rb.req);
+ RetrieveFieldsOptimizer retrieveFieldsOptimizer = RetrieveFieldsOptimizer.create(docFetcher, srf);
while (iter.hasNext()) {
Integer docId = iter.next();
NamedList<Object> docNL = new NamedList<>();
if (keyField != null) {
- reader.document(docId, getUniqValue);
- String uniqVal = null;
- if (uniqValues.size() != 0) {
- uniqVal = uniqValues.get(0);
- uniqValues.clear();
- docNL.add("uniqueKey", uniqVal);
- termVectors.add(uniqVal, docNL);
+ SolrDocument sdoc = null;
+ try {
+ if (retrieveFieldsOptimizer.returnStoredFields()) {
+ Document doc = docFetcher.doc(docId, retrieveFieldsOptimizer.getStoredFields());
+ // make sure to use the schema from the searcher and not the request (cross-core)
+ sdoc = DocsStreamer.convertLuceneDocToSolrDoc(doc, searcher.getSchema(), srf);
+ } else {
+ // no need to get stored fields of the document, see SOLR-5968
+ sdoc = new SolrDocument();
+ }
+
+ // decorate the document with non-stored docValues fields
+ if (retrieveFieldsOptimizer.returnDVFields()) {
+ docFetcher.decorateDocValueFields(sdoc, docId, retrieveFieldsOptimizer.getDvFields());
+ }
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading document with docId " + docId, e);
}
+ Object val = sdoc.getFieldValue(uniqFieldName);
+ String uniqVal = "";
+ if (val instanceof StoredField) {
+ uniqVal = ((StoredField) val).stringValue();
+ } else {
+ uniqVal = val.toString();
+ }
+
+ docNL.add("uniqueKey", uniqVal);
+ termVectors.add(uniqVal, docNL);
} else {
// support for schemas w/o a unique key,
termVectors.add("doc-" + docId, docNL);
}
- if ( null != fields ) {
+ if (null != fields) {
for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
final String field = entry.getKey();
final Terms vector = reader.getTermVector(docId, field);
@@ -321,11 +319,14 @@ public class TermVectorComponent extends SearchComponent implements SolrCoreAwar
} else {
// extract all fields
final Fields vectors = reader.getTermVectors(docId);
- for (String field : vectors) {
- Terms terms = vectors.terms(field);
- if (terms != null) {
- TermsEnum termsEnum = terms.iterator();
- mapOneVector(docNL, allFields, reader, docId, termsEnum, field);
+ // There can be no documents with vectors
+ if (vectors != null) {
+ for (String field : vectors) {
+ Terms terms = vectors.terms(field);
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator();
+ mapOneVector(docNL, allFields, reader, docId, termsEnum, field);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96e83929/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java b/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java
index f143b94..fcd47a7 100644
--- a/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java
+++ b/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java
@@ -53,19 +53,19 @@ public class RetrieveFieldsOptimizer {
storedFields.clear();
}
- boolean returnStoredFields() {
+ public boolean returnStoredFields() {
return !(storedFields != null && storedFields.isEmpty());
}
- boolean returnDVFields() {
+ public boolean returnDVFields() {
return !dvFields.isEmpty();
}
- Set<String> getStoredFields() {
+ public Set<String> getStoredFields() {
return storedFields;
}
- Set<String> getDvFields() {
+ public Set<String> getDvFields() {
return dvFields;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96e83929/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java
index 6324d8a..d238131 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/BaseEditorialTransformer.java
@@ -73,6 +73,8 @@ public abstract class BaseEditorialTransformer extends DocTransformer {
ft.readableToIndexed(f.stringValue(), bytesRefBuilder);
}
return bytesRefBuilder.get();
+ } else if (obj instanceof String) { // Allows the idField to be stored=false, docValues=true
+ return new BytesRef(((String)obj));
}
throw new AssertionError("Expected an IndexableField but got: " + obj.getClass());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96e83929/solr/core/src/test-files/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml
index 64d6fdd..b1a261b 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml
@@ -502,7 +502,7 @@
</fieldType>
<fieldType name="severityType" class="${solr.tests.EnumFieldType}" enumsConfig="enumsConfig.xml" enumName="severity"/>
- <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="id" type="string" indexed="true" stored="${solr.tests.id.stored:true}" multiValued="false" required="false" docValues="${solr.tests.id.docValues:false}"/>
<field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="signatureField" type="string" indexed="true" stored="false"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96e83929/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
index 892d51f..0db6b0f 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/QueryElevationComponentTest.java
@@ -23,6 +23,7 @@ import java.io.PrintWriter;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
+import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.SolrTestCaseJ4;
@@ -36,14 +37,43 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.FileUtils;
import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class QueryElevationComponentTest extends SolrTestCaseJ4 {
+ @Rule
+ public TestRule solrTestRules = RuleChain.outerRule(new SystemPropertiesRestoreRule());
+
+
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ switch (random().nextInt(3)) {
+ case 0:
+ System.setProperty("solr.tests.id.stored", "true");
+ System.setProperty("solr.tests.id.docValues", "true");
+ break;
+ case 1:
+ System.setProperty("solr.tests.id.stored", "true");
+ System.setProperty("solr.tests.id.docValues", "false");
+ break;
+ case 2:
+ System.setProperty("solr.tests.id.stored", "false");
+ System.setProperty("solr.tests.id.docValues", "true");
+ break;
+ default:
+ fail("Bad random number generatged not between 0-2 iunclusive");
+ break;
+ }
+ }
+
@Before
@Override
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/96e83929/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java
index 91a5869..c083a0a 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java
@@ -20,110 +20,147 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.TermVectorParams;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
/**
*
*
**/
public class TermVectorComponentTest extends SolrTestCaseJ4 {
+
+ @Rule
+ public TestRule solrTestRules = RuleChain.outerRule(new SystemPropertiesRestoreRule());
+
+ // ensure that we operate correctly with all valid combinations of the uniqueKey being
+ // stored and/or in docValues.
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml","schema.xml");
+ switch (random().nextInt(3)) {
+ case 0:
+ System.setProperty("solr.tests.id.stored", "true");
+ System.setProperty("solr.tests.id.docValues", "true");
+ break;
+ case 1:
+ System.setProperty("solr.tests.id.stored", "true");
+ System.setProperty("solr.tests.id.docValues", "false");
+ break;
+ case 2:
+ System.setProperty("solr.tests.id.stored", "false");
+ System.setProperty("solr.tests.id.docValues", "true");
+ break;
+ default:
+ fail("Bad random number generatged not between 0-2 iunclusive");
+ break;
+ }
+ initCore("solrconfig.xml", "schema.xml");
+ }
+
+ static String tv = "/tvrh";
+ @Test
+ public void testCanned() throws Exception {
+
+ clearIndex();
assertU(adoc("id", "0",
- "test_posoffpaytv", "This is a title and another title",
- "test_posofftv", "This is a title and another title",
- "test_basictv", "This is a title and another title",
- "test_notv", "This is a title and another title",
- "test_postv", "This is a title and another title",
- "test_offtv", "This is a title and another title"
+ "test_posoffpaytv", "This is a title and another title",
+ "test_posofftv", "This is a title and another title",
+ "test_basictv", "This is a title and another title",
+ "test_notv", "This is a title and another title",
+ "test_postv", "This is a title and another title",
+ "test_offtv", "This is a title and another title"
));
assertU(adoc("id", "1",
- "test_posoffpaytv", "The quick reb fox jumped over the lazy brown dogs.",
- "test_posofftv", "The quick reb fox jumped over the lazy brown dogs.",
- "test_basictv", "The quick reb fox jumped over the lazy brown dogs.",
- "test_notv", "The quick reb fox jumped over the lazy brown dogs.",
- "test_postv", "The quick reb fox jumped over the lazy brown dogs.",
- "test_offtv", "The quick reb fox jumped over the lazy brown dogs."
+ "test_posoffpaytv", "The quick reb fox jumped over the lazy brown dogs.",
+ "test_posofftv", "The quick reb fox jumped over the lazy brown dogs.",
+ "test_basictv", "The quick reb fox jumped over the lazy brown dogs.",
+ "test_notv", "The quick reb fox jumped over the lazy brown dogs.",
+ "test_postv", "The quick reb fox jumped over the lazy brown dogs.",
+ "test_offtv", "The quick reb fox jumped over the lazy brown dogs."
));
assertU(adoc("id", "2",
- "test_posoffpaytv", "This is a document",
- "test_posofftv", "This is a document",
- "test_basictv", "This is a document",
- "test_notv", "This is a document",
- "test_postv", "This is a document",
- "test_offtv", "This is a document"
+ "test_posoffpaytv", "This is a document",
+ "test_posofftv", "This is a document",
+ "test_basictv", "This is a document",
+ "test_notv", "This is a document",
+ "test_postv", "This is a document",
+ "test_offtv", "This is a document"
));
assertU(adoc("id", "3",
- "test_posoffpaytv", "another document",
- "test_posofftv", "another document",
- "test_basictv", "another document",
- "test_notv", "another document",
- "test_postv", "another document",
- "test_offtv", "another document"
+ "test_posoffpaytv", "another document",
+ "test_posofftv", "another document",
+ "test_basictv", "another document",
+ "test_notv", "another document",
+ "test_postv", "another document",
+ "test_offtv", "another document"
));
//bunch of docs that are variants on blue
assertU(adoc("id", "4",
- "test_posoffpaytv", "blue",
- "test_posofftv", "blue",
- "test_basictv", "blue",
- "test_notv", "blue",
- "test_postv", "blue",
- "test_offtv", "blue"
+ "test_posoffpaytv", "blue",
+ "test_posofftv", "blue",
+ "test_basictv", "blue",
+ "test_notv", "blue",
+ "test_postv", "blue",
+ "test_offtv", "blue"
));
assertU(adoc("id", "5",
- "test_posoffpaytv", "blud",
- "test_posofftv", "blud",
- "test_basictv", "blud",
- "test_notv", "blud",
- "test_postv", "blud",
- "test_offtv", "blud"
+ "test_posoffpaytv", "blud",
+ "test_posofftv", "blud",
+ "test_basictv", "blud",
+ "test_notv", "blud",
+ "test_postv", "blud",
+ "test_offtv", "blud"
));
assertU(adoc("id", "6",
- "test_posoffpaytv", "boue",
- "test_posofftv", "boue",
- "test_basictv", "boue",
- "test_notv", "boue",
- "test_postv", "boue",
- "test_offtv", "boue"
+ "test_posoffpaytv", "boue",
+ "test_posofftv", "boue",
+ "test_basictv", "boue",
+ "test_notv", "boue",
+ "test_postv", "boue",
+ "test_offtv", "boue"
));
assertU(adoc("id", "7",
- "test_posoffpaytv", "glue",
- "test_posofftv", "glue",
- "test_basictv", "glue",
- "test_notv", "glue",
- "test_postv", "glue",
- "test_offtv", "glue"
+ "test_posoffpaytv", "glue",
+ "test_posofftv", "glue",
+ "test_basictv", "glue",
+ "test_notv", "glue",
+ "test_postv", "glue",
+ "test_offtv", "glue"
));
assertU(adoc("id", "8",
- "test_posoffpaytv", "blee",
- "test_posofftv", "blee",
- "test_basictv", "blee",
- "test_notv", "blee",
- "test_postv", "blee",
- "test_offtv", "blee"
+ "test_posoffpaytv", "blee",
+ "test_posofftv", "blee",
+ "test_basictv", "blee",
+ "test_notv", "blee",
+ "test_postv", "blee",
+ "test_offtv", "blee"
));
assertU(adoc("id", "9",
- "test_posoffpaytv", "blah",
- "test_posofftv", "blah",
- "test_basictv", "blah",
- "test_notv", "blah",
- "test_postv", "blah",
- "test_offtv", "blah"
+ "test_posoffpaytv", "blah",
+ "test_posofftv", "blah",
+ "test_basictv", "blah",
+ "test_notv", "blah",
+ "test_postv", "blah",
+ "test_offtv", "blah"
));
assertNull(h.validateUpdate(commit()));
+ doBasics();
+ doOptions();
+ doPerField();
+ doPayloads();
}
- static String tv = "/tvrh";
- @Test
- public void testBasics() throws Exception {
- assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true", TermVectorParams.TF, "true")
- ,"/termVectors=={'0':{'uniqueKey':'0'," +
+ private void doBasics() throws Exception {
+ assertJQ(req("json.nl", "map", "qt", tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true", TermVectorParams.TF, "true")
+ , "/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_posofftv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
@@ -131,38 +168,38 @@ public class TermVectorComponentTest extends SolrTestCaseJ4 {
" 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}}"
);
// tv.fl diff from fl
- assertJQ(req("json.nl","map",
- "qt",tv,
- "q", "id:0",
- "fl", "*,score",
- "tv.fl", "test_basictv,test_offtv",
- TermVectorComponent.COMPONENT_NAME, "true",
- TermVectorParams.TF, "true")
- ,"/termVectors=={'0':{'uniqueKey':'0'," +
+ assertJQ(req("json.nl", "map",
+ "qt", tv,
+ "q", "id:0",
+ "fl", "*,score",
+ "tv.fl", "test_basictv,test_offtv",
+ TermVectorComponent.COMPONENT_NAME, "true",
+ TermVectorParams.TF, "true")
+ , "/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}}}"
);
// multi-valued tv.fl
- assertJQ(req("json.nl","map",
- "qt",tv,
- "q", "id:0",
- "fl", "*,score",
- "tv.fl", "test_basictv",
- "tv.fl","test_offtv",
- TermVectorComponent.COMPONENT_NAME, "true",
- TermVectorParams.TF, "true")
- ,"/termVectors=={'0':{'uniqueKey':'0'," +
+ assertJQ(req("json.nl", "map",
+ "qt", tv,
+ "q", "id:0",
+ "fl", "*,score",
+ "tv.fl", "test_basictv",
+ "tv.fl", "test_offtv",
+ TermVectorComponent.COMPONENT_NAME, "true",
+ TermVectorParams.TF, "true")
+ , "/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}}}"
);
// re-use fl glob
- assertJQ(req("json.nl","map",
- "qt",tv,
- "q", "id:0",
- "fl", "*,score",
- TermVectorComponent.COMPONENT_NAME, "true",
- TermVectorParams.TF, "true")
- ,"/termVectors=={'0':{'uniqueKey':'0'," +
+ assertJQ(req("json.nl", "map",
+ "qt", tv,
+ "q", "id:0",
+ "fl", "*,score",
+ TermVectorComponent.COMPONENT_NAME, "true",
+ TermVectorParams.TF, "true")
+ , "/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_posofftv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
@@ -170,51 +207,50 @@ public class TermVectorComponentTest extends SolrTestCaseJ4 {
" 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}}"
);
// re-use fl, ignore things we can't handle
- assertJQ(req("json.nl","map",
- "qt",tv,
- "q", "id:0",
- "fl", "score,test_basictv,[docid],test_postv,val:sum(3,4)",
- TermVectorComponent.COMPONENT_NAME, "true",
- TermVectorParams.TF, "true")
- ,"/termVectors=={'0':{'uniqueKey':'0'," +
+ assertJQ(req("json.nl", "map",
+ "qt", tv,
+ "q", "id:0",
+ "fl", "score,test_basictv,[docid],test_postv,val:sum(3,4)",
+ TermVectorComponent.COMPONENT_NAME, "true",
+ TermVectorParams.TF, "true")
+ , "/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}}"
);
// re-use (multi-valued) fl, ignore things we can't handle
- assertJQ(req("json.nl","map",
- "qt",tv,
- "q", "id:0",
- "fl", "score,test_basictv",
- "fl", "[docid],test_postv,val:sum(3,4)",
- TermVectorComponent.COMPONENT_NAME, "true",
- TermVectorParams.TF, "true")
- ,"/termVectors=={'0':{'uniqueKey':'0'," +
+ assertJQ(req("json.nl", "map",
+ "qt", tv,
+ "q", "id:0",
+ "fl", "score,test_basictv",
+ "fl", "[docid],test_postv,val:sum(3,4)",
+ TermVectorComponent.COMPONENT_NAME, "true",
+ TermVectorParams.TF, "true")
+ , "/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}}"
);
}
- @Test
- public void testOptions() throws Exception {
- assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
- , TermVectorParams.TF, "true", TermVectorParams.DF, "true", TermVectorParams.OFFSETS, "true", TermVectorParams.POSITIONS, "true", TermVectorParams.TF_IDF, "true")
- ,"/termVectors/0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':5}, 'df':2, 'tf-idf':0.5}"
+ private void doOptions() throws Exception {
+ assertJQ(req("json.nl", "map", "qt", tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
+ , TermVectorParams.TF, "true", TermVectorParams.DF, "true", TermVectorParams.OFFSETS, "true", TermVectorParams.POSITIONS, "true", TermVectorParams.TF_IDF, "true")
+ , "/termVectors/0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':5}, 'df':2, 'tf-idf':0.5}"
);
-
- assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
+
+ assertJQ(req("json.nl", "map", "qt", tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
, TermVectorParams.ALL, "true")
- ,"/termVectors/0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':5}, 'df':2, 'tf-idf':0.5}"
- );
-
+ , "/termVectors/0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':5}, 'df':2, 'tf-idf':0.5}"
+ );
+
// test each combination at random
final List<String> list = new ArrayList<>();
- list.addAll(Arrays.asList("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"));
- String[][] options = new String[][] { { TermVectorParams.TF, "'tf':1" },
- { TermVectorParams.OFFSETS, "'offsets':{'start':20, 'end':27}" },
- { TermVectorParams.POSITIONS, "'positions':{'position':5}" },
- { TermVectorParams.DF, "'df':2" },
- { TermVectorParams.TF_IDF, "'tf-idf':0.5" } };
+ list.addAll(Arrays.asList("json.nl", "map", "qt", tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"));
+ String[][] options = new String[][]{{TermVectorParams.TF, "'tf':1"},
+ {TermVectorParams.OFFSETS, "'offsets':{'start':20, 'end':27}"},
+ {TermVectorParams.POSITIONS, "'positions':{'position':5}"},
+ {TermVectorParams.DF, "'df':2"},
+ {TermVectorParams.TF_IDF, "'tf-idf':0.5"}};
StringBuilder expected = new StringBuilder("/termVectors/0/test_posofftv/anoth=={");
boolean first = true;
for (int i = 0; i < options.length; i++) {
@@ -225,46 +261,62 @@ public class TermVectorComponentTest extends SolrTestCaseJ4 {
}
first = false;
expected.append(options[i][1]);
-
+
}
list.add(options[i][0]);
list.add(use ? "true" : "false");
}
-
+
expected.append("}");
assertJQ(req(list.toArray(new String[0])), expected.toString());
}
- @Test
- public void testPerField() throws Exception {
- assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
- ,TermVectorParams.TF, "true", TermVectorParams.DF, "true", TermVectorParams.OFFSETS, "true", TermVectorParams.POSITIONS, "true", TermVectorParams.TF_IDF, "true"
- ,TermVectorParams.FIELDS, "test_basictv,test_notv,test_postv,test_offtv,test_posofftv,test_posoffpaytv"
- ,"f.test_posoffpaytv." + TermVectorParams.PAYLOADS, "false"
- ,"f.test_posofftv." + TermVectorParams.POSITIONS, "false"
- ,"f.test_offtv." + TermVectorParams.OFFSETS, "false"
- ,"f.test_basictv." + TermVectorParams.DF, "false"
- ,"f.test_basictv." + TermVectorParams.TF, "false"
- ,"f.test_basictv." + TermVectorParams.TF_IDF, "false"
+ private void doPerField() throws Exception {
+ assertJQ(req("json.nl", "map", "qt", tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
+ , TermVectorParams.TF, "true", TermVectorParams.DF, "true", TermVectorParams.OFFSETS, "true", TermVectorParams.POSITIONS, "true", TermVectorParams.TF_IDF, "true"
+ , TermVectorParams.FIELDS, "test_basictv,test_notv,test_postv,test_offtv,test_posofftv,test_posoffpaytv"
+ , "f.test_posoffpaytv." + TermVectorParams.PAYLOADS, "false"
+ , "f.test_posofftv." + TermVectorParams.POSITIONS, "false"
+ , "f.test_offtv." + TermVectorParams.OFFSETS, "false"
+ , "f.test_basictv." + TermVectorParams.DF, "false"
+ , "f.test_basictv." + TermVectorParams.TF, "false"
+ , "f.test_basictv." + TermVectorParams.TF_IDF, "false"
)
- ,"/termVectors/0/test_basictv=={'anoth':{},'titl':{}}"
- ,"/termVectors/0/test_postv/anoth=={'tf':1, 'positions':{'position':5}, 'df':2, 'tf-idf':0.5}"
- ,"/termVectors/0/test_offtv/anoth=={'tf':1, 'df':2, 'tf-idf':0.5}"
- ,"/termVectors/warnings=={ 'noTermVectors':['test_notv'], 'noPositions':['test_basictv', 'test_offtv'], 'noOffsets':['test_basictv', 'test_postv']}"
+ , "/termVectors/0/test_basictv=={'anoth':{},'titl':{}}"
+ , "/termVectors/0/test_postv/anoth=={'tf':1, 'positions':{'position':5}, 'df':2, 'tf-idf':0.5}"
+ , "/termVectors/0/test_offtv/anoth=={'tf':1, 'df':2, 'tf-idf':0.5}"
+ , "/termVectors/warnings=={ 'noTermVectors':['test_notv'], 'noPositions':['test_basictv', 'test_offtv'], 'noOffsets':['test_basictv', 'test_postv']}"
);
}
- @Test
- public void testPayloads() throws Exception {
+ private void doPayloads() throws Exception {
// This field uses TokenOffsetPayloadTokenFilter, which
// stuffs start (20) and end offset (27) into the
// payload:
- assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
- , TermVectorParams.TF, "true", TermVectorParams.DF, "true", TermVectorParams.OFFSETS, "true", TermVectorParams.POSITIONS, "true", TermVectorParams.TF_IDF, "true",
- TermVectorParams.PAYLOADS, "true")
- ,"/termVectors/0/test_posoffpaytv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':5}, 'payloads':{'payload': 'AAAAFAAAABs='}, 'df':2, 'tf-idf':0.5}"
+ assertJQ(req("json.nl", "map", "qt", tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
+ , TermVectorParams.TF, "true", TermVectorParams.DF, "true", TermVectorParams.OFFSETS, "true", TermVectorParams.POSITIONS, "true", TermVectorParams.TF_IDF, "true",
+ TermVectorParams.PAYLOADS, "true")
+ , "/termVectors/0/test_posoffpaytv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':5}, 'payloads':{'payload': 'AAAAFAAAABs='}, 'df':2, 'tf-idf':0.5}"
+ );
+ }
+
+ @Test
+ public void testNoVectors() throws Exception {
+ clearIndex();
+ assertU(adoc("id", "0"));
+ assertU(adoc("id", "1"));
+ assertU(adoc("id", "2"));
+ assertU(adoc("id", "3"));
+ assertNull(h.validateUpdate(commit()));
+
+ // Kind of an odd test, but we just want to know if we don't generate an NPE when there is nothing to give back in the term vectors.
+ assertJQ(req("json.nl", "map", "qt", tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true", TermVectorParams.TF, "true")
+ , "/termVectors=={'0':{'uniqueKey':'0'}}}"
);
+
+
}
+
}
[25/31] lucene-solr:jira/http2: SOLR-11834: rollback [subquery]
fields documentation
Posted by da...@apache.org.
SOLR-11834: rollback [subquery] fields documentation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/63fc1246
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/63fc1246
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/63fc1246
Branch: refs/heads/jira/http2
Commit: 63fc1246f7d30c73bff0fdbf344216a34efdc554
Parents: 6c11284
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Thu Aug 9 20:27:35 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Thu Aug 9 20:28:45 2018 +0300
----------------------------------------------------------------------
solr/solr-ref-guide/src/transforming-result-documents.adoc | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63fc1246/solr/solr-ref-guide/src/transforming-result-documents.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/transforming-result-documents.adoc b/solr/solr-ref-guide/src/transforming-result-documents.adoc
index f837baf..f9fe805 100644
--- a/solr/solr-ref-guide/src/transforming-result-documents.adoc
+++ b/solr/solr-ref-guide/src/transforming-result-documents.adoc
@@ -251,12 +251,15 @@ Here is how it looks like in various formats:
==== Subquery Result Fields
-To appear in subquery document list, a field should be specified in subquery's `fl` parameter e.g., `foo.fl` (it is not necessary to specify in main one's `fl`). Of course, you can use wildcard in this parameter. For example, if field title should appear in categories subquery, it can be done via one of these ways.
+To appear in subquery document list, a field should be specified both fl parameters, in main one fl (despite the main result documents have no this field) and in subquery's one e.g., `foo.fl`.
+Of course, you can use wildcard in any or both of these parameters. For example, if field `title` should appear in categories subquery, it can be done via one of these ways.
[source,plain]
----
-fl=...id,categories:[subquery]&categories.fl=title&categories.q=...
-fl=...id,categories:[subquery]&categories.fl=*&categories.q=...
+fl=...title,categories:[subquery]&categories.fl=title&categories.q=...
+fl=...title,categories:[subquery]&categories.fl=*&categories.q=...
+fl=...*,categories:[subquery]&categories.fl=title&categories.q=...
+fl=...*,categories:[subquery]&categories.fl=*&categories.q=...
----
==== Subquery Parameters Shift
[29/31] lucene-solr:jira/http2: LUCENE-8451: Add disabled regression
test
Posted by da...@apache.org.
LUCENE-8451: Add disabled regression test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9c02bf09
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9c02bf09
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9c02bf09
Branch: refs/heads/jira/http2
Commit: 9c02bf09b765197cc1360baa831b55f511059f42
Parents: 71310a4
Author: Karl Wright <Da...@gmail.com>
Authored: Thu Aug 9 20:39:38 2018 -0400
Committer: Karl Wright <Da...@gmail.com>
Committed: Thu Aug 9 20:39:38 2018 -0400
----------------------------------------------------------------------
.../lucene/spatial3d/geom/GeoPolygonTest.java | 22 ++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9c02bf09/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index 1aae820..d68db13 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -1889,4 +1889,26 @@ shape:
assertTrue(polygon.isWithin(point) == largePolygon.isWithin(point));
}
+ @Test
+ @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/LUCENE-8451")
+ public void testLUCENE8451() {
+ //POLYGON((-2.5185339401969213 -24.093993739745027,0.0 8.828539494442529E-27,5.495998489568957E-11 -8.321407453133E-11,2.7174659198424288E-11 1.0260761462208114E-10,88.32137548549387 16.934529875343244,-87.97237709688223 39.919704493657484,-88.0876897472551 34.91204903885665,-2.5185339401969213 -24.093993739745027))
+ final List<GeoPoint> points = new ArrayList<>();
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(-24.093993739745027), Geo3DUtil.fromDegrees(-2.5185339401969213)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(8.828539494442529E-27), Geo3DUtil.fromDegrees(0.0)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(-8.321407453133E-11), Geo3DUtil.fromDegrees(5.495998489568957E-11)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(1.0260761462208114E-10), Geo3DUtil.fromDegrees(2.7174659198424288E-11)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(16.934529875343244), Geo3DUtil.fromDegrees(88.32137548549387)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(39.919704493657484), Geo3DUtil.fromDegrees(-87.97237709688223)));
+ points.add(new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(34.91204903885665), Geo3DUtil.fromDegrees(-88.0876897472551)));
+ final GeoPolygonFactory.PolygonDescription description = new GeoPolygonFactory.PolygonDescription(points);
+ final GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(PlanetModel.SPHERE, description);
+ final GeoPolygon largePolygon = GeoPolygonFactory.makeLargeGeoPolygon(PlanetModel.SPHERE, Collections.singletonList(description));
+
+ //POINT(179.99999999999983 -5.021400461974724E-11)
+ final GeoPoint point = new GeoPoint(PlanetModel.SPHERE, Geo3DUtil.fromDegrees(-5.021400461974724E-11), Geo3DUtil.fromDegrees(179.99999999999983));
+ assertTrue(polygon.isWithin(point) == largePolygon.isWithin(point));
+
+ }
+
}
[17/31] lucene-solr:jira/http2: Ref Guide: fix typo in
MODIFYCOLLECTION example
Posted by da...@apache.org.
Ref Guide: fix typo in MODIFYCOLLECTION example
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2674c538
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2674c538
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2674c538
Branch: refs/heads/jira/http2
Commit: 2674c53809a24527be007000e1753ec1121a9a20
Parents: 49e3cca
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Aug 8 08:52:18 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Aug 8 08:53:15 2018 -0500
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-auto-add-replicas.adoc | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2674c538/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
index 87e50f5..622ce41 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
@@ -27,11 +27,11 @@ The boolean `autoAddReplicas` parameter can be passed to the CREATE command of t
[source,text]
http://localhost:8983/solr/admin/collections?action=CREATE&name=my_collection&numShards=1&replicationFactor=5&autoAddReplicas=true
-The MODIFYCOLLECTION command can be used to enable/disable this feature for any collection.
+The MODIFYCOLLECTION command can be used to enable or disable this feature for any collection.
.Modify collection to disable autoAddReplicas
[source,text]
-http://localhost:8983/solr/admin/collections?action=MODIFYCOLLECTION&name=my_collection&autoAddReplicas=false
+http://localhost:8983/solr/admin/collections?action=MODIFYCOLLECTION&collection=my_collection&autoAddReplicas=false
== Implementation Using .autoAddReplicas Trigger
@@ -39,7 +39,7 @@ A Trigger named `.autoAddReplicas` is automatically created whenever any collect
Only one trigger is sufficient to serve all collections having this feature enabled. The `.autoAddReplicas` trigger watches for nodes that are lost from the cluster and uses the default `TriggerActions` to create new replicas to replace the ones which were hosted by the lost node. If the old node comes back online, it unloads the moved replicas and the node is free to host other replicas as and when required.
-Since the trigger provides the autoAddReplicas feature for all collections, the `suspend-trigger` and `resume-trigger` Autoscaling API commands can be used to disable and enable this feature for all collections in one API call.
+Since the trigger provides the autoAddReplicas feature for all collections, the `suspend-trigger` and `resume-trigger` Autoscaling API commands can be used to disable and enable this feature for all collections in one API call.
.Suspending autoAddReplicas for all collections
[source,json]
[19/31] lucene-solr:jira/http2: LUCENE-8447: Add DISJOINT and WITHIN
support to LatLonShape queries
Posted by da...@apache.org.
LUCENE-8447: Add DISJOINT and WITHIN support to LatLonShape queries
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cbaedb47
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cbaedb47
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cbaedb47
Branch: refs/heads/jira/http2
Commit: cbaedb470c8ddc7b21e8ff0b2729c6cf97fbd3d0
Parents: 9b418a4
Author: Nicholas Knize <nk...@gmail.com>
Authored: Mon Aug 6 16:31:21 2018 -0500
Committer: Nicholas Knize <nk...@gmail.com>
Committed: Wed Aug 8 12:50:05 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
.../org/apache/lucene/document/LatLonShape.java | 13 +-
.../document/LatLonShapeBoundingBoxQuery.java | 420 +++++--------------
.../document/LatLonShapePolygonQuery.java | 301 ++-----------
.../lucene/document/LatLonShapeQuery.java | 394 +++++++++++++++++
.../document/BaseLatLonShapeTestCase.java | 46 +-
.../document/TestLatLonLineShapeQueries.java | 40 +-
.../document/TestLatLonPointShapeQueries.java | 20 +-
.../document/TestLatLonPolygonShapeQueries.java | 36 +-
.../apache/lucene/document/TestLatLonShape.java | 3 +-
10 files changed, 659 insertions(+), 616 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 31c8988..f1e6ebb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -225,6 +225,8 @@ Changes in Runtime Behavior:
Improvements
+* LUCENE-8447: Add DISJOINT and WITHIN support to LatLonShape queries. (Nick Knize)
+
* LUCENE-8440: Add support for indexing and searching Line and Point shapes using LatLonShape encoding (Nick Knize)
* LUCENE-8435: Add new LatLonShapePolygonQuery for querying indexed LatLonShape fields by arbitrary polygons (Nick Knize)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
index 01a31ad..3ac171f9 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
@@ -131,12 +131,12 @@ public class LatLonShape {
* note: does not currently support dateline crossing boxes
* todo split dateline crossing boxes into two queries like {@link LatLonPoint#newBoxQuery}
**/
- public static Query newBoxQuery(String field, double minLatitude, double maxLatitude, double minLongitude, double maxLongitude) {
- return new LatLonShapeBoundingBoxQuery(field, minLatitude, maxLatitude, minLongitude, maxLongitude);
+ public static Query newBoxQuery(String field, QueryRelation queryRelation, double minLatitude, double maxLatitude, double minLongitude, double maxLongitude) {
+ return new LatLonShapeBoundingBoxQuery(field, queryRelation, minLatitude, maxLatitude, minLongitude, maxLongitude);
}
- public static Query newPolygonQuery(String field, Polygon... polygons) {
- return new LatLonShapePolygonQuery(field, polygons);
+ public static Query newPolygonQuery(String field, QueryRelation queryRelation, Polygon... polygons) {
+ return new LatLonShapePolygonQuery(field, queryRelation, polygons);
}
/** polygons are decomposed into tessellated triangles using {@link org.apache.lucene.geo.Tessellator}
@@ -167,4 +167,9 @@ public class LatLonShape {
NumericUtils.intToSortableBytes(cX, bytes, BYTES * 5);
}
}
+
+ /** Query Relation Types **/
+ public enum QueryRelation {
+ INTERSECTS, WITHIN, DISJOINT
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
index 9779210..e797043 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
@@ -16,29 +16,11 @@
*/
package org.apache.lucene.document;
-import java.io.IOException;
import java.util.Arrays;
-import java.util.Objects;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Tessellator;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.ScorerSupplier;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.DocIdSetBuilder;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FutureArrays;
import org.apache.lucene.util.NumericUtils;
@@ -58,19 +40,18 @@ import static org.apache.lucene.geo.GeoUtils.orient;
*
* @lucene.experimental
**/
-class LatLonShapeBoundingBoxQuery extends Query {
- final String field;
+final class LatLonShapeBoundingBoxQuery extends LatLonShapeQuery {
final byte[] bbox;
final int minX;
final int maxX;
final int minY;
final int maxY;
- public LatLonShapeBoundingBoxQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
+ public LatLonShapeBoundingBoxQuery(String field, LatLonShape.QueryRelation queryRelation, double minLat, double maxLat, double minLon, double maxLon) {
+ super(field, queryRelation);
if (minLon > maxLon) {
throw new IllegalArgumentException("dateline crossing bounding box queries are not supported for [" + field + "]");
}
- this.field = field;
this.bbox = new byte[4 * LatLonPoint.BYTES];
this.minX = encodeLongitudeCeil(minLon);
this.maxX = encodeLongitude(maxLon);
@@ -83,301 +64,129 @@ class LatLonShapeBoundingBoxQuery extends Query {
}
@Override
- public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- return new ConstantScoreWeight(this, boost) {
-
- private boolean edgeIntersectsQuery(double ax, double ay, double bx, double by) {
- // top
- if (orient(ax, ay, bx, by, minX, maxY) * orient(ax, ay, bx, by, maxX, maxY) <= 0 &&
- orient(minX, maxY, maxX, maxY, ax, ay) * orient(minX, maxY, maxX, maxY, bx, by) <= 0) {
- return true;
- }
-
- // right
- if (orient(ax, ay, bx, by, maxX, maxY) * orient(ax, ay, bx, by, maxX, minY) <= 0 &&
- orient(maxX, maxY, maxX, minY, ax, ay) * orient(maxX, maxY, maxX, minY, bx, by) <= 0) {
- return true;
- }
-
- // bottom
- if (orient(ax, ay, bx, by, maxX, minY) * orient(ax, ay, bx, by, minX, minY) <= 0 &&
- orient(maxX, minY, minX, minY, ax, ay) * orient(maxX, minY, minX, minY, bx, by) <= 0) {
- return true;
- }
-
- // left
- if (orient(ax, ay, bx, by, minX, minY) * orient(ax, ay, bx, by, minX, maxY) <= 0 &&
- orient(minX, minY, minX, maxY, ax, ay) * orient(minX, minY, minX, maxY, bx, by) <= 0) {
- return true;
- }
- return false;
- }
-
- private boolean queryContains(byte[] t, int point) {
- final int yIdx = 2 * LatLonPoint.BYTES * point;
- final int xIdx = yIdx + LatLonPoint.BYTES;
-
- if (FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 0, LatLonPoint.BYTES) < 0 || //minY
- FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 || //maxY
- FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 || // minX
- FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, bbox.length) > 0) {
- return false;
- }
- return true;
- }
-
- private boolean queryIntersects(int ax, int ay, int bx, int by, int cx, int cy) {
- // check each edge of the triangle against the query
- if (edgeIntersectsQuery(ax, ay, bx, by) ||
- edgeIntersectsQuery(bx, by, cx, cy) ||
- edgeIntersectsQuery(cx, cy, ax, ay)) {
- return true;
- }
- return false;
- }
-
- private boolean queryCrossesTriangle(byte[] t) {
- // 1. query contains any triangle points
- if (queryContains(t, 0) || queryContains(t, 1) || queryContains(t, 2)) {
- return true;
- }
-
- int aY = NumericUtils.sortableBytesToInt(t, 0);
- int aX = NumericUtils.sortableBytesToInt(t, LatLonPoint.BYTES);
- int bY = NumericUtils.sortableBytesToInt(t, 2 * LatLonPoint.BYTES);
- int bX = NumericUtils.sortableBytesToInt(t, 3 * LatLonPoint.BYTES);
- int cY = NumericUtils.sortableBytesToInt(t, 4 * LatLonPoint.BYTES);
- int cX = NumericUtils.sortableBytesToInt(t, 5 * LatLonPoint.BYTES);
-
- int tMinX = StrictMath.min(StrictMath.min(aX, bX), cX);
- int tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX);
- int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY);
- int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY);
-
- // 2. check bounding boxes are disjoint
- if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) {
- return false;
- }
-
- // 3. check triangle contains any query points
- if (Tessellator.pointInTriangle(minX, minY, aX, aY, bX, bY, cX, cY)) {
- return true;
- } else if (Tessellator.pointInTriangle(maxX, minY, aX, aY, bX, bY, cX, cY)) {
- return true;
- } else if (Tessellator.pointInTriangle(maxX, maxY, aX, aY, bX, bY, cX, cY)) {
- return true;
- } else if (Tessellator.pointInTriangle(minX, maxY, aX, aY, bX, bY, cX, cY)) {
- return true;
- }
-
-
- // 4. last ditch effort: check crossings
- if (queryIntersects(aX, aY, bX, bY, cX, cY)) {
- return true;
- }
- return false;
- }
-
- private Relation relateRangeToQuery(byte[] minTriangle, byte[] maxTriangle) {
- // compute bounding box
- int minXOfs = 0;
- int minYOfs = 0;
- int maxXOfs = 0;
- int maxYOfs = 0;
- for (int d = 1; d < 3; ++d) {
- // check minX
- int aOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- int bOfs = (d * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minXOfs = d;
- }
- // check maxX
- aOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxXOfs = d;
- }
- // check minY
- aOfs = minYOfs * 2 * LatLonPoint.BYTES;
- bOfs = d * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minYOfs = d;
- }
- // check maxY
- aOfs = maxYOfs * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxYOfs = d;
- }
- }
- minXOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- maxXOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- minYOfs *= 2 * LatLonPoint.BYTES;
- maxYOfs *= 2 * LatLonPoint.BYTES;
-
- // check bounding box (DISJOINT)
- if (FutureArrays.compareUnsigned(minTriangle, minXOfs, minXOfs + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) > 0 ||
- FutureArrays.compareUnsigned(maxTriangle, maxXOfs, maxXOfs + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 ||
- FutureArrays.compareUnsigned(minTriangle, minYOfs, minYOfs + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 ||
- FutureArrays.compareUnsigned(maxTriangle, maxYOfs, maxYOfs + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) < 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
-
- if (FutureArrays.compareUnsigned(minTriangle, minXOfs, minXOfs + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) > 0 &&
- FutureArrays.compareUnsigned(maxTriangle, maxXOfs, maxXOfs + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) < 0 &&
- FutureArrays.compareUnsigned(minTriangle, minYOfs, minYOfs + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) > 0 &&
- FutureArrays.compareUnsigned(maxTriangle, maxYOfs, maxYOfs + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0) {
- return Relation.CELL_INSIDE_QUERY;
- }
- return Relation.CELL_CROSSES_QUERY;
- }
-
- private IntersectVisitor getIntersectVisitor(DocIdSetBuilder result) {
- return new IntersectVisitor() {
-
- DocIdSetBuilder.BulkAdder adder;
-
- @Override
- public void grow(int count) {
- adder = result.grow(count);
- }
-
- @Override
- public void visit(int docID) throws IOException {
- adder.add(docID);
- }
-
- @Override
- public void visit(int docID, byte[] t) throws IOException {
- if (queryCrossesTriangle(t)) {
- adder.add(docID);
- }
- }
+ protected Relation relateRangeBBoxToQuery(int minXOffset, int minYOffset, byte[] minTriangle,
+ int maxXOffset, int maxYOffset, byte[] maxTriangle) {
+ // check bounding box (DISJOINT)
+ if (FutureArrays.compareUnsigned(minTriangle, minXOffset, minXOffset + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(maxTriangle, maxXOffset, maxXOffset + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 ||
+ FutureArrays.compareUnsigned(minTriangle, minYOffset, minYOffset + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(maxTriangle, maxYOffset, maxYOffset + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
- @Override
- public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
- return relateRangeToQuery(minTriangle, maxTriangle);
- }
- };
- }
+ if (FutureArrays.compareUnsigned(minTriangle, minXOffset, minXOffset + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) > 0 &&
+ FutureArrays.compareUnsigned(maxTriangle, maxXOffset, maxXOffset + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) < 0 &&
+ FutureArrays.compareUnsigned(minTriangle, minYOffset, minYOffset + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) > 0 &&
+ FutureArrays.compareUnsigned(maxTriangle, maxYOffset, maxYOffset + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0) {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ return Relation.CELL_CROSSES_QUERY;
+ }
- /**
- * Create a visitor that clears documents that do NOT match the bounding box query.
- */
- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {
- return new IntersectVisitor() {
+ /** returns true if the query matches the encoded triangle */
+ @Override
+ protected boolean queryMatches(byte[] t) {
+ if (queryRelation == LatLonShape.QueryRelation.WITHIN) {
+ return queryContains(t, 0) && queryContains(t, 1) && queryContains(t, 2);
+ }
+ return queryIntersects(t);
+ }
- @Override
- public void visit(int docID) {
- result.clear(docID);
- cost[0]--;
- }
+ /** returns true if the query intersects the encoded triangle */
+ protected boolean queryIntersects(byte[] t) {
- @Override
- public void visit(int docID, byte[] packedTriangle) {
- if (queryCrossesTriangle(packedTriangle) == false) {
- result.clear(docID);
- cost[0]--;
- }
- }
+ // 1. query contains any triangle points
+ if (queryContains(t, 0) || queryContains(t, 1) || queryContains(t, 2)) {
+ return true;
+ }
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- Relation r = relateRangeToQuery(minPackedValue, maxPackedValue);
- if (r == Relation.CELL_OUTSIDE_QUERY) {
- return Relation.CELL_INSIDE_QUERY;
- } else if (r == Relation.CELL_INSIDE_QUERY) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- return r;
- }
- };
- }
+ int aY = NumericUtils.sortableBytesToInt(t, 0);
+ int aX = NumericUtils.sortableBytesToInt(t, LatLonPoint.BYTES);
+ int bY = NumericUtils.sortableBytesToInt(t, 2 * LatLonPoint.BYTES);
+ int bX = NumericUtils.sortableBytesToInt(t, 3 * LatLonPoint.BYTES);
+ int cY = NumericUtils.sortableBytesToInt(t, 4 * LatLonPoint.BYTES);
+ int cX = NumericUtils.sortableBytesToInt(t, 5 * LatLonPoint.BYTES);
+
+ int tMinX = StrictMath.min(StrictMath.min(aX, bX), cX);
+ int tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX);
+ int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY);
+ int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY);
+
+ // 2. check bounding boxes are disjoint
+ if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) {
+ return false;
+ }
- @Override
- public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
- LeafReader reader = context.reader();
- PointValues values = reader.getPointValues(field);
- if (values == null) {
- return null;
- }
+ // 3. check triangle contains any query points
+ if (Tessellator.pointInTriangle(minX, minY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ } else if (Tessellator.pointInTriangle(maxX, minY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ } else if (Tessellator.pointInTriangle(maxX, maxY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ } else if (Tessellator.pointInTriangle(minX, maxY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ }
- boolean allDocsMatch = true;
- if (values.getDocCount() != reader.maxDoc() ||
- relateRangeToQuery(values.getMinPackedValue(), values.getMaxPackedValue()) != Relation.CELL_INSIDE_QUERY) {
- allDocsMatch = false;
- }
- final Weight weight = this;
- if (allDocsMatch) {
- return new ScorerSupplier() {
- @Override
- public Scorer get(long leadCost) throws IOException {
- return new ConstantScoreScorer(weight, score(),
- DocIdSetIterator.all(reader.maxDoc()));
- }
+ // 4. last ditch effort: check crossings
+ if (queryIntersects(aX, aY, bX, bY, cX, cY)) {
+ return true;
+ }
+ return false;
+ }
- @Override
- public long cost() {
- return reader.maxDoc();
- }
- };
- } else {
- return new ScorerSupplier() {
- final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
- final IntersectVisitor visitor = getIntersectVisitor(result);
- long cost = -1;
+ /** returns true if the edge (defined by (ax, ay) (bx, by)) intersects the query */
+ private boolean edgeIntersectsQuery(double ax, double ay, double bx, double by) {
+ // top
+ if (orient(ax, ay, bx, by, minX, maxY) * orient(ax, ay, bx, by, maxX, maxY) <= 0 &&
+ orient(minX, maxY, maxX, maxY, ax, ay) * orient(minX, maxY, maxX, maxY, bx, by) <= 0) {
+ return true;
+ }
- @Override
- public Scorer get(long leadCost) throws IOException {
- if (values.getDocCount() == reader.maxDoc()
- && values.getDocCount() == values.size()
- && cost() > reader.maxDoc() / 2) {
- // If all docs have exactly one value and the cost is greater
- // than half the leaf size then maybe we can make things faster
- // by computing the set of documents that do NOT match the query
- final FixedBitSet result = new FixedBitSet(reader.maxDoc());
- result.set(0, reader.maxDoc());
- int[] cost = new int[]{reader.maxDoc()};
- values.intersect(getInverseIntersectVisitor(result, cost));
- final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
- return new ConstantScoreScorer(weight, score(), iterator);
- }
+ // right
+ if (orient(ax, ay, bx, by, maxX, maxY) * orient(ax, ay, bx, by, maxX, minY) <= 0 &&
+ orient(maxX, maxY, maxX, minY, ax, ay) * orient(maxX, maxY, maxX, minY, bx, by) <= 0) {
+ return true;
+ }
- values.intersect(visitor);
- DocIdSetIterator iterator = result.build().iterator();
- return new ConstantScoreScorer(weight, score(), iterator);
- }
+ // bottom
+ if (orient(ax, ay, bx, by, maxX, minY) * orient(ax, ay, bx, by, minX, minY) <= 0 &&
+ orient(maxX, minY, minX, minY, ax, ay) * orient(maxX, minY, minX, minY, bx, by) <= 0) {
+ return true;
+ }
- @Override
- public long cost() {
- if (cost == -1) {
- // Computing the cost may be expensive, so only do it if necessary
- cost = values.estimatePointCount(visitor);
- assert cost >= 0;
- }
- return cost;
- }
- };
- }
- }
+ // left
+ if (orient(ax, ay, bx, by, minX, minY) * orient(ax, ay, bx, by, minX, maxY) <= 0 &&
+ orient(minX, minY, minX, maxY, ax, ay) * orient(minX, minY, minX, maxY, bx, by) <= 0) {
+ return true;
+ }
+ return false;
+ }
- @Override
- public Scorer scorer(LeafReaderContext context) throws IOException {
- ScorerSupplier scorerSupplier = scorerSupplier(context);
- if (scorerSupplier == null) {
- return null;
- }
- return scorerSupplier.get(Long.MAX_VALUE);
- }
+ /** returns true if the query contains the triangle vertex */
+ private boolean queryContains(byte[] t, int point) {
+ final int yIdx = 2 * LatLonPoint.BYTES * point;
+ final int xIdx = yIdx + LatLonPoint.BYTES;
- @Override
- public boolean isCacheable(LeafReaderContext ctx) {
- return true;
- }
- };
+ if (FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 0, LatLonPoint.BYTES) < 0 || //minY
+ FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 || //maxY
+ FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 || // minX
+ FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, bbox.length) > 0) {
+ return false;
+ }
+ return true;
}
- public String getField() {
- return field;
+ /** returns true if the query intersects the provided triangle (in encoded space) */
+ private boolean queryIntersects(int ax, int ay, int bx, int by, int cx, int cy) {
+ // check each edge of the triangle against the query
+ if (edgeIntersectsQuery(ax, ay, bx, by) ||
+ edgeIntersectsQuery(bx, by, cx, cy) ||
+ edgeIntersectsQuery(cx, cy, ax, ay)) {
+ return true;
+ }
+ return false;
}
@Override
@@ -385,15 +194,14 @@ class LatLonShapeBoundingBoxQuery extends Query {
return sameClassAs(o) && equalsTo(getClass().cast(o));
}
- private boolean equalsTo(LatLonShapeBoundingBoxQuery o) {
- return Objects.equals(field, o.field) &&
- Arrays.equals(bbox, o.bbox);
+ @Override
+ protected boolean equalsTo(Object o) {
+ return super.equalsTo(o) && Arrays.equals(bbox, ((LatLonShapeBoundingBoxQuery)o).bbox);
}
@Override
public int hashCode() {
- int hash = classHash();
- hash = 31 * hash + field.hashCode();
+ int hash = super.hashCode();
hash = 31 * hash + Arrays.hashCode(bbox);
return hash;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
index be47971..755946e 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
@@ -16,39 +16,13 @@
*/
package org.apache.lucene.document;
-import java.io.IOException;
import java.util.Arrays;
-import java.util.Objects;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.GeoEncodingUtils;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
-import org.apache.lucene.geo.Rectangle;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.ScorerSupplier;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.DocIdSetBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.FutureArrays;
-import org.apache.lucene.util.NumericUtils;
-
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitudeCeil;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil;
/**
* Finds all previously indexed shapes that intersect the specified arbitrary.
@@ -58,15 +32,15 @@ import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil;
*
* @lucene.experimental
**/
-public class LatLonShapePolygonQuery extends Query {
- final String field;
+final class LatLonShapePolygonQuery extends LatLonShapeQuery {
final Polygon[] polygons;
+ final private Polygon2D poly2D;
-
- public LatLonShapePolygonQuery(String field, Polygon... polygons) {
- if (field == null) {
- throw new IllegalArgumentException("field must not be null");
- }
+ /**
+ * Creates a query that matches all indexed shapes to the provided polygons
+ */
+ public LatLonShapePolygonQuery(String field, QueryRelation queryRelation, Polygon... polygons) {
+ super(field, queryRelation);
if (polygons == null) {
throw new IllegalArgumentException("polygons must not be null");
}
@@ -76,236 +50,40 @@ public class LatLonShapePolygonQuery extends Query {
for (int i = 0; i < polygons.length; i++) {
if (polygons[i] == null) {
throw new IllegalArgumentException("polygon[" + i + "] must not be null");
+ } else if (polygons[i].minLon > polygons[i].maxLon) {
+ throw new IllegalArgumentException("LatLonShapePolygonQuery does not currently support querying across dateline.");
}
}
- this.field = field;
this.polygons = polygons.clone();
+ this.poly2D = Polygon2D.create(polygons);
}
@Override
- public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- final Rectangle box = Rectangle.fromPolygon(polygons);
- final byte minLat[] = new byte[Integer.BYTES];
- final byte maxLat[] = new byte[Integer.BYTES];
- final byte minLon[] = new byte[Integer.BYTES];
- final byte maxLon[] = new byte[Integer.BYTES];
- NumericUtils.intToSortableBytes(encodeLatitudeCeil(box.minLat), minLat, 0);
- NumericUtils.intToSortableBytes(encodeLatitude(box.maxLat), maxLat, 0);
- NumericUtils.intToSortableBytes(encodeLongitudeCeil(box.minLon), minLon, 0);
- NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0);
-
- final Polygon2D polygon = Polygon2D.create(polygons);
-
- return new ConstantScoreWeight(this, boost) {
-
- private Relation relateRangeToQuery(byte[] minTriangle, byte[] maxTriangle) {
- // compute bounding box
- int minXOfs = 0;
- int minYOfs = 0;
- int maxXOfs = 0;
- int maxYOfs = 0;
- for (int d = 1; d < 3; ++d) {
- // check minX
- int aOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- int bOfs = (d * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minXOfs = d;
- }
- // check maxX
- aOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxXOfs = d;
- }
- // check minY
- aOfs = minYOfs * 2 * LatLonPoint.BYTES;
- bOfs = d * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minYOfs = d;
- }
- // check maxY
- aOfs = maxYOfs * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxYOfs = d;
- }
- }
- minXOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- maxXOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- minYOfs *= 2 * LatLonPoint.BYTES;
- maxYOfs *= 2 * LatLonPoint.BYTES;
-
- double minLat = GeoEncodingUtils.decodeLatitude(minTriangle, minYOfs);
- double minLon = GeoEncodingUtils.decodeLongitude(minTriangle, minXOfs);
- double maxLat = GeoEncodingUtils.decodeLatitude(maxTriangle, maxYOfs);
- double maxLon = GeoEncodingUtils.decodeLongitude(maxTriangle, maxXOfs);
-
- // check internal node against query
- return polygon.relate(minLat, maxLat, minLon, maxLon);
- }
-
- private boolean queryCrossesTriangle(byte[] t) {
- double ay = GeoEncodingUtils.decodeLatitude(t, 0);
- double ax = GeoEncodingUtils.decodeLongitude(t, LatLonPoint.BYTES);
- double by = GeoEncodingUtils.decodeLatitude(t, 2 * LatLonPoint.BYTES);
- double bx = GeoEncodingUtils.decodeLongitude(t, 3 * LatLonPoint.BYTES);
- double cy = GeoEncodingUtils.decodeLatitude(t, 4 * LatLonPoint.BYTES);
- double cx = GeoEncodingUtils.decodeLongitude(t, 5 * LatLonPoint.BYTES);
- return polygon.relateTriangle(ax, ay, bx, by, cx, cy) != Relation.CELL_OUTSIDE_QUERY;
- }
-
- private IntersectVisitor getIntersectVisitor(DocIdSetBuilder result) {
- return new IntersectVisitor() {
-
- DocIdSetBuilder.BulkAdder adder;
-
- @Override
- public void grow(int count) {
- adder = result.grow(count);
- }
-
- @Override
- public void visit(int docID) throws IOException {
- adder.add(docID);
- }
-
- @Override
- public void visit(int docID, byte[] t) throws IOException {
- if (queryCrossesTriangle(t)) {
- adder.add(docID);
- }
- }
-
- @Override
- public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
- return relateRangeToQuery(minTriangle, maxTriangle);
- }
- };
- }
-
- /**
- * Create a visitor that clears documents that do NOT match the polygon query.
- */
- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {
- return new IntersectVisitor() {
-
- @Override
- public void visit(int docID) {
- result.clear(docID);
- cost[0]--;
- }
-
- @Override
- public void visit(int docID, byte[] packedTriangle) {
- if (queryCrossesTriangle(packedTriangle) == false) {
- result.clear(docID);
- cost[0]--;
- }
- }
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- Relation r = relateRangeToQuery(minPackedValue, maxPackedValue);
- if (r == Relation.CELL_OUTSIDE_QUERY) {
- return Relation.CELL_INSIDE_QUERY;
- } else if (r == Relation.CELL_INSIDE_QUERY) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- return r;
- }
- };
- }
-
- @Override
- public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
- LeafReader reader = context.reader();
- PointValues values = reader.getPointValues(field);
- if (values == null) {
- // No docs in this segment had any points fields
- return null;
- }
- FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
- if (fieldInfo == null) {
- // No docs in this segment indexed this field at all
- return null;
- }
-
- boolean allDocsMatch = true;
- if (values.getDocCount() != reader.maxDoc() ||
- relateRangeToQuery(values.getMinPackedValue(), values.getMaxPackedValue()) != Relation.CELL_INSIDE_QUERY) {
- allDocsMatch = false;
- }
-
- final Weight weight = this;
- if (allDocsMatch) {
- return new ScorerSupplier() {
- @Override
- public Scorer get(long leadCost) throws IOException {
- return new ConstantScoreScorer(weight, score(),
- DocIdSetIterator.all(reader.maxDoc()));
- }
-
- @Override
- public long cost() {
- return reader.maxDoc();
- }
- };
- } else {
- return new ScorerSupplier() {
- final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
- final IntersectVisitor visitor = getIntersectVisitor(result);
- long cost = -1;
-
- @Override
- public Scorer get(long leadCost) throws IOException {
- if (values.getDocCount() == reader.maxDoc()
- && values.getDocCount() == values.size()
- && cost() > reader.maxDoc() / 2) {
- // If all docs have exactly one value and the cost is greater
- // than half the leaf size then maybe we can make things faster
- // by computing the set of documents that do NOT match the query
- final FixedBitSet result = new FixedBitSet(reader.maxDoc());
- result.set(0, reader.maxDoc());
- int[] cost = new int[]{reader.maxDoc()};
- values.intersect(getInverseIntersectVisitor(result, cost));
- final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
- return new ConstantScoreScorer(weight, score(), iterator);
- }
-
- values.intersect(visitor);
- DocIdSetIterator iterator = result.build().iterator();
- return new ConstantScoreScorer(weight, score(), iterator);
- }
-
- @Override
- public long cost() {
- if (cost == -1) {
- // Computing the cost may be expensive, so only do it if necessary
- cost = values.estimatePointCount(visitor);
- assert cost >= 0;
- }
- return cost;
- }
- };
- }
- }
-
- @Override
- public Scorer scorer(LeafReaderContext context) throws IOException {
- ScorerSupplier scorerSupplier = scorerSupplier(context);
- if (scorerSupplier == null) {
- return null;
- }
- return scorerSupplier.get(Long.MAX_VALUE);
- }
-
- @Override
- public boolean isCacheable(LeafReaderContext ctx) {
- return true;
- }
- };
+ protected Relation relateRangeBBoxToQuery(int minXOffset, int minYOffset, byte[] minTriangle,
+ int maxXOffset, int maxYOffset, byte[] maxTriangle) {
+ double minLat = GeoEncodingUtils.decodeLatitude(minTriangle, minYOffset);
+ double minLon = GeoEncodingUtils.decodeLongitude(minTriangle, minXOffset);
+ double maxLat = GeoEncodingUtils.decodeLatitude(maxTriangle, maxYOffset);
+ double maxLon = GeoEncodingUtils.decodeLongitude(maxTriangle, maxXOffset);
+
+ // check internal node against query
+ return poly2D.relate(minLat, maxLat, minLon, maxLon);
}
- public String getField() {
- return field;
+ @Override
+ protected boolean queryMatches(byte[] triangle) {
+ double ay = GeoEncodingUtils.decodeLatitude(triangle, 0);
+ double ax = GeoEncodingUtils.decodeLongitude(triangle, LatLonPoint.BYTES);
+ double by = GeoEncodingUtils.decodeLatitude(triangle, 2 * LatLonPoint.BYTES);
+ double bx = GeoEncodingUtils.decodeLongitude(triangle, 3 * LatLonPoint.BYTES);
+ double cy = GeoEncodingUtils.decodeLatitude(triangle, 4 * LatLonPoint.BYTES);
+ double cx = GeoEncodingUtils.decodeLongitude(triangle, 5 * LatLonPoint.BYTES);
+
+ if (queryRelation == QueryRelation.WITHIN) {
+ return poly2D.relateTriangle(ax, ay, bx, by, cx, cy) == Relation.CELL_INSIDE_QUERY;
+ }
+ // INTERSECTS
+ return poly2D.relateTriangle(ax, ay, bx, by, cx, cy) != Relation.CELL_OUTSIDE_QUERY;
}
@Override
@@ -323,18 +101,13 @@ public class LatLonShapePolygonQuery extends Query {
}
@Override
- public boolean equals(Object o) {
- return sameClassAs(o) && equalsTo(getClass().cast(o));
- }
-
- private boolean equalsTo(LatLonShapePolygonQuery o) {
- return Objects.equals(field, o.field) && Arrays.equals(polygons, o.polygons);
+ protected boolean equalsTo(Object o) {
+ return super.equalsTo(o) && Arrays.equals(polygons, ((LatLonShapePolygonQuery)o).polygons);
}
@Override
public int hashCode() {
- int hash = classHash();
- hash = 31 * hash + field.hashCode();
+ int hash = super.hashCode();
hash = 31 * hash + Arrays.hashCode(polygons);
return hash;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java
new file mode 100644
index 0000000..c678941
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.lucene.document.LatLonShape.QueryRelation;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.PointValues.IntersectVisitor;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.DocIdSetBuilder;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.FutureArrays;
+
+/**
+ * Base LatLonShape Query class providing common query logic for
+ * {@link LatLonShapeBoundingBoxQuery} and {@link LatLonShapePolygonQuery}
+ *
+ * Note: this class implements the majority of the INTERSECTS, WITHIN, DISJOINT relation logic
+ *
+ * @lucene.experimental
+ **/
+abstract class LatLonShapeQuery extends Query {
+ /** field name */
+ final String field;
+ /** query relation
+ * disjoint: {@code CELL_OUTSIDE_QUERY}
+ * intersects: {@code CELL_CROSSES_QUERY},
+ * within: {@code CELL_WITHIN_QUERY} */
+ final LatLonShape.QueryRelation queryRelation;
+
+ protected LatLonShapeQuery(String field, final QueryRelation queryType) {
+ if (field == null) {
+ throw new IllegalArgumentException("field must not be null");
+ }
+ this.field = field;
+ this.queryRelation = queryType;
+ }
+
+ /**
+ * relates an internal node (bounding box of a range of triangles) to the target query
+ * Note: logic is specific to query type
+ * see {@link LatLonShapeBoundingBoxQuery#relateRangeToQuery} and {@link LatLonShapePolygonQuery#relateRangeToQuery}
+ */
+ protected abstract Relation relateRangeBBoxToQuery(int minXOffset, int minYOffset, byte[] minTriangle,
+ int maxXOffset, int maxYOffset, byte[] maxTriangle);
+
+ /** returns true if the provided triangle matches the query */
+ protected abstract boolean queryMatches(byte[] triangle);
+
+ /** relates a range of triangles (internal node) to the query */
+ protected Relation relateRangeToQuery(byte[] minTriangle, byte[] maxTriangle) {
+ // compute bounding box of internal node
+ int minXOfs = 0;
+ int minYOfs = 0;
+ int maxXOfs = 0;
+ int maxYOfs = 0;
+ for (int d = 1; d < 3; ++d) {
+ // check minX
+ int aOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ int bOfs = (d * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
+ minXOfs = d;
+ }
+ // check maxX
+ aOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
+ maxXOfs = d;
+ }
+ // check minY
+ aOfs = minYOfs * 2 * LatLonPoint.BYTES;
+ bOfs = d * 2 * LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
+ minYOfs = d;
+ }
+ // check maxY
+ aOfs = maxYOfs * 2 * LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
+ maxYOfs = d;
+ }
+ }
+ minXOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ maxXOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ minYOfs *= 2 * LatLonPoint.BYTES;
+ maxYOfs *= 2 * LatLonPoint.BYTES;
+
+ Relation r = relateRangeBBoxToQuery(minXOfs, minYOfs, minTriangle, maxXOfs, maxYOfs, maxTriangle);
+
+ if (queryRelation == QueryRelation.DISJOINT) {
+ return transposeRelation(r);
+ }
+ return r;
+ }
+
+ @Override
+ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+
+ return new ConstantScoreWeight(this, boost) {
+
+ /** create a visitor that adds documents that match the query using a sparse bitset. (Used by INTERSECT) */
+ protected IntersectVisitor getSparseIntersectVisitor(DocIdSetBuilder result) {
+ return new IntersectVisitor() {
+ DocIdSetBuilder.BulkAdder adder;
+
+ @Override
+ public void grow(int count) {
+ adder = result.grow(count);
+ }
+
+ @Override
+ public void visit(int docID) throws IOException {
+ adder.add(docID);
+ }
+
+ @Override
+ public void visit(int docID, byte[] t) throws IOException {
+ if (queryMatches(t)) {
+ adder.add(docID);
+ }
+ }
+
+ @Override
+ public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
+ return relateRangeToQuery(minTriangle, maxTriangle);
+ }
+ };
+ }
+
+ /** create a visitor that adds documents that match the query using a dense bitset. (Used by WITHIN, DISJOINT) */
+ protected IntersectVisitor getDenseIntersectVisitor(FixedBitSet intersect, FixedBitSet disjoint) {
+ return new IntersectVisitor() {
+
+ @Override
+ public void visit(int docID) throws IOException {
+ if (queryRelation == QueryRelation.DISJOINT) {
+ // if DISJOINT query set the doc in the disjoint bitset
+ disjoint.set(docID);
+ } else {
+ // for INTERSECT, and WITHIN queries we set the intersect bitset
+ intersect.set(docID);
+ }
+ }
+
+ @Override
+ public void visit(int docID, byte[] t) throws IOException {
+ if (queryMatches(t)) {
+ intersect.set(docID);
+ } else {
+ disjoint.set(docID);
+ }
+ }
+
+ @Override
+ public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
+ return relateRangeToQuery(minTriangle, maxTriangle);
+ }
+ };
+ }
+
+ /** get a scorer supplier for INTERSECT queries */
+ protected ScorerSupplier getIntersectScorerSupplier(LeafReader reader, PointValues values, Weight weight) throws IOException {
+ DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
+ IntersectVisitor visitor = getSparseIntersectVisitor(result);
+ return new RelationScorerSupplier(values, visitor) {
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ return getIntersectsScorer(LatLonShapeQuery.this, reader, weight, result, score());
+ }
+ };
+ }
+
+ /** get a scorer supplier for all other queries (DISJOINT, WITHIN) */
+ protected ScorerSupplier getScorerSupplier(LeafReader reader, PointValues values, Weight weight) throws IOException {
+ if (queryRelation == QueryRelation.INTERSECTS) {
+ return getIntersectScorerSupplier(reader, values, weight);
+ }
+
+ FixedBitSet intersect = new FixedBitSet(reader.maxDoc());
+ FixedBitSet disjoint = new FixedBitSet(reader.maxDoc());
+ IntersectVisitor visitor = getDenseIntersectVisitor(intersect, disjoint);
+ return new RelationScorerSupplier(values, visitor) {
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ return getScorer(LatLonShapeQuery.this, weight, intersect, disjoint, score());
+ }
+ };
+ }
+
+ @Override
+ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+ LeafReader reader = context.reader();
+ PointValues values = reader.getPointValues(field);
+ if (values == null) {
+ // No docs in this segment had any points fields
+ return null;
+ }
+ FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+ if (fieldInfo == null) {
+ // No docs in this segment indexed this field at all
+ return null;
+ }
+
+ boolean allDocsMatch = true;
+ if (values.getDocCount() != reader.maxDoc() ||
+ relateRangeToQuery(values.getMinPackedValue(), values.getMaxPackedValue()) != Relation.CELL_INSIDE_QUERY) {
+ allDocsMatch = false;
+ }
+
+ final Weight weight = this;
+ if (allDocsMatch) {
+ return new ScorerSupplier() {
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ return new ConstantScoreScorer(weight, score(),
+ DocIdSetIterator.all(reader.maxDoc()));
+ }
+
+ @Override
+ public long cost() {
+ return reader.maxDoc();
+ }
+ };
+ } else {
+ return getScorerSupplier(reader, values, weight);
+ }
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ ScorerSupplier scorerSupplier = scorerSupplier(context);
+ if (scorerSupplier == null) {
+ return null;
+ }
+ return scorerSupplier.get(Long.MAX_VALUE);
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return true;
+ }
+ };
+ }
+
+ /** returns the field name */
+ public String getField() {
+ return field;
+ }
+
+ /** returns the query relation */
+ public QueryRelation getQueryRelation() {
+ return queryRelation;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = classHash();
+ hash = 31 * hash + field.hashCode();
+ hash = 31 * hash + queryRelation.hashCode();
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return sameClassAs(o) && equalsTo(o);
+ }
+
+ protected boolean equalsTo(Object o) {
+ return Objects.equals(field, ((LatLonShapeQuery)o).field) && this.queryRelation == ((LatLonShapeQuery)o).queryRelation;
+ }
+
+ /** transpose the relation; INSIDE becomes OUTSIDE, OUTSIDE becomes INSIDE, CROSSES remains unchanged */
+ private static Relation transposeRelation(Relation r) {
+ if (r == Relation.CELL_INSIDE_QUERY) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ } else if (r == Relation.CELL_OUTSIDE_QUERY) {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ return Relation.CELL_CROSSES_QUERY;
+ }
+
+ /** utility class for implementing constant score logic specifig to INTERSECT, WITHIN, and DISJOINT */
+ protected static abstract class RelationScorerSupplier extends ScorerSupplier {
+ PointValues values;
+ IntersectVisitor visitor;
+ long cost = -1;
+
+ RelationScorerSupplier(PointValues values, IntersectVisitor visitor) {
+ this.values = values;
+ this.visitor = visitor;
+ }
+
+ /** create a visitor that clears documents that do NOT match the polygon query; used with INTERSECTS */
+ private IntersectVisitor getInverseIntersectVisitor(LatLonShapeQuery query, FixedBitSet result, int[] cost) {
+ return new IntersectVisitor() {
+
+ @Override
+ public void visit(int docID) {
+ result.clear(docID);
+ cost[0]--;
+ }
+
+ @Override
+ public void visit(int docID, byte[] packedTriangle) {
+ if (query.queryMatches(packedTriangle) == false) {
+ result.clear(docID);
+ cost[0]--;
+ }
+ }
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ return transposeRelation(query.relateRangeToQuery(minPackedValue, maxPackedValue));
+ }
+ };
+ }
+
+ /** returns a Scorer for INTERSECT queries that uses a sparse bitset */
+ protected Scorer getIntersectsScorer(LatLonShapeQuery query, LeafReader reader, Weight weight,
+ DocIdSetBuilder docIdSetBuilder, final float boost) throws IOException {
+ if (values.getDocCount() == reader.maxDoc()
+ && values.getDocCount() == values.size()
+ && cost() > reader.maxDoc() / 2) {
+ // If all docs have exactly one value and the cost is greater
+ // than half the leaf size then maybe we can make things faster
+ // by computing the set of documents that do NOT match the query
+ final FixedBitSet result = new FixedBitSet(reader.maxDoc());
+ result.set(0, reader.maxDoc());
+ int[] cost = new int[]{reader.maxDoc()};
+ values.intersect(getInverseIntersectVisitor(query, result, cost));
+ final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
+ return new ConstantScoreScorer(weight, boost, iterator);
+ }
+
+ values.intersect(visitor);
+ DocIdSetIterator iterator = docIdSetBuilder.build().iterator();
+ return new ConstantScoreScorer(weight, boost, iterator);
+ }
+
+ /** returns a Scorer for all other (non INTERSECT) queries */
+ protected Scorer getScorer(LatLonShapeQuery query, Weight weight,
+ FixedBitSet intersect, FixedBitSet disjoint, final float boost) throws IOException {
+ values.intersect(visitor);
+ DocIdSetIterator iterator;
+ if (query.queryRelation == QueryRelation.DISJOINT) {
+ disjoint.andNot(intersect);
+ iterator = new BitSetIterator(disjoint, cost());
+ } else if (query.queryRelation == QueryRelation.WITHIN) {
+ intersect.andNot(disjoint);
+ iterator = new BitSetIterator(intersect, cost());
+ } else {
+ iterator = new BitSetIterator(intersect, cost());
+ }
+ return new ConstantScoreScorer(weight, boost, iterator);
+ }
+
+ @Override
+ public long cost() {
+ if (cost == -1) {
+ // Computing the cost may be expensive, so only do it if necessary
+ cost = values.estimatePointCount(visitor);
+ assert cost >= 0;
+ }
+ return cost;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java b/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
index a7560ee..191e2cb 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
@@ -22,6 +22,7 @@ import java.util.HashSet;
import java.util.Set;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
@@ -94,6 +95,16 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
return new Polygon(lats, lons);
}
+ protected Line quantizeLine(Line line) {
+ double[] lats = new double[line.numPoints()];
+ double[] lons = new double[line.numPoints()];
+ for (int i = 0; i < lats.length; ++i) {
+ lats[i] = quantizeLat(line.getLat(i));
+ lons[i] = quantizeLon(line.getLon(i));
+ }
+ return new Line(lats, lons);
+ }
+
protected abstract Field[] createIndexableFields(String field, Object shape);
private void addShapeToDoc(String field, Document doc, Object shape) {
@@ -103,12 +114,12 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
}
}
- protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
- return LatLonShape.newBoxQuery(field, minLat, maxLat, minLon, maxLon);
+ protected Query newRectQuery(String field, QueryRelation queryRelation, double minLat, double maxLat, double minLon, double maxLon) {
+ return LatLonShape.newBoxQuery(field, queryRelation, minLat, maxLat, minLon, maxLon);
}
- protected Query newPolygonQuery(String field, Polygon... polygons) {
- return LatLonShape.newPolygonQuery(field, polygons);
+ protected Query newPolygonQuery(String field, QueryRelation queryRelation, Polygon... polygons) {
+ return LatLonShape.newPolygonQuery(field, queryRelation, polygons);
}
// A particularly tricky adversary for BKD tree:
@@ -240,7 +251,8 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
break;
}
}
- Query query = newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon);
+ QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
+ Query query = newRectQuery(FIELD_NAME, queryRelation, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon);
if (VERBOSE) {
System.out.println(" query=" + query);
@@ -280,7 +292,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
expected = false;
} else {
// check quantized poly against quantized query
- expected = getValidator().testBBoxQuery(quantizeLatCeil(rect.minLat), quantizeLat(rect.maxLat),
+ expected = getValidator(queryRelation).testBBoxQuery(quantizeLatCeil(rect.minLat), quantizeLat(rect.maxLat),
quantizeLonCeil(rect.minLon), quantizeLon(rect.maxLon), shapes[id]);
}
@@ -292,10 +304,11 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
+ b.append(" relation=" + queryRelation + "\n");
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" shape=" + shapes[id] + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
- b.append(" rect=Rectangle(" + quantizeLatCeil(rect.minLat) + " TO " + quantizeLat(rect.maxLat) + " lon=" + quantizeLonCeil(rect.minLon) + " TO " + quantizeLon(rect.maxLon) + ")");
+ b.append(" rect=Rectangle(" + quantizeLatCeil(rect.minLat) + " TO " + quantizeLat(rect.maxLat) + " lon=" + quantizeLonCeil(rect.minLon) + " TO " + quantizeLon(rect.maxLon) + ")\n");
if (true) {
fail("wrong hit (first of possibly more):\n\n" + b);
} else {
@@ -326,7 +339,8 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
// Polygon
Polygon queryPolygon = GeoTestUtil.nextPolygon();
Polygon2D queryPoly2D = Polygon2D.create(queryPolygon);
- Query query = newPolygonQuery(FIELD_NAME, queryPolygon);
+ QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
+ Query query = newPolygonQuery(FIELD_NAME, queryRelation, queryPolygon);
if (VERBOSE) {
System.out.println(" query=" + query);
@@ -365,7 +379,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
} else if (shapes[id] == null) {
expected = false;
} else {
- expected = getValidator().testPolygonQuery(queryPoly2D, shapes[id]);
+ expected = getValidator(queryRelation).testPolygonQuery(queryPoly2D, shapes[id]);
}
if (hits.get(docID) != expected) {
@@ -376,6 +390,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
+ b.append(" relation=" + queryRelation + "\n");
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" shape=" + shapes[id] + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
@@ -394,7 +409,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
}
}
- protected abstract Validator getValidator();
+ protected abstract Validator getValidator(QueryRelation relation);
/** internal point class for testing point shapes */
protected static class Point {
@@ -466,8 +481,13 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
}
}
- protected interface Validator {
- boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape);
- boolean testPolygonQuery(Polygon2D poly2d, Object shape);
+ protected abstract class Validator {
+ protected QueryRelation queryRelation = QueryRelation.INTERSECTS;
+ public abstract boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape);
+ public abstract boolean testPolygonQuery(Polygon2D poly2d, Object shape);
+
+ public void setRelation(QueryRelation relation) {
+ this.queryRelation = relation;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
index 21367dc..9a91232 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
@@ -16,16 +16,12 @@
*/
package org.apache.lucene.document;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.index.PointValues.Relation;
-import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
-
/** random bounding box and polygon query tests for random generated {@link Line} types */
public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
@@ -42,17 +38,25 @@ public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
}
@Override
- protected Validator getValidator() {
+ protected Validator getValidator(QueryRelation queryRelation) {
+ VALIDATOR.setRelation(queryRelation);
return VALIDATOR;
}
- protected class LineValidator implements Validator {
+ protected class LineValidator extends Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
+ Line l = (Line)shape;
+ if (queryRelation == QueryRelation.WITHIN) {
+ // within: bounding box of shape should be within query box
+ return minLat <= quantizeLat(l.minLat) && maxLat >= quantizeLat(l.maxLat)
+ && minLon <= quantizeLon(l.minLon) && maxLon >= quantizeLon(l.maxLon);
+ }
+
// to keep it simple we convert the bbox into a polygon and use poly2d
Polygon2D p = Polygon2D.create(new Polygon[] {new Polygon(new double[] {minLat, minLat, maxLat, maxLat, minLat},
new double[] {minLon, maxLon, maxLon, minLon, minLon})});
- return testLine(p, (Line)shape);
+ return testLine(p, l);
}
@Override
@@ -62,11 +66,12 @@ public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
private boolean testLine(Polygon2D queryPoly, Line line) {
double ax, ay, bx, by, temp;
+ Relation r;
for (int i = 0, j = 1; j < line.numPoints(); ++i, ++j) {
- ay = decodeLatitude(encodeLatitude(line.getLat(i)));
- ax = decodeLongitude(encodeLongitude(line.getLon(i)));
- by = decodeLatitude(encodeLatitude(line.getLat(j)));
- bx = decodeLongitude(encodeLongitude(line.getLon(j)));
+ ay = quantizeLat(line.getLat(i));
+ ax = quantizeLon(line.getLon(i));
+ by = quantizeLat(line.getLat(j));
+ bx = quantizeLon(line.getLon(j));
if (ay > by) {
temp = ay;
ay = by;
@@ -84,11 +89,16 @@ public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
bx = temp;
}
}
- if (queryPoly.relateTriangle(ax, ay, bx, by, ax, ay) != Relation.CELL_OUTSIDE_QUERY) {
- return true;
+ r = queryPoly.relateTriangle(ax, ay, bx, by, ax, ay);
+ if (queryRelation == QueryRelation.DISJOINT) {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return false;
+ } else if (queryRelation == QueryRelation.WITHIN) {
+ if (r != Relation.CELL_INSIDE_QUERY) return false;
+ } else {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return true;
}
}
- return false;
+ return queryRelation == QueryRelation.INTERSECTS ? false : true;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
index 3adb26b..df924fe 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.document;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.index.PointValues.Relation;
@@ -41,17 +42,22 @@ public class TestLatLonPointShapeQueries extends BaseLatLonShapeTestCase {
}
@Override
- protected Validator getValidator() {
+ protected Validator getValidator(QueryRelation relation) {
+ VALIDATOR.setRelation(relation);
return VALIDATOR;
}
- protected class PointValidator implements Validator {
+ protected class PointValidator extends Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
Point p = (Point)shape;
double lat = decodeLatitude(encodeLatitude(p.lat));
double lon = decodeLongitude(encodeLongitude(p.lon));
- return (lat < minLat || lat > maxLat || lon < minLon || lon > maxLon) == false;
+ boolean isDisjoint = lat < minLat || lat > maxLat || lon < minLon || lon > maxLon;
+ if (queryRelation == QueryRelation.DISJOINT) {
+ return isDisjoint;
+ }
+ return isDisjoint == false;
}
@Override
@@ -60,7 +66,13 @@ public class TestLatLonPointShapeQueries extends BaseLatLonShapeTestCase {
double lat = decodeLatitude(encodeLatitude(p.lat));
double lon = decodeLongitude(encodeLongitude(p.lon));
// for consistency w/ the query we test the point as a triangle
- return poly2d.relateTriangle(lon, lat, lon, lat, lon, lat) != Relation.CELL_OUTSIDE_QUERY;
+ Relation r = poly2d.relateTriangle(lon, lat, lon, lat, lon, lat);
+ if (queryRelation == QueryRelation.WITHIN) {
+ return r == Relation.CELL_INSIDE_QUERY;
+ } else if (queryRelation == QueryRelation.DISJOINT) {
+ return r == Relation.CELL_OUTSIDE_QUERY;
+ }
+ return r != Relation.CELL_OUTSIDE_QUERY;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
index 17eb6e8..9b844ab 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
@@ -18,11 +18,13 @@ package org.apache.lucene.document;
import java.util.List;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.geo.Tessellator;
import org.apache.lucene.index.PointValues.Relation;
+/** random bounding box and polygon query tests for random indexed {@link Polygon} types */
public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
protected final PolygonValidator VALIDATOR = new PolygonValidator();
@@ -53,30 +55,46 @@ public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
}
@Override
- protected Validator getValidator() {
+ protected Validator getValidator(QueryRelation relation) {
+ VALIDATOR.setRelation(relation);
return VALIDATOR;
}
- protected class PolygonValidator implements Validator {
+ protected class PolygonValidator extends Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
- Polygon2D poly = Polygon2D.create(quantizePolygon((Polygon)shape));
- return poly.relate(minLat, maxLat, minLon, maxLon) != Relation.CELL_OUTSIDE_QUERY;
+ Polygon p = (Polygon)shape;
+ if (queryRelation == QueryRelation.WITHIN) {
+ // within: bounding box of shape should be within query box
+ return minLat <= quantizeLat(p.minLat) && maxLat >= quantizeLat(p.maxLat)
+ && minLon <= quantizeLon(p.minLon) && maxLon >= quantizeLon(p.maxLon);
+ }
+
+ Polygon2D poly = Polygon2D.create(quantizePolygon(p));
+ Relation r = poly.relate(minLat, maxLat, minLon, maxLon);
+ if (queryRelation == QueryRelation.DISJOINT) {
+ return r == Relation.CELL_OUTSIDE_QUERY;
+ }
+ return r != Relation.CELL_OUTSIDE_QUERY;
}
@Override
public boolean testPolygonQuery(Polygon2D query, Object shape) {
-
List<Tessellator.Triangle> tessellation = Tessellator.tessellate((Polygon) shape);
for (Tessellator.Triangle t : tessellation) {
// we quantize the triangle for consistency with the index
- if (query.relateTriangle(quantizeLon(t.getLon(0)), quantizeLat(t.getLat(0)),
+ Relation r = query.relateTriangle(quantizeLon(t.getLon(0)), quantizeLat(t.getLat(0)),
quantizeLon(t.getLon(1)), quantizeLat(t.getLat(1)),
- quantizeLon(t.getLon(2)), quantizeLat(t.getLat(2))) != Relation.CELL_OUTSIDE_QUERY) {
- return true;
+ quantizeLon(t.getLon(2)), quantizeLat(t.getLat(2)));
+ if (queryRelation == QueryRelation.DISJOINT) {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return false;
+ } else if (queryRelation == QueryRelation.WITHIN) {
+ if (r != Relation.CELL_INSIDE_QUERY) return false;
+ } else {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return true;
}
}
- return false;
+ return queryRelation == QueryRelation.INTERSECTS ? false : true;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
index 3aa5ace..2f6e5e1 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
@@ -17,6 +17,7 @@
package org.apache.lucene.document;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
@@ -52,7 +53,7 @@ public class TestLatLonShape extends LuceneTestCase {
}
protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
- return LatLonShape.newBoxQuery(field, minLat, maxLat, minLon, maxLon);
+ return LatLonShape.newBoxQuery(field, QueryRelation.INTERSECTS, minLat, maxLat, minLon, maxLon);
}
@Ignore
[09/31] lucene-solr:jira/http2: LUCENE-8442: Avoid merge in
testPendingDeleteDVGeneration
Posted by da...@apache.org.
LUCENE-8442: Avoid merge in testPendingDeleteDVGeneration
If a merge is triggered, it might reclaim the randomFile which we picked
up but haven't opened yet, the test will fail with NoSuchFileException.
This commit disables merges to avoid such scenarios.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/38986531
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/38986531
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/38986531
Branch: refs/heads/jira/http2
Commit: 3898653195b24987b8ba53ea9acf8713316b66f3
Parents: 208d59b
Author: Nhat Nguyen <nh...@elastic.co>
Authored: Tue Aug 7 08:39:46 2018 -0400
Committer: Nhat Nguyen <nh...@elastic.co>
Committed: Tue Aug 7 09:19:56 2018 -0400
----------------------------------------------------------------------
lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38986531/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
index b640298..08b3dc1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -2664,6 +2664,7 @@ public class TestIndexWriter extends LuceneTestCase {
Closeable closeable = () -> IOUtils.close(toClose)) {
IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()))
.setUseCompoundFile(false)
+ .setMergePolicy(NoMergePolicy.INSTANCE) // avoid merging away the randomFile
.setMaxBufferedDocs(2)
.setRAMBufferSizeMB(-1);
IndexWriter w = new IndexWriter(dir, iwc);
@@ -2702,6 +2703,7 @@ public class TestIndexWriter extends LuceneTestCase {
w.rollback();
iwc = new IndexWriterConfig(new MockAnalyzer(random()))
.setUseCompoundFile(false)
+ .setMergePolicy(NoMergePolicy.INSTANCE)
.setMaxBufferedDocs(2)
.setRAMBufferSizeMB(-1);
w = new IndexWriter(dir, iwc);
[02/31] lucene-solr:jira/http2: LUCENE-8444: Prevent building
functionally identical plane bounds when constructing
DualCrossingEdgeIterator
Posted by da...@apache.org.
LUCENE-8444: Prevent building functionally identical plane bounds when constructing DualCrossingEdgeIterator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ea2e564d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ea2e564d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ea2e564d
Branch: refs/heads/jira/http2
Commit: ea2e564dad694af50bb8dd022ddb2d7f94cb7a2e
Parents: 70a4939
Author: Ignacio Vera <iv...@apache.org>
Authored: Mon Aug 6 12:40:08 2018 +0200
Committer: Ignacio Vera <iv...@apache.org>
Committed: Mon Aug 6 12:40:08 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../lucene/spatial3d/geom/GeoComplexPolygon.java | 8 ++++----
.../lucene/spatial3d/geom/GeoPolygonTest.java | 18 ++++++++++++++++++
3 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ea2e564d/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 14e2b02..865f5e2 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -179,6 +179,9 @@ Bug Fixes:
* LUCENE-8445: Tighten condition when two planes are identical to prevent constructing
bogus tiles when building GeoPolygons. (Ignacio Vera)
+* LUCENE-8444: Prevent building functionally identical plane bounds when constructing
+ DualCrossingEdgeIterator . (Ignacio Vera)
+
* LUCENE-8380: UTF8TaxonomyWriterCache inconsistency. (Ruslan Torobaev, Dawid Weiss)
* LUCENE-8164: IndexWriter silently accepts broken payload. This has been fixed
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ea2e564d/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
index 580338d..a7c55dd 100644
--- a/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
+++ b/lucene/spatial3d/src/java/org/apache/lucene/spatial3d/geom/GeoComplexPolygon.java
@@ -1504,16 +1504,16 @@ class GeoComplexPolygon extends GeoBasePolygon {
final SidedPlane testPointBound1 = new SidedPlane(intersectionPoint, testPointPlane, testPoint);
final SidedPlane testPointBound2 = new SidedPlane(testPoint, testPointPlane, intersectionPoint);
- if (testPointBound1.isNumericallyIdentical(testPointBound2)) {
- throw new IllegalArgumentException("Dual iterator unreliable when bounds planes are numerically identical");
+ if (testPointBound1.isFunctionallyIdentical(testPointBound2)) {
+ throw new IllegalArgumentException("Dual iterator unreliable when bounds planes are functionally identical");
}
this.testPointCutoffPlane = testPointBound1;
this.testPointOtherCutoffPlane = testPointBound2;
final SidedPlane checkPointBound1 = new SidedPlane(intersectionPoint, travelPlane, thePointX, thePointY, thePointZ);
final SidedPlane checkPointBound2 = new SidedPlane(thePointX, thePointY, thePointZ, travelPlane, intersectionPoint);
- if (checkPointBound1.isNumericallyIdentical(checkPointBound2)) {
- throw new IllegalArgumentException("Dual iterator unreliable when bounds planes are numerically identical");
+ if (checkPointBound1.isFunctionallyIdentical(checkPointBound2)) {
+ throw new IllegalArgumentException("Dual iterator unreliable when bounds planes are functionally identical");
}
this.checkPointCutoffPlane = checkPointBound1;
this.checkPointOtherCutoffPlane = checkPointBound2;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ea2e564d/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
index c8cbc1b..1aae820 100755
--- a/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
+++ b/lucene/spatial3d/src/test/org/apache/lucene/spatial3d/geom/GeoPolygonTest.java
@@ -1851,6 +1851,24 @@ shape:
}
@Test
+ public void testLUCENE8444() {
+ //POLYGON((0.0 -67.68132244526963,-1.2477695347678826E-95 -88.11137674490907, 1.7059188343238906E-9 7.009654350320916,0.0 -67.68132244526963))
+ final List<GeoPoint> points = new ArrayList<>();
+ points.add(new GeoPoint(PlanetModel.WGS84, Geo3DUtil.fromDegrees(-67.68132244526963), Geo3DUtil.fromDegrees(0.0)));
+ points.add(new GeoPoint(PlanetModel.WGS84, Geo3DUtil.fromDegrees(-88.11137674490907), Geo3DUtil.fromDegrees(-1.2477695347678826E-95)));
+ points.add(new GeoPoint(PlanetModel.WGS84, Geo3DUtil.fromDegrees(7.009654350320916), Geo3DUtil.fromDegrees(1.7059188343238906E-9)));
+
+ final GeoPolygonFactory.PolygonDescription description = new GeoPolygonFactory.PolygonDescription(points);
+ final GeoPolygon polygon = GeoPolygonFactory.makeGeoPolygon(PlanetModel.WGS84, description);
+ final GeoPolygon largePolygon = GeoPolygonFactory.makeLargeGeoPolygon(PlanetModel.WGS84, Collections.singletonList(description));
+
+ //POINT(180.0 -61.73229670194638)
+ final GeoPoint point = new GeoPoint(PlanetModel.WGS84, Geo3DUtil.fromDegrees(-61.73229670194638), Geo3DUtil.fromDegrees(180.0));
+ //assertTrue(polygon.isWithin(point) == largePolygon.isWithin(point));
+ assertTrue(false == largePolygon.isWithin(point));
+ }
+
+ @Test
public void testLUCENE8445() {
//POLYGON((32.18017946378854 -17.397683785381247,49.51018758330871 -9.870219317504647,58.77903721991479 33.90553510354402,2.640604559432277 9.363173880050821,3.1673235739886286E-10 8.853669066894417E-11,0.0 -5.7E-322,4.820339742500488E-5 5.99784517213369E-7,32.18017946378854 -17.397683785381247))
final List<GeoPoint> points = new ArrayList<>();
[27/31] lucene-solr:jira/http2: SOLR-12555: Use `expectThrows` for
expected exceptions
Posted by da...@apache.org.
SOLR-12555: Use `expectThrows` for expected exceptions
This commit replaces the `try { doX(); fail(); }` pattern with the
`expectThrows` test helper, which was created for this purpose. This
commit makes these changes in the core package: `o.a.solr.cloud`.
Closes #425
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/00aeb64c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/00aeb64c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/00aeb64c
Branch: refs/heads/jira/http2
Commit: 00aeb64c10290780256476de2266a9bb600c336d
Parents: 63fc124
Author: Jason Gerlowski <ge...@apache.org>
Authored: Thu Aug 9 15:42:17 2018 -0400
Committer: Jason Gerlowski <ge...@apache.org>
Committed: Thu Aug 9 15:46:50 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/BasicFunctionalityTest.java | 158 ++++++------
.../test/org/apache/solr/CursorPagingTest.java | 10 +-
.../solr/TestCursorMarkWithoutUniqueKey.java | 9 +-
.../org/apache/solr/TestDistributedSearch.java | 45 ++--
.../org/apache/solr/TestTolerantSearch.java | 18 +-
.../solr/cloud/BasicDistributedZk2Test.java | 30 +--
.../solr/cloud/BasicDistributedZkTest.java | 14 +-
.../test/org/apache/solr/cloud/BasicZkTest.java | 14 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 46 ++--
.../apache/solr/cloud/DeleteReplicaTest.java | 19 +-
.../org/apache/solr/cloud/ForceLeaderTest.java | 10 +-
.../apache/solr/cloud/OverseerStatusTest.java | 17 +-
.../org/apache/solr/cloud/SolrXmlInZkTest.java | 10 +-
.../solr/cloud/TestAuthenticationFramework.java | 10 +-
.../solr/cloud/TestCloudDeleteByQuery.java | 11 +-
.../apache/solr/cloud/TestCloudInspectUtil.java | 19 +-
.../apache/solr/cloud/TestConfigSetsAPI.java | 11 +-
.../solr/cloud/TestConfigSetsAPIZkFailure.java | 14 +-
.../solr/cloud/TestDownShardTolerantSearch.java | 17 +-
.../TestLeaderInitiatedRecoveryThread.java | 64 +++--
.../cloud/TestPullReplicaErrorHandling.java | 13 +-
.../TestSolrCloudWithDelegationTokens.java | 25 +-
.../TestSolrCloudWithSecureImpersonation.java | 101 +++-----
.../cloud/TestTolerantUpdateProcessorCloud.java | 238 +++++++++----------
.../org/apache/solr/cloud/TestZkChroot.java | 23 +-
.../test/org/apache/solr/cloud/ZkCLITest.java | 18 +-
.../org/apache/solr/cloud/ZkSolrClientTest.java | 59 ++---
27 files changed, 449 insertions(+), 574 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
index ed1f663..18c4f69 100644
--- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
+++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
@@ -356,42 +356,39 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
// test that malformed numerics cause client error not server error
for (String field : FIELDS) {
- try {
- h.update(add( doc("id","100", field, BAD_VALUE)));
- fail("Didn't encounter an error trying to add a bad date: " + field);
- } catch (SolrException e) {
- String msg = e.toString();
- assertTrue("not an (update) client error on field: " + field +" : "+ msg,
- 400 <= e.code() && e.code() < 500);
- assertTrue("(update) client error does not mention bad value: " + msg,
- msg.contains(BAD_VALUE));
- assertTrue("client error does not mention document id: " + msg,
- msg.contains("[doc=100]"));
- }
- SchemaField sf = h.getCore().getLatestSchema().getField(field);
+ SolrException e1 = expectThrows(SolrException.class,
+ "Didn't encounter an error trying to add a bad date: " + field,
+ () -> h.update(add( doc("id","100", field, BAD_VALUE))));
+ String msg1 = e1.getMessage();
+ assertTrue("not an (update) client error on field: " + field +" : "+ msg1,
+ 400 <= e1.code() && e1.code() < 500);
+ assertTrue("(update) client error does not mention bad value: " + msg1,
+ msg1.contains(BAD_VALUE));
+ assertTrue("client error does not mention document id: " + msg1,
+ msg1.contains("[doc=100]"));
+ SchemaField sf = h.getCore().getLatestSchema().getField(field);
if (!sf.hasDocValues() && !sf.indexed()) {
continue;
}
- try {
- h.query(req("q",field + ":" + BAD_VALUE));
- fail("Didn't encounter an error trying to query a bad date: " + field);
- } catch (SolrException e) {
- String msg = e.toString();
- assertTrue("not a (search) client error on field: " + field +" : "+ msg,
- 400 <= e.code() && e.code() < 500);
- assertTrue("(search) client error does not mention bad value: " + msg,
- msg.contains(BAD_VALUE));
- }
- try {
- h.query(req("q",field + ":[NOW TO " + BAD_VALUE + "]"));
- fail("Didn't encounter an error trying to query a bad date: " + field);
- } catch (SolrException e) {
- String msg = e.toString();
- assertTrue("not a (search) client error on field: " + field +" : "+ msg,
- 400 <= e.code() && e.code() < 500);
- assertTrue("(search) client error does not mention bad value: " + msg,
- msg.contains(BAD_VALUE));
- }
+ SolrException e2 = expectThrows(SolrException.class,
+ "Didn't encounter an error trying to add a bad date: " + field,
+ () -> h.query(req("q",field + ":" + BAD_VALUE))
+ );
+ String msg2 = e2.toString();
+ assertTrue("not a (search) client error on field: " + field +" : "+ msg2,
+ 400 <= e2.code() && e2.code() < 500);
+ assertTrue("(search) client error does not mention bad value: " + msg2,
+ msg2.contains(BAD_VALUE));
+
+ SolrException e3 = expectThrows(SolrException.class,
+ "Didn't encounter an error trying to add a bad date: " + field,
+ () -> h.query(req("q",field + ":[NOW TO " + BAD_VALUE + "]"))
+ );
+ String msg3 = e3.toString();
+ assertTrue("not a (search) client error on field: " + field +" : "+ msg3,
+ 400 <= e3.code() && e3.code() < 500);
+ assertTrue("(search) client error does not mention bad value: " + msg3,
+ msg3.contains(BAD_VALUE));
}
}
@@ -414,42 +411,40 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
// test that malformed numerics cause client error not server error
for (String field : FIELDS) {
- try {
- h.update(add( doc("id","100", field, BAD_VALUE)));
- fail("Didn't encounter an error trying to add a non-number: " + field);
- } catch (SolrException e) {
- String msg = e.toString();
- assertTrue("not an (update) client error on field: " + field +" : "+ msg,
- 400 <= e.code() && e.code() < 500);
- assertTrue("(update) client error does not mention bad value: " + msg,
- msg.contains(BAD_VALUE));
- assertTrue("client error does not mention document id",
- msg.contains("[doc=100]"));
- }
+ SolrException e1 = expectThrows(SolrException.class,
+ "Didn't encounter an error trying to add a non-number: " + field,
+ () -> h.update(add( doc("id","100", field, BAD_VALUE))));
+ String msg1 = e1.toString();
+ assertTrue("not an (update) client error on field: " + field +" : "+ msg1,
+ 400 <= e1.code() && e1.code() < 500);
+ assertTrue("(update) client error does not mention bad value: " + msg1,
+ msg1.contains(BAD_VALUE));
+ assertTrue("client error does not mention document id",
+ msg1.contains("[doc=100]"));
SchemaField sf = h.getCore().getLatestSchema().getField(field);
if (!sf.hasDocValues() && !sf.indexed()) {
continue;
}
- try {
- h.query(req("q",field + ":" + BAD_VALUE));
- fail("Didn't encounter an error trying to query a non-number: " + field);
- } catch (SolrException e) {
- String msg = e.toString();
- assertTrue("not a (search) client error on field: " + field +" : "+ msg,
- 400 <= e.code() && e.code() < 500);
- assertTrue("(search) client error does not mention bad value: " + msg,
- msg.contains(BAD_VALUE));
- }
- try {
- h.query(req("q",field + ":[10 TO " + BAD_VALUE + "]"));
- fail("Didn't encounter an error trying to query a non-number: " + field);
- } catch (SolrException e) {
- String msg = e.toString();
- assertTrue("not a (search) client error on field: " + field +" : "+ msg,
- 400 <= e.code() && e.code() < 500);
- assertTrue("(search) client error does not mention bad value: " + msg,
- msg.contains(BAD_VALUE));
- }
+
+ SolrException e2 = expectThrows(SolrException.class,
+ "Didn't encounter an error trying to add a non-number: " + field,
+ () -> h.query(req("q",field + ":" + BAD_VALUE))
+ );
+ String msg2 = e2.toString();
+ assertTrue("not a (search) client error on field: " + field +" : "+ msg2,
+ 400 <= e2.code() && e2.code() < 500);
+ assertTrue("(search) client error does not mention bad value: " + msg2,
+ msg2.contains(BAD_VALUE));
+
+ SolrException e3 = expectThrows(SolrException.class,
+ "Didn't encounter an error trying to add a non-number: " + field,
+ () -> h.query(req("q",field + ":[10 TO " + BAD_VALUE + "]"))
+ );
+ String msg3 = e3.toString();
+ assertTrue("not a (search) client error on field: " + field +" : "+ msg3,
+ 400 <= e3.code() && e3.code() < 500);
+ assertTrue("(search) client error does not mention bad value: " + msg3,
+ msg3.contains(BAD_VALUE));
}
}
@@ -1001,26 +996,23 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
"sortabuse_t", "zzz xxx ccc vvv bbb nnn qqq www eee rrr ttt"));
assertU(commit());
-
- try {
+
+ RuntimeException outerEx = expectThrows(RuntimeException.class, () -> {
ignoreException("can not sort on multivalued field: sortabuse_t");
assertQ("sort on something that shouldn't work",
- req("q", "sortabuse_b:true",
- "sort", "sortabuse_t asc"),
- "*[count(//doc)=2]");
- fail("no error encountered when sorting on sortabuse_t");
- } catch (Exception outer) {
- // EXPECTED
- Throwable root = getRootCause(outer);
- assertEquals("sort exception root cause",
- SolrException.class, root.getClass());
- SolrException e = (SolrException) root;
- assertEquals("incorrect error type",
- SolrException.ErrorCode.BAD_REQUEST,
- SolrException.ErrorCode.getErrorCode(e.code()));
- assertTrue("exception doesn't contain field name",
- -1 != e.getMessage().indexOf("sortabuse_t"));
- }
+ req("q", "sortabuse_b:true",
+ "sort", "sortabuse_t asc"),
+ "*[count(//doc)=2]");
+ });
+ Throwable root = getRootCause(outerEx);
+ assertEquals("sort exception root cause",
+ SolrException.class, root.getClass());
+ SolrException e = (SolrException) root;
+ assertEquals("incorrect error type",
+ SolrException.ErrorCode.BAD_REQUEST,
+ SolrException.ErrorCode.getErrorCode(e.code()));
+ assertTrue("exception doesn't contain field name",
+ e.getMessage().contains("sortabuse_t"));
}
// /** this doesn't work, but if it did, this is how we'd test it. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/CursorPagingTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/CursorPagingTest.java b/solr/core/src/test/org/apache/solr/CursorPagingTest.java
index 257b846..7ad00e7 100644
--- a/solr/core/src/test/org/apache/solr/CursorPagingTest.java
+++ b/solr/core/src/test/org/apache/solr/CursorPagingTest.java
@@ -842,13 +842,13 @@ public class CursorPagingTest extends SolrTestCaseJ4 {
throws Exception {
try {
- ignoreException(expSubstr);
- assertJQ(req(p));
- fail("no exception matching expected: " + expCode.code + ": " + expSubstr);
- } catch (SolrException e) {
+ SolrException e = expectThrows(SolrException.class, () -> {
+ ignoreException(expSubstr);
+ assertJQ(req(p));
+ });
assertEquals(expCode.code, e.code());
assertTrue("Expected substr not found: " + expSubstr + " <!< " + e.getMessage(),
- e.getMessage().contains(expSubstr));
+ e.getMessage().contains(expSubstr));
} finally {
unIgnoreException(expSubstr);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/TestCursorMarkWithoutUniqueKey.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/TestCursorMarkWithoutUniqueKey.java b/solr/core/src/test/org/apache/solr/TestCursorMarkWithoutUniqueKey.java
index 8eed87e..cedc9c8 100644
--- a/solr/core/src/test/org/apache/solr/TestCursorMarkWithoutUniqueKey.java
+++ b/solr/core/src/test/org/apache/solr/TestCursorMarkWithoutUniqueKey.java
@@ -53,11 +53,12 @@ public class TestCursorMarkWithoutUniqueKey extends SolrTestCaseJ4 {
try {
ignoreException("Cursor functionality is not available unless the IndexSchema defines a uniqueKey field");
- assertQ(req("q", "*:*", "sort", "fld desc", "cursorMark", CURSOR_MARK_START));
- fail("No exception when querying with a cursorMark with no uniqueKey defined.");
- } catch (Exception e) {
+ expectThrows(RuntimeException.class,
+ "No exception when querying with a cursorMark with no uniqueKey defined.",
+ () -> assertQ(req("q", "*:*", "sort", "fld desc", "cursorMark", CURSOR_MARK_START))
+ );
+ } finally {
unIgnoreException("Cursor functionality is not available unless the IndexSchema defines a uniqueKey field");
}
-
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
index 3ec1c6d..ee1c60e 100644
--- a/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
+++ b/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
@@ -911,13 +911,11 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
//SOLR 3161 ensure shards.qt=/update fails (anything but search handler really)
// Also see TestRemoteStreaming#testQtUpdateFails()
- try {
- ignoreException("isShard is only acceptable");
- // query("q","*:*","shards.qt","/update","stream.body","<delete><query>*:*</query></delete>");
- // fail();
- } catch (SolrException e) {
- //expected
- }
+
+ //SolrException e = expectThrows(SolrException.class, () -> {
+ // ignoreException("isShard is only acceptable");
+ // query("q","*:*","shards.qt","/update","stream.body","<delete><query>*:*</query></delete>");
+ //});
unIgnoreException("isShard is only acceptable");
// test debugging
@@ -1219,41 +1217,34 @@ public class TestDistributedSearch extends BaseDistributedSearchTestCase {
private void validateCommonQueryParameters() throws Exception {
ignoreException("parameter cannot be negative");
- try {
+ SolrException e1 = expectThrows(SolrException.class, () -> {
SolrQuery query = new SolrQuery();
query.setParam("start", "non_numeric_value").setQuery("*");
QueryResponse resp = query(query);
- fail("Expected the last query to fail, but got response: " + resp);
- } catch (SolrException e) {
- assertEquals(ErrorCode.BAD_REQUEST.code, e.code());
- }
+ });
+ assertEquals(ErrorCode.BAD_REQUEST.code, e1.code());
- try {
+ SolrException e2 = expectThrows(SolrException.class, () -> {
SolrQuery query = new SolrQuery();
query.setStart(-1).setQuery("*");
QueryResponse resp = query(query);
- fail("Expected the last query to fail, but got response: " + resp);
- } catch (SolrException e) {
- assertEquals(ErrorCode.BAD_REQUEST.code, e.code());
- }
+ });
+ assertEquals(ErrorCode.BAD_REQUEST.code, e2.code());
- try {
+ SolrException e3 = expectThrows(SolrException.class, () -> {
SolrQuery query = new SolrQuery();
query.setRows(-1).setStart(0).setQuery("*");
QueryResponse resp = query(query);
- fail("Expected the last query to fail, but got response: " + resp);
- } catch (SolrException e) {
- assertEquals(ErrorCode.BAD_REQUEST.code, e.code());
- }
+ });
+ assertEquals(ErrorCode.BAD_REQUEST.code, e3.code());
- try {
+ SolrException e4 = expectThrows(SolrException.class, () -> {
SolrQuery query = new SolrQuery();
query.setParam("rows", "non_numeric_value").setQuery("*");
QueryResponse resp = query(query);
- fail("Expected the last query to fail, but got response: " + resp);
- } catch (SolrException e) {
- assertEquals(ErrorCode.BAD_REQUEST.code, e.code());
- }
+ });
+ assertEquals(ErrorCode.BAD_REQUEST.code, e4.code());
+
resetExceptionIgnores();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/TestTolerantSearch.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java
index ec1f16c..61a11f0 100644
--- a/solr/core/src/test/org/apache/solr/TestTolerantSearch.java
+++ b/solr/core/src/test/org/apache/solr/TestTolerantSearch.java
@@ -131,12 +131,9 @@ public class TestTolerantSearch extends SolrJettyTestBase {
query.setFacet(true);
ignoreException("Dummy exception in BadResponseWriter");
- try {
- collection1.query(query);
- fail("Should get an exception");
- } catch (Exception e) {
- //expected
- }
+
+ expectThrows(SolrException.class, () -> collection1.query(query));
+
query.set(ShardParams.SHARDS_TOLERANT, "true");
QueryResponse response = collection1.query(query);
assertTrue(response.getResponseHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
@@ -179,12 +176,9 @@ public class TestTolerantSearch extends SolrJettyTestBase {
query.setFacet(true);
ignoreException("Dummy exception in BadResponseWriter");
- try {
- collection1.query(query);
- fail("Should get an exception");
- } catch (Exception e) {
- //expected
- }
+
+ expectThrows(Exception.class, () -> collection1.query(query));
+
query.set(ShardParams.SHARDS_TOLERANT, "true");
QueryResponse response = collection1.query(query);
assertTrue(response.getResponseHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
index ad9f7254..6b03824 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
@@ -116,12 +116,7 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
long docId = testUpdateAndDelete();
// index a bad doc...
- try {
- indexr(t1, "a doc with no id");
- fail("this should fail");
- } catch (SolrException e) {
- // expected
- }
+ expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id"));
// TODO: bring this to its own method?
// try indexing to a leader that has no replicas up
@@ -255,29 +250,26 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
private void brindDownShardIndexSomeDocsAndRecover() throws Exception {
SolrQuery query = new SolrQuery("*:*");
query.set("distrib", false);
-
+
commit();
-
+
long deadShardCount = shardToJetty.get(SHARD2).get(0).client.solrClient
.query(query).getResults().getNumFound();
query("q", "*:*", "sort", "n_tl1 desc");
-
+
int oldLiveNodes = cloudClient.getZkStateReader().getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true).size();
-
+
assertEquals(5, oldLiveNodes);
-
+
// kill a shard
CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0);
-
+
// ensure shard is dead
- try {
- index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1,
- "specific doc!");
- fail("This server should be down and this update should have failed");
- } catch (SolrServerException e) {
- // expected..
- }
+ expectThrows(SolrServerException.class,
+ "This server should be down and this update should have failed",
+ () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!")
+ );
commit();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
index b6de031..a46187f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java
@@ -713,24 +713,20 @@ public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase {
log.info("### STARTING doOptimisticLockingAndUpdating");
printLayout();
- SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);
+ final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1);
indexDoc(sd);
ignoreException("version conflict");
for (SolrClient client : clients) {
- try {
- client.add(sd);
- fail();
- } catch (SolrException e) {
- assertEquals(409, e.code());
- }
+ SolrException e = expectThrows(SolrException.class, () -> client.add(sd));
+ assertEquals(409, e.code());
}
unIgnoreException("version conflict");
// TODO: test deletes. SolrJ needs a good way to pass version for delete...
- sd = sdoc("id", 1000, "foo_i",5);
- clients.get(0).add(sd);
+ final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i",5);
+ clients.get(0).add(sd2);
List<Integer> expected = new ArrayList<>();
int val = 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
index 79b49dc..af3174d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicZkTest.java
@@ -150,15 +150,15 @@ public class BasicZkTest extends AbstractZkTestCase {
zkController.getZkClient().setData("/configs/conf1/solrconfig.xml", new byte[0], true);
// we set the solrconfig to nothing, so this reload should fail
- try {
+ SolrException e = expectThrows(SolrException.class,
+ "The reloaded SolrCore did not pick up configs from zookeeper",
+ () -> {
ignoreException("solrconfig.xml");
h.getCoreContainer().reload(h.getCore().getName());
- fail("The reloaded SolrCore did not pick up configs from zookeeper");
- } catch(SolrException e) {
- resetExceptionIgnores();
- assertTrue(e.getMessage().contains("Unable to reload core [collection1]"));
- assertTrue(e.getCause().getMessage().contains("Error loading solr config from solrconfig.xml"));
- }
+ });
+ resetExceptionIgnores();
+ assertTrue(e.getMessage().contains("Unable to reload core [collection1]"));
+ assertTrue(e.getCause().getMessage().contains("Error loading solr config from solrconfig.xml"));
// test stats call
Map<String, Metric> metrics = h.getCore().getCoreMetricManager().getRegistry().getMetrics();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
index 0a1b1ed..f61fa5e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CollectionsAPISolrJTest.java
@@ -514,31 +514,25 @@ public class CollectionsAPISolrJTest extends SolrCloudTestCase {
waitForState("Expecting attribute 'maxShardsPerNode' to be deleted", collection,
(n, c) -> null == c.get("maxShardsPerNode"));
- try {
- CollectionAdminRequest.modifyCollection(collection, null)
- .setAttribute("non_existent_attr", 25)
- .process(cluster.getSolrClient());
- fail("An attempt to set unknown collection attribute should have failed");
- } catch (IllegalArgumentException e) {
- // expected
- }
-
- try {
- CollectionAdminRequest.modifyCollection(collection, null)
- .setAttribute("non_existent_attr", null)
- .process(cluster.getSolrClient());
- fail("An attempt to set null value should have failed");
- } catch (IllegalArgumentException e) {
- // expected
- }
-
- try {
- CollectionAdminRequest.modifyCollection(collection, null)
- .unsetAttribute("non_existent_attr")
- .process(cluster.getSolrClient());
- fail("An attempt to unset unknown collection attribute should have failed");
- } catch (IllegalArgumentException e) {
- // expected
- }
+ expectThrows(IllegalArgumentException.class,
+ "An attempt to set unknown collection attribute should have failed",
+ () -> CollectionAdminRequest.modifyCollection(collection, null)
+ .setAttribute("non_existent_attr", 25)
+ .process(cluster.getSolrClient())
+ );
+
+ expectThrows(IllegalArgumentException.class,
+ "An attempt to set null value should have failed",
+ () -> CollectionAdminRequest.modifyCollection(collection, null)
+ .setAttribute("non_existent_attr", null)
+ .process(cluster.getSolrClient())
+ );
+
+ expectThrows(IllegalArgumentException.class,
+ "An attempt to unset unknown collection attribute should have failed",
+ () -> CollectionAdminRequest.modifyCollection(collection, null)
+ .unsetAttribute("non_existent_attr")
+ .process(cluster.getSolrClient())
+ );
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
index 30124d1..232735a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
@@ -144,20 +144,16 @@ public class DeleteReplicaTest extends SolrCloudTestCase {
CollectionAdminRequest.deleteReplicasFromShard(collectionName, "shard1", 2).process(cluster.getSolrClient());
waitForState("Expected a single shard with a single replica", collectionName, clusterShape(1, 1));
-
- try {
- CollectionAdminRequest.deleteReplicasFromShard(collectionName, "shard1", 1).process(cluster.getSolrClient());
- fail("Expected Exception, Can't delete the last replica by count");
- } catch (SolrException e) {
- // expected
- assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
- assertTrue(e.getMessage().contains("There is only one replica available"));
- }
+
+ SolrException e = expectThrows(SolrException.class,
+ "Can't delete the last replica by count",
+ () -> CollectionAdminRequest.deleteReplicasFromShard(collectionName, "shard1", 1).process(cluster.getSolrClient())
+ );
+ assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
+ assertTrue(e.getMessage().contains("There is only one replica available"));
DocCollection docCollection = getCollectionState(collectionName);
// We know that since leaders are preserved, PULL replicas should not be left alone in the shard
assertEquals(0, docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).size());
-
-
}
@Test
@@ -257,6 +253,7 @@ public class DeleteReplicaTest extends SolrCloudTestCase {
return false;
}
LOG.info("Running delete core {}",cd);
+
try {
ZkNodeProps m = new ZkNodeProps(
Overseer.QUEUE_OPERATION, OverseerAction.DELETECORE.toLower(),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
index b25ff5f..793dcb3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ForceLeaderTest.java
@@ -317,13 +317,9 @@ public class ForceLeaderTest extends HttpPartitionTest {
private void assertSendDocFails(int docId) throws Exception {
// sending a doc in this state fails
- try {
- sendDoc(docId);
- log.error("Should've failed indexing during a down state. Cluster state: " + printClusterStateInfo());
- fail("Should've failed indexing during a down state.");
- } catch (SolrException ex) {
- log.info("Document couldn't be sent, which is expected.");
- }
+ expectThrows(SolrException.class,
+ "Should've failed indexing during a down state.",
+ () -> sendDoc(docId));
}
private void putNonLeadersIntoLIR(String collectionName, String shard, ZkController zkController, Replica leader, List<Replica> notLeaders) throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
index 80fd38e..6d169e3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.cloud;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.NamedList;
@@ -58,14 +59,14 @@ public class OverseerStatusTest extends SolrCloudTestCase {
SimpleOrderedMap<Object> reload = (SimpleOrderedMap<Object>) collection_operations.get(CollectionParams.CollectionAction.RELOAD.toLower());
assertEquals("No stats for reload in OverseerCollectionProcessor", 1, reload.get("requests"));
- try {
- CollectionAdminRequest.splitShard("non_existent_collection")
- .setShardName("non_existent_shard")
- .process(cluster.getSolrClient());
- fail("Split shard for non existent collection should have failed");
- } catch (Exception e) {
- // expected because we did not correctly specify required params for split
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ "Split shard for non existent collection should have failed",
+ () -> CollectionAdminRequest
+ .splitShard("non_existent_collection")
+ .setShardName("non_existent_shard")
+ .process(cluster.getSolrClient())
+ );
+
resp = new CollectionAdminRequest.OverseerStatus().process(cluster.getSolrClient()).getResponse();
collection_operations = (NamedList<Object>) resp.get("collection_operations");
SimpleOrderedMap<Object> split = (SimpleOrderedMap<Object>) collection_operations.get(CollectionParams.CollectionAction.SPLITSHARD.toLower());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java b/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java
index 5e9e180..519b978 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SolrXmlInZkTest.java
@@ -140,12 +140,12 @@ public class SolrXmlInZkTest extends SolrTestCaseJ4 {
@Test
public void testNotInZkOrOnDisk() throws Exception {
try {
- System.setProperty("hostPort", "8787");
- setUpZkAndDiskXml(false, false); // solr.xml not on disk either
- fail("Should have thrown an exception here");
- } catch (SolrException solre) {
+ SolrException e = expectThrows(SolrException.class, () -> {
+ System.setProperty("hostPort", "8787");
+ setUpZkAndDiskXml(false, false); // solr.xml not on disk either
+ });
assertTrue("Should be failing to create default solr.xml in code",
- solre.getMessage().contains("solr.xml does not exist"));
+ e.getMessage().contains("solr.xml does not exist"));
} finally {
closeZK();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java b/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java
index 0615289..c795b14 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestAuthenticationFramework.java
@@ -29,6 +29,7 @@ import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpClientUtil;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.SolrHttpClientBuilder;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -79,12 +80,9 @@ public class TestAuthenticationFramework extends SolrCloudTestCase {
// Should fail with 401
try {
- collectionCreateSearchDeleteTwice();
- fail("Should've returned a 401 error");
- } catch (Exception ex) {
- if (!ex.getMessage().contains("Error 401")) {
- fail("Should've returned a 401 error");
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ this::collectionCreateSearchDeleteTwice);
+ assertTrue("Should've returned a 401 error", e.getMessage().contains("Error 401"));
} finally {
MockAuthenticationPlugin.expectedUsername = null;
MockAuthenticationPlugin.expectedPassword = null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java
index d85b139..f210d1c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudDeleteByQuery.java
@@ -30,7 +30,6 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@@ -199,12 +198,10 @@ public class TestCloudDeleteByQuery extends SolrCloudTestCase {
public void testMalformedDBQ(SolrClient client) throws Exception {
assertNotNull("client not initialized", client);
- try {
- UpdateResponse rsp = update(params()).deleteByQuery("foo_i:not_a_num").process(client);
- fail("Expected DBQ failure: " + rsp.toString());
- } catch (SolrException e) {
- assertEquals("not the expected DBQ failure: " + e.getMessage(), 400, e.code());
- }
+ SolrException e = expectThrows(SolrException.class,
+ "Expected DBQ failure",
+ () -> update(params()).deleteByQuery("foo_i:not_a_num").process(client));
+ assertEquals("not the expected DBQ failure: " + e.getMessage(), 400, e.code());
}
//
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestCloudInspectUtil.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudInspectUtil.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudInspectUtil.java
index 6de9f2d..6031e03 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudInspectUtil.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudInspectUtil.java
@@ -100,20 +100,15 @@ public class TestCloudInspectUtil extends SolrTestCaseJ4 {
// ################################
- addFails = new HashSet<String>();
- deleteFails = new HashSet<String>();
+ final HashSet<String> addFailsExpectEx = new HashSet<String>();
+ final HashSet<String> deleteFailsExpectEx = new HashSet<String>();
- a = getDocList("2", "3", "4");
- b = getDocList("2", "3", "4");
-
- try {
- legal = CloudInspectUtil.checkIfDiffIsLegal(a, b, "control", "cloud",
- addFails, deleteFails);
- fail("Expected exception because lists have no diff");
- } catch (IllegalArgumentException e) {
- // expected
- }
+ final SolrDocumentList aExpectEx = getDocList("2", "3", "4");
+ final SolrDocumentList bExpectEx = getDocList("2", "3", "4");
+ expectThrows(IllegalArgumentException.class, "Expected exception because lists have no diff",
+ () -> CloudInspectUtil.checkIfDiffIsLegal(aExpectEx, bExpectEx,
+ "control", "cloud", addFailsExpectEx, deleteFailsExpectEx));
}
private SolrDocumentList getDocList(String ... ids) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index c76b3d6..71a0c2f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -92,7 +92,6 @@ import org.apache.solr.util.ExternalPaths;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.noggit.JSONParser;
@@ -632,13 +631,9 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 {
private void verifyException(SolrClient solrClient, ConfigSetAdminRequest request,
String errorContains) throws Exception {
- try {
- solrClient.request(request);
- Assert.fail("Expected exception");
- } catch (Exception e) {
- assertTrue("Expected exception message to contain: " + errorContains
- + " got: " + e.getMessage(), e.getMessage().contains(errorContains));
- }
+ Exception e = expectThrows(Exception.class, () -> solrClient.request(request));
+ assertTrue("Expected exception message to contain: " + errorContains
+ + " got: " + e.getMessage(), e.getMessage().contains(errorContains));
}
@Test
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java
index 34acb250..41c5463 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java
@@ -38,7 +38,6 @@ import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Create;
-import org.apache.solr.client.solrj.response.ConfigSetAdminResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
@@ -58,7 +57,6 @@ import org.apache.zookeeper.server.ZKDatabase;
import org.apache.zookeeper.server.quorum.Leader.Proposal;
import org.apache.zookeeper.txn.TxnHeader;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -112,14 +110,10 @@ public class TestConfigSetsAPIZkFailure extends SolrTestCaseJ4 {
Create create = new Create();
create.setBaseConfigSetName(BASE_CONFIGSET_NAME).setConfigSetName(CONFIGSET_NAME);
- try {
- ConfigSetAdminResponse response = create.process(solrClient);
- Assert.fail("Expected solr exception");
- } catch (RemoteSolrException se) {
- // partial creation should have been cleaned up
- assertFalse(configManager.configExists(CONFIGSET_NAME));
- assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, se.code());
- }
+ RemoteSolrException se = expectThrows(RemoteSolrException.class, () -> create.process(solrClient));
+ // partial creation should have been cleaned up
+ assertFalse(configManager.configExists(CONFIGSET_NAME));
+ assertEquals(SolrException.ErrorCode.SERVER_ERROR.code, se.code());
} finally {
zkClient.close();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestDownShardTolerantSearch.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestDownShardTolerantSearch.java b/solr/core/src/test/org/apache/solr/cloud/TestDownShardTolerantSearch.java
index 415d4e4..2686ccf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestDownShardTolerantSearch.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestDownShardTolerantSearch.java
@@ -69,14 +69,13 @@ public class TestDownShardTolerantSearch extends SolrCloudTestCase {
assertThat(response.getStatus(), is(0));
assertTrue(response.getResults().getNumFound() > 0);
- try {
- cluster.getSolrClient().query("tolerant", new SolrQuery("*:*").setRows(1).setParam(ShardParams.SHARDS_TOLERANT, false));
- fail("Request should have failed because we killed shard1 jetty");
- } catch (SolrServerException e) {
- log.info("error from server", e);
- assertNotNull(e.getCause());
- assertTrue("Error message from server should have the name of the down shard",
- e.getCause().getMessage().contains("shard"));
- }
+ SolrServerException e = expectThrows(SolrServerException.class,
+ "Request should have failed because we killed shard1 jetty",
+ () -> cluster.getSolrClient().query("tolerant", new SolrQuery("*:*").setRows(1)
+ .setParam(ShardParams.SHARDS_TOLERANT, false))
+ );
+ assertNotNull(e.getCause());
+ assertTrue("Error message from server should have the name of the down shard",
+ e.getCause().getMessage().contains("shard"));
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java b/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java
index e3ee8bb..20ac056 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestLeaderInitiatedRecoveryThread.java
@@ -51,8 +51,8 @@ public class TestLeaderInitiatedRecoveryThread extends AbstractFullDistribZkTest
final String leaderCoreNodeName = shardToLeaderJetty.get(SHARD1).coreNodeName;
final CloudJettyRunner leaderRunner = shardToLeaderJetty.get(SHARD1);
- CoreContainer coreContainer = leaderRunner.jetty.getCoreContainer();
- ZkController zkController = coreContainer.getZkController();
+ final CoreContainer coreContainer1 = leaderRunner.jetty.getCoreContainer();
+ final ZkController zkController1 = coreContainer1.getZkController();
CloudJettyRunner notLeader = null;
for (CloudJettyRunner cloudJettyRunner : shardToJetty.get(SHARD1)) {
@@ -83,22 +83,21 @@ public class TestLeaderInitiatedRecoveryThread extends AbstractFullDistribZkTest
/*
1. Test that publishDownState throws exception when zkController.isReplicaInRecoveryHandling == false
*/
- try {
-
- LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer,
+
+ SolrException e = expectThrows(SolrException.class,
+ "publishDownState should not have succeeded because replica url is not marked in leader initiated recovery in ZkController",
+ () -> {
+ LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController1, coreContainer1,
DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd);
- assertFalse(zkController.isReplicaInRecoveryHandling(replicaCoreNodeProps.getCoreUrl()));
+ assertFalse(zkController1.isReplicaInRecoveryHandling(replicaCoreNodeProps.getCoreUrl()));
thread.run();
- fail("publishDownState should not have succeeded because replica url is not marked in leader initiated recovery in ZkController");
- } catch (SolrException e) {
- assertTrue(e.code() == SolrException.ErrorCode.INVALID_STATE.code);
- }
-
+ });
+ assertEquals(e.code(), SolrException.ErrorCode.INVALID_STATE.code);
/*
2. Test that a non-live replica cannot be put into LIR or down state
*/
- LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer,
+ LeaderInitiatedRecoveryThread thread = new LeaderInitiatedRecoveryThread(zkController1, coreContainer1,
DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd);
// kill the replica
int children = cloudClient.getZkStateReader().getZkClient().getChildren("/live_nodes", null, true).size();
@@ -129,7 +128,7 @@ public class TestLeaderInitiatedRecoveryThread extends AbstractFullDistribZkTest
ChaosMonkey.start(notLeader.jetty);
waitForRecoveriesToFinish(true);
- thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer,
+ thread = new LeaderInitiatedRecoveryThread(zkController1, coreContainer1,
DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {
@Override
protected void updateLIRState(String replicaCoreNodeName) {
@@ -138,13 +137,13 @@ public class TestLeaderInitiatedRecoveryThread extends AbstractFullDistribZkTest
};
assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
- assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
+ assertNull(zkController1.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
/*
4. Test that if ZK connection loss or session expired then thread should not attempt to publish down state even if forcePublish=true
*/
- thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer,
+ thread = new LeaderInitiatedRecoveryThread(zkController1, coreContainer1,
DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {
@Override
protected void updateLIRState(String replicaCoreNodeName) {
@@ -153,13 +152,13 @@ public class TestLeaderInitiatedRecoveryThread extends AbstractFullDistribZkTest
};
assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false));
assertFalse(thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), true));
- assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
+ assertNull(zkController1.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
/*
5. Test that any exception other then ZK connection loss or session expired should publish down state only if forcePublish=true
*/
- thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer,
+ thread = new LeaderInitiatedRecoveryThread(zkController1, coreContainer1,
DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, cd) {
@Override
protected void updateLIRState(String replicaCoreNodeName) {
@@ -187,26 +186,21 @@ public class TestLeaderInitiatedRecoveryThread extends AbstractFullDistribZkTest
Thread.sleep(500);
}
- assertNull(zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
+ assertNull(zkController1.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
assertEquals(Replica.State.DOWN, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getReplica(replica.getName()).getState());
/*
6. Test that non-leader cannot set LIR nodes
*/
- coreContainer = notLeader.jetty.getCoreContainer();
- zkController = coreContainer.getZkController();
+ final CoreContainer coreContainer2 = notLeader.jetty.getCoreContainer();
+ final ZkController zkController2 = coreContainer2.getZkController();
- thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer,
- DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer.getCores().iterator().next().getCoreDescriptor()) {
+ thread = new LeaderInitiatedRecoveryThread(zkController2, coreContainer2,
+ DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer2.getCores().iterator().next().getCoreDescriptor()) {
@Override
protected void updateLIRState(String replicaCoreNodeName) {
- try {
- super.updateLIRState(replicaCoreNodeName);
- } catch (Exception e) {
- assertTrue(e instanceof ZkController.NotLeaderException);
- throw e;
- }
+ throw expectThrows(ZkController.NotLeaderException.class, () -> super.updateLIRState(replicaCoreNodeName));
}
};
cversion = getOverseerCversion();
@@ -217,21 +211,21 @@ public class TestLeaderInitiatedRecoveryThread extends AbstractFullDistribZkTest
7. assert that we can write a LIR state if everything else is fine
*/
// reset the zkcontroller to the one from the leader
- coreContainer = leaderRunner.jetty.getCoreContainer();
- zkController = coreContainer.getZkController();
- thread = new LeaderInitiatedRecoveryThread(zkController, coreContainer,
- DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer.getCores().iterator().next().getCoreDescriptor());
+ final CoreContainer coreContainer3 = leaderRunner.jetty.getCoreContainer();
+ final ZkController zkController3 = coreContainer3.getZkController();
+ thread = new LeaderInitiatedRecoveryThread(zkController3, coreContainer3,
+ DEFAULT_COLLECTION, SHARD1, replicaCoreNodeProps, 1, coreContainer3.getCores().iterator().next().getCoreDescriptor());
thread.publishDownState(replicaCoreNodeProps.getCoreName(), replica.getName(), replica.getNodeName(), replicaCoreNodeProps.getCoreUrl(), false);
timeOut = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
while (!timeOut.hasTimedOut()) {
- Replica.State state = zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName());
+ Replica.State state = zkController3.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName());
if (state == Replica.State.DOWN) {
break;
}
Thread.sleep(500);
}
- assertNotNull(zkController.getLeaderInitiatedRecoveryStateObject(DEFAULT_COLLECTION, SHARD1, replica.getName()));
- assertEquals(Replica.State.DOWN, zkController.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
+ assertNotNull(zkController3.getLeaderInitiatedRecoveryStateObject(DEFAULT_COLLECTION, SHARD1, replica.getName()));
+ assertEquals(Replica.State.DOWN, zkController3.getLeaderInitiatedRecoveryState(DEFAULT_COLLECTION, SHARD1, replica.getName()));
/*
7. Test that
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
index 263f227..93235f9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplicaErrorHandling.java
@@ -152,12 +152,13 @@ public void testCantConnectToPullReplica() throws Exception {
assertNumDocs(10 + i, leaderClient);
}
}
- try (HttpSolrClient pullReplicaClient = getHttpSolrClient(s.getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getCoreUrl())) {
- pullReplicaClient.query(new SolrQuery("*:*")).getResults().getNumFound();
- fail("Shouldn't be able to query the pull replica");
- } catch (SolrServerException e) {
- //expected
- }
+
+ SolrServerException e = expectThrows(SolrServerException.class, () -> {
+ try(HttpSolrClient pullReplicaClient = getHttpSolrClient(s.getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getCoreUrl())) {
+ pullReplicaClient.query(new SolrQuery("*:*")).getResults().getNumFound();
+ }
+ });
+
assertNumberOfReplicas(numShards, 0, numShards, true, true);// Replica should still be active, since it doesn't disconnect from ZooKeeper
{
long numFound = 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java
index 9a66a4a..7ba11cf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithDelegationTokens.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.cloud;
-import junit.framework.Assert;
import org.apache.hadoop.util.Time;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
@@ -383,33 +382,31 @@ public class TestSolrCloudWithDelegationTokens extends SolrTestCaseJ4 {
SolrRequest request = getAdminRequest(new ModifiableSolrParams());
// test without token
- HttpSolrClient ss =
- new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString())
- .withResponseParser(solrClientPrimary.getParser())
- .build();
+ final HttpSolrClient ssWoToken =
+ new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString())
+ .withResponseParser(solrClientPrimary.getParser())
+ .build();
try {
- doSolrRequest(ss, request, ErrorCode.UNAUTHORIZED.code);
+ doSolrRequest(ssWoToken, request, ErrorCode.UNAUTHORIZED.code);
} finally {
- ss.close();
+ ssWoToken.close();
}
- ss = new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString())
+ final HttpSolrClient ssWToken = new HttpSolrClient.Builder(solrClientPrimary.getBaseURL().toString())
.withKerberosDelegationToken(token)
.withResponseParser(solrClientPrimary.getParser())
.build();
try {
// test with token via property
- doSolrRequest(ss, request, HttpStatus.SC_OK);
+ doSolrRequest(ssWToken, request, HttpStatus.SC_OK);
// test with param -- should throw an exception
ModifiableSolrParams tokenParam = new ModifiableSolrParams();
tokenParam.set("delegation", "invalidToken");
- try {
- doSolrRequest(ss, getAdminRequest(tokenParam), ErrorCode.FORBIDDEN.code);
- Assert.fail("Expected exception");
- } catch (IllegalArgumentException ex) {}
+ expectThrows(IllegalArgumentException.class,
+ () -> doSolrRequest(ssWToken, getAdminRequest(tokenParam), ErrorCode.FORBIDDEN.code));
} finally {
- ss.close();
+ ssWToken.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
index 184cd90..8d6684d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
@@ -223,37 +223,28 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
@Test
public void testProxyNoConfigGroups() throws Exception {
- try {
- solrClient.request(getProxyRequest("noGroups","bar"));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("noGroups", "bar")));
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> solrClient.request(getProxyRequest("noGroups","bar"))
+ );
+ assertTrue(e.getMessage().contains(getExpectedGroupExMsg("noGroups", "bar")));
}
@Test
public void testProxyWrongHost() throws Exception {
- try {
- solrClient.request(getProxyRequest("wrongHost","bar"));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- assertTrue(ex.getMessage().contains(getExpectedHostExMsg("wrongHost")));
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> solrClient.request(getProxyRequest("wrongHost","bar"))
+ );
+ assertTrue(e.getMessage().contains(getExpectedHostExMsg("wrongHost")));
}
@Test
public void testProxyNoConfigHosts() throws Exception {
- try {
- solrClient.request(getProxyRequest("noHosts","bar"));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- // FixMe: this should return an exception about the host being invalid,
- // but a bug (HADOOP-11077) causes an NPE instead.
- //assertTrue(ex.getMessage().contains(getExpectedHostExMsg("noHosts")));
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> solrClient.request(getProxyRequest("noHosts","bar"))
+ );
+ // FixMe: this should return an exception about the host being invalid,
+ // but a bug (HADOOP-11077) causes an NPE instead.
+ // assertTrue(ex.getMessage().contains(getExpectedHostExMsg("noHosts")));
}
@Test
@@ -264,14 +255,11 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
@Test
public void testProxyInvalidProxyUser() throws Exception {
- try {
- // wrong direction, should fail
- solrClient.request(getProxyRequest("bar","anyHostAnyUser"));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bar", "anyHostAnyUser")));
- }
+ // wrong direction, should fail
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> solrClient.request(getProxyRequest("bar","anyHostAnyUser"))
+ );
+ assertTrue(e.getMessage().contains(getExpectedGroupExMsg("bar", "anyHostAnyUser")));
}
@Test
@@ -290,49 +278,38 @@ public class TestSolrCloudWithSecureImpersonation extends SolrTestCaseJ4 {
@Test
public void testProxyUnknownRemote() throws Exception {
- try {
- // Use a reserved ip address
- String nonProxyUserConfiguredIpAddress = "255.255.255.255";
- solrClient.request(getProxyRequest("localHostAnyGroup", "bar", "unknownhost.bar.foo", nonProxyUserConfiguredIpAddress));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- assertTrue(ex.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup")));
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> {
+ // Use a reserved ip address
+ String nonProxyUserConfiguredIpAddress = "255.255.255.255";
+ solrClient.request(getProxyRequest("localHostAnyGroup", "bar", "unknownhost.bar.foo", nonProxyUserConfiguredIpAddress));
+ });
+ assertTrue(e.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup")));
}
@Test
public void testProxyInvalidRemote() throws Exception {
- try {
- String invalidIpAddress = "-127.-128";
- solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", invalidIpAddress));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- assertTrue(ex.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup")));
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> {
+ String invalidIpAddress = "-127.-128";
+ solrClient.request(getProxyRequest("localHostAnyGroup","bar", "[ff01::114]", invalidIpAddress));
+ });
+ assertTrue(e.getMessage().contains(getExpectedHostExMsg("localHostAnyGroup")));
}
@Test
public void testProxyInvalidGroup() throws Exception {
- try {
- solrClient.request(getProxyRequest("bogusGroup","bar", null));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- assertTrue(ex.getMessage().contains(getExpectedGroupExMsg("bogusGroup", "bar")));
- }
+ HttpSolrClient.RemoteSolrException e = expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> solrClient.request(getProxyRequest("bogusGroup","bar", null))
+ );
+ assertTrue(e.getMessage().contains(getExpectedGroupExMsg("bogusGroup", "bar")));
}
@Test
public void testProxyNullProxyUser() throws Exception {
- try {
- solrClient.request(getProxyRequest("","bar"));
- fail("Expected RemoteSolrException");
- }
- catch (HttpSolrClient.RemoteSolrException ex) {
- // this exception is specific to our implementation, don't check a specific message.
- }
+ expectThrows(HttpSolrClient.RemoteSolrException.class,
+ () -> solrClient.request(getProxyRequest("","bar"))
+ );
}
@Test
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
index 8a84724..87bab84 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestTolerantUpdateProcessorCloud.java
@@ -239,38 +239,32 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
assertQueryDocIds(c, false, "id_not_exists");
// verify adding 2 broken docs causes a clint exception
- try {
- UpdateResponse rsp = update(params(),
- doc(f("id", S_ONE_PRE + "X"), f("foo_i", "bogus_val_X")),
- doc(f("id", S_TWO_PRE + "Y"), f("foo_i", "bogus_val_Y"))
- ).process(c);
- fail("did not get a top level exception when more then 10 docs failed: " + rsp.toString());
- } catch (SolrException e) {
- assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(),
- 400, e.code());
- }
+ SolrException e = expectThrows(SolrException.class,
+ "did not get a top level exception when more then 10 docs failed", () ->
+ update(params(),
+ doc(f("id", S_ONE_PRE + "X"), f("foo_i", "bogus_val_X")),
+ doc(f("id", S_TWO_PRE + "Y"), f("foo_i", "bogus_val_Y"))
+ ).process(c)
+ );
+ assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(),
+ 400, e.code());
// verify malformed deleteByQuerys fail
- try {
- UpdateResponse rsp = update(params()).deleteByQuery("foo_i:not_a_num").process(c);
- fail("sanity check for malformed DBQ didn't fail: " + rsp.toString());
- } catch (SolrException e) {
- assertEquals("not the expected DBQ failure: " + e.getMessage(), 400, e.code());
- }
+ e = expectThrows(SolrException.class,
+ "sanity check for malformed DBQ didn't fail",
+ () -> update(params()).deleteByQuery("foo_i:not_a_num").process(c));
+ assertEquals("not the expected DBQ failure: " + e.getMessage(), 400, e.code());
// verify opportunistic concurrency deletions fail as we expect when docs are / aren't present
for (UpdateRequest r : new UpdateRequest[] {
update(params("commit", "true")).deleteById(S_ONE_PRE + "1", -1L),
update(params("commit", "true")).deleteById(S_TWO_PRE + "2", -1L),
update(params("commit", "true")).deleteById("id_not_exists", 1L) }) {
- try {
- UpdateResponse rsp = r.process(c);
- fail("sanity check for opportunistic concurrency delete didn't fail: "
- + r.toString() + " => " + rsp.toString());
- } catch (SolrException e) {
- assertEquals("not the expected opportunistic concurrency failure code: "
- + r.toString() + " => " + e.getMessage(), 409, e.code());
- }
+ e = expectThrows(SolrException.class, "sanity check for opportunistic concurrency delete didn't fail",
+ () -> r.process(c)
+ );
+ assertEquals("not the expected opportunistic concurrency failure code: "
+ + r.toString() + " => " + e.getMessage(), 409, e.code());
}
}
}
@@ -538,51 +532,49 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
assertEquals(0, client.deleteByQuery("*:*").getStatus());
// many docs from diff shards, more then 10 (total) should fail
-
- try {
- rsp = update(params("update.chain", "tolerant-chain-max-errors-10",
- "commit", "true"),
- doc(f("id", S_ONE_PRE + "11")),
- doc(f("id", S_TWO_PRE + "21"), f("foo_i", "bogus_val")),
- doc(f("id", S_ONE_PRE + "12")),
- doc(f("id", S_TWO_PRE + "22"), f("foo_i", "bogus_val")),
- doc(f("id", S_ONE_PRE + "13")),
- doc(f("id", S_TWO_PRE + "23"), f("foo_i", "bogus_val")),
- doc(f("id", S_ONE_PRE + "14"), f("foo_i", "bogus_val")),
- doc(f("id", S_TWO_PRE + "24")),
- doc(f("id", S_ONE_PRE + "15"), f("foo_i", "bogus_val")),
- doc(f("id", S_TWO_PRE + "25")),
- doc(f("id", S_ONE_PRE + "16"), f("foo_i", "bogus_val")),
- doc(f("id", S_TWO_PRE + "26"), f("foo_i", "bogus_val")),
- doc(f("id", S_ONE_PRE + "17")),
- doc(f("id", S_TWO_PRE + "27")),
- doc(f("id", S_ONE_PRE + "18"), f("foo_i", "bogus_val")),
- doc(f("id", S_TWO_PRE + "28"), f("foo_i", "bogus_val")),
- doc(f("id", S_ONE_PRE + "19"), f("foo_i", "bogus_val")),
- doc(f("id", S_TWO_PRE + "29"), f("foo_i", "bogus_val")),
- doc(f("id", S_ONE_PRE + "10")), // may be skipped, more then 10 fails
- doc(f("id", S_TWO_PRE + "20")) // may be skipped, more then 10 fails
- ).process(client);
-
- fail("did not get a top level exception when more then 10 docs failed: " + rsp.toString());
- } catch (SolrException e) {
+ SolrException e = expectThrows(SolrException.class,
+ "did not get a top level exception when more then 10 docs failed",
+ () -> update(params("update.chain", "tolerant-chain-max-errors-10", "commit", "true"),
+ doc(f("id", S_ONE_PRE + "11")),
+ doc(f("id", S_TWO_PRE + "21"), f("foo_i", "bogus_val")),
+ doc(f("id", S_ONE_PRE + "12")),
+ doc(f("id", S_TWO_PRE + "22"), f("foo_i", "bogus_val")),
+ doc(f("id", S_ONE_PRE + "13")),
+ doc(f("id", S_TWO_PRE + "23"), f("foo_i", "bogus_val")),
+ doc(f("id", S_ONE_PRE + "14"), f("foo_i", "bogus_val")),
+ doc(f("id", S_TWO_PRE + "24")),
+ doc(f("id", S_ONE_PRE + "15"), f("foo_i", "bogus_val")),
+ doc(f("id", S_TWO_PRE + "25")),
+ doc(f("id", S_ONE_PRE + "16"), f("foo_i", "bogus_val")),
+ doc(f("id", S_TWO_PRE + "26"), f("foo_i", "bogus_val")),
+ doc(f("id", S_ONE_PRE + "17")),
+ doc(f("id", S_TWO_PRE + "27")),
+ doc(f("id", S_ONE_PRE + "18"), f("foo_i", "bogus_val")),
+ doc(f("id", S_TWO_PRE + "28"), f("foo_i", "bogus_val")),
+ doc(f("id", S_ONE_PRE + "19"), f("foo_i", "bogus_val")),
+ doc(f("id", S_TWO_PRE + "29"), f("foo_i", "bogus_val")),
+ doc(f("id", S_ONE_PRE + "10")), // may be skipped, more then 10 fails
+ doc(f("id", S_TWO_PRE + "20")) // may be skipped, more then 10 fails
+ ).process(client)
+ );
+ {
// we can't make any reliable assertions about the error message, because
// it varies based on how the request was routed -- see SOLR-8830
- assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(),
- // NOTE: we always expect a 400 because we know that's what we would get from these types of errors
- // on a single node setup -- a 5xx type error isn't something we should have triggered
- 400, e.code());
+ assertEquals("not the type of error we were expecting (" + e.code() + "): " + e.toString(),
+ // NOTE: we always expect a 400 because we know that's what we would get from these types of errors
+ // on a single node setup -- a 5xx type error isn't something we should have triggered
+ 400, e.code());
// verify that the Exceptions metadata can tell us what failed.
NamedList<String> remoteErrMetadata = e.getMetadata();
assertNotNull("no metadata in: " + e.toString(), remoteErrMetadata);
Set<ToleratedUpdateError> actualKnownErrs
- = new LinkedHashSet<ToleratedUpdateError>(remoteErrMetadata.size());
+ = new LinkedHashSet<ToleratedUpdateError>(remoteErrMetadata.size());
int actualKnownErrsCount = 0;
for (int i = 0; i < remoteErrMetadata.size(); i++) {
ToleratedUpdateError err =
- ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
- remoteErrMetadata.getVal(i));
+ ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
+ remoteErrMetadata.getVal(i));
if (null == err) {
// some metadata unrelated to this update processor
continue;
@@ -591,16 +583,17 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
actualKnownErrs.add(err);
}
assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(),
- 11, actualKnownErrsCount);
+ 11, actualKnownErrsCount);
assertEquals("at least one dup error in metadata: " + remoteErrMetadata.toString(),
- actualKnownErrsCount, actualKnownErrs.size());
+ actualKnownErrsCount, actualKnownErrs.size());
for (ToleratedUpdateError err : actualKnownErrs) {
assertEquals("only expected type of error is ADD: " + err,
- CmdType.ADD, err.getType());
+ CmdType.ADD, err.getType());
assertTrue("failed err msg didn't match expected value: " + err,
- err.getMessage().contains("bogus_val"));
+ err.getMessage().contains("bogus_val"));
}
}
+
assertEquals(0, client.commit().getStatus()); // need to force since update didn't finish
assertQueryDocIds(client, false
// explicitly failed
@@ -621,7 +614,8 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
// many docs from diff shards, more then 10 from a single shard (two) should fail
- try {
+ e = expectThrows(SolrException.class, "did not get a top level exception when more then 10 docs failed",
+ () -> {
ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(30);
docs.add(doc(f("id", S_ONE_PRE + "z")));
docs.add(doc(f("id", S_TWO_PRE + "z")));
@@ -633,30 +627,30 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
}
docs.add(doc(f("id", S_ONE_PRE + "x"))); // may be skipped, more then 10 fails
docs.add(doc(f("id", S_TWO_PRE + "x"))); // may be skipped, more then 10 fails
-
- rsp = update(params("update.chain", "tolerant-chain-max-errors-10",
- "commit", "true"),
- docs.toArray(new SolrInputDocument[docs.size()])).process(client);
-
- fail("did not get a top level exception when more then 10 docs failed: " + rsp.toString());
- } catch (SolrException e) {
+
+ update(params("update.chain", "tolerant-chain-max-errors-10",
+ "commit", "true"),
+ docs.toArray(new SolrInputDocument[docs.size()])).process(client);
+ });
+
+ {
// we can't make any reliable assertions about the error message, because
// it varies based on how the request was routed -- see SOLR-8830
assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(),
- // NOTE: we always expect a 400 because we know that's what we would get from these types of errors
- // on a single node setup -- a 5xx type error isn't something we should have triggered
- 400, e.code());
+ // NOTE: we always expect a 400 because we know that's what we would get from these types of errors
+ // on a single node setup -- a 5xx type error isn't something we should have triggered
+ 400, e.code());
// verify that the Exceptions metadata can tell us what failed.
NamedList<String> remoteErrMetadata = e.getMetadata();
assertNotNull("no metadata in: " + e.toString(), remoteErrMetadata);
Set<ToleratedUpdateError> actualKnownErrs
- = new LinkedHashSet<ToleratedUpdateError>(remoteErrMetadata.size());
+ = new LinkedHashSet<ToleratedUpdateError>(remoteErrMetadata.size());
int actualKnownErrsCount = 0;
for (int i = 0; i < remoteErrMetadata.size(); i++) {
ToleratedUpdateError err =
- ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
- remoteErrMetadata.getVal(i));
+ ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
+ remoteErrMetadata.getVal(i));
if (null == err) {
// some metadata unrelated to this update processor
continue;
@@ -665,19 +659,19 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
actualKnownErrs.add(err);
}
assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(),
- 11, actualKnownErrsCount);
+ 11, actualKnownErrsCount);
assertEquals("at least one dup error in metadata: " + remoteErrMetadata.toString(),
- actualKnownErrsCount, actualKnownErrs.size());
+ actualKnownErrsCount, actualKnownErrs.size());
for (ToleratedUpdateError err : actualKnownErrs) {
assertEquals("only expected type of error is ADD: " + err,
- CmdType.ADD, err.getType());
+ CmdType.ADD, err.getType());
assertTrue("failed id had unexpected prefix: " + err,
- err.getId().startsWith(S_TWO_PRE));
+ err.getId().startsWith(S_TWO_PRE));
assertTrue("failed err msg didn't match expected value: " + err,
- err.getMessage().contains("bogus_val"));
+ err.getMessage().contains("bogus_val"));
}
-
}
+
assertEquals(0, client.commit().getStatus()); // need to force since update didn't finish
assertQueryDocIds(client, true
, S_ONE_PRE + "z", S_ONE_PRE + "y", S_TWO_PRE + "z", S_TWO_PRE + "y" // first
@@ -700,7 +694,9 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
// many docs from diff shards, more then 10 don't have any uniqueKey specified
- try {
+ e = expectThrows(SolrException.class,
+ "did not get a top level exception when more then 10 docs mising uniqueKey",
+ () -> {
ArrayList<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(30);
docs.add(doc(f("id", S_ONE_PRE + "z")));
docs.add(doc(f("id", S_TWO_PRE + "z")));
@@ -712,19 +708,19 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
}
docs.add(doc(f("id", S_ONE_PRE + "x"))); // may be skipped, more then 10 fails
docs.add(doc(f("id", S_TWO_PRE + "x"))); // may be skipped, more then 10 fails
-
- rsp = update(params("update.chain", "tolerant-chain-max-errors-10",
- "commit", "true"),
- docs.toArray(new SolrInputDocument[docs.size()])).process(client);
-
- fail("did not get a top level exception when more then 10 docs mising uniqueKey: " + rsp.toString());
- } catch (SolrException e) {
+
+ update(params("update.chain", "tolerant-chain-max-errors-10",
+ "commit", "true"),
+ docs.toArray(new SolrInputDocument[docs.size()])).process(client);
+ });
+
+ {
// we can't make any reliable assertions about the error message, because
// it varies based on how the request was routed -- see SOLR-8830
assertEquals("not the type of error we were expecting ("+e.code()+"): " + e.toString(),
- // NOTE: we always expect a 400 because we know that's what we would get from these types of errors
- // on a single node setup -- a 5xx type error isn't something we should have triggered
- 400, e.code());
+ // NOTE: we always expect a 400 because we know that's what we would get from these types of errors
+ // on a single node setup -- a 5xx type error isn't something we should have triggered
+ 400, e.code());
// verify that the Exceptions metadata can tell us what failed.
NamedList<String> remoteErrMetadata = e.getMetadata();
@@ -732,21 +728,22 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
int actualKnownErrsCount = 0;
for (int i = 0; i < remoteErrMetadata.size(); i++) {
ToleratedUpdateError err =
- ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
- remoteErrMetadata.getVal(i));
+ ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
+ remoteErrMetadata.getVal(i));
if (null == err) {
// some metadata unrelated to this update processor
continue;
}
actualKnownErrsCount++;
assertEquals("only expected type of error is ADD: " + err,
- CmdType.ADD, err.getType());
+ CmdType.ADD, err.getType());
assertTrue("failed id didn't match 'unknown': " + err,
- err.getId().contains("unknown"));
+ err.getId().contains("unknown"));
}
assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(),
- 11, actualKnownErrsCount);
+ 11, actualKnownErrsCount);
}
+
assertEquals(0, client.commit().getStatus()); // need to force since update didn't finish
assertQueryDocIds(client, true
, S_ONE_PRE + "z", S_ONE_PRE + "y", S_TWO_PRE + "z", S_TWO_PRE + "y" // first
@@ -857,36 +854,39 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
}
// attempt a request containing 4 errors of various types (add, delI, delQ) .. 1 too many
- try {
- rsp = update(params("update.chain", "tolerant-chain-max-errors-10",
- "maxErrors", "3",
- "commit", "true"),
- doc(f("id", docId22), f("foo_i", "bogus_val")))
- .deleteById(docId1, -1L)
- .deleteByQuery("malformed:[")
- .deleteById(docId21, -1L)
- .process(client);
- fail("did not get a top level exception when more then 4 updates failed: " + rsp.toString());
- } catch (SolrException e) {
+
+ SolrException e = expectThrows(SolrException.class,
+ "did not get a top level exception when more then 4 updates failed",
+ () -> update(params("update.chain", "tolerant-chain-max-errors-10",
+ "maxErrors", "3",
+ "commit", "true"),
+ doc(f("id", docId22), f("foo_i", "bogus_val")))
+ .deleteById(docId1, -1L)
+ .deleteByQuery("malformed:[")
+ .deleteById(docId21, -1L)
+ .process(client)
+ );
+
+ {
// we can't make any reliable assertions about the error message, because
// it varies based on how the request was routed -- see SOLR-8830
-
+
// likewise, we can't make a firm(er) assertion about the response code...
assertTrue("not the type of error we were expecting ("+e.code()+"): " + e.toString(),
- // should be one these 2 depending on order that the async errors were hit...
- // on a single node setup -- a 5xx type error isn't something we should have triggered
- 400 == e.code() || 409 == e.code());
+ // should be one these 2 depending on order that the async errors were hit...
+ // on a single node setup -- a 5xx type error isn't something we should have triggered
+ 400 == e.code() || 409 == e.code());
// verify that the Exceptions metadata can tell us what failed.
NamedList<String> remoteErrMetadata = e.getMetadata();
assertNotNull("no metadata in: " + e.toString(), remoteErrMetadata);
Set<ToleratedUpdateError> actualKnownErrs
- = new LinkedHashSet<ToleratedUpdateError>(remoteErrMetadata.size());
+ = new LinkedHashSet<ToleratedUpdateError>(remoteErrMetadata.size());
int actualKnownErrsCount = 0;
for (int i = 0; i < remoteErrMetadata.size(); i++) {
ToleratedUpdateError err =
- ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
- remoteErrMetadata.getVal(i));
+ ToleratedUpdateError.parseMetadataIfToleratedUpdateError(remoteErrMetadata.getName(i),
+ remoteErrMetadata.getVal(i));
if (null == err) {
// some metadata unrelated to this update processor
continue;
@@ -895,9 +895,9 @@ public class TestTolerantUpdateProcessorCloud extends SolrCloudTestCase {
actualKnownErrs.add(err);
}
assertEquals("wrong number of errors in metadata: " + remoteErrMetadata.toString(),
- 4, actualKnownErrsCount);
+ 4, actualKnownErrsCount);
assertEquals("at least one dup error in metadata: " + remoteErrMetadata.toString(),
- actualKnownErrsCount, actualKnownErrs.size());
+ actualKnownErrsCount, actualKnownErrs.size());
}
// sanity check our 2 existing docs are still here
[23/31] lucene-solr:jira/http2: SOLR-11863: Add knnRegress Stream
Evaluator to support nearest neighbor regression
Posted by da...@apache.org.
SOLR-11863: Add knnRegress Stream Evaluator to support nearest neighbor regression
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cb1db482
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cb1db482
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cb1db482
Branch: refs/heads/jira/http2
Commit: cb1db482523cf33b7927b5155d506202d8ddbd89
Parents: e9f3a3c
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Aug 8 21:05:02 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Aug 8 21:05:21 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/Lang.java | 1 +
.../solr/client/solrj/io/eval/KnnEvaluator.java | 18 +-
.../solrj/io/eval/KnnRegressionEvaluator.java | 194 +++++++++++++++++++
.../solrj/io/eval/MinMaxScaleEvaluator.java | 2 +-
.../client/solrj/io/eval/PredictEvaluator.java | 30 ++-
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 71 +++++++
7 files changed, 311 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index a01a841..6f170c4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -246,6 +246,7 @@ public class Lang {
.withFunctionName("zeros", ZerosEvaluator.class)
.withFunctionName("getValue", GetValueEvaluator.class)
.withFunctionName("setValue", SetValueEvaluator.class)
+ .withFunctionName("knnRegress", KnnRegressionEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
index 251e092..81607cf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
@@ -67,8 +67,6 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
throw new IOException("The third parameter for knn should be k.");
}
- double[][] data = matrix.getData();
-
DistanceMeasure distanceMeasure = null;
if(values.length == 4) {
@@ -77,6 +75,15 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
distanceMeasure = new EuclideanDistance();
}
+ return search(matrix, vec, k, distanceMeasure);
+ }
+
+ public static Matrix search(Matrix observations,
+ double[] vec,
+ int k,
+ DistanceMeasure distanceMeasure) {
+
+ double[][] data = observations.getData();
TreeSet<Neighbor> neighbors = new TreeSet();
for(int i=0; i<data.length; i++) {
double distance = distanceMeasure.compute(vec, data[i]);
@@ -87,8 +94,9 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
}
double[][] out = new double[neighbors.size()][];
- List<String> rowLabels = matrix.getRowLabels();
+ List<String> rowLabels = observations.getRowLabels();
List<String> newRowLabels = new ArrayList();
+ List<Number> indexes = new ArrayList();
List<Number> distances = new ArrayList();
int i=-1;
@@ -102,6 +110,7 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
out[++i] = data[rowIndex];
distances.add(neighbor.getDistance());
+ indexes.add(rowIndex);
}
Matrix knn = new Matrix(out);
@@ -110,8 +119,9 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
knn.setRowLabels(newRowLabels);
}
- knn.setColumnLabels(matrix.getColumnLabels());
+ knn.setColumnLabels(observations.getColumnLabels());
knn.setAttribute("distances", distances);
+ knn.setAttribute("indexes", indexes);
return knn;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
new file mode 100644
index 0000000..957936e
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.ml.distance.DistanceMeasure;
+import org.apache.commons.math3.ml.distance.EuclideanDistance;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class KnnRegressionEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public KnnRegressionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object ... values) throws IOException {
+
+ if(values.length < 3) {
+ throw new IOException("knnRegress expects atleast three parameters: an observation matrix, an outcomes vector and k.");
+ }
+
+ Matrix observations = null;
+ List<Number> outcomes = null;
+ int k = 5;
+ DistanceMeasure distanceMeasure = new EuclideanDistance();
+
+ if(values[0] instanceof Matrix) {
+ observations = (Matrix)values[0];
+ } else {
+ throw new IOException("The first parameter for knnRegress should be the observation matrix.");
+ }
+
+ if(values[1] instanceof List) {
+ outcomes = (List) values[1];
+ } else {
+ throw new IOException("The second parameter for knnRegress should be outcome array. ");
+ }
+
+ if(values[2] instanceof Number) {
+ k = ((Number) values[2]).intValue();
+ } else {
+ throw new IOException("The third parameter for knnRegress should be k. ");
+ }
+
+ if(values.length == 4) {
+ if(values[3] instanceof DistanceMeasure) {
+ distanceMeasure = (DistanceMeasure) values[3];
+ throw new IOException("The fourth parameter for knnRegress should be a distance measure. ");
+ }
+ }
+
+ double[] outcomeData = new double[outcomes.size()];
+ for(int i=0; i<outcomeData.length; i++) {
+ outcomeData[i] = outcomes.get(i).doubleValue();
+ }
+
+ Map map = new HashMap();
+ map.put("k", k);
+ map.put("observations", observations.getRowCount());
+ map.put("features", observations.getColumnCount());
+ map.put("distance", distanceMeasure.getClass().getSimpleName());
+
+ return new KnnRegressionTuple(observations, outcomeData, k, distanceMeasure, map);
+ }
+
+
+ public static class KnnRegressionTuple extends Tuple {
+
+ private Matrix observations;
+ private Matrix scaledObservations;
+ private double[] outcomes;
+ private int k;
+ private DistanceMeasure distanceMeasure;
+
+ public KnnRegressionTuple(Matrix observations,
+ double[] outcomes,
+ int k,
+ DistanceMeasure distanceMeasure,
+ Map<?,?> map) {
+ super(map);
+ this.observations = observations;
+ this.outcomes = outcomes;
+ this.k = k;
+ this.distanceMeasure = distanceMeasure;
+ }
+
+ //MinMax Scale both the observations and the predictors
+
+ public double[] scale(double[] predictors) {
+ double[][] data = observations.getData();
+ //We need to scale the columns of the data matrix with along with the predictors
+ Array2DRowRealMatrix matrix = new Array2DRowRealMatrix(data);
+ Array2DRowRealMatrix transposed = (Array2DRowRealMatrix) matrix.transpose();
+ double[][] featureRows = transposed.getDataRef();
+
+ double[] scaledPredictors = new double[predictors.length];
+
+ for(int i=0; i<featureRows.length; i++) {
+ double[] featureRow = featureRows[i];
+ double[] combinedFeatureRow = new double[featureRow.length+1];
+ System.arraycopy(featureRow, 0, combinedFeatureRow, 0, featureRow.length);
+ combinedFeatureRow[featureRow.length] = predictors[i]; // Add the last feature from the predictor
+ double[] scaledFeatures = MinMaxScaleEvaluator.scale(combinedFeatureRow, 0, 1);
+ scaledPredictors[i] = scaledFeatures[featureRow.length];
+ System.arraycopy(scaledFeatures, 0, featureRow, 0, featureRow.length);
+ }
+
+ Array2DRowRealMatrix scaledFeatureMatrix = new Array2DRowRealMatrix(featureRows);
+
+
+ Array2DRowRealMatrix scaledObservationsMatrix= (Array2DRowRealMatrix)scaledFeatureMatrix.transpose();
+ this.scaledObservations = new Matrix(scaledObservationsMatrix.getDataRef());
+ return scaledPredictors;
+ }
+
+
+ public Matrix scale(Matrix predictors) {
+ double[][] observationData = observations.getData();
+ //We need to scale the columns of the data matrix with along with the predictors
+ Array2DRowRealMatrix observationMatrix = new Array2DRowRealMatrix(observationData);
+ Array2DRowRealMatrix observationTransposed = (Array2DRowRealMatrix) observationMatrix.transpose();
+ double[][] observationFeatureRows = observationTransposed.getDataRef();
+
+ double[][] predictorsData = predictors.getData();
+ //We need to scale the columns of the data matrix with along with the predictors
+ Array2DRowRealMatrix predictorMatrix = new Array2DRowRealMatrix(predictorsData);
+ Array2DRowRealMatrix predictorTransposed = (Array2DRowRealMatrix) predictorMatrix.transpose();
+ double[][] predictorFeatureRows = predictorTransposed.getDataRef();
+
+ for(int i=0; i<observationFeatureRows.length; i++) {
+ double[] observationFeatureRow = observationFeatureRows[i];
+ double[] predictorFeatureRow = predictorFeatureRows[i];
+ double[] combinedFeatureRow = new double[observationFeatureRow.length+predictorFeatureRow.length];
+ System.arraycopy(observationFeatureRow, 0, combinedFeatureRow, 0, observationFeatureRow.length);
+ System.arraycopy(predictorFeatureRow, 0, combinedFeatureRow, observationFeatureRow.length, predictorFeatureRow.length);
+
+ double[] scaledFeatures = MinMaxScaleEvaluator.scale(combinedFeatureRow, 0, 1);
+ System.arraycopy(scaledFeatures, 0, observationFeatureRow, 0, observationFeatureRow.length);
+ System.arraycopy(scaledFeatures, observationFeatureRow.length, predictorFeatureRow, 0, predictorFeatureRow.length);
+ }
+
+ Array2DRowRealMatrix scaledFeatureMatrix = new Array2DRowRealMatrix(observationFeatureRows);
+ Array2DRowRealMatrix scaledObservationsMatrix= (Array2DRowRealMatrix)scaledFeatureMatrix.transpose();
+ this.scaledObservations = new Matrix(scaledObservationsMatrix.getDataRef());
+
+ Array2DRowRealMatrix scaledPredictorMatrix = new Array2DRowRealMatrix(predictorFeatureRows);
+ Array2DRowRealMatrix scaledTransposedPredictorMatrix= (Array2DRowRealMatrix)scaledPredictorMatrix.transpose();
+ return new Matrix(scaledTransposedPredictorMatrix.getDataRef());
+ }
+
+
+ public double predict(double[] values) {
+
+ Matrix knn = KnnEvaluator.search(scaledObservations, values, k, distanceMeasure);
+ List<Number> indexes = (List<Number>)knn.getAttribute("indexes");
+
+ double sum = 0;
+
+ //Collect the outcomes for the nearest neighbors
+ for(Number n : indexes) {
+ sum += outcomes[n.intValue()];
+ }
+
+ //Return the average of the outcomes as the prediction.
+
+ return sum/((double)indexes.size());
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
index 60c6377..3996910 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
@@ -76,7 +76,7 @@ public class MinMaxScaleEvaluator extends RecursiveObjectEvaluator implements Ma
}
}
- private double[] scale(double[] values, double min, double max) {
+ public static double[] scale(double[] values, double min, double max) {
double localMin = Double.MAX_VALUE;
double localMax = Double.MIN_VALUE;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
index 2444370..9385928 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
@@ -43,7 +43,11 @@ public class PredictEvaluator extends RecursiveObjectEvaluator implements ManyVa
Object first = objects[0];
Object second = objects[1];
- if (!(first instanceof BivariateFunction) && !(first instanceof VectorFunction) && !(first instanceof RegressionEvaluator.RegressionTuple) && !(first instanceof OLSRegressionEvaluator.MultipleRegressionTuple)) {
+ if (!(first instanceof BivariateFunction) &&
+ !(first instanceof VectorFunction) &&
+ !(first instanceof RegressionEvaluator.RegressionTuple) &&
+ !(first instanceof OLSRegressionEvaluator.MultipleRegressionTuple) &&
+ !(first instanceof KnnRegressionEvaluator.KnnRegressionTuple)) {
throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found type %s for the first value, expecting a RegressionTuple", toExpression(constructingFactory), first.getClass().getSimpleName()));
}
@@ -83,6 +87,30 @@ public class PredictEvaluator extends RecursiveObjectEvaluator implements ManyVa
return predictions;
}
+ } else if (first instanceof KnnRegressionEvaluator.KnnRegressionTuple) {
+ KnnRegressionEvaluator.KnnRegressionTuple regressedTuple = (KnnRegressionEvaluator.KnnRegressionTuple) first;
+ if (second instanceof List) {
+ List<Number> list = (List<Number>) second;
+ double[] predictors = new double[list.size()];
+
+ for (int i = 0; i < list.size(); i++) {
+ predictors[i] = list.get(i).doubleValue();
+ }
+
+ predictors = regressedTuple.scale(predictors);
+
+ return regressedTuple.predict(predictors);
+ } else if (second instanceof Matrix) {
+
+ Matrix m = (Matrix) second;
+ m = regressedTuple.scale(m);
+ double[][] data = m.getData();
+ List<Number> predictions = new ArrayList();
+ for (double[] predictors : data) {
+ predictions.add(regressedTuple.predict(predictors));
+ }
+ return predictions;
+ }
} else if (first instanceof VectorFunction) {
VectorFunction vectorFunction = (VectorFunction) first;
UnivariateFunction univariateFunction = (UnivariateFunction)vectorFunction.getFunction();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 22b432f..df56844 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -69,7 +69,7 @@ public class TestLang extends LuceneTestCase {
TemporalEvaluatorDayOfQuarter.FUNCTION_NAME, "abs", "add", "div", "mult", "sub", "log", "pow",
"mod", "ceil", "floor", "sin", "asin", "sinh", "cos", "acos", "cosh", "tan", "atan", "tanh", "round", "sqrt",
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft", "euclidean","manhattan",
- "earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue"};
+ "earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue", "knnRegress"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 98a52a6..a9be57e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -3395,6 +3395,77 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
@Test
+ public void testKnnRegress() throws Exception {
+ String cexpr = "let(echo=true, a=array(8.5, 12.89999962, 5.199999809, 10.69999981, 3.099999905, 3.5, 9.199999809, 9, 15.10000038, 10.19999981), " +
+ "b=array(5.099999905, 5.800000191, 2.099999905, 8.399998665, 2.900000095, 1.200000048, 3.700000048, 7.599999905, 7.699999809, 4.5)," +
+ "c=array(4.699999809, 8.800000191, 15.10000038, 12.19999981, 10.60000038, 3.5, 9.699999809, 5.900000095, 20.79999924, 7.900000095)," +
+ "d=array(85.09999847, 106.3000031, 50.20000076, 130.6000061, 54.79999924, 30.29999924, 79.40000153, 91, 135.3999939, 89.30000305)," +
+ "e=transpose(matrix(a, b, c))," +
+ "f=knnRegress(e, d, 1)," +
+ "g=predict(f, e))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> predictions = (List<Number>)tuples.get(0).get("g");
+ assertEquals(predictions.size(), 10);
+ //k=1 should bring back only one prediction for the exact match in the training set
+ assertEquals(predictions.get(0).doubleValue(), 85.09999847, 0);
+ assertEquals(predictions.get(1).doubleValue(), 106.3000031, 0);
+ assertEquals(predictions.get(2).doubleValue(), 50.20000076, 0);
+ assertEquals(predictions.get(3).doubleValue(), 130.6000061, 0);
+ assertEquals(predictions.get(4).doubleValue(), 54.79999924, 0);
+ assertEquals(predictions.get(5).doubleValue(), 30.29999924, 0);
+ assertEquals(predictions.get(6).doubleValue(), 79.40000153, 0);
+ assertEquals(predictions.get(7).doubleValue(), 91, 0);
+ assertEquals(predictions.get(8).doubleValue(), 135.3999939, 0);
+ assertEquals(predictions.get(9).doubleValue(), 89.30000305, 0);
+
+ cexpr = "let(echo=true, a=array(8.5, 12.89999962, 5.199999809, 10.69999981, 3.099999905, 3.5, 9.199999809, 9, 15.10000038, 10.19999981), " +
+ "b=array(5.099999905, 5.800000191, 2.099999905, 8.399998665, 2.900000095, 1.200000048, 3.700000048, 7.599999905, 7.699999809, 4.5)," +
+ "c=array(4.699999809, 8.800000191, 15.10000038, 12.19999981, 10.60000038, 3.5, 9.699999809, 5.900000095, 20.79999924, 7.900000095)," +
+ "d=array(85.09999847, 106.3000031, 50.20000076, 130.6000061, 54.79999924, 30.29999924, 79.40000153, 91, 135.3999939, 89.30000305)," +
+ "e=transpose(matrix(a, b, c))," +
+ "f=knnRegress(e, d, 1)," +
+ "g=predict(f, array(8, 5, 4)))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Number prediction = (Number)tuples.get(0).get("g");
+ assertEquals(prediction.doubleValue(), 85.09999847, 0);
+
+ cexpr = "let(echo=true, a=array(8.5, 12.89999962, 5.199999809, 10.69999981, 3.099999905, 3.5, 9.199999809, 9, 15.10000038, 8.19999981), " +
+ "b=array(5.099999905, 5.800000191, 2.099999905, 8.399998665, 2.900000095, 1.200000048, 3.700000048, 7.599999905, 5.699999809, 4.5)," +
+ "c=array(4.699999809, 8.800000191, 15.10000038, 12.19999981, 10.60000038, 3.5, 9.699999809, 5.900000095, 20.79999924, 4.900000095)," +
+ "d=array(85.09999847, 106.3000031, 50.20000076, 130.6000061, 54.79999924, 30.29999924, 79.40000153, 91, 135.3999939, 89.30000305)," +
+ "e=transpose(matrix(a, b, c))," +
+ "f=knnRegress(e, d, 2)," +
+ "g=predict(f, array(8, 5, 4)))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ prediction = (Number)tuples.get(0).get("g");
+ assertEquals(prediction.doubleValue(), 87.20000076, 0);
+ }
+
+ @Test
public void testPlot() throws Exception {
String cexpr = "let(a=array(3,2,3), plot(type=scatter, x=a, y=array(5,6,3)))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
[08/31] lucene-solr:jira/http2: SOLR-12541: Metrics handler throws an
error if there are transient cores.
Posted by da...@apache.org.
SOLR-12541: Metrics handler throws an error if there are transient cores.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/208d59b5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/208d59b5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/208d59b5
Branch: refs/heads/jira/http2
Commit: 208d59b5e3dbc5207d9b150d56d70e53f81c0e3f
Parents: 08ca54f
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Aug 7 13:20:44 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Aug 7 13:21:41 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../org/apache/solr/handler/ReplicationHandler.java | 4 ++--
.../src/test/org/apache/solr/core/TestLazyCores.java | 15 +++++++++++++++
3 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/208d59b5/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ac8ff3c..d5edef3 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -213,6 +213,8 @@ Bug Fixes
* SOLR-11770: NPE in tvrh if no field is specified and document doesn't contain any fields with term vectors
+* SOLR-12541: Metrics handler throws an error if there are transient cores. (ab)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/208d59b5/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
index 93b8f25..5041841 100644
--- a/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
@@ -837,13 +837,13 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
public void initializeMetrics(SolrMetricManager manager, String registry, String tag, String scope) {
super.initializeMetrics(manager, registry, tag, scope);
- manager.registerGauge(this, registry, () -> core != null ? NumberUtils.readableSize(core.getIndexSize()) : "",
+ manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? NumberUtils.readableSize(core.getIndexSize()) : ""),
tag, true, "indexSize", getCategory().toString(), scope);
manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? getIndexVersion().toString() : ""),
tag, true, "indexVersion", getCategory().toString(), scope);
manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? getIndexVersion().generation : 0),
tag, true, GENERATION, getCategory().toString(), scope);
- manager.registerGauge(this, registry, () -> core != null ? core.getIndexDir() : "",
+ manager.registerGauge(this, registry, () -> (core != null && !core.isClosed() ? core.getIndexDir() : ""),
tag, true, "indexPath", getCategory().toString(), scope);
manager.registerGauge(this, registry, () -> isMaster,
tag, true, "isMaster", getCategory().toString(), scope);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/208d59b5/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
index 9d03ca4..f8ffb84 100644
--- a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
+++ b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
@@ -33,9 +33,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.handler.admin.CoreAdminHandler;
+import org.apache.solr.handler.admin.MetricsHandler;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -240,6 +242,19 @@ public class TestLazyCores extends SolrTestCaseJ4 {
"collection8", "collection9");
checkNotInCores(cc, Arrays.asList("collection2", "collection3"));
+ // verify that getting metrics from an unloaded core doesn't cause exceptions (SOLR-12541)
+ MetricsHandler handler = new MetricsHandler(h.getCoreContainer());
+
+ SolrQueryResponse resp = new SolrQueryResponse();
+ handler.handleRequest(makeReq(core1, CommonParams.QT, "/admin/metrics"), resp);
+ NamedList values = resp.getValues();
+ assertNotNull(values.get("metrics"));
+ values = (NamedList) values.get("metrics");
+ NamedList nl = (NamedList) values.get("solr.core.collection2");
+ assertNotNull(nl);
+ Object o = nl.get("REPLICATION./replication.indexPath");
+ assertNotNull(o);
+
// Note decrementing the count when the core is removed from the lazyCores list is appropriate, since the
// refcount is 1 when constructed. anyone _else_ who's opened up one has to close it.
[04/31] lucene-solr:jira/http2: SOLR-12626:
TestInjection.waitForInSyncWithLeader retries in case of errors
Posted by da...@apache.org.
SOLR-12626: TestInjection.waitForInSyncWithLeader retries in case of errors
When waiting for follower replicas to be in sync with the leader, TestInjection.waitForInSyncWithLeader
now keeps waiting even if it gets an exception while trying to get leader's replication details. The
method will only exit when it exhausts the number of retries or succeeds.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/db1e0642
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/db1e0642
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/db1e0642
Branch: refs/heads/jira/http2
Commit: db1e0642cb4ed454709aba06678d5a23fc0af6ee
Parents: c338cf6
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Mon Aug 6 13:33:53 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Mon Aug 6 15:59:25 2018 -0700
----------------------------------------------------------------------
.../org/apache/solr/util/TestInjection.java | 22 ++++++++++++--------
1 file changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/db1e0642/solr/core/src/java/org/apache/solr/util/TestInjection.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java
index 34915fb..4642eac 100644
--- a/solr/core/src/java/org/apache/solr/util/TestInjection.java
+++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java
@@ -414,7 +414,7 @@ public class TestInjection {
}
@SuppressForbidden(reason = "Need currentTimeMillis, because COMMIT_TIME_MSEC_KEY use currentTimeMillis as value")
- public static boolean waitForInSyncWithLeader(SolrCore core, ZkController zkController, String collection, String shardId) throws InterruptedException {
+ public static boolean waitForInSyncWithLeader(SolrCore core, ZkController zkController, String collection, String shardId) {
if (waitForReplicasInSync == null) return true;
log.info("Start waiting for replica in sync with leader");
long currentTime = System.currentTimeMillis();
@@ -422,8 +422,8 @@ public class TestInjection {
boolean enabled = pair.first();
if (!enabled) return true;
long t = System.currentTimeMillis() - 200;
- try {
- for (int i = 0; i < pair.second(); i++) {
+ for (int i = 0; i < pair.second(); i++) {
+ try {
if (core.isClosed()) return true;
Replica leaderReplica = zkController.getZkStateReader().getLeaderRetry(
collection, shardId);
@@ -431,7 +431,7 @@ public class TestInjection {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CommonParams.QT, ReplicationHandler.PATH);
params.set(COMMAND, CMD_DETAILS);
-
+
NamedList<Object> response = leaderClient.request(new QueryRequest(params));
long leaderVersion = (long) ((NamedList)response.get("details")).get("indexVersion");
String localVersion = core.withSearcher(searcher ->
@@ -445,14 +445,18 @@ public class TestInjection {
log.debug("Tlog replica not in sync with leader yet. Attempt: {}. Local Version={}, leader Version={}", i, localVersion, leaderVersion);
Thread.sleep(500);
}
-
+
}
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ if (core.isClosed()) return true;
+ log.error("Thread interrupted while waiting for core {} to be in sync with leader", core.getName());
+ return false;
+ } catch (Exception e) {
+ if (core.isClosed()) return true;
+ log.error("Exception when wait for replicas in sync with master. Will retry until timeout.", e);
}
-
- } catch (Exception e) {
- log.error("Exception when wait for replicas in sync with master");
}
-
return false;
}
[31/31] lucene-solr:jira/http2: Resolved merge conflict caused by
SOLR-11881
Posted by da...@apache.org.
Resolved merge conflict caused by SOLR-11881
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9f554f6d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9f554f6d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9f554f6d
Branch: refs/heads/jira/http2
Commit: 9f554f6de0420e39128ce366e6e8e028c2ff66e7
Parents: 51b2a1b 9c02bf0
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Aug 10 16:13:18 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Aug 10 16:13:18 2018 +0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 14 +
.../org/apache/lucene/search/BlockMaxDISI.java | 94 ++++
.../lucene/search/Boolean2ScorerSupplier.java | 4 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../lucene/search/DisjunctionMaxQuery.java | 2 +-
.../lucene/search/DisjunctionMaxScorer.java | 80 ++--
.../apache/lucene/search/DisjunctionScorer.java | 22 +-
.../lucene/search/DisjunctionSumScorer.java | 4 +-
.../apache/lucene/search/ReqOptSumScorer.java | 175 ++++---
.../apache/lucene/index/TestIndexWriter.java | 2 +
.../lucene/search/TestDisjunctionMaxQuery.java | 77 ++-
.../lucene/search/TestReqOptSumScorer.java | 241 +++++++++-
lucene/ivy-versions.properties | 1 -
.../org/apache/lucene/document/LatLonShape.java | 13 +-
.../document/LatLonShapeBoundingBoxQuery.java | 420 +++++------------
.../document/LatLonShapePolygonQuery.java | 301 ++----------
.../lucene/document/LatLonShapeQuery.java | 394 ++++++++++++++++
.../document/BaseLatLonShapeTestCase.java | 46 +-
.../document/TestLatLonLineShapeQueries.java | 40 +-
.../document/TestLatLonPointShapeQueries.java | 20 +-
.../document/TestLatLonPolygonShapeQueries.java | 36 +-
.../apache/lucene/document/TestLatLonShape.java | 3 +-
.../spatial3d/geom/GeoComplexPolygon.java | 8 +-
.../org/apache/lucene/spatial3d/geom/Plane.java | 2 +-
.../lucene/spatial3d/geom/GeoPolygonTest.java | 63 ++-
.../lucene/search/RandomApproximationQuery.java | 10 +-
solr/CHANGES.txt | 25 +
.../configsets/collection1/conf/solrconfig.xml | 20 +-
solr/core/ivy.xml | 1 -
.../org/apache/solr/core/ConfigSetService.java | 7 +-
.../org/apache/solr/core/CoreContainer.java | 3 +
.../apache/solr/handler/ReplicationHandler.java | 4 +-
.../handler/admin/ZookeeperStatusHandler.java | 222 +++++++++
.../handler/component/TermVectorComponent.java | 89 ++--
.../solr/handler/export/DoubleValueSortDoc.java | 93 ++++
.../solr/handler/export/ExportWriter.java | 13 +-
.../solr/handler/export/QuadValueSortDoc.java | 126 +++++
.../solr/handler/export/SingleValueSortDoc.java | 82 ++++
.../org/apache/solr/handler/export/SortDoc.java | 7 +-
.../solr/handler/export/TripleValueSortDoc.java | 110 +++++
.../apache/solr/handler/loader/XMLLoader.java | 18 +-
.../org/apache/solr/handler/sql/SolrSchema.java | 4 +
.../solr/response/RetrieveFieldsOptimizer.java | 8 +-
.../transform/BaseEditorialTransformer.java | 2 +
.../apache/solr/update/SolrCmdDistributor.java | 194 +++++---
.../processor/DistributedUpdateProcessor.java | 36 +-
.../ParseDateFieldUpdateProcessorFactory.java | 82 +++-
.../TimeRoutedAliasUpdateProcessor.java | 4 +-
.../org/apache/solr/util/TestInjection.java | 22 +-
.../solr/collection1/conf/schema-sql.xml | 27 ++
.../test-files/solr/collection1/conf/schema.xml | 2 +-
...dd-schema-fields-update-processor-chains.xml | 20 +-
...lrconfig-parsing-update-processor-chains.xml | 49 +-
.../collection1/conf/solrconfig-schemaless.xml | 20 +-
.../configsets/_default/conf/solrconfig.xml | 20 +-
.../org/apache/solr/BasicFunctionalityTest.java | 158 +++----
.../test/org/apache/solr/CursorPagingTest.java | 10 +-
.../solr/TestCursorMarkWithoutUniqueKey.java | 9 +-
.../org/apache/solr/TestDistributedSearch.java | 61 ++-
.../org/apache/solr/TestTolerantSearch.java | 18 +-
.../solr/cloud/BasicDistributedZk2Test.java | 30 +-
.../solr/cloud/BasicDistributedZkTest.java | 14 +-
.../test/org/apache/solr/cloud/BasicZkTest.java | 14 +-
.../solr/cloud/CollectionsAPISolrJTest.java | 46 +-
.../apache/solr/cloud/DeleteReplicaTest.java | 19 +-
.../org/apache/solr/cloud/ForceLeaderTest.java | 10 +-
.../apache/solr/cloud/OverseerStatusTest.java | 17 +-
.../org/apache/solr/cloud/SolrXmlInZkTest.java | 10 +-
.../solr/cloud/TestAuthenticationFramework.java | 10 +-
.../solr/cloud/TestCloudDeleteByQuery.java | 11 +-
.../apache/solr/cloud/TestCloudInspectUtil.java | 19 +-
.../apache/solr/cloud/TestConfigSetsAPI.java | 11 +-
.../solr/cloud/TestConfigSetsAPIZkFailure.java | 14 +-
.../solr/cloud/TestDownShardTolerantSearch.java | 17 +-
.../TestLeaderInitiatedRecoveryThread.java | 64 ++-
.../cloud/TestPullReplicaErrorHandling.java | 13 +-
.../TestSolrCloudWithDelegationTokens.java | 25 +-
.../TestSolrCloudWithSecureImpersonation.java | 101 ++--
.../cloud/TestTolerantUpdateProcessorCloud.java | 238 +++++-----
.../org/apache/solr/cloud/TestZkChroot.java | 23 +-
.../test/org/apache/solr/cloud/ZkCLITest.java | 18 +-
.../org/apache/solr/cloud/ZkSolrClientTest.java | 59 +--
.../autoscaling/AutoScalingHandlerTest.java | 43 +-
.../org/apache/solr/core/TestLazyCores.java | 15 +
.../org/apache/solr/handler/TestSQLHandler.java | 243 ++++++++--
.../solr/handler/admin/MetricsHandlerTest.java | 4 +-
.../admin/ZookeeperStatusHandlerTest.java | 85 ++++
.../component/QueryElevationComponentTest.java | 30 ++
.../component/TermVectorComponentTest.java | 340 +++++++------
.../apache/solr/update/AddBlockUpdateTest.java | 148 ++++++
.../solr/update/MockingHttp2SolrClient.java | 87 +++-
.../solr/update/SolrCmdDistributorTest.java | 472 +++++++++++++++----
...dSchemaFieldsUpdateProcessorFactoryTest.java | 23 +-
.../ParsingFieldUpdateProcessorsTest.java | 157 +++---
solr/example/files/conf/solrconfig.xml | 20 +-
solr/licenses/joda-time-2.2.jar.sha1 | 1 -
solr/licenses/joda-time-LICENSE-ASL.txt | 202 --------
solr/licenses/joda-time-NOTICE.txt | 5 -
.../configsets/_default/conf/solrconfig.xml | 20 +-
solr/solr-ref-guide/src/cloud-screens.adoc | 5 +
.../src/images/cloud-screens/cloud-zkstatus.png | Bin 0 -> 175359 bytes
solr/solr-ref-guide/src/schemaless-mode.adoc | 20 +-
...solrcloud-autoscaling-auto-add-replicas.adoc | 6 +-
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
...olrcloud-autoscaling-policy-preferences.adoc | 167 +++++--
.../src/transforming-result-documents.adoc | 9 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 10 +-
.../client/solrj/cloud/autoscaling/Clause.java | 18 +-
.../cloud/autoscaling/MoveReplicaSuggester.java | 3 +-
.../solrj/cloud/autoscaling/NodeVariable.java | 5 +-
.../client/solrj/cloud/autoscaling/Policy.java | 2 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 38 +-
.../solrj/cloud/autoscaling/RangeVal.java | 10 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 6 +-
.../solrj/cloud/autoscaling/ReplicaInfo.java | 17 +-
.../solrj/cloud/autoscaling/Suggester.java | 14 +-
.../solrj/cloud/autoscaling/VariableBase.java | 1 +
.../solrj/cloud/autoscaling/Violation.java | 10 +-
.../solr/client/solrj/impl/Http2SolrClient.java | 14 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 2 +-
.../org/apache/solr/client/solrj/io/Lang.java | 1 +
.../solr/client/solrj/io/eval/KnnEvaluator.java | 18 +-
.../solrj/io/eval/KnnRegressionEvaluator.java | 194 ++++++++
.../solrj/io/eval/MinMaxScaleEvaluator.java | 2 +-
.../client/solrj/io/eval/PolyFitEvaluator.java | 7 +-
.../client/solrj/io/eval/PredictEvaluator.java | 30 +-
.../solr/client/solrj/util/ClientUtils.java | 20 +-
.../solr/common/ConditionalMapWriter.java | 75 +++
.../java/org/apache/solr/common/MapWriter.java | 6 +
.../apache/solr/common/params/CommonParams.java | 1 +
.../java/org/apache/solr/common/util/XML.java | 98 +---
.../solrj/cloud/autoscaling/TestPolicy.java | 16 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 240 ++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 86 +++-
.../apache/solr/cloud/SolrCloudTestCase.java | 19 +
.../org/apache/solr/util/BaseTestHarness.java | 23 +-
solr/webapp/web/css/angular/cloud.css | 71 ++-
solr/webapp/web/css/angular/menu.css | 1 +
solr/webapp/web/index.html | 1 +
solr/webapp/web/js/angular/controllers/cloud.js | 42 +-
solr/webapp/web/js/angular/services.js | 6 +
solr/webapp/web/partials/cloud.html | 59 +++
143 files changed, 5253 insertions(+), 2442 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/lucene/ivy-versions.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 7437042,d5aafec..710f03f
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@@ -17,15 -17,28 +17,18 @@@
package org.apache.solr.update;
-import java.io.Closeable;
import java.io.IOException;
-import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.ConnectException;
+ import java.net.SocketException;
+ import java.net.SocketTimeoutException;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import org.apache.http.HttpResponse;
+import java.util.concurrent.Phaser;
+
+ import org.apache.http.NoHttpResponseException;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.BinaryResponseParser;
-import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; // jdoc
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
@@@ -43,28 -56,133 +46,27 @@@ import org.slf4j.LoggerFactory
/**
* Used for distributing commands from a shard leader to its replicas.
*/
-public class SolrCmdDistributor implements Closeable {
+public class SolrCmdDistributor {
- private static final int MAX_RETRIES_ON_FORWARD = 25;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private StreamingSolrClients clients;
- private boolean finished = false; // see finish()
- private int retryPause = 500;
-
private final List<Error> allErrors = new ArrayList<>();
- private final List<Error> errors = Collections.synchronizedList(new ArrayList<Error>());
-
- private final CompletionService<Object> completionService;
- private final Set<Future<Object>> pending = new HashSet<>();
-
- public static interface AbortCheck {
- public boolean abortCheck();
- }
-
+ private Http2SolrClient client;
+ private Phaser pendingTasksPhaser = new Phaser(1);
- private int maxRetriesOnForward = MAX_RETRIES_ON_FORWARD;
++ private int retryPause = 500;
+
public SolrCmdDistributor(UpdateShardHandler updateShardHandler) {
- this.clients = new StreamingSolrClients(updateShardHandler);
- this.completionService = new ExecutorCompletionService<>(updateShardHandler.getUpdateExecutor());
+ this.client = updateShardHandler.getUpdateOnlyHttpClient();
}
--
- public SolrCmdDistributor(Http2SolrClient client, int maxRetriesOnForward) {
- public SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
- this.clients = clients;
++
++ public SolrCmdDistributor(Http2SolrClient client, int retryPause) {
+ this.client = client;
- this.maxRetriesOnForward = maxRetriesOnForward;
+ this.retryPause = retryPause;
- completionService = new ExecutorCompletionService<>(clients.getUpdateExecutor());
}
- public void finish() {
- try {
- assert ! finished : "lifecycle sanity check";
- finished = true;
-
- blockAndDoRetries();
- } finally {
- clients.shutdown();
- }
+ public void finish() {
+ blockUntilFinished();
}
-
- public void close() {
- clients.shutdown();
- }
-
- private void doRetriesIfNeeded() {
- // NOTE: retries will be forwards to a single url
-
- List<Error> errors = new ArrayList<>(this.errors);
- errors.addAll(clients.getErrors());
- List<Error> resubmitList = new ArrayList<>();
-
- if (log.isInfoEnabled() && errors.size() > 0) {
- log.info("SolrCmdDistributor found {} errors", errors.size());
- }
-
- if (log.isDebugEnabled() && errors.size() > 0) {
- StringBuilder builder = new StringBuilder("SolrCmdDistributor found:");
- int maxErrorsToShow = 10;
- for (Error e:errors) {
- if (maxErrorsToShow-- <= 0) break;
- builder.append("\n" + e);
- }
- if (errors.size() > 10) {
- builder.append("\n... and ");
- builder.append(errors.size() - 10);
- builder.append(" more");
- }
- log.debug(builder.toString());
- }
- for (Error err : errors) {
- try {
- String oldNodeUrl = err.req.node.getUrl();
-
- /*
- * if this is a retryable request we may want to retry, depending on the error we received and
- * the number of times we have already retried
- */
- boolean isRetry = err.req.shouldRetry(err);
-
- if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
- err.e);
-
- // this can happen in certain situations such as close
- if (isRetry) {
- err.req.retries++;
-
- if (err.req.node instanceof ForwardNode) {
- SolrException.log(SolrCmdDistributor.log, "forwarding update to "
- + oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
- + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
- } else {
- SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
- + oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
- + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
- }
- try {
- Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- log.warn(null, e);
- }
- resubmitList.add(err);
- } else {
- allErrors.add(err);
- }
- } catch (Exception e) {
- // continue on
- log.error("Unexpected Error while doing request retries", e);
- }
- }
-
- clients.clearErrors();
- this.errors.clear();
- for (Error err : resubmitList) {
- submit(err.req, false);
- }
-
- if (resubmitList.size() > 0) {
- blockAndDoRetries();
- }
- }
-
public void distribDelete(DeleteUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {
distribDelete(cmd, nodes, params, false, null, null);
}
@@@ -73,6 -191,10 +75,10 @@@
RollupRequestReplicationTracker rollupTracker,
LeaderRequestReplicationTracker leaderTracker) throws IOException {
+ if (!cmd.isDeleteById()) {
- blockAndDoRetries(); // For DBQ, flush all writes before submitting
++ blockUntilFinished(); // For DBQ, flush all writes before submitting
+ }
+
for (Node node : nodes) {
UpdateRequest uReq = new UpdateRequest();
uReq.setParams(params);
@@@ -153,96 -302,36 +158,97 @@@
+ req.node.getUrl() + " retry:"
+ req.retries + " " + req.cmd + " params:" + req.uReq.getParams());
}
-
- if (isCommit) {
- // a commit using ConncurrentUpdateSolrServer is not async,
- // so we make it async to prevent commits from happening
- // serially across multiple nodes
- pending.add(completionService.submit(() -> {
- doRequest(req);
- return null;
- }));
+
+ try {
+ req.uReq.setBasePath(req.node.getUrl());
+ if (req.synchronous) {
+ NamedList rsp = client.request(req.uReq);
+ req.trackRequestResult(rsp, true);
+ pendingTasksPhaser.arriveAndDeregister();
+ } else {
+ //TODO write add cmds in single outputstream
+ client.request(req.uReq, null, new Http2SolrClient.OnComplete<NamedList>() {
+ @Override
+ public void onSuccess(NamedList result) {
+ req.trackRequestResult(result, true);
+ pendingTasksPhaser.arriveAndDeregister();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ handleAndRetry(req, t, isCommit);
+ }
+ });
+ }
+ } catch (Exception e) {
+ handleAndRetry(req, e, isCommit);
+ }
+ }
+
+ private void handleAndRetry(Req req, Throwable t, boolean isCommit) {
+ SolrException.log(log, t);
+ Error error = new Error();
+ error.t = t;
+ error.req = req;
+ if (t instanceof SolrException) {
+ error.statusCode = ((SolrException) t).code();
+ }
+ if (checkRetry(error)) {
+ submit0(req, isCommit);
} else {
- doRequest(req);
+ req.trackRequestResult(null, false);
+ allErrors.add(error);
+ pendingTasksPhaser.arriveAndDeregister();
}
}
-
- private void doRequest(final Req req) {
+
+ private boolean checkRetry(Error err) {
- String oldNodeUrl = err.req.node.getUrl();
-
- // if there is a retry url, we want to retry...
- boolean isRetry = err.req.node.checkRetry();
-
- boolean doRetry = false;
- int rspCode = err.statusCode;
-
- if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
- err.t);
-
- // this can happen in certain situations such as close
- if (isRetry) {
- if (rspCode == 404 || rspCode == 403 || rspCode == 503) {
- doRetry = true;
- }
-
- // if it's a connect exception, lets try again
- if (err.t instanceof SolrServerException) {
- if (((SolrServerException) err.t).getRootCause() instanceof ConnectException) {
- doRetry = true;
- }
- }
++ log.info("SolrCmdDistributor got error", err);
+ try {
- SolrClient solrClient = clients.getSolrClient(req);
- solrClient.request(req.uReq);
- } catch (Exception e) {
- SolrException.log(log, e);
- Error error = new Error();
- error.e = e;
- error.req = req;
- if (e instanceof SolrException) {
- error.statusCode = ((SolrException) e).code();
++ String oldNodeUrl = err.req.node.getUrl();
+
- if (err.t instanceof ConnectException) {
- doRetry = true;
- }
++ /*
++ * if this is a retryable request we may want to retry, depending on the error we received and
++ * the number of times we have already retried
++ */
++ boolean isRetry = err.req.shouldRetry(err);
++ if (testing_errorHook != null) Diagnostics.call(testing_errorHook, err.t);
+
- if (err.req.retries < maxRetriesOnForward && doRetry) {
++ // this can happen in certain situations such as close
++ if (isRetry) {
+ err.req.retries++;
+
- SolrException.log(SolrCmdDistributor.log, "forwarding update to "
- + oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + " " + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + rspCode, err.t);
-
++ if (err.req.node instanceof ForwardNode) {
++ SolrException.log(SolrCmdDistributor.log, "forwarding update to "
++ + oldNodeUrl + " failed - retrying ... retries: "
++ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
++ + err.req.cmd.toString() + " params:"
++ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.t);
++ } else {
++ SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
++ + oldNodeUrl + " failed - retrying ... retries: "
++ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
++ + err.req.cmd.toString() + " params:"
++ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.t);
++ }
++ try {
++ Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
++ } catch (InterruptedException e) {
++ Thread.currentThread().interrupt();
++ log.warn(null, e);
++ }
+ return true;
+ } else {
+ return false;
}
- } else {
- errors.add(error);
++ } catch (Exception e) {
++ // continue on
++ log.error("Unexpected Error while doing request retries", e);
++ // avoid infinite loop
+ return false;
}
}
-
+
public static class Req {
public Node node;
public UpdateRequest uReq;
@@@ -267,6 -356,16 +273,16 @@@
this.leaderTracker = leaderTracker;
}
+ /**
+ * @return true if this request should be retried after receiving a particular error
+ * false otherwise
+ */
- public boolean shouldRetry(Error err) {
++ boolean shouldRetry(Error err) {
+ boolean isRetry = node.checkRetry(err);
+ isRetry &= uReq.getDeleteQuery() == null || uReq.getDeleteQuery().isEmpty(); //Don't retry DBQs
+ return isRetry && retries < node.getMaxRetries();
+ }
+
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("SolrCmdDistributor$Req: cmd=").append(cmd.toString());
@@@ -380,9 -511,33 +405,33 @@@
}
@Override
- public boolean checkRetry() {
+ public boolean checkRetry(Error err) {
+ if (!retry) return false;
-
++
+ if (err.statusCode == 404 || err.statusCode == 403 || err.statusCode == 503) {
+ return true;
+ }
-
++
+ // if it's a connect exception, lets try again
- if (err.e instanceof SolrServerException) {
- if (isRetriableException(((SolrServerException) err.e).getRootCause())) {
++ if (err.t instanceof SolrServerException) {
++ if (isRetriableException(((SolrServerException) err.t).getRootCause())) {
+ return true;
+ }
+ } else {
- if (isRetriableException(err.e)) {
++ if (isRetriableException(err.t)) {
+ return true;
+ }
+ }
return false;
}
+
+ /**
+ * @return true if Solr should retry in case of hitting this exception
+ * false otherwise
+ */
+ private boolean isRetriableException(Throwable t) {
+ return t instanceof SocketException || t instanceof NoHttpResponseException || t instanceof SocketTimeoutException;
+ }
@Override
public String getBaseUrl() {
@@@ -448,23 -612,35 +506,35 @@@
}
@Override
- public boolean checkRetry() {
- ZkCoreNodeProps leaderProps;
- try {
- leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderRetry(
- collection, shardId));
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return false;
- } catch (Exception e) {
- // we retry with same info
- log.warn(null, e);
- return true;
+ public boolean checkRetry(Error err) {
+ boolean doRetry = false;
+ if (err.statusCode == 404 || err.statusCode == 403 || err.statusCode == 503) {
+ doRetry = true;
}
-
- this.nodeProps = leaderProps;
- return true;
+ // if it's a connect exception, lets try again
- if (err.e instanceof SolrServerException && ((SolrServerException) err.e).getRootCause() instanceof ConnectException) {
++ if (err.t instanceof SolrServerException && ((SolrServerException) err.t).getRootCause() instanceof ConnectException) {
+ doRetry = true;
- } else if (err.e instanceof ConnectException) {
++ } else if (err.t instanceof ConnectException) {
+ doRetry = true;
+ }
+ if (doRetry) {
+ ZkCoreNodeProps leaderProps;
+ try {
+ leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderRetry(
+ collection, shardId));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ } catch (Exception e) {
+ // we retry with same info
+ log.warn(null, e);
+ return true;
+ }
+
+ this.nodeProps = leaderProps;
+ }
+ return doRetry;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/test/org/apache/solr/cloud/TestSolrCloudWithSecureImpersonation.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
index b843709,0000000..2b8d8a2
mode 100644,000000..100644
--- a/solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
+++ b/solr/core/src/test/org/apache/solr/update/MockingHttp2SolrClient.java
@@@ -1,95 -1,0 +1,154 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update;
+
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketException;
++import java.util.HashSet;
++import java.util.Set;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
++import org.apache.solr.client.solrj.request.UpdateRequest;
++import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+
+public class MockingHttp2SolrClient extends Http2SolrClient {
+
- public enum Exp {CONNECT_EXCEPTION, SOCKET_EXCEPTION};
++ public enum Exp {CONNECT_EXCEPTION, SOCKET_EXCEPTION, BAD_REQUEST};
+
+ private volatile Exp exp = null;
++ private boolean oneExpPerReq;
++ private Set<SolrRequest> reqGotException;
+
+ public MockingHttp2SolrClient(String baseSolrUrl, Builder builder) {
+ super(baseSolrUrl, builder);
++ this.oneExpPerReq = builder.oneExpPerReq;
++ this.reqGotException = new HashSet<>();
+ }
+
+ public static class Builder extends Http2SolrClient.Builder {
++ private boolean oneExpPerReq = false;
+
- public Builder(String baseSolrUrl, UpdateShardHandlerConfig config) {
- super(baseSolrUrl);
++ public Builder(UpdateShardHandlerConfig config) {
++ super();
+ this.connectionTimeout(config.getDistributedConnectionTimeout());
+ this.idleTimeout(config.getDistributedSocketTimeout());
+ }
+
+ public MockingHttp2SolrClient build() {
- return new MockingHttp2SolrClient(baseSolrUrl, this);
++ return new MockingHttp2SolrClient(null, this);
++ }
++
++ // DBQ won't cause exception
++ Builder oneExpPerReq() {
++ oneExpPerReq = true;
++ return this;
+ }
+ }
+
+
+ public void setExp(Exp exp) {
+ this.exp = exp;
+ }
+
- private IOException exception() {
++ private Exception exception() {
+ switch (exp) {
+ case CONNECT_EXCEPTION:
+ return new ConnectException();
+ case SOCKET_EXCEPTION:
+ return new SocketException();
++ case BAD_REQUEST:
++ return new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bad Request");
+ default:
+ break;
+ }
+ return null;
+ }
+
+ @Override
+ public NamedList<Object> request(SolrRequest request, String collection)
+ throws SolrServerException, IOException {
++ if (request instanceof UpdateRequest) {
++ UpdateRequest ur = (UpdateRequest) request;
++ if (!ur.getDeleteQuery().isEmpty())
++ return super.request(request, collection);
++ }
++
+ if (exp != null) {
- if (LuceneTestCase.random().nextBoolean()) {
- throw exception();
++ if (oneExpPerReq) {
++ if (reqGotException.contains(request))
++ return super.request(request, collection);
++ else
++ reqGotException.add(request);
++ }
++
++ Exception e = exception();
++ if (e instanceof IOException) {
++ if (LuceneTestCase.random().nextBoolean()) {
++ throw (IOException) e;
++ } else {
++ throw new SolrServerException(e);
++ }
++ } else if (e instanceof SolrServerException) {
++ throw (SolrServerException) e;
+ } else {
- throw new SolrServerException(exception());
++ throw new SolrServerException(e);
+ }
+ }
+
- return super.request(request);
++ return super.request(request, collection);
+ }
+
- public Http2ClientResponse request(SolrRequest solrRequest, String collection, OnComplete onComplete)
++ public Http2ClientResponse request(SolrRequest request, String collection, OnComplete onComplete)
+ throws SolrServerException, IOException {
++ if (request instanceof UpdateRequest) {
++ UpdateRequest ur = (UpdateRequest) request;
++ // won't throw exception if request is DBQ
++ if (ur.getDeleteQuery() != null && !ur.getDeleteQuery().isEmpty())
++ return super.request(request, collection, onComplete);
++ }
++
+ if (exp != null) {
- if (LuceneTestCase.random().nextBoolean()) {
- throw exception();
++ if (oneExpPerReq) {
++ if (reqGotException.contains(request))
++ return super.request(request, collection, onComplete);
++ else
++ reqGotException.add(request);
++ }
++
++ Exception e = exception();
++ if (e instanceof IOException) {
++ if (LuceneTestCase.random().nextBoolean()) {
++ throw (IOException) e;
++ } else {
++ throw new SolrServerException(e);
++ }
++ } else if (e instanceof SolrServerException) {
++ throw (SolrServerException) e;
+ } else {
- throw new SolrServerException(exception());
++ throw new SolrServerException(e);
+ }
+ }
- return super.request(solrRequest, collection, onComplete);
++
++ return super.request(request, collection, onComplete);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9f554f6d/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
index 551aed5,24cf717..08b8ef9
--- a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
@@@ -16,9 -16,9 +16,10 @@@
*/
package org.apache.solr.update;
+import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
+ import java.net.SocketException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
@@@ -45,21 -46,26 +47,29 @@@ import org.apache.solr.core.SolrCore
import org.apache.solr.core.SolrEventListener;
import org.apache.solr.index.LogDocMergePolicyFactory;
import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.update.MockStreamingSolrClients.Exp;
import org.apache.solr.update.SolrCmdDistributor.Error;
+ import org.apache.solr.update.SolrCmdDistributor.ForwardNode;
import org.apache.solr.update.SolrCmdDistributor.Node;
- import org.apache.solr.update.SolrCmdDistributor.RetryNode;
import org.apache.solr.update.SolrCmdDistributor.StdNode;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
+ import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker;
+ import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.xml.sax.SAXException;
++import static org.apache.solr.update.MockingHttp2SolrClient.Exp.BAD_REQUEST;
++import static org.apache.solr.update.MockingHttp2SolrClient.Exp.CONNECT_EXCEPTION;
++import static org.apache.solr.update.MockingHttp2SolrClient.Exp.SOCKET_EXCEPTION;
++
// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
-
- private static enum NodeType {FORWARD, STANDARD};
-
+
++ private enum NodeType {FORWARD, STANDARD};
++
private AtomicInteger id = new AtomicInteger();
-
+
@BeforeClass
public static void beforeClass() throws Exception {
// we can't use the Randomized merge policy because the test depends on
@@@ -323,33 -330,178 +334,181 @@@
((NamedList<Object>) resp.get("index")).get("maxDoc"));
}
}
-
+
- testMaxRetries();
- testOneRetry();
+ testMaxRetries(NodeType.FORWARD);
+ testMaxRetries(NodeType.STANDARD);
+ testOneRetry(NodeType.FORWARD);
+ testOneRetry(NodeType.STANDARD);
testRetryNodeAgainstBadAddress();
- testRetryNodeWontRetrySocketError();
-
+ testStdNodeRetriesSocketError();
+ testForwardNodeWontRetrySocketError();
+ testNodeWontRetryBadRequest(NodeType.FORWARD);
+ testNodeWontRetryBadRequest(NodeType.STANDARD);
+ testMinRfOnRetries(NodeType.FORWARD);
+ testMinRfOnRetries(NodeType.STANDARD);
testDistribOpenSearcher();
+ testReqShouldRetryNoRetries();
+ testReqShouldRetryMaxRetries();
+ testReqShouldRetryBadRequest();
+ testReqShouldRetryNotFound();
+ testReqShouldRetryDBQ();
+ testDeletes(false, true);
+ testDeletes(false, false);
+ testDeletes(true, true);
+ testDeletes(true, false);
}
-
+
- private void testMaxRetries() throws IOException {
- final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder("", UpdateShardHandlerConfig.DEFAULT);
- try (MockingHttp2SolrClient solrClient = clientBuilder.build()) {
- SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(solrClient, 5);
- solrClient.setExp(MockingHttp2SolrClient.Exp.CONNECT_EXCEPTION);
+ private void testDeletes(boolean dbq, boolean withFailures) throws Exception {
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+ solrclient.commit(true, true);
+ long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
- final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
++ final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT)
++ .oneExpPerReq();
++ try (MockingHttp2SolrClient streamingClients = clientBuilder.build()) {
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
+ if (withFailures) {
- streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++ streamingClients.setExp(CONNECT_EXCEPTION);
+ }
ArrayList<Node> nodes = new ArrayList<>();
- final HttpSolrClient solrclient1 = (HttpSolrClient) clients.get(0);
- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
- ZkStateReader.CORE_NAME_PROP, "");
-
final AtomicInteger retries = new AtomicInteger();
- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient1.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
- nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
++ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ Node retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
@Override
- public boolean checkRetry() {
+ public boolean checkRetry(Error err) {
- streamingClients.setExp(null);
retries.incrementAndGet();
- return true;
+ return super.checkRetry(err);
}
};
+
+ nodes.add(retryNode);
+
+ for (int i = 0 ; i < 5 ; i++) {
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ int currentId = id.incrementAndGet();
+ cmd.solrDoc = sdoc("id", currentId);
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ cmdDistrib.distribAdd(cmd, nodes, params);
++ cmdDistrib.blockUntilFinished();
++
+ DeleteUpdateCommand dcmd = new DeleteUpdateCommand(null);
+ if (dbq) {
+ dcmd.setQuery("id:" + currentId);
+ } else {
+ dcmd.setId(String.valueOf(currentId));
+ }
+ cmdDistrib.distribDelete(dcmd, nodes, params, false, null, null);
+ }
-
+
++
++ streamingClients.setExp(null);
+ CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
+ cmdDistrib.distribCommit(ccmd, nodes, new ModifiableSolrParams());
+ cmdDistrib.finish();
-
++
+ int expectedRetryCount = 0;
+ if (withFailures) {
+ if (dbq) {
- expectedRetryCount = 1; // just the first cmd would be retried
++ expectedRetryCount = 5;
+ } else {
+ expectedRetryCount = 10;
+ }
+ }
+ assertEquals(expectedRetryCount, retries.get());
+
+
+ long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
+
+ // we will get java.net.ConnectException which we retry on
+ assertEquals(numFoundBefore, numFoundAfter);
+ assertEquals(0, cmdDistrib.getErrors().size());
+ }
+ }
+
+ private void testMinRfOnRetries(NodeType nodeType) throws Exception {
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
- final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
- streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++ final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT);
++ try (MockingHttp2SolrClient streamingClients = clientBuilder.build()) {
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++ streamingClients.setExp(CONNECT_EXCEPTION);
+ ArrayList<Node> nodes = new ArrayList<>();
+
+ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
+ ZkStateReader.CORE_NAME_PROP, "");
+
+ final AtomicInteger retries = new AtomicInteger();
+ nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ if (nodeType == NodeType.FORWARD) {
+ nodes.add(new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ if (retries.incrementAndGet() >= 3) {
+ streamingClients.setExp(null);
+ }
+ return super.checkRetry(err);
+ }
+ });
+ } else {
+ nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ if (retries.incrementAndGet() >= 3) {
+ streamingClients.setExp(null);
+ }
+ return super.checkRetry(err);
+ }
+ });
+ }
+
+
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ cmd.solrDoc = sdoc("id", id.incrementAndGet());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ RollupRequestReplicationTracker rollupReqTracker = new RollupRequestReplicationTracker("2");
+ LeaderRequestReplicationTracker leaderReqTracker = new LeaderRequestReplicationTracker("shard1", 2);
+
+ cmdDistrib.distribAdd(cmd, nodes, params, false, rollupReqTracker, leaderReqTracker);
+ cmdDistrib.finish();
+ assertEquals(3, retries.get());
+ assertEquals(2, leaderReqTracker.getAchievedRf());// "2" here is because one would be the leader, that creates the instance of LeaderRequestReplicationTracker, the second one is the node
+
+ assertEquals(0, cmdDistrib.getErrors().size());
+ }
+ }
+
+ private void testMaxRetries(NodeType nodeType) throws IOException {
- final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
- streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++ final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT);
++ try (MockingHttp2SolrClient streamingClients = clientBuilder.build()) {
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++ streamingClients.setExp(CONNECT_EXCEPTION);
+ ArrayList<Node> nodes = new ArrayList<>();
+ final HttpSolrClient solrclient1 = (HttpSolrClient) clients.get(0);
+
+ final AtomicInteger retries = new AtomicInteger();
+ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient1.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ Node retryNode;
+ if (nodeType == NodeType.FORWARD) {
+ retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 6) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ } else {
+ retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 6) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ }
-
++
+
nodes.add(retryNode);
AddUpdateCommand cmd = new AddUpdateCommand(null);
@@@ -364,30 -516,84 +523,81 @@@
assertEquals(1, cmdDistrib.getErrors().size());
}
}
-
+
- private void testOneRetry() throws Exception {
+ private void testReqShouldRetryNoRetries() {
- Error err = getError(new SocketException());
++ Error err = getError(new SocketException());
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 0), new UpdateRequest(), true);
+ assertFalse(req.shouldRetry(err));
+ }
-
++
+ private void testReqShouldRetryDBQ() {
- Error err = getError(new SocketException());
++ Error err = getError(new SocketException());
+ UpdateRequest dbqReq = new UpdateRequest();
+ dbqReq.deleteByQuery("*:*");
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), dbqReq, true);
+ assertFalse(req.shouldRetry(err));
+ }
-
++
+ private void testReqShouldRetryMaxRetries() {
- Error err = getError(new SocketException());
++ Error err = getError(new SocketException());
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+ assertTrue(req.shouldRetry(err));
+ req.retries++;
+ assertFalse(req.shouldRetry(err));
+ }
-
++
+ private void testReqShouldRetryBadRequest() {
- Error err = getError(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "bad request"));
++ Error err = getError(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "bad request"));
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+ assertFalse(req.shouldRetry(err));
+ }
-
++
+ private void testReqShouldRetryNotFound() {
+ Error err = getError(new SolrException(SolrException.ErrorCode.NOT_FOUND, "not found"));
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+ assertTrue(req.shouldRetry(err));
+ }
-
++
+ private Error getError(Exception e) {
+ Error err = new Error();
- err.e = e;
++ err.t = e;
+ if (e instanceof SolrException) {
+ err.statusCode = ((SolrException)e).code();
+ }
+ return err;
+ }
-
++
+ private void testOneRetry(NodeType nodeType) throws Exception {
final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
.getNumFound();
- final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder("", UpdateShardHandlerConfig.DEFAULT);
- try (MockingHttp2SolrClient solrClient = clientBuilder.build()) {
- SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(solrClient, 5);
- solrClient.setExp(MockingHttp2SolrClient.Exp.CONNECT_EXCEPTION);
- final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
- streamingClients.setExp(Exp.CONNECT_EXCEPTION);
++ try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++ streamingClients.setExp(CONNECT_EXCEPTION);
ArrayList<Node> nodes = new ArrayList<>();
-- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
-- ZkStateReader.CORE_NAME_PROP, "");
--
final AtomicInteger retries = new AtomicInteger();
-- nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
- @Override
- public boolean checkRetry() {
- solrClient.setExp(null);
- retries.incrementAndGet();
- return true;
- }
- };
++ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ Node retryNode;
+ if (nodeType == NodeType.FORWARD) {
+ retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ streamingClients.setExp(null);
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ } else {
+ retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ streamingClients.setExp(null);
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ }
nodes.add(retryNode);
@@@ -417,22 -624,78 +628,75 @@@
final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
.getNumFound();
- final MockingHttp2SolrClient.Builder clientBuilder = new MockingHttp2SolrClient.Builder("", UpdateShardHandlerConfig.DEFAULT);
- try (MockingHttp2SolrClient solrClient = clientBuilder.build()) {
- SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(solrClient, 5);
- solrClient.setExp(MockingHttp2SolrClient.Exp.SOCKET_EXCEPTION);
- final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
- streamingClients.setExp(Exp.BAD_REQUEST);
++ try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++ streamingClients.setExp(BAD_REQUEST);
ArrayList<Node> nodes = new ArrayList<>();
-
ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
ZkStateReader.CORE_NAME_PROP, "");
final AtomicInteger retries = new AtomicInteger();
- nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
+ Node retryNode;
+ if (nodeType == NodeType.FORWARD) {
+ retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ } else {
+ retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ }
+ nodes.add(retryNode);
+
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ cmd.solrDoc = sdoc("id", id.incrementAndGet());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
+ cmdDistrib.distribAdd(cmd, nodes, params);
+
+ streamingClients.setExp(null);
+ cmdDistrib.distribCommit(ccmd, nodes, params);
+ cmdDistrib.finish();
+
+ // it will checkRetry, but not actually do it...
+ assertEquals(1, retries.get());
+
+
+ long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
+
+ // we will get java.net.SocketException: Network is unreachable, which we don't retry on
+ assertEquals(numFoundBefore, numFoundAfter);
+ assertEquals(1, cmdDistrib.getErrors().size());
+ unIgnoreException("Bad Request");
+ }
+ }
-
++
+ private void testForwardNodeWontRetrySocketError() throws Exception {
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+ long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
- final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
- streamingClients.setExp(Exp.SOCKET_EXCEPTION);
++ try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++ streamingClients.setExp(SOCKET_EXCEPTION);
+ ArrayList<Node> nodes = new ArrayList<>();
+
- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
- ZkStateReader.CORE_NAME_PROP, "");
-
+ final AtomicInteger retries = new AtomicInteger();
- nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
++ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
@Override
- public boolean checkRetry() {
+ public boolean checkRetry(Error err) {
retries.incrementAndGet();
- return true;
+ return super.checkRetry(err);
}
};
@@@ -462,26 -725,24 +726,21 @@@
assertEquals(1, cmdDistrib.getErrors().size());
}
}
-
+
- private void testRetryNodeAgainstBadAddress() throws SolrServerException, IOException {
- // Test RetryNode
- {
- SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
- final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
- long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
- .getNumFound();
-
+ private void testStdNodeRetriesSocketError() throws Exception {
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
- final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
- streamingClients.setExp(Exp.SOCKET_EXCEPTION);
++ try (MockingHttp2SolrClient streamingClients = new MockingHttp2SolrClient.Builder(UpdateShardHandlerConfig.DEFAULT).build()) {
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0);
++ streamingClients.setExp(SOCKET_EXCEPTION);
ArrayList<Node> nodes = new ArrayList<>();
- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
- ZkStateReader.CORE_NAME_PROP, "");
-
+ final AtomicInteger retries = new AtomicInteger();
- nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
++ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ Node retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
@Override
- public boolean checkRetry() {
- ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
- ZkStateReader.CORE_NAME_PROP, "");
- this.nodeProps = new ZkCoreNodeProps(leaderProps);
-
- return true;
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
}
};
@@@ -494,29 -754,66 +752,65 @@@
ModifiableSolrParams params = new ModifiableSolrParams();
cmdDistrib.distribAdd(cmd, nodes, params);
-
- CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
- params = new ModifiableSolrParams();
- params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
- cmdDistrib.distribCommit(ccmd, nodes, params);
cmdDistrib.finish();
- long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
- .getNumFound();
+ // it will checkRetry, but not actually do it...
+ assertEquals(6, retries.get());
+ }
+ }
- // different OS's will throw different exceptions for the bad address above
- if (numFoundBefore != numFoundAfter) {
- assertEquals(0, cmdDistrib.getErrors().size());
- assertEquals(numFoundBefore + 1, numFoundAfter);
- } else {
- // we will get java.net.SocketException: Network is unreachable and not retry
- assertEquals(numFoundBefore, numFoundAfter);
+ private void testRetryNodeAgainstBadAddress() throws SolrServerException, IOException {
+ // Test RetryNode
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler)) {
- final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
- long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
- .getNumFound();
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
++ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
++ long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
++ .getNumFound();
+
- ArrayList<Node> nodes = new ArrayList<>();
++ ArrayList<Node> nodes = new ArrayList<>();
- assertEquals(1, cmdDistrib.getErrors().size());
- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
- ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
- @Override
- public boolean checkRetry(Error err) {
- ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
- ZkStateReader.CORE_NAME_PROP, "");
- this.nodeProps = new ZkCoreNodeProps(leaderProps);
++ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
++ ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
++ @Override
++ public boolean checkRetry(Error err) {
++ ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
++ ZkStateReader.CORE_NAME_PROP, "");
++ this.nodeProps = new ZkCoreNodeProps(leaderProps);
+
- return super.checkRetry(err);
- }
- };
++ return super.checkRetry(err);
+ }
++ };
+
+
- nodes.add(retryNode);
++ nodes.add(retryNode);
+
+
- AddUpdateCommand cmd = new AddUpdateCommand(null);
- cmd.solrDoc = sdoc("id", id.incrementAndGet());
- ModifiableSolrParams params = new ModifiableSolrParams();
++ AddUpdateCommand cmd = new AddUpdateCommand(null);
++ cmd.solrDoc = sdoc("id", id.incrementAndGet());
++ ModifiableSolrParams params = new ModifiableSolrParams();
+
- cmdDistrib.distribAdd(cmd, nodes, params);
++ cmdDistrib.distribAdd(cmd, nodes, params);
+
- CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
- params = new ModifiableSolrParams();
- params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
- cmdDistrib.distribCommit(ccmd, nodes, params);
- cmdDistrib.finish();
++ CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
++ params = new ModifiableSolrParams();
++ params.set(DistributedUpdateProcessor.COMMIT_END_POINT, true);
++ cmdDistrib.distribCommit(ccmd, nodes, params);
++ cmdDistrib.finish();
+
- long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
- .getNumFound();
++ long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
++ .getNumFound();
+
- // different OS's will throw different exceptions for the bad address above
- if (numFoundBefore != numFoundAfter) {
- assertEquals(0, cmdDistrib.getErrors().size());
- assertEquals(numFoundBefore + 1, numFoundAfter);
- } else {
- // we will get java.net.SocketException: Network is unreachable and not retry
- assertEquals(numFoundBefore, numFoundAfter);
++ // different OS's will throw different exceptions for the bad address above
++ if (numFoundBefore != numFoundAfter) {
++ assertEquals(0, cmdDistrib.getErrors().size());
++ assertEquals(numFoundBefore + 1, numFoundAfter);
++ } else {
++ // we will get java.net.SocketException: Network is unreachable and not retry
++ assertEquals(numFoundBefore, numFoundAfter);
+
- assertEquals(1, cmdDistrib.getErrors().size());
- }
++ assertEquals(1, cmdDistrib.getErrors().size());
}
}
-
+
@Override
public void distribTearDown() throws Exception {
updateShardHandler.close();
@@@ -524,23 -821,22 +818,21 @@@
}
private void testDistribOpenSearcher() {
- {
- SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler)) {
-- UpdateRequest updateRequest = new UpdateRequest();
++ SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(updateShardHandler);
++ UpdateRequest updateRequest = new UpdateRequest();
-- CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
++ CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
-- //test default value (should be true)
-- cmdDistrib.addCommit(updateRequest, ccmd);
-- boolean openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, false);
-- assertTrue(openSearcher);
++ //test default value (should be true)
++ cmdDistrib.addCommit(updateRequest, ccmd);
++ boolean openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, false);
++ assertTrue(openSearcher);
-- //test openSearcher = false
-- ccmd.openSearcher = false;
++ //test openSearcher = false
++ ccmd.openSearcher = false;
-- cmdDistrib.addCommit(updateRequest, ccmd);
-- openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, true);
-- assertFalse(openSearcher);
-- }
++ cmdDistrib.addCommit(updateRequest, ccmd);
++ openSearcher = updateRequest.getParams().getBool(UpdateParams.OPEN_SEARCHER, true);
++ assertFalse(openSearcher);
}
}
[18/31] lucene-solr:jira/http2: SOLR-12618: AutoScalingHandlerTest
failing in jenkins
Posted by da...@apache.org.
SOLR-12618: AutoScalingHandlerTest failing in jenkins
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9b418a45
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9b418a45
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9b418a45
Branch: refs/heads/jira/http2
Commit: 9b418a4593c2cd4f49aab2485c5614d11e1bb5eb
Parents: 2674c53
Author: Noble Paul <no...@apache.org>
Authored: Thu Aug 9 01:24:03 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Thu Aug 9 01:24:03 2018 +1000
----------------------------------------------------------------------
.../autoscaling/AutoScalingHandlerTest.java | 43 ++--
.../cloud/autoscaling/AddReplicaSuggester.java | 10 +-
.../client/solrj/cloud/autoscaling/Clause.java | 18 +-
.../cloud/autoscaling/MoveReplicaSuggester.java | 3 +-
.../solrj/cloud/autoscaling/NodeVariable.java | 5 +-
.../client/solrj/cloud/autoscaling/Policy.java | 2 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 38 ++-
.../solrj/cloud/autoscaling/RangeVal.java | 10 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 6 +-
.../solrj/cloud/autoscaling/ReplicaInfo.java | 17 +-
.../solrj/cloud/autoscaling/Suggester.java | 14 +-
.../solrj/cloud/autoscaling/VariableBase.java | 1 +
.../solrj/cloud/autoscaling/Violation.java | 10 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 2 +-
.../solr/common/ConditionalMapWriter.java | 75 ++++++
.../java/org/apache/solr/common/MapWriter.java | 6 +
.../solrj/cloud/autoscaling/TestPolicy.java | 16 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 240 +++++++++++++++++++
.../apache/solr/cloud/SolrCloudTestCase.java | 19 ++
19 files changed, 462 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 5b8ec21..b9e8e66 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -675,7 +676,6 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
public void testReadApi() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
// first trigger
@@ -693,9 +693,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
String setClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
+ " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}]" +
"}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
response = solrClient.request(req);
@@ -713,17 +711,9 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertEquals(response.get("result").toString(), "success");
String setPolicyCommand = "{'set-policy': {" +
- " 'xyz':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]," +
- " 'policy1':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]," +
- " 'policy2':[" +
- " {'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]" +
- "}}";
+ " 'xyz':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
+ " 'policy1':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
+ " 'policy2':[{'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}]}}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setPolicyCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -747,7 +737,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
List<Map> clusterPolicy = (List<Map>) response.get("cluster-policy");
assertNotNull(clusterPolicy);
- assertEquals(3, clusterPolicy.size());
+ assertEquals(2, clusterPolicy.size());
Map policies = (Map) response.get("policies");
assertNotNull(policies);
@@ -766,11 +756,11 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
for (int i = 0; i < 2; i++) {
Map node = (Map) sortedNodes.get(i);
assertNotNull(node);
- assertEquals(6, node.size());
assertNotNull(node.get("node"));
assertNotNull(node.get("cores"));
assertEquals(0d, node.get("cores"));
assertNotNull(node.get("freedisk"));
+ assertNotNull(node.get("replicas"));
assertTrue(node.get("freedisk") instanceof Double);
assertNotNull(node.get("sysLoadAvg"));
assertTrue(node.get("sysLoadAvg") instanceof Double);
@@ -800,8 +790,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
String tempClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
+ " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}"+
" ]" +
"}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, tempClusterPolicyCommand);
@@ -831,12 +820,26 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
for (Map<String, Object> violation : violations) {
assertEquals("readApiTestViolations", violation.get("collection"));
assertEquals("shard1", violation.get("shard"));
- assertEquals(2d, getObjectByPath(violation, true, "violation/delta"));
+ assertEquals(1.0d, getObjectByPath(violation, true, "violation/delta"));
assertEquals(3l, getObjectByPath(violation, true, "violation/replica/NRT"));
assertNotNull(violation.get("clause"));
}
+ log.info("Before starting new jetty ,{}", cluster.getJettySolrRunners()
+ .stream()
+ .map(jettySolrRunner -> jettySolrRunner.getNodeName()).collect(Collectors.toList()));
JettySolrRunner runner1 = cluster.startJettySolrRunner();
cluster.waitForAllNodes(30);
+ log.info("started new jetty {}", runner1.getNodeName());
+
+ response = waitForResponse(namedList -> {
+ List l = (List) Utils.getObjectByPath(namedList, false, "diagnostics/liveNodes");
+ if (l != null && l.contains(runner1.getNodeName())) return true;
+ return false;
+ },
+ createAutoScalingRequest(SolrRequest.METHOD.GET, "/diagnostics", null),
+ 200,
+ 20,
+ runner1.getNodeName() + " could not come up ");
req = createAutoScalingRequest(SolrRequest.METHOD.GET, "/suggestions", null);
response = solrClient.request(req);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
index 2a31e71..a498932 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@ -47,16 +47,19 @@ class AddReplicaSuggester extends Suggester {
//iterate through nodes and identify the least loaded
List<Violation> leastSeriousViolation = null;
Row bestNode = null;
+ double[] bestDeviation = null;
for (int i = getMatrix().size() - 1; i >= 0; i--) {
Row row = getMatrix().get(i);
if (!isNodeSuitableForReplicaAddition(row)) continue;
Row tmpRow = row.addReplica(shard.first(), shard.second(), type, strict);
- List<Violation> errs = testChangedMatrix(strict, tmpRow.session);
-
+ double[] deviation = new double[1];
+ List<Violation> errs = testChangedMatrix(strict, tmpRow.session, deviation);
if (!containsNewErrors(errs)) {
- if (isLessSerious(errs, leastSeriousViolation)) {
+ if ((errs.isEmpty() && isLessDeviant(bestDeviation, deviation)) ||//there are no violations but this is deviating less
+ isLessSerious(errs, leastSeriousViolation)) {//there are errors , but this has less serious violation
leastSeriousViolation = errs;
bestNode = tmpRow;
+ bestDeviation = deviation;
}
}
}
@@ -73,6 +76,7 @@ class AddReplicaSuggester extends Suggester {
return null;
}
+
@Override
public CollectionParams.CollectionAction getAction() {
return ADDREPLICA;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index cd9212b..e325d10 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -42,6 +42,7 @@ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.RANGE_EQUAL;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
@@ -215,7 +216,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
if (v == 0) {// higher the number of replicas , harder to satisfy
- v = Preference.compareWithTolerance((Double) this.replica.val, (Double) that.replica.val, 1);
+ Double thisVal = this.replica.val instanceof RangeVal ? ((RangeVal) this.replica.val).max.doubleValue() : (Double) this.replica.val;
+ Double thatVal = that.replica.val instanceof RangeVal ? ((RangeVal) that.replica.val).max.doubleValue() : (Double) that.replica.val;
+ v = Preference.compareWithTolerance(thisVal, thatVal, 1);
v = this.replica.op == LESS_THAN ? v : v * -1;
}
if (v == 0) v = compareTypes(this.type, that.type);
@@ -240,7 +243,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
//replica value is zero
boolean isReplicaZero() {
- return replica != null && replica.getOperand() == Operand.EQUAL &&
+ return replica != null && replica.getOperand() == Operand.EQUAL && replica.val instanceof Double &&
Preference.compareWithTolerance(0d, (Double) replica.val, 1) == 0;
}
@@ -353,7 +356,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
return operand;
}
- public List<Violation> test(Policy.Session session) {
+ public List<Violation> test(Policy.Session session, double[] deviations) {
ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator);
if (isPerCollectiontag()) {
@@ -377,6 +380,13 @@ public class Clause implements MapWriter, Comparable<Clause> {
sealedClause.getReplica().delta(replicas),
tag.varType.meta.isNodeSpecificVal() ? null : counts.getKey());
tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
+ } else {
+ if (deviations != null && sealedClause.replica.op == RANGE_EQUAL) {
+ Number actualCount = replicas.getVal(type);
+ Double realDelta = ((RangeVal) sealedClause.replica.val).realDelta(actualCount.doubleValue());
+ realDelta = this.isReplicaZero() ? -1 * realDelta : realDelta;
+ deviations[0] += Math.abs(realDelta);
+ }
}
}
}
@@ -422,7 +432,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
tagVsCount.get(row.node).increment(shards.getValue());
}
} else {
+ tagVsCount.computeIfAbsent(String.valueOf(t.getValue()), s -> new ReplicaCount());
boolean pass = sealedClause.getTag().isPass(tagVal);
+ if(!pass && !isReplicaZero()) continue;
tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
if (pass) {
tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
index fffaee2..9e778c0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@ -62,7 +62,8 @@ public class MoveReplicaSuggester extends Suggester {
if (!isNodeSuitableForReplicaAddition(targetRow)) continue;
targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first
Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node
- List<Violation> errs = testChangedMatrix(strict, srcRowModified.session);
+ double[] deviation = new double[1];
+ List<Violation> errs = testChangedMatrix(strict, srcRowModified.session, deviation);
srcRowModified.session.applyRules(); // now resort the nodes with the new values
Policy.Session tmpSession = srcRowModified.session;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
index 3292800..48dcd5e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
@@ -33,13 +33,12 @@ public class NodeVariable extends VariableBase {
if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .forceOperation(true)
.hint(Suggester.Hint.SRC_NODE, ctx.violation.node)
.hint(ctx.violation.shard.equals(ANY) ? Suggester.Hint.COLL : Suggester.Hint.COLL_SHARD,
ctx.violation.shard.equals(ANY) ? ctx.violation.coll : new Pair<>(ctx.violation.coll, ctx.violation.shard));
- ctx.addSuggestion(suggester);
+ if(ctx.addSuggestion(suggester) == null) break;
}
}
-
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index b398caf..8b0a956 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -572,7 +572,7 @@ public class Policy implements MapWriter {
setApproxValuesAndSortNodes(clusterPreferences, matrix);
for (Clause clause : expandedClauses) {
- List<Violation> errs = clause.test(this);
+ List<Violation> errs = clause.test(this, null);
violations.addAll(errs);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index f662eff..a88617c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -28,12 +28,14 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
@@ -183,30 +185,20 @@ public class PolicyHelper {
public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) {
Policy.Session session = policy.createSession(cloudManager);
List<Row> sorted = session.getSortedNodes();
- List<Violation> violations = session.getViolations();
-
- List<Preference> clusterPreferences = policy.getClusterPreferences();
-
- List<Map<String, Object>> sortedNodes = new ArrayList<>(sorted.size());
- for (Row row : sorted) {
- Map<String, Object> map = Utils.makeMap("node", row.node);
- map.put("isLive", row.isLive);
- for (Cell cell : row.getCells()) {
- for (Preference clusterPreference : clusterPreferences) {
- Policy.SortParam name = clusterPreference.getName();
- if (cell.getName().equalsIgnoreCase(name.name())) {
- map.put(name.name(), cell.getValue());
- break;
- }
- }
+ return ew -> ew.put("sortedNodes", (IteratorWriter) iw -> {
+ for (Row row : sorted) {
+ iw.add((MapWriter) ew1 -> {
+ ew1.put("node", row.node).
+ put("isLive", row.isLive);
+ for (Cell cell : row.getCells())
+ ew1.put(cell.name, cell.val,
+ (Predicate) o -> o != null && (!(o instanceof Map) || !((Map) o).isEmpty()));
+ ew1.put("replicas", row.collectionVsShardVsReplicas);
+ });
}
- sortedNodes.add(map);
- }
-
- return ew -> {
- ew.put("sortedNodes", sortedNodes);
- ew.put("violations", violations);
- };
+ }).put("liveNodes", cloudManager.getClusterStateProvider().getLiveNodes())
+ .put("violations", session.getViolations())
+ .put("config", session.getPolicy());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
index 16cfe6a..b9f6b8d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
@@ -35,8 +35,12 @@ class RangeVal implements MapWriter {
Double.compare(testVal.doubleValue(), max.doubleValue()) <= 0;
}
+ public Double realDelta(double v) {
+ if (actual != null) return v - actual.doubleValue();
+ else return delta(v);
+ }
+
public Double delta(double v) {
-// if (actual != null) return v - actual.doubleValue();
if (v >= max.doubleValue()) return v - max.doubleValue();
if (v <= min.doubleValue()) return v - min.doubleValue();
return 0d;
@@ -49,6 +53,8 @@ class RangeVal implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put("min", min).put("max", max).putIfNotNull("actual", actual);
+ ew.put("min", min)
+ .put("max", max)
+ .putIfNotNull("actual", actual);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
index 588508f..7e17df1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
@@ -33,9 +33,9 @@ class ReplicaCount implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put(Replica.Type.NRT.name(), nrt);
- ew.put(Replica.Type.PULL.name(), pull);
- ew.put(Replica.Type.TLOG.name(), tlog);
+ if (nrt > 0) ew.put(Replica.Type.NRT.name(), nrt);
+ if (pull > 0) ew.put(Replica.Type.PULL.name(), pull);
+ if (tlog > 0) ew.put(Replica.Type.TLOG.name(), tlog);
ew.put("count", total());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
index edc7269..25f12bd 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
@@ -53,7 +53,7 @@ public class ReplicaInfo implements MapWriter {
}
public ReplicaInfo(String name, String core, String coll, String shard, Replica.Type type, String node, Map<String, Object> vals) {
- if(vals==null) vals = Collections.emptyMap();
+ if (vals == null) vals = Collections.emptyMap();
this.name = name;
if (vals != null) {
this.variables.putAll(vals);
@@ -66,6 +66,21 @@ public class ReplicaInfo implements MapWriter {
this.node = node;
}
+ ReplicaInfo(Map<String, Object> map) {
+ this.name = map.keySet().iterator().next();
+ Map details = (Map) map.get(name);
+ details = Utils.getDeepCopy(details, 4);
+ this.collection = (String) details.remove("collection");
+ this.shard = (String) details.remove("shard");
+ this.core = (String) details.remove("core");
+ this.node = (String) details.remove("node_name");
+ this.isLeader = Boolean.parseBoolean((String) details.getOrDefault("leader", "false"));
+ details.remove("leader");
+ type = Replica.Type.valueOf((String) details.getOrDefault("type", "NRT"));
+ details.remove("type");
+ this.variables.putAll(details);
+ }
+
public Object clone() {
return new ReplicaInfo(name, core, collection, shard, type, node, variables);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index 9b9b60e..db0aab4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -35,6 +35,7 @@ import java.util.function.Predicate;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.common.ConditionalMapWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
@@ -70,6 +71,10 @@ public abstract class Suggester implements MapWriter {
this.session = session.copy();
}
+ boolean isLessDeviant(double[] previousBest, double[] newDeviation) {
+ if (previousBest == null) return true;
+ return newDeviation[0] < previousBest[0];
+ }
public Suggester hint(Hint hint, Object value) {
hint.validator.accept(value);
if (hint.multiValued) {
@@ -203,7 +208,9 @@ public abstract class Suggester implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put("type", violation == null ? "improvement" : "violation");
- ew.putIfNotNull("violation", violation);
+ ew.putIfNotNull("violation",
+ new ConditionalMapWriter(violation,
+ (k, v) -> !"violatingReplicas".equals(k)));
ew.put("operation", operation);
}
@@ -237,6 +244,7 @@ public abstract class Suggester implements MapWriter {
boolean containsNewErrors(List<Violation> violations) {
boolean isTxOpen = session.transaction != null && session.transaction.isOpen();
+ if (violations.size() > originalViolations.size()) return true;
for (Violation v : violations) {
//the computed value can change over time. So it's better to evaluate it in the end
if (isTxOpen && v.getClause().hasComputedValue) continue;
@@ -272,12 +280,12 @@ public abstract class Suggester implements MapWriter {
}
}
- List<Violation> testChangedMatrix(boolean strict, Policy.Session session) {
+ List<Violation> testChangedMatrix(boolean strict, Policy.Session session, double[] deviation) {
Policy.setApproxValuesAndSortNodes(session.getPolicy().clusterPreferences, session.matrix);
List<Violation> errors = new ArrayList<>();
for (Clause clause : session.expandedClauses) {
if (strict || clause.strict) {
- List<Violation> errs = clause.test(session);
+ List<Violation> errs = clause.test(session, deviation);
if (!errs.isEmpty()) {
errors.addAll(errs);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
index 8b3f1e1..4387eb1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
@@ -107,6 +107,7 @@ public class VariableBase implements Variable {
val = condition.op.readRuleValue(condition);
if (val != condition.val) return val;
}
+ if (!isRuleVal && "".equals(val) && this.varType.type != String.class) val = -1;
if (name == null) name = this.varType.tagName;
if (varType.type == Double.class) {
Double num = Clause.parseDouble(name, val);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 39b3c0b..53f7924 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Objects;
import java.util.function.Function;
+import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.Utils;
@@ -160,13 +161,20 @@ public class Violation implements MapWriter {
ew.putIfNotNull("collection", coll);
ew.putIfNotNull("shard", shard);
ew.putIfNotNull("node", node);
- ew.putIfNotNull("tagKey", String.valueOf(tagKey));
+ ew.putStringIfNotNull("tagKey", tagKey);
ew.putIfNotNull("violation", (MapWriter) ew1 -> {
if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal);
else ew1.put(clause.tag.name, String.valueOf(actualVal));
ew1.putIfNotNull("delta", replicaCountDelta);
});
ew.put("clause", getClause());
+ if (!replicaInfoAndErrs.isEmpty()) {
+ ew.put("violatingReplicas", (IteratorWriter) iw -> {
+ for (ReplicaInfoAndErr replicaInfoAndErr : replicaInfoAndErrs) {
+ iw.add(replicaInfoAndErr.replicaInfo);
+ }
+ });
+ }
}
static class Ctx {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 83fb25a..d148e21 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -94,7 +94,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return solrClient.getClusterStateProvider();
}
- private void readReplicaDetails() throws IOException {
+ protected void readReplicaDetails() throws IOException {
ClusterStateProvider clusterStateProvider = getClusterStateProvider();
ClusterState clusterState = clusterStateProvider.getClusterState();
if (clusterState == null) { // zkStateReader still initializing
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
new file mode 100644
index 0000000..9a1106f
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.IOException;
+import java.util.function.BiPredicate;
+
+public class ConditionalMapWriter implements MapWriter {
+ private final MapWriter delegate;
+ private final BiPredicate<String, Object> predicate;
+
+ public ConditionalMapWriter(MapWriter delegate, BiPredicate<String, Object> predicate) {
+ this.delegate = delegate;
+ this.predicate = predicate;
+ }
+
+ private class EntryWriterWrapper implements EntryWriter {
+ private final EntryWriter delegate;
+
+ EntryWriterWrapper(EntryWriter delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public EntryWriter put(String k, Object v) throws IOException {
+ if (predicate.test(k, v)) delegate.put(k, v);
+ return this;
+ }
+
+ @Override
+ public EntryWriter put(String k, int v) throws IOException {
+ return put(k, Integer.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, long v) throws IOException {
+ return put(k, Long.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, float v) throws IOException {
+ return put(k, Float.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, double v) throws IOException {
+ return put(k, Double.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, boolean v) throws IOException {
+ return put(k, Boolean.valueOf(v));
+ }
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ delegate.writeMap(new EntryWriterWrapper(ew));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index fc57e3d..56f3b8d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Predicate;
import org.apache.solr.common.util.Utils;
@@ -102,6 +103,11 @@ public interface MapWriter extends MapSerializable {
return this;
}
+ default EntryWriter put(String k, Object v, Predicate<Object> p) throws IOException {
+ if (p.test(v)) put(k, v);
+ return this;
+ }
+
default EntryWriter putIfNotNull(String k, Object v) throws IOException {
if(v != null) put(k,v);
return this;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 3bd58aa..19db8fc 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -2467,8 +2467,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
" }}}";
String autoScalingjson = " { cluster-policy:[" +
- " { replica :'<34%', shard: '#EACH', sysprop.az : east}," +
- " { replica :'<67%', shard: '#EACH', sysprop.az : west}" +
+ " { replica :'33%', shard: '#EACH', sysprop.az : east}," +
+ " { replica :'67%', shard: '#EACH', sysprop.az : west}" +
" ]," +
" cluster-preferences :[{ minimize : cores }]}";
@@ -2480,6 +2480,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<String> nodes = new ArrayList<>();
+ int westCount = 0, eastCount = 0;
for (int i = 0; i < 12; i++) {
SolrRequest suggestion = txn.getCurrentSession()
.getSuggester(ADDREPLICA)
@@ -2488,9 +2489,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(suggestion);
String node = suggestion.getParams().get("node");
nodes.add(node);
+ if ("10.0.0.6:8983_solr".equals(node)) eastCount++;
+ if ("10.0.0.6:7574_solr".equals(node)) westCount++;
if (i % 3 == 1) assertEquals("10.0.0.6:8983_solr", node);
else assertEquals("10.0.0.6:7574_solr", node);
}
+ assertEquals(8, westCount);
+ assertEquals(4, eastCount);
List<Violation> violations = txn.close();
assertTrue(violations.isEmpty());
@@ -2655,7 +2660,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}";
AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
- assertEquals(2, violations.size());
+ assertEquals("expected 2 violations", 2, violations.size());
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(2, suggestions.size());
for (Suggester.SuggestionInfo suggestion : suggestions) {
@@ -3562,11 +3567,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
Object root = Utils.fromJSONString(writer.toString());
assertEquals(2l,
Utils.getObjectByPath(root, true, "violations[0]/violation/replica/NRT"));
- assertEquals(0l,
- Utils.getObjectByPath(root, true, "violations[0]/violation/replica/PULL"));
- assertEquals(0l,
- Utils.getObjectByPath(root, true, "violations[0]/violation/replica/TLOG"));
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index c1d69b9..d6a5d51 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -288,4 +289,243 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
};
}
+ public void testAutoScalingHandlerFailure() {
+ String diagnostics = "{" +
+ " 'diagnostics': {" +
+ " 'sortedNodes': [" +
+ " {" +
+ " 'node': '127.0.0.1:63191_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.97510064011647," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'readApiTestViolations': {" +
+ " 'shard1': [" +
+ " {" +
+ " 'core_node5': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n2'," +
+ " 'leader': 'true'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node7': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n4'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node12': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n10'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63192_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'readApiTestViolations': {" +
+ " 'shard1': [" +
+ " {" +
+ " 'core_node3': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n1'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node9': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n6'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node11': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n8'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63219_solr'," +
+ " 'isLive': true," +
+ " 'cores': 0.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {}" +
+ " }" +
+ " ]," +
+ " 'liveNodes': [" +
+ " '127.0.0.1:63191_solr'," +
+ " '127.0.0.1:63192_solr'," +
+ " '127.0.0.1:63219_solr'" +
+ " ]," +
+ " 'violations': [" +
+ " {" +
+ " 'collection': 'readApiTestViolations'," +
+ " 'shard': 'shard1'," +
+ " 'node': '127.0.0.1:63191_solr'," +
+ " 'violation': {" +
+ " 'replica': {'NRT': 3, 'count': 3}," +
+ " 'delta': 2.0" +
+ " }," +
+ " 'clause': {" +
+ " 'replica': '<3'," +
+ " 'shard': '#EACH'," +
+ " 'node': '#ANY'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'collection': 'readApiTestViolations'," +
+ " 'shard': 'shard1'," +
+ " 'node': '127.0.0.1:63192_solr'," +
+ " 'violation': {" +
+ " 'replica': {'NRT': 3, 'count': 3}," +
+ " 'delta': 2.0" +
+ " }," +
+ " 'clause': {" +
+ " 'replica': '<2'," +
+ " 'shard': '#EACH'," +
+ " 'node': '#ANY'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]," +
+ " 'config': {" +
+ " 'cluster-preferences': [" +
+ " {'minimize': 'cores', 'precision': 3}," +
+ " {'maximize': 'freedisk', 'precision': 100}," +
+ " {'minimize': 'sysLoadAvg', 'precision': 10}," +
+ " {'minimize': 'heapUsage', 'precision': 10}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " {'cores': '<10', 'node': '#ANY'}," +
+ " {'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole': 'overseer', 'replica': 0}" +
+ " ]" +
+ " }" +
+ " }}";
+ Map<String, Object> m = (Map<String, Object>) Utils.fromJSONString(diagnostics);
+
+ Policy policy = new Policy((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+ Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config")), cloudManagerFromDiagnostics);
+ assertEquals(2, suggestions.size());
+
+ }
+
+ static SolrCloudManager createCloudManagerFromDiagnostics(Map<String, Object> m) {
+ List<Map> sortedNodes = (List<Map>) Utils.getObjectByPath(m, false, "diagnostics/sortedNodes");
+ Set<String> liveNodes = new HashSet<>();
+ SolrClientNodeStateProvider nodeStateProvider = new SolrClientNodeStateProvider(null) {
+ @Override
+ protected void readReplicaDetails() {
+ for (Object o : sortedNodes) {
+ String node = (String) ((Map) o).get("node");
+ liveNodes.add(node);
+ Map nodeDetails = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, s -> new LinkedHashMap<>());
+ Map<String, Map<String, List<Map>>> replicas = (Map<String, Map<String, List<Map>>>) ((Map) o).get("replicas");
+ replicas.forEach((coll, shardVsReplicas) -> shardVsReplicas
+ .forEach((shard, repDetails) -> {
+ List<ReplicaInfo> reps = (List) ((Map) nodeDetails
+ .computeIfAbsent(coll, o1 -> new LinkedHashMap<>()))
+ .computeIfAbsent(shard, o12 -> new ArrayList<ReplicaInfo>());
+ for (Map map : repDetails) reps.add(new ReplicaInfo(map));
+ }));
+ }
+
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return nodeVsCollectionVsShardVsReplicaInfo.get(node) == null ?
+ Collections.emptyMap() :
+ nodeVsCollectionVsShardVsReplicaInfo.get(node);
+ }
+
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ for (Map n : sortedNodes) if (n.get("node").equals(node)) return n;
+ return Collections.emptyMap();
+ }
+ };
+ return new DelegatingCloudManager(null) {
+ @Override
+ public NodeStateProvider getNodeStateProvider() {
+ return nodeStateProvider;
+ }
+
+ @Override
+ public ClusterStateProvider getClusterStateProvider() {
+ return new DelegatingClusterStateProvider(null) {
+ @Override
+ public Set<String> getLiveNodes() {
+ return liveNodes;
+ }
+ };
+ }
+ };
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index a4c3b6d..d0cc4c5 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -49,6 +50,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
import org.junit.AfterClass;
import org.junit.Before;
@@ -352,4 +354,21 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
}
}
+ protected NamedList waitForResponse(Predicate<NamedList> predicate, SolrRequest request, int intervalInMillis, int numRetries, String messageOnFail) {
+ int i = 0;
+ for (; i < numRetries; i++) {
+ try {
+ NamedList<Object> response = cluster.getSolrClient().request(request);
+ if (predicate.test(response)) return response;
+ Thread.sleep(intervalInMillis);
+ } catch (RuntimeException rte) {
+ throw rte;
+ } catch (Exception e) {
+ throw new RuntimeException("error executing request", e);
+ }
+ }
+ fail("Tried " + i + " times , could not succeed. " + messageOnFail);
+ return null;
+ }
+
}
[14/31] lucene-solr:jira/http2: SOLR-7767: "ZK Status" sub menu under
"Cloud" tab to see status of zookeeper ensemble
Posted by da...@apache.org.
SOLR-7767: "ZK Status" sub menu under "Cloud" tab to see status of zookeeper ensemble
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9306922d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9306922d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9306922d
Branch: refs/heads/jira/http2
Commit: 9306922d65a3995d5816ffd61ab59fedb5202e06
Parents: ba9b18f
Author: Jan Høydahl <ja...@apache.org>
Authored: Wed Aug 8 12:43:19 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Wed Aug 8 12:43:19 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/core/CoreContainer.java | 3 +
.../handler/admin/ZookeeperStatusHandler.java | 222 +++++++++++++++++++
.../admin/ZookeeperStatusHandlerTest.java | 85 +++++++
solr/solr-ref-guide/src/cloud-screens.adoc | 5 +
.../src/images/cloud-screens/cloud-zkstatus.png | Bin 0 -> 175359 bytes
.../apache/solr/common/params/CommonParams.java | 1 +
solr/webapp/web/css/angular/cloud.css | 71 +++++-
solr/webapp/web/css/angular/menu.css | 1 +
solr/webapp/web/index.html | 1 +
solr/webapp/web/js/angular/controllers/cloud.js | 38 +++-
solr/webapp/web/js/angular/services.js | 6 +
solr/webapp/web/partials/cloud.html | 59 +++++
13 files changed, 491 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3ede98b..7468451 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -140,6 +140,8 @@ New Features
* SOLR-8207: Add "Nodes" view to the Admin UI "Cloud" tab, listing nodes and key metrics (janhoy)
+* SOLR-7767: "ZK Status" sub menu under "Cloud" tab to see status of zookeeper ensemble (janhoy)
+
* SOLR-11990: Make it possible to co-locate replicas of multiple collections together in a node. A collection may be
co-located with another collection during collection creation time by specifying a 'withCollection' parameter. It can
also be co-located afterwards by using the modify collection API. The co-location guarantee is enforced regardless of
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 0583101..83384fb 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -85,6 +85,7 @@ import org.apache.solr.handler.admin.SecurityConfHandler;
import org.apache.solr.handler.admin.SecurityConfHandlerLocal;
import org.apache.solr.handler.admin.SecurityConfHandlerZk;
import org.apache.solr.handler.admin.ZookeeperInfoHandler;
+import org.apache.solr.handler.admin.ZookeeperStatusHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.logging.LogWatcher;
import org.apache.solr.logging.MDCLoggingContext;
@@ -120,6 +121,7 @@ import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
+import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
@@ -564,6 +566,7 @@ public class CoreContainer {
this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());
createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);
+ createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class);
collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
healthCheckHandler = createHandler(HEALTH_CHECK_HANDLER_PATH, cfg.getHealthCheckHandlerClass(), HealthCheckHandler.class);
infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
new file mode 100644
index 0000000..8842437
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.lang.invoke.MethodHandles;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Zookeeper Status handler, talks to ZK using sockets and four-letter words
+ *
+ * @since solr 7.5
+ */
+public final class ZookeeperStatusHandler extends RequestHandlerBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final int ZOOKEEPER_DEFAULT_PORT = 2181;
+ private static final String STATUS_RED = "red";
+ private static final String STATUS_GREEN = "green";
+ private static final String STATUS_YELLOW = "yellow";
+ private static final String STATUS_NA = "N/A";
+ private CoreContainer cores;
+
+ public ZookeeperStatusHandler(CoreContainer cc) {
+ this.cores = cc;
+ }
+
+ @Override
+ public String getDescription() {
+ return "Fetch Zookeeper status";
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.ADMIN;
+ }
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ final SolrParams params = req.getParams();
+ Map<String, String> map = new HashMap<>(1);
+ NamedList values = rsp.getValues();
+ values.add("zkStatus", getZkStatus(cores.getZkController().getZkServerAddress()));
+ }
+
+ /*
+ Gets all info from ZK API and returns as a map
+ */
+ protected Map<String, Object> getZkStatus(String zkHost) {
+ Map<String, Object> zkStatus = new HashMap<>();
+ List<String> zookeepers = Arrays.asList(zkHost.split("/")[0].split(","));
+ List<Object> details = new ArrayList<>();
+ int numOk = 0;
+ String status = STATUS_NA;
+ int standalone = 0;
+ int followers = 0;
+ int reportedFollowers = 0;
+ int leaders = 0;
+ List<String> errors = new ArrayList<>();
+ for (String zk : zookeepers) {
+ try {
+ Map<String, Object> stat = monitorZookeeper(zk);
+ details.add(stat);
+ if ("true".equals(String.valueOf(stat.get("ok")))) {
+ numOk++;
+ }
+ String state = String.valueOf(stat.get("zk_server_state"));
+ if ("follower".equals(state)) {
+ followers++;
+ } else if ("leader".equals(state)) {
+ leaders++;
+ reportedFollowers = Integer.parseInt(String.valueOf(stat.get("zk_followers")));
+ } else if ("standalone".equals(state)) {
+ standalone++;
+ }
+ } catch (SolrException se) {
+ log.warn("Failed talking to zookeeper" + zk, se);
+ errors.add(se.getMessage());
+ Map<String, Object> stat = new HashMap<>();
+ stat.put("host", zk);
+ stat.put("ok", false);
+ details.add(stat);
+ }
+ }
+ zkStatus.put("ensembleSize", zookeepers.size());
+ zkStatus.put("zkHost", zkHost);
+ zkStatus.put("details", details);
+ if (followers+leaders > 0 && standalone > 0) {
+ status = STATUS_RED;
+ errors.add("The zk nodes do not agree on their mode, check details");
+ }
+ if (standalone > 1) {
+ status = STATUS_RED;
+ errors.add("Only one zk allowed in standalone mode");
+ }
+ if (leaders > 1) {
+ zkStatus.put("mode", "ensemble");
+ status = STATUS_RED;
+ errors.add("Only one leader allowed, got " + leaders);
+ }
+ if (followers > 0 && leaders == 0) {
+ zkStatus.put("mode", "ensemble");
+ status = STATUS_RED;
+ errors.add("We do not have a leader");
+ }
+ if (leaders > 0 && followers != reportedFollowers) {
+ zkStatus.put("mode", "ensemble");
+ status = STATUS_RED;
+ errors.add("Leader reports " + reportedFollowers + " followers, but we only found " + followers +
+ ". Please check zkHost configuration");
+ }
+ if (followers+leaders == 0 && standalone == 1) {
+ zkStatus.put("mode", "standalone");
+ }
+ if (followers+leaders > 0 && (zookeepers.size())%2 == 0) {
+ if (!STATUS_RED.equals(status)) {
+ status = STATUS_YELLOW;
+ }
+ errors.add("We have an even number of zookeepers which is not recommended");
+ }
+ if (followers+leaders > 0 && standalone == 0) {
+ zkStatus.put("mode", "ensemble");
+ }
+ if (status.equals(STATUS_NA)) {
+ if (numOk == zookeepers.size()) {
+ status = STATUS_GREEN;
+ } else if (numOk < zookeepers.size() && numOk > zookeepers.size() / 2) {
+ status = STATUS_YELLOW;
+ errors.add("Some zookeepers are down: " + numOk + "/" + zookeepers.size());
+ } else {
+ status = STATUS_RED;
+ errors.add("Mismatch in number of zookeeper nodes live. numOK=" + numOk + ", expected " + zookeepers.size());
+ }
+ }
+ zkStatus.put("status", status);
+ if (!errors.isEmpty()) {
+ zkStatus.put("errors", errors);
+ }
+ return zkStatus;
+ }
+
+ private Map<String, Object> monitorZookeeper(String zkHostPort) {
+ List<String> lines = getZkRawResponse(zkHostPort, "mntr");
+ Map<String, Object> obj = new HashMap<>();
+ obj.put("host", zkHostPort);
+ obj.put("ok", "imok".equals(getZkRawResponse(zkHostPort, "ruok").get(0)));
+ for (String line : lines) {
+ obj.put(line.split("\t")[0], line.split("\t")[1]);
+ }
+ lines = getZkRawResponse(zkHostPort, "conf");
+ for (String line : lines) {
+ obj.put(line.split("=")[0], line.split("=")[1]);
+ }
+ return obj;
+ }
+
+ /**
+ * Sends a four-letter-word command to one particular Zookeeper server and returns the response as list of strings
+ * @param zkHostPort the host:port for one zookeeper server to access
+ * @param fourLetterWordCommand the custom 4-letter command to send to Zookeeper
+ * @return a list of lines returned from Zookeeper
+ */
+ private List<String> getZkRawResponse(String zkHostPort, String fourLetterWordCommand) {
+ String[] hostPort = zkHostPort.split(":");
+ String host = hostPort[0];
+ int port = ZOOKEEPER_DEFAULT_PORT;
+ if (hostPort.length > 1) {
+ port = Integer.parseInt(hostPort[1]);
+ }
+ try (
+ Socket socket = new Socket(host, port);
+ Writer writer = new OutputStreamWriter(socket.getOutputStream(), "utf-8");
+ PrintWriter out = new PrintWriter(writer, true);
+ BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream(), "utf-8"));) {
+ out.println(fourLetterWordCommand);
+ List<String> response = in.lines().collect(Collectors.toList());
+ log.debug("Got response from ZK on host {} and port {}: {}", host, port, response);
+ if (response == null || response.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Empty response from Zookeeper " + zkHostPort);
+ }
+ return response;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed talking to Zookeeper " + zkHostPort, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
new file mode 100644
index 0000000..7ec8bf2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.admin;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.response.DelegationTokenResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ZookeeperStatusHandlerTest extends SolrCloudTestCase {
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(1)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /*
+ Test the monitoring endpoint, used in the Cloud => ZkStatus Admin UI screen
+ NOTE: We do not currently test with multiple zookeepers, but the only difference is that there are multiple "details" objects and mode is "ensemble"...
+ */
+ @Test
+ public void monitorZookeeper() throws IOException, SolrServerException, InterruptedException, ExecutionException, TimeoutException {
+ URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl();
+ HttpSolrClient solr = new HttpSolrClient.Builder(baseUrl.toString()).build();
+ GenericSolrRequest mntrReq = new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/zookeeper/status", new ModifiableSolrParams());
+ mntrReq.setResponseParser(new DelegationTokenResponse.JsonMapResponseParser());
+ NamedList<Object> nl = solr.httpUriRequest(mntrReq).future.get(1000, TimeUnit.MILLISECONDS);
+
+ assertEquals("zkStatus", nl.getName(1));
+ Map<String,Object> zkStatus = (Map<String,Object>) nl.get("zkStatus");
+ assertEquals("green", zkStatus.get("status"));
+ assertEquals("standalone", zkStatus.get("mode"));
+ assertEquals(1L, zkStatus.get("ensembleSize"));
+ List<Object> detailsList = (List<Object>)zkStatus.get("details");
+ assertEquals(1, detailsList.size());
+ Map<String,Object> details = (Map<String,Object>) detailsList.get(0);
+ assertEquals(true, details.get("ok"));
+ assertTrue(Integer.parseInt((String) details.get("zk_znode_count")) > 50);
+ solr.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/solr-ref-guide/src/cloud-screens.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/cloud-screens.adoc b/solr/solr-ref-guide/src/cloud-screens.adoc
index 34982ee..cc5a5eb 100644
--- a/solr/solr-ref-guide/src/cloud-screens.adoc
+++ b/solr/solr-ref-guide/src/cloud-screens.adoc
@@ -42,6 +42,11 @@ image::images/cloud-screens/cloud-tree.png[image,width=487,height=250]
As an aid to debugging, the data shown in the "Tree" view can be exported locally using the following command `bin/solr zk ls -r /`
+== ZK Status view
+The "ZK Status" view gives an overview over the Zookeepers used by Solr. It lists whether running in `standalone` or `ensemble` mode, shows how many zookeepers are configured, and then displays a table listing detailed monitoring status for each of the zookeepers, inlcuding who is the leader, configuration parameters and more.
+
+image::images/cloud-screens/cloud-zkstatus.png[image,width=512,height=509]
+
== Graph views
The "Graph" view shows a graph of each collection, the shards that make up those collections, and the addresses and type ("NRT", "TLOG" or "PULL") of each replica for each shard.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png b/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png
new file mode 100644
index 0000000..b1b7452
Binary files /dev/null and b/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index 63a763c..798410d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -182,6 +182,7 @@ public interface CommonParams {
String AUTHZ_PATH = "/admin/authorization";
String AUTHC_PATH = "/admin/authentication";
String ZK_PATH = "/admin/zookeeper";
+ String ZK_STATUS_PATH = "/admin/zookeeper/status";
String SYSTEM_INFO_PATH = "/admin/info/system";
String METRICS_PATH = "/admin/metrics";
String METRICS_HISTORY_PATH = "/admin/metrics/history";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/css/angular/cloud.css
----------------------------------------------------------------------
diff --git a/solr/webapp/web/css/angular/cloud.css b/solr/webapp/web/css/angular/cloud.css
index c3d54a5..5c8ce45 100644
--- a/solr/webapp/web/css/angular/cloud.css
+++ b/solr/webapp/web/css/angular/cloud.css
@@ -536,7 +536,8 @@ limitations under the License.
}
/* Styling of reload and details buttons */
-#content #cloud #controls
+#content #cloud #controls,
+#content #cloud #frame #zk-status-content #zk-controls
{
display: block;
height: 30px;
@@ -641,4 +642,70 @@ limitations under the License.
border-radius: 4px;
background-color: rgba(0,0,0,.5);
-webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
-}
\ No newline at end of file
+}
+#content #cloud #zk-table td,
+#content #cloud #zk-table th
+{
+ border: 0px solid #ddd;
+ border-bottom: 0.50px solid #eee;
+ padding-right: 5px;
+ padding-left: 5px;
+}
+
+#content #cloud #zk-table th
+{
+ border-bottom: 1px solid #ddd;
+ border-top: 1px solid #ddd;
+ font-weight: bolder;
+ font-stretch: extra-expanded;
+ background: #F8F8F8;
+}
+
+#content #cloud #zk-table
+{
+ border-top: 1px solid #c0c0c0;
+ margin-top: 10px;
+ border-collapse: collapse;
+
+ font-weight: bold;
+}
+
+#content #cloud #zk-table #detail-divider
+{
+ background-color: #f8f8f8;
+ height: 10px;
+}
+
+.zookeeper-status
+{
+ font-size: large;
+}
+
+.zookeeper-errors
+{
+ background-color: lightpink;
+ padding: 10px;
+ border: 1px;
+ margin-top: 10px;
+ margin-bottom: 10px;
+}
+
+.zookeeper-errors li::before
+{
+ content: "- ";
+}
+
+.zkstatus-green
+{
+ color: darkgreen;
+}
+
+.zkstatus-yellow
+{
+ color: orange;
+}
+
+.zkstatus-red
+{
+ color: red;
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/css/angular/menu.css
----------------------------------------------------------------------
diff --git a/solr/webapp/web/css/angular/menu.css b/solr/webapp/web/css/angular/menu.css
index ba5e0b6..f4e04c1 100644
--- a/solr/webapp/web/css/angular/menu.css
+++ b/solr/webapp/web/css/angular/menu.css
@@ -261,6 +261,7 @@ limitations under the License.
#menu #cloud.global p a { background-image: url( ../../img/ico/network-cloud.png ); }
#menu #cloud.global .tree a { background-image: url( ../../img/ico/folder-tree.png ); }
#menu #cloud.global .nodes a { background-image: url( ../../img/solr-ico.png ); }
+#menu #cloud.global .zkstatus a { background-image: url( ../../img/ico/node-master.png ); }
#menu #cloud.global .graph a { background-image: url( ../../img/ico/molecule.png ); }
#menu #cloud.global .rgraph a { background-image: url( ../../img/ico/asterisk.png ); }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/index.html
----------------------------------------------------------------------
diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html
index 256af89..0663805 100644
--- a/solr/webapp/web/index.html
+++ b/solr/webapp/web/index.html
@@ -152,6 +152,7 @@ limitations under the License.
<ul ng-show="showingCloud">
<li class="nodes" ng-class="{active:page=='cloud-nodes'}"><a href="#/~cloud?view=nodes">Nodes</a></li>
<li class="tree" ng-class="{active:page=='cloud-tree'}"><a href="#/~cloud?view=tree">Tree</a></li>
+ <li class="zkstatus" ng-class="{active:page=='cloud-zkstatus'}"><a href="#/~cloud?view=zkstatus">ZK Status</a></li>
<li class="graph" ng-class="{active:page=='cloud-graph'}"><a href="#/~cloud?view=graph">Graph</a></li>
<li class="rgraph" ng-class="{active:page=='cloud-rgraph'}"><a href="#/~cloud?view=rgraph">Graph (Radial)</a></li>
</ul>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/js/angular/controllers/cloud.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js
index a033ce5..8cbd4db 100644
--- a/solr/webapp/web/js/angular/controllers/cloud.js
+++ b/solr/webapp/web/js/angular/controllers/cloud.js
@@ -16,7 +16,7 @@
*/
solrAdminApp.controller('CloudController',
- function($scope, $location, Zookeeper, Constants, Collections, System, Metrics) {
+ function($scope, $location, Zookeeper, Constants, Collections, System, Metrics, ZookeeperStatus) {
$scope.showDebug = false;
@@ -41,6 +41,9 @@ solrAdminApp.controller('CloudController',
} else if (view === "nodes") {
$scope.resetMenu("cloud-nodes", Constants.IS_ROOT_PAGE);
nodesSubController($scope, Collections, System, Metrics);
+ } else if (view === "zkstatus") {
+ $scope.resetMenu("cloud-zkstatus", Constants.IS_ROOT_PAGE);
+ zkStatusSubController($scope, ZookeeperStatus, false);
}
}
);
@@ -486,7 +489,39 @@ var nodesSubController = function($scope, Collections, System, Metrics) {
$scope.initClusterState();
};
+var zkStatusSubController = function($scope, ZookeeperStatus) {
+ $scope.showZkStatus = true;
+ $scope.showNodes = false;
+ $scope.showTree = false;
+ $scope.showGraph = false;
+ $scope.tree = {};
+ $scope.showData = false;
+ $scope.showDetails = false;
+
+ $scope.toggleDetails = function() {
+ $scope.showDetails = !$scope.showDetails === true;
+ };
+
+ $scope.initZookeeper = function() {
+ ZookeeperStatus.monitor({}, function(data) {
+ $scope.zkState = data.zkStatus;
+ $scope.mainKeys = ["ok", "clientPort", "zk_server_state", "zk_version",
+ "zk_approximate_data_size", "zk_znode_count", "zk_num_alive_connections"];
+ $scope.detailKeys = ["dataDir", "dataLogDir",
+ "zk_avg_latency", "zk_max_file_descriptor_count", "zk_watch_count",
+ "zk_packets_sent", "zk_packets_received",
+ "tickTime", "maxClientCnxns", "minSessionTimeout", "maxSessionTimeout"];
+ $scope.ensembleMainKeys = ["serverId", "electionPort", "quorumPort"];
+ $scope.ensembleDetailKeys = ["peerType", "electionAlg", "initLimit", "syncLimit",
+ "zk_followers", "zk_synced_followers", "zk_pending_syncs"];
+ });
+ };
+
+ $scope.initZookeeper();
+};
+
var treeSubController = function($scope, Zookeeper) {
+ $scope.showZkStatus = false;
$scope.showTree = true;
$scope.showGraph = false;
$scope.tree = {};
@@ -545,6 +580,7 @@ function secondsForHumans ( seconds ) {
}
var graphSubController = function ($scope, Zookeeper, isRadial) {
+ $scope.showZkStatus = false;
$scope.showTree = false;
$scope.showGraph = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/js/angular/services.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js
index 66f2654..e3dcd3f 100644
--- a/solr/webapp/web/js/angular/services.js
+++ b/solr/webapp/web/js/angular/services.js
@@ -81,6 +81,12 @@ solrAdminServices.factory('System',
}}
});
}])
+.factory('ZookeeperStatus',
+ ['$resource', function($resource) {
+ return $resource('admin/zookeeper/status', {wt:'json', _:Date.now()}, {
+ "monitor": {}
+ });
+ }])
.factory('Properties',
['$resource', function($resource) {
return $resource('admin/info/properties', {'wt':'json', '_':Date.now()});
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/partials/cloud.html
----------------------------------------------------------------------
diff --git a/solr/webapp/web/partials/cloud.html b/solr/webapp/web/partials/cloud.html
index 078c9af..825e195 100644
--- a/solr/webapp/web/partials/cloud.html
+++ b/solr/webapp/web/partials/cloud.html
@@ -18,6 +18,65 @@ limitations under the License.
<div id="frame">
+ <div id="zk-status-content" class="content clearfix" ng-show="showZkStatus">
+ <div id="zk-controls">
+ <a class="reload" ng-click="initZookeeper()"><span>Refresh</span></a>
+ <a class="details-button" ng-click="toggleDetails()" ng-class="{on:showDetails}">
+ <span>Toggle details</span>
+ </a>
+ </div>
+
+ <div class="zookeeper-status">Status: <span class="zkstatus-{{zkState.status}}">{{zkState.status}}</span></div>
+ <div class="zookeeper-errors" ng-show="zkState.errors">
+ Errors:
+ <ul>
+ <li ng-repeat="error in zkState.errors">{{error}}</li>
+ </ul>
+ </div>
+ <div>ZK connection string: {{zkState.zkHost}}</div>
+ <div>Ensemble size: {{zkState.ensembleSize}}</div>
+ <div>Ensemble mode: {{zkState.mode}}</div>
+
+ <table id="zk-table">
+ <thead>
+ <tr>
+ <th></th>
+ <th ng-repeat="host in zkState.details">{{host.host}}</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="key in mainKeys">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details" ng-style="key === 'zk_server_state' && host[key] === 'leader' ? {'font-weight': 'bold'} : {'font-weight': 'normal'}">
+ {{key === 'zk_version' ? host[key].split("-")[0] : host[key]}}
+ </td>
+ </tr>
+ <tr ng-repeat="key in ensembleMainKeys" ng-show="zkState.mode === 'ensemble'">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details">
+ {{host[key]}}
+ </td>
+ </tr>
+ <tr id="detail-divider" ng-show="showDetails" >
+ <td ng-class="details"></td>
+ <td ng-repeat="host in zkState.details" ng-class="details"></td>
+ </tr>
+ <tr ng-repeat="key in detailKeys" ng-show="showDetails">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details">
+ {{host[key]}}
+ </td>
+ </tr>
+ <tr ng-repeat="key in ensembleDetailKeys" ng-show="showDetails && zkState.mode === 'ensemble'">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details">
+ {{host[key]}}
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+
<div id="tree-content" class="content clearfix" ng-show="showTree">
<jstree class="tree" on-select="showTreeLink(url)" id="tree" data="tree"></jstree>
[28/31] lucene-solr:jira/http2: SOLR-11585: Solr SQL does not work
with point numeric fields
Posted by da...@apache.org.
SOLR-11585: Solr SQL does not work with point numeric fields
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/71310a47
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/71310a47
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/71310a47
Branch: refs/heads/jira/http2
Commit: 71310a4722f3e95a0e10a4ca4297d090ace3efd4
Parents: 00aeb64
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Aug 9 19:57:24 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Aug 9 19:57:50 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/handler/sql/SolrSchema.java | 4 +
.../solr/collection1/conf/schema-sql.xml | 27 +++
.../org/apache/solr/handler/TestSQLHandler.java | 243 ++++++++++++++++---
3 files changed, 246 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71310a47/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
index f7318cf..c4ef72c 100644
--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java
@@ -108,12 +108,16 @@ class SolrSchema extends AbstractSchema {
case "tlong":
case "int":
case "long":
+ case "pint":
+ case "plong":
type = typeFactory.createJavaType(Long.class);
break;
case "tfloat":
case "tdouble":
case "float":
case "double":
+ case "pfloat":
+ case "pdouble":
type = typeFactory.createJavaType(Double.class);
break;
default:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71310a47/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
index 430d966..40bbe5a 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
@@ -52,6 +52,14 @@
<fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
<fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="8" omitNorms="true" positionIncrementGap="0"/>
+ <!-- Point Fields -->
+ <fieldType name="pint" class="solr.IntPointField" docValues="true"/>
+ <fieldType name="plong" class="solr.LongPointField" docValues="true"/>
+ <fieldType name="pdouble" class="solr.DoublePointField" docValues="true"/>
+ <fieldType name="pfloat" class="solr.FloatPointField" docValues="true"/>
+ <fieldType name="pdate" class="solr.DatePointField" docValues="true"/>
+
+
<!-- numeric field types that manipulate the value into
a string value that isn't human readable in it's internal form,
but sorts correctly and supports range queries.
@@ -605,6 +613,25 @@
<dynamicField name="*_mfacet" type="string" indexed="true" stored="false" multiValued="true"/>
+
+ <!-- Test point fields explicitly -->
+ <dynamicField name="*_i_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_is_p" type="pint" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_i_ni_p" type="pint" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_is_ni_p" type="pint" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_l_p" type="plong" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ls_p" type="plong" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_l_ni_p" type="plong" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ls_ni_p" type="plong" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_f_p" type="pfloat" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_fs_p" type="pfloat" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_f_ni_p" type="pfloat" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_fs_ni_p" type="pfloat" indexed="false" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_d_p" type="pdouble" indexed="true" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ds_p" type="pdouble" indexed="true" stored="true" docValues="true" multiValued="true"/>
+ <dynamicField name="*_d_ni_p" type="pdouble" indexed="false" stored="true" docValues="true" multiValued="false"/>
+ <dynamicField name="*_ds_ni_p" type="pdouble" indexed="false" stored="true" docValues="true" multiValued="true"/>
+
<!-- make sure custom sims work with dynamic fields -->
<!--
<dynamicField name="*_sim1" type="sim1" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71310a47/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
index 8315868..b860d6e 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
@@ -22,8 +22,8 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
+import java.util.Random;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
@@ -78,7 +78,7 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
@Test
//28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ //@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void doTest() throws Exception {
waitForRecoveriesToFinish(false);
@@ -87,6 +87,8 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
testMixedCaseFields();
testBasicGrouping();
testBasicGroupingTint();
+ testBasicGroupingIntLongPoints();
+ testBasicGroupingFloatDoublePoints();
testBasicGroupingFacets();
testSelectDistinct();
testSelectDistinctFacets();
@@ -107,21 +109,29 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
commit();
- indexDoc(sdoc("id", "1", "text", "XXXX XXXX", "str_s", "a", "field_i", "7"));
- indexDoc(sdoc("id", "2", "text", "XXXX XXXX", "str_s", "b", "field_i", "8"));
- indexDoc(sdoc("id", "3", "text", "XXXX XXXX", "str_s", "a", "field_i", "20"));
- indexDoc(sdoc("id", "4", "text", "XXXX XXXX", "str_s", "b", "field_i", "11"));
- indexDoc(sdoc("id", "5", "text", "XXXX XXXX", "str_s", "c", "field_i", "30"));
- indexDoc(sdoc("id", "6", "text", "XXXX XXXX", "str_s", "c", "field_i", "40"));
- indexDoc(sdoc("id", "7", "text", "XXXX XXXX", "str_s", "c", "field_i", "50"));
- indexDoc(sdoc("id", "8", "text", "XXXX XXXX", "str_s", "c", "field_i", "60"));
+ indexDoc(sdoc("id", "1", "text", "XXXX XXXX", "str_s", "a", "field_i", "7", "field_i_p", "7",
+ "field_f_p", "7.5", "field_d_p", "7.5", "field_l_p", "7"));
+ indexDoc(sdoc("id", "2", "text", "XXXX XXXX", "str_s", "b", "field_i", "8", "field_i_p", "8",
+ "field_f_p", "8.5", "field_d_p", "8.5","field_l_p", "8" ));
+ indexDoc(sdoc("id", "3", "text", "XXXX XXXX", "str_s", "a", "field_i", "20", "field_i_p", "20",
+ "field_f_p", "20.5", "field_d_p", "20.5", "field_l_p", "20"));
+ indexDoc(sdoc("id", "4", "text", "XXXX XXXX", "str_s", "b", "field_i", "11", "field_i_p", "11",
+ "field_f_p", "11.5", "field_d_p", "11.5", "field_l_p", "11"));
+ indexDoc(sdoc("id", "5", "text", "XXXX XXXX", "str_s", "c", "field_i", "30","field_i_p", "30", "" +
+ "field_f_p", "30.5", "field_d_p", "30.5", "field_l_p", "30"));
+ indexDoc(sdoc("id", "6", "text", "XXXX XXXX", "str_s", "c", "field_i", "40", "field_i_p", "40",
+ "field_f_p", "40.5", "field_d_p", "40.5", "field_l_p", "40"));
+ indexDoc(sdoc("id", "7", "text", "XXXX XXXX", "str_s", "c", "field_i", "50", "field_i_p", "50",
+ "field_f_p", "50.5", "field_d_p", "50.5", "field_l_p", "50"));
+ indexDoc(sdoc("id", "8", "text", "XXXX XXXX", "str_s", "c", "field_i", "60", "field_i_p", "60",
+ "field_f_p", "60.5", "field_d_p", "60.5", "field_l_p", "60"));
commit();
System.out.println("############# testBasicSelect() ############");
SolrParams sParams = mapParams(CommonParams.QT, "/sql",
- "stmt", "select id, field_i, str_s from collection1 where (text='(XXXX)' OR text='XXXX') AND text='XXXX' order by field_i desc");
+ "stmt", "select id, field_i, str_s, field_i_p, field_f_p, field_d_p, field_l_p from collection1 where (text='(XXXX)' OR text='XXXX') AND text='XXXX' order by field_i desc");
SolrStream solrStream = new SolrStream(jetty.url, sParams);
List<Tuple> tuples = getTuples(solrStream);
@@ -130,47 +140,103 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
Tuple tuple;
tuple = tuples.get(0);
- assert(tuple.getLong("id") == 8);
- assert(tuple.getLong("field_i") == 60);
+ assertEquals(tuple.getLong("id").longValue(),8);
+ assertEquals(tuple.getLong("field_i").longValue(), 60);
assert(tuple.get("str_s").equals("c"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 60L);
+ assertEquals(tuple.getDouble("field_f_p"), 60.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 60.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 60);
+
+
+
tuple = tuples.get(1);
- assert(tuple.getLong("id") == 7);
- assert(tuple.getLong("field_i") == 50);
+ assertEquals(tuple.getLong("id").longValue(), 7);
+ assertEquals(tuple.getLong("field_i").longValue(), 50);
assert(tuple.get("str_s").equals("c"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 50);
+ assertEquals(tuple.getDouble("field_f_p"), 50.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 50.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 50);
+
+
+
tuple = tuples.get(2);
- assert(tuple.getLong("id") == 6);
- assert(tuple.getLong("field_i") == 40);
+ assertEquals(tuple.getLong("id").longValue(),6);
+ assertEquals(tuple.getLong("field_i").longValue(), 40);
assert(tuple.get("str_s").equals("c"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 40);
+ assertEquals(tuple.getDouble("field_f_p"), 40.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 40.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 40);
+
+
+
+
tuple = tuples.get(3);
- assert(tuple.getLong("id") == 5);
- assert(tuple.getLong("field_i") == 30);
+ assertEquals(tuple.getLong("id").longValue(), 5);
+ assertEquals(tuple.getLong("field_i").longValue(), 30);
assert(tuple.get("str_s").equals("c"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 30);
+ assertEquals(tuple.getDouble("field_f_p"), 30.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 30.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 30);
+
+
+
tuple = tuples.get(4);
- assert(tuple.getLong("id") == 3);
- assert(tuple.getLong("field_i") == 20);
+ assertEquals(tuple.getLong("id").longValue(),3);
+ assertEquals(tuple.getLong("field_i").longValue(), 20);
assert(tuple.get("str_s").equals("a"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 20);
+ assertEquals(tuple.getDouble("field_f_p"), 20.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 20.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 20);
+
+
+
tuple = tuples.get(5);
- assert(tuple.getLong("id") == 4);
- assert(tuple.getLong("field_i") == 11);
+ assertEquals(tuple.getLong("id").longValue(), 4);
+ assertEquals(tuple.getLong("field_i").longValue(), 11);
assert(tuple.get("str_s").equals("b"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 11);
+ assertEquals(tuple.getDouble("field_f_p"), 11.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 11.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 11);
+
+
+
tuple = tuples.get(6);
- assert(tuple.getLong("id") == 2);
- assert(tuple.getLong("field_i") == 8);
+ assertEquals(tuple.getLong("id").longValue(), 2);
+ assertEquals(tuple.getLong("field_i").longValue(), 8);
assert(tuple.get("str_s").equals("b"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 8);
+ assertEquals(tuple.getDouble("field_f_p"), 8.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 8.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 8);
+
+
+
tuple = tuples.get(7);
- assert(tuple.getLong("id") == 1);
- assert(tuple.getLong("field_i") == 7);
+ assertEquals(tuple.getLong("id").longValue(), 1);
+ assertEquals(tuple.getLong("field_i").longValue(), 7);
assert(tuple.get("str_s").equals("a"));
+ assertEquals(tuple.getLong("field_i_p").longValue(), 7);
+ assertEquals(tuple.getDouble("field_f_p"), 7.5, 0.0);
+ assertEquals(tuple.getDouble("field_d_p"), 7.5, 0.0);
+ assertEquals(tuple.getLong("field_l_p").longValue(), 7);
+
+
//Assert field order
- assertResponseContains(clients.get(0), sParams, "{\"docs\":[{\"id\":\"8\",\"field_i\":60,\"str_s\":\"c\"}");
+ assertResponseContains(clients.get(0), sParams, "{\"docs\":[{\"id\":\"8\",\"field_i\":60,\"str_s\":\"c\",\"field_i_p\":60,\"field_f_p\":60.5,\"field_d_p\":60.5,\"field_l_p\":60}");
//Test unlimited unsorted result. Should sort on _version_ desc
sParams = mapParams(CommonParams.QT, "/sql", "stmt", "select id, field_i, str_s from collection1 where text='XXXX'");
@@ -939,6 +1005,127 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
}
}
+ private void testBasicGroupingIntLongPoints() throws Exception {
+ try {
+
+ Random random = random();
+ int r = random.nextInt(2);
+ String[] intOrLong = {"field_i_p", "field_l_p"};
+ String[] facetOrMap = {"facet", "map_reduce"};
+ String field = intOrLong[r];
+ r = random.nextInt(2);
+ String mode = facetOrMap[r];
+ CloudJettyRunner jetty = this.cloudJettys.get(0);
+
+ del("*:*");
+
+ commit();
+
+ indexr("id", "1", "text", "XXXX XXXX", "str_s", "a", field, "7");
+ indexr("id", "2", "text", "XXXX XXXX", "str_s", "b", field, "8");
+ indexr("id", "3", "text", "XXXX XXXX", "str_s", "a", field, "20");
+ indexr("id", "4", "text", "XXXX XXXX", "str_s", "b", field, "11");
+ indexr("id", "5", "text", "XXXX XXXX", "str_s", "c", field, "30");
+ indexr("id", "6", "text", "XXXX XXXX", "str_s", "c", field, "40");
+ indexr("id", "7", "text", "XXXX XXXX", "str_s", "c", field, "50");
+ indexr("id", "8", "text", "XXXX XXXX", "str_s", "c", field, "60");
+ indexr("id", "9", "text", "XXXX XXXY", "str_s", "d", field, "70");
+ commit();
+
+ SolrParams sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", mode,
+ "stmt", "select str_s, count(*), sum("+field+"), min("+field+"), max("+field+"), avg("+field+") from collection1 where text='XXXX' group by str_s order by sum("+field+") asc limit 2");
+
+ SolrStream solrStream = new SolrStream(jetty.url, sParams);
+ List<Tuple> tuples = getTuples(solrStream);
+
+ //Only two results because of the limit.
+ assert(tuples.size() == 2);
+ Tuple tuple;
+
+ tuple = tuples.get(0);
+ assert(tuple.get("str_s").equals("b"));
+ assert(tuple.getDouble("EXPR$1") == 2); //count(*)
+ assert(tuple.getDouble("EXPR$2") == 19); //sum(field_i)
+ assert(tuple.getDouble("EXPR$3") == 8); //min(field_i)
+ assert(tuple.getDouble("EXPR$4") == 11); //max(field_i)
+ assert(tuple.getDouble("EXPR$5") == 10); //avg(field_i)
+
+ tuple = tuples.get(1);
+ assert(tuple.get("str_s").equals("a"));
+ assert(tuple.getDouble("EXPR$1") == 2); //count(*)
+ assert(tuple.getDouble("EXPR$2") == 27); //sum(field_i)
+ assert(tuple.getDouble("EXPR$3") == 7); //min(field_i)
+ assert(tuple.getDouble("EXPR$4") == 20); //max(field_i)
+ assert(tuple.getDouble("EXPR$5") == 14); //avg(field_i)
+
+
+
+ } finally {
+ delete();
+ }
+ }
+
+ private void testBasicGroupingFloatDoublePoints() throws Exception {
+ try {
+
+ Random random = random();
+ int r = random.nextInt(2);
+ String[] intOrLong = {"field_f_p", "field_d_p"};
+ String[] facetOrMap = {"facet", "map_reduce"};
+ String field = intOrLong[r];
+ r = random.nextInt(2);
+ String mode = facetOrMap[r];
+
+ CloudJettyRunner jetty = this.cloudJettys.get(0);
+
+ del("*:*");
+
+ commit();
+
+ indexr("id", "1", "text", "XXXX XXXX", "str_s", "a", field, "7.0");
+ indexr("id", "2", "text", "XXXX XXXX", "str_s", "b", field, "8.0");
+ indexr("id", "3", "text", "XXXX XXXX", "str_s", "a", field, "20.0");
+ indexr("id", "4", "text", "XXXX XXXX", "str_s", "b", field, "11.0");
+ indexr("id", "5", "text", "XXXX XXXX", "str_s", "c", field, "30.0");
+ indexr("id", "6", "text", "XXXX XXXX", "str_s", "c", field, "40.0");
+ indexr("id", "7", "text", "XXXX XXXX", "str_s", "c", field, "50.0");
+ indexr("id", "8", "text", "XXXX XXXX", "str_s", "c", field, "60.0");
+ indexr("id", "9", "text", "XXXX XXXY", "str_s", "d", field, "70.0");
+ commit();
+
+ SolrParams sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", mode,
+ "stmt", "select str_s, count(*), sum("+field+"), min("+field+"), max("+field+"), avg("+field+") from collection1 where text='XXXX' group by str_s order by sum("+field+") asc limit 2");
+
+ SolrStream solrStream = new SolrStream(jetty.url, sParams);
+ List<Tuple> tuples = getTuples(solrStream);
+
+ //Only two results because of the limit.
+ assert(tuples.size() == 2);
+ Tuple tuple;
+
+ tuple = tuples.get(0);
+ assert(tuple.get("str_s").equals("b"));
+ assertEquals(tuple.getDouble("EXPR$1"), 2, 0.0); //count(*)
+ assertEquals(tuple.getDouble("EXPR$2"), 19, 0.0); //sum(field_i)
+ assertEquals(tuple.getDouble("EXPR$3"), 8, 0.0); //min(field_i)
+ assertEquals(tuple.getDouble("EXPR$4"), 11, 0.0); //max(field_i)
+ assertEquals(tuple.getDouble("EXPR$5"), 9.5, 0.0); //avg(field_i)
+
+ tuple = tuples.get(1);
+ assert(tuple.get("str_s").equals("a"));
+ assertEquals(tuple.getDouble("EXPR$1"), 2, 0.0); //count(*)
+ assertEquals(tuple.getDouble("EXPR$2"), 27, 0.0); //sum(field_i)
+ assertEquals(tuple.getDouble("EXPR$3"), 7, 0.0); //min(field_i)
+ assertEquals(tuple.getDouble("EXPR$4"), 20, 0.0); //max(field_i)
+ assertEquals(tuple.getDouble("EXPR$5"), 13.5, 0.0); //avg(field_i)
+
+
+
+ } finally {
+ delete();
+ }
+ }
+
private void testSelectDistinctFacets() throws Exception {
try {
CloudJettyRunner jetty = this.cloudJettys.get(0);
[11/31] lucene-solr:jira/http2: SOLR-12586: Change
ParseDateFieldUpdateProcessorFactory to use java.time.DateTimeFormatter,
not Joda Time. Note: slightly different pattern language! Remove Joda Time.
Closes #428
Posted by da...@apache.org.
SOLR-12586: Change ParseDateFieldUpdateProcessorFactory to use java.time.DateTimeFormatter, not Joda Time.
Note: slightly different pattern language!
Remove Joda Time.
Closes #428
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c3887b35
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c3887b35
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c3887b35
Branch: refs/heads/jira/http2
Commit: c3887b351e77ab6d23ddd662564b3d0d977895e4
Parents: 2ca5a6d
Author: David Smiley <ds...@apache.org>
Authored: Tue Aug 7 11:51:16 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Aug 7 11:51:16 2018 -0400
----------------------------------------------------------------------
lucene/ivy-versions.properties | 1 -
solr/CHANGES.txt | 10 +
.../configsets/collection1/conf/solrconfig.xml | 20 +-
solr/core/ivy.xml | 1 -
.../org/apache/solr/core/ConfigSetService.java | 7 +-
.../ParseDateFieldUpdateProcessorFactory.java | 82 ++++++--
...dd-schema-fields-update-processor-chains.xml | 20 +-
...lrconfig-parsing-update-processor-chains.xml | 49 ++---
.../collection1/conf/solrconfig-schemaless.xml | 20 +-
.../configsets/_default/conf/solrconfig.xml | 20 +-
...dSchemaFieldsUpdateProcessorFactoryTest.java | 23 +--
.../ParsingFieldUpdateProcessorsTest.java | 157 +++++++-------
solr/example/files/conf/solrconfig.xml | 20 +-
solr/licenses/joda-time-2.2.jar.sha1 | 1 -
solr/licenses/joda-time-LICENSE-ASL.txt | 202 -------------------
solr/licenses/joda-time-NOTICE.txt | 5 -
.../configsets/_default/conf/solrconfig.xml | 20 +-
solr/solr-ref-guide/src/schemaless-mode.adoc | 20 +-
18 files changed, 218 insertions(+), 460 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 3a1ef2b..d7141bd 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -77,7 +77,6 @@ io.prometheus.version = 0.2.0
/javax.activation/activation = 1.1.1
/javax.servlet/javax.servlet-api = 3.1.0
-/joda-time/joda-time = 2.2
/junit/junit = 4.10
/mecab/mecab-ipadic = 2.7.0-20070801
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d5edef3..94960bb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -44,11 +44,21 @@ Upgrade Notes
MemoryDocValues). If you used postingsFormat="Memory" or docValuesFormat="Memory"
switch to "Direct" instead. (Dawid Weiss)
+* SOLR-12586: The date format patterns used by ParseDateFieldUpdateProcessorFactory (present in "schemaless mode")
+ are now interpreted by Java 8's java.time.DateTimeFormatter instead of Joda Time. The pattern language is very
+ similar but not the same. Typically, simply update the pattern by changing an uppercase 'Z' to lowercase 'z' and
+ that's it. For the current recommended set of patterns in schemaless mode, see "Schemaless Mode" in the ref guide.
+ (David Smiley, Bar Rotstein)
+
Other Changes
----------------------
* SOLR-12614: Make "Nodes" view the default in AdminUI "Cloud" tab (janhoy)
+* SOLR-12586: Remove Joda Time dependency. Upgrade ParseDateFieldUpdateProcessorFactory (present in "schemaless mode")
+ to use Java 8's java.time.DateTimeFormatter instead (see upgrade notes).
+ (David Smiley, Bar Rotstein)
+
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml b/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
index 72c5430..37c41d8 100644
--- a/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
+++ b/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
@@ -164,22 +164,10 @@
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index ee6fe80..5a0fd09 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -44,7 +44,6 @@
<dependency org="org.ow2.asm" name="asm-commons" rev="${/org.ow2.asm/asm-commons}" conf="compile"/>
<dependency org="org.restlet.jee" name="org.restlet" rev="${/org.restlet.jee/org.restlet}" conf="compile"/>
<dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="${/org.restlet.jee/org.restlet.ext.servlet}" conf="compile"/>
- <dependency org="joda-time" name="joda-time" rev="${/joda-time/joda-time}" conf="compile"/>
<dependency org="dom4j" name="dom4j" rev="${/dom4j/dom4j}" conf="compile"/>
<dependency org="com.carrotsearch" name="hppc" rev="${/com.carrotsearch/hppc}" conf="compile"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
index 69e160b..7ce1a52 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
@@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.time.Instant;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
@@ -33,8 +34,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -212,12 +211,10 @@ public abstract class ConfigSetService {
super(loader, configSetBase);
}
- public static final DateTimeFormatter cacheKeyFormatter = DateTimeFormat.forPattern("yyyyMMddHHmmss");
-
public static String cacheName(Path schemaFile) throws IOException {
long lastModified = Files.getLastModifiedTime(schemaFile).toMillis();
return String.format(Locale.ROOT, "%s:%s",
- schemaFile.toString(), cacheKeyFormatter.print(lastModified));
+ schemaFile.toString(), Instant.ofEpochMilli(lastModified).toString());
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
index 234b48b..f0ea5d2 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
@@ -17,6 +17,17 @@
package org.apache.solr.update.processor;
import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalQueries;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedHashMap;
@@ -24,6 +35,7 @@ import java.util.Locale;
import java.util.Map;
import org.apache.commons.lang.LocaleUtils;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
@@ -31,10 +43,6 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.DateValueFieldType;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,8 +65,8 @@ import org.slf4j.LoggerFactory;
* </p>
* <p>
* One or more date "format" specifiers must be specified. See
- * <a href="http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html"
- * >Joda-time's DateTimeFormat javadocs</a> for a description of format strings.
+ * <a href="https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html"
+ * >Java 8's DateTimeFormatter javadocs</a> for a description of format strings.
* </p>
* <p>
* A default time zone name or offset may optionally be specified for those dates
@@ -115,9 +123,8 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
for (Map.Entry<String,DateTimeFormatter> format : formats.entrySet()) {
DateTimeFormatter parser = format.getValue();
try {
- DateTime dateTime = parser.parseDateTime(srcStringVal);
- return dateTime.withZone(DateTimeZone.UTC).toDate();
- } catch (IllegalArgumentException e) {
+ return Date.from(parseInstant(parser, srcStringVal));
+ } catch (DateTimeParseException e) {
log.debug("value '{}' is not parseable with format '{}'",
new Object[] { srcStringVal, format.getKey() });
}
@@ -144,15 +151,18 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
}
Object defaultTimeZoneParam = args.remove(DEFAULT_TIME_ZONE_PARAM);
- DateTimeZone defaultTimeZone = DateTimeZone.UTC;
+ ZoneId defaultTimeZone = ZoneOffset.UTC;
if (null != defaultTimeZoneParam) {
- defaultTimeZone = DateTimeZone.forID(defaultTimeZoneParam.toString());
+ defaultTimeZone = ZoneId.of(defaultTimeZoneParam.toString());
}
Collection<String> formatsParam = args.removeConfigArgs(FORMATS_PARAM);
if (null != formatsParam) {
for (String value : formatsParam) {
- formats.put(value, DateTimeFormat.forPattern(value).withZone(defaultTimeZone).withLocale(locale));
+ DateTimeFormatter formatter = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .appendPattern(value).toFormatter(locale).withZone(defaultTimeZone);
+ validateFormatter(formatter);
+ formats.put(value, formatter);
}
}
super.init(args);
@@ -172,4 +182,52 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
return (null == type) || type instanceof DateValueFieldType;
};
}
+
+ public static void validateFormatter(DateTimeFormatter formatter) {
+ // check it's valid via round-trip
+ try {
+ parseInstant(formatter, formatter.format(Instant.now()));
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Bad or unsupported pattern: " + formatter.toFormat().toString(), e);
+ }
+ }
+
+ // see https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8177021 which is fixed in Java 9.
+ // The upshot is that trying to use parse(Instant::from) is unreliable in the event that
+ // the input string contains a timezone/offset that differs from the "override zone"
+ // (which we configure in DEFAULT_TIME_ZONE). Besides, we need the code below which handles
+ // the optionality of time. Were it not for that, we truly could do formatter.parse(Instant::from).
+ private static Instant parseInstant(DateTimeFormatter formatter, String dateStr) {
+ final TemporalAccessor temporal = formatter.parse(dateStr);
+ // Get Date; mandatory
+ LocalDate date = temporal.query(TemporalQueries.localDate());//mandatory
+ if (date == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Date (year, month, day) is mandatory: " + formatter.toFormat().toString());
+ }
+ // Get Time; optional
+ LocalTime time = temporal.query(TemporalQueries.localTime());
+ if (time == null) {
+ time = LocalTime.MIN;
+ }
+
+ final LocalDateTime localDateTime = LocalDateTime.of(date, time);
+
+ // Get Zone Offset; optional
+ ZoneOffset offset = temporal.query(TemporalQueries.offset());
+ if (offset == null) {
+ // no Zone offset; get Zone ID
+ ZoneId zoneId = temporal.query(TemporalQueries.zone());
+ if (zoneId == null) {
+ zoneId = formatter.getZone();
+ if (zoneId == null) {
+ zoneId = ZoneOffset.UTC;
+ }
+ }
+ return localDateTime.atZone(zoneId).toInstant();
+ } else {
+ return localDateTime.toInstant(offset);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
index b0b63df..4394d5c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
@@ -179,22 +179,10 @@
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory"/>
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
index 5930d37..83be4ee 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
@@ -29,35 +29,35 @@
<updateRequestProcessorChain name="parse-date">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-explicit-not-in-schema-selector-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<bool name="fieldNameMatchesSchemaField">false</bool>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-explicit-typeclass-point-selector-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="typeClass">solr.DatePointField</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-explicit-typeclass-selector-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="typeClass">solr.TrieDateField</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
@@ -65,8 +65,7 @@
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="defaultTimeZone">America/New_York</str>
<str name="locale">en_US</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSS</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSS[z]</str>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
@@ -75,7 +74,7 @@
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="defaultTimeZone">America/Los_Angeles</str>
<arr name="format">
- <str>MM/dd/yyyy</str>
+ <str>M/d/yyyy</str>
</arr>
</processor>
</updateRequestProcessorChain>
@@ -85,30 +84,16 @@
<str name="defaultTimeZone">UTC</str>
<str name="locale">en_US</str>
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
- <str>yyyy-MM-dd hh:mm a</str>
- <str>yyyy-MM-dd hh:mma</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
- <str>EEE MMM dd HH:mm:ss Z yyyy</str>
- <str>EEE MMM dd HH:mm:ss yyyy Z</str>
- <str>EEE MMM dd HH:mm:ss yyyy</str>
- <str>EEE, dd MMM yyyy HH:mm:ss Z</str>
- <str>EEEE, dd-MMM-yy HH:mm:ss Z</str>
+ <str>yyyy-MM-dd hh:mm[ ]a</str>
+ <str>EEE MMM dd HH:mm:ss ZZZ yyyy</str>
+ <str>EEE MMM dd HH:mm:ss yyyy[ z]</str>
+ <str>EEE, dd MMM yyyy HH:mm:ss z</str>
+ <str>EEEE, dd-MMM-yy HH:mm:ss z</str>
<str>EEEE, MMMM dd, yyyy</str>
<str>MMMM dd, yyyy</str>
<str>MMM. dd, yyyy</str>
@@ -229,7 +214,7 @@
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
<str>yyyy-MM-dd</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str>yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
<str>yyyy-MM-dd'T'HH:mm</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
index 4f10a8e..446f3f0 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
@@ -61,22 +61,10 @@
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory"/>
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
index 51f2c63..d393316 100644
--- a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
@@ -1141,22 +1141,10 @@
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
index d1a3a53..96cc676 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
@@ -17,17 +17,18 @@
package org.apache.solr.update.processor;
import java.io.File;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Date;
+import java.util.Locale;
import org.apache.commons.io.FileUtils;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.schema.IndexSchema;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
import org.junit.After;
import org.junit.Before;
@@ -62,9 +63,7 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe
IndexSchema schema = h.getCore().getLatestSchema();
final String fieldName = "newfield1";
assertNull(schema.getFieldOrNull(fieldName));
- String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- Date date = dateTimeFormatter.parseDateTime(dateString).toDate();
+ Date date = Date.from(Instant.now());
SolrInputDocument d = processAdd("add-fields-no-run-processor", doc(f("id", "1"), f(fieldName, date)));
assertNotNull(d);
schema = h.getCore().getLatestSchema();
@@ -202,11 +201,11 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe
String field3String2 = "-5.28E-3";
Double field3Value2 = -5.28E-3;
String field4String1 = "1999-04-17 17:42";
- DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm").withZoneUTC();
- DateTime dateTime = dateTimeFormatter.parseDateTime(field4String1);
- Date field4Value1 = dateTime.toDate();
- DateTimeFormatter dateTimeFormatter2 = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZoneUTC();
- String field4Value1String = dateTimeFormatter2.print(dateTime) + "Z";
+ DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm", Locale.ROOT).withZone(ZoneOffset.UTC);
+ LocalDateTime dateTime = LocalDateTime.parse(field4String1, dateTimeFormatter);
+ Date field4Value1 = Date.from(dateTime.atZone(ZoneOffset.UTC).toInstant());
+ DateTimeFormatter dateTimeFormatter2 = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT).withZone(ZoneOffset.UTC);
+ String field4Value1String = dateTime.format(dateTimeFormatter2) + "Z";
SolrInputDocument d = processAdd
("parse-and-add-fields", doc(f("id", "6"), f(fieldName1, field1String1, field1String2, field1String3),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java b/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
index 3aeb1fb..e26ca41 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
@@ -16,28 +16,35 @@
*/
package org.apache.solr.update.processor;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.schema.IndexSchema;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-import org.junit.BeforeClass;
-
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.schema.IndexSchema;
+import org.junit.BeforeClass;
+
/**
* Tests for the field mutating update processors
* that parse Dates, Longs, Doubles, and Booleans.
*/
public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
private static final double EPSILON = 1E-15;
+ private static final DateTimeFormatter isoDateOptionalTimeFormatter =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd['T'HH:mm[:ss[.SSS]]][z", Locale.ROOT).withZone(ZoneOffset.UTC);
@BeforeClass
public static void beforeClass() throws Exception {
@@ -50,10 +57,8 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
String dateString = "2010-11-12T13:14:15.168Z";
SolrInputDocument d = processAdd("parse-date", doc(f("id", "9"), f("date_dt", dateString)));
assertNotNull(d);
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
assertTrue(d.getFieldValue("date_dt") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_dt")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date) d.getFieldValue("date_dt")).toInstant());
assertU(commit());
assertQ(req("id:9"), "//date[@name='date_dt'][.='" + dateString + "']");
}
@@ -64,10 +69,8 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
String dateString = "2010-11-12T13:14:15.168Z";
SolrInputDocument d = processAdd("parse-date", doc(f("id", "39"), f("date_tdt", dateString)));
assertNotNull(d);
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
assertTrue(d.getFieldValue("date_tdt") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_tdt")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date) d.getFieldValue("date_tdt")).toInstant());
assertU(commit());
assertQ(req("id:39"), "//date[@name='date_tdt'][.='" + dateString + "']");
}
@@ -77,14 +80,12 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-date-no-run-processor",
doc(f("id", "18"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("not_in_schema")).toInstant());
d = processAdd("parse-date-no-run-processor",
doc(f("id", "36"), f("not_in_schema", "not a date", dateString)));
@@ -116,12 +117,8 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
("parse-date-non-UTC-defaultTimeZone", doc(f("id", "99"), f("dateUTC_dt", dateStringUTC),
f("dateNoTimeZone_dt", dateStringNoTimeZone)));
assertNotNull(d);
- String pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
- DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(pattern);
- DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(dateStringUTC);
assertTrue(d.getFieldValue("dateUTC_dt") instanceof Date);
assertTrue(d.getFieldValue("dateNoTimeZone_dt") instanceof Date);
- assertEquals(dateTimeUTC.getMillis(), ((Date) d.getFieldValue("dateUTC_dt")).getTime());
assertU(commit());
assertQ(req("id:99")
,"//date[@name='dateUTC_dt'][.='" + dateStringUTC + "']"
@@ -132,22 +129,18 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-date-explicit-not-in-schema-selector-no-run-processor",
doc(f("id", "88"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("not_in_schema")).toInstant());
}
public void testParseDateExplicitTypeClassSelector() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("date_dt"));
String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d;
if (schema.getField("date_dt").getType().isPointField()) {
d = processAdd("parse-date-explicit-typeclass-point-selector-no-run-processor",
@@ -159,71 +152,66 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
assertNotNull(d);
assertTrue(d.getFieldValue("date_dt") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("date_dt")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("date_dt")).toInstant());
}
public void testParseUSPacificDate() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "8/9/2010"; // Interpreted as 00:00 US Pacific Daylight Time = UTC+07:00
- String dateStringUTC = "2010-08-09T07:00:00.000Z";
SolrInputDocument d = processAdd("US-Pacific-parse-date-no-run-processor",
doc(f("id", "288"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateStringUTC,
- (new DateTime(((Date)d.getFieldValue("not_in_schema")).getTime(),DateTimeZone.UTC)).toString());
+ assertEquals(Instant.parse("2010-08-09T07:00:00.000Z"), ((Date)d.getFieldValue("not_in_schema")).toInstant());
}
public void testParseDateFormats() throws Exception {
String[] formatExamples = {
- "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "2010-01-15T00:00:00.000Z",
- "yyyy-MM-dd'T'HH:mm:ss,SSSZ", "2010-01-15T00:00:00,000Z",
- "yyyy-MM-dd'T'HH:mm:ss.SSS", "2010-01-15T00:00:00.000",
- "yyyy-MM-dd'T'HH:mm:ss,SSS", "2010-01-15T00:00:00,000",
- "yyyy-MM-dd'T'HH:mm:ssZ", "2010-01-15T00:00:00Z",
- "yyyy-MM-dd'T'HH:mm:ss", "2010-01-15T00:00:00",
- "yyyy-MM-dd'T'HH:mmZ", "2010-01-15T00:00Z",
- "yyyy-MM-dd'T'HH:mm", "2010-01-15T00:00",
- "yyyy-MM-dd HH:mm:ss.SSSZ", "2010-01-15 00:00:00.000Z",
- "yyyy-MM-dd HH:mm:ss,SSSZ", "2010-01-15 00:00:00,000Z",
- "yyyy-MM-dd HH:mm:ss.SSS", "2010-01-15 00:00:00.000",
- "yyyy-MM-dd HH:mm:ss,SSS", "2010-01-15 00:00:00,000",
- "yyyy-MM-dd HH:mm:ssZ", "2010-01-15 00:00:00Z",
- "yyyy-MM-dd HH:mm:ss", "2010-01-15 00:00:00",
- "yyyy-MM-dd HH:mmZ", "2010-01-15 00:00Z",
- "yyyy-MM-dd HH:mm", "2010-01-15 00:00",
- "yyyy-MM-dd hh:mm a", "2010-01-15 12:00 AM",
- "yyyy-MM-dd hh:mma", "2010-01-15 12:00AM",
- "yyyy-MM-dd", "2010-01-15",
- "EEE MMM dd HH:mm:ss Z yyyy", "Fri Jan 15 00:00:00 +0000 2010",
- "EEE MMM dd HH:mm:ss yyyy Z", "Fri Jan 15 00:00:00 2010 +00:00",
- "EEE MMM dd HH:mm:ss yyyy", "Fri Jan 15 00:00:00 2010",
- "EEE, dd MMM yyyy HH:mm:ss Z", "Fri, 15 Jan 2010 00:00:00 +00:00",
- "EEEE, dd-MMM-yy HH:mm:ss Z", "Friday, 15-Jan-10 00:00:00 +00:00",
- "EEEE, MMMM dd, yyyy", "Friday, January 15, 2010",
- "MMMM dd, yyyy", "January 15, 2010",
- "MMM. dd, yyyy", "Jan. 15, 2010"
+ "2010-01-15T00:00:00.000Z",
+ "2010-01-15T00:00:00,000Z",
+ "2010-01-15T00:00:00.000",
+ "2010-01-15T00:00:00,000",
+ "2010-01-15T00:00:00Z",
+ "2010-01-15T00:00:00",
+ "2010-01-15T00:00Z",
+ "2010-01-15T00:00",
+ "2010-01-15 00:00:00.000Z",
+ "2010-01-15 00:00:00,000Z",
+ "2010-01-15 00:00:00.000",
+ "2010-01-15 00:00:00,000",
+ "2010-01-15 00:00:00Z",
+ "2010-01-15 00:00:00",
+ "2010-01-15 00:00Z",
+ "2010-01-15 00:00",
+ "2010-01-15 12:00 AM",
+ "2010-01-15 12:00AM",
+ "2010-01-15",
+ "Fri Jan 15 00:00:00 +0000 2010",
+ "Fri Jan 15 00:00:00 2010 +00:00",
+ "Fri Jan 15 00:00:00 2010",
+ "Fri, 15 Jan 2010 00:00:00 +00:00",
+ "Friday, 15-Jan-10 00:00:00 +00:00",
+ "Friday, January 15, 2010",
+ "January 15, 2010",
+ "Jan. 15, 2010"
};
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("dateUTC_dt")); // should match "*_dt" dynamic field
- String dateTimePattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
- DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(dateTimePattern);
- DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(formatExamples[1]);
+ Instant expectedInstant = Instant.parse(formatExamples[0]);
- for (int i = 0 ; i < formatExamples.length ; i += 2) {
- String format = formatExamples[i];
- String dateString = formatExamples[i + 1];
+ for (int i = 0 ; i < formatExamples.length ; ++i) {
+ String dateString = formatExamples[i];
String id = "95" + i;
SolrInputDocument d = processAdd("parse-date-UTC-defaultTimeZone-no-run-processor",
doc(f("id", id), f("dateUTC_dt", dateString)));
assertNotNull(d);
- assertTrue("date '" + dateString + "' with format '" + format + "' is not mutated to a Date",
+ assertTrue("index: " + i + " date '" + dateString + "' is not mutated to a Date",
d.getFieldValue("dateUTC_dt") instanceof Date);
- assertEquals("date '" + dateString + "' with format '" + format + "' mismatched milliseconds",
- dateTimeUTC.getMillis(), ((Date)d.getFieldValue("dateUTC_dt")).getTime());
+ assertEquals("date '" + dateString + "' mismatched milliseconds",
+ expectedInstant, ((Date)d.getFieldValue("dateUTC_dt")).toInstant());
}
}
@@ -232,23 +220,20 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
assertNull(schema.getFieldOrNull("not_in_schema"));
String frenchDateString = "le vendredi 15 janvier 2010";
String dateString = "2010-01-15T00:00:00.000Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-french-date-UTC-defaultTimeZone-no-run-processor",
doc(f("id", "88"), f("not_in_schema", frenchDateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("not_in_schema")).toInstant());
}
public void testFailedParseMixedDate() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Object,Object> mixed = new HashMap<>();
String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
- mixed.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
+ mixed.put(parse(isoDateOptionalTimeFormatter, dateString), dateString);
}
Double extraDouble = 29.554d;
mixed.put(extraDouble, extraDouble); // Double-typed field value
@@ -805,11 +790,10 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
longs.remove(o);
}
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Date,String> dates = new HashMap<>();
String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
- dates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
+ dates.put(parse(isoDateOptionalTimeFormatter, dateString), dateString);
}
d = processAdd(chain, doc(f("id", "343"), f(fieldName, dates.values())));
assertNotNull(d);
@@ -896,13 +880,12 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
}
assertTrue(mixedBooleans.isEmpty());
- dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Date,Object> mixedDates = new HashMap<>();
dateStrings = new String[] { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
- mixedDates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
+ mixedDates.put(parse(isoDateOptionalTimeFormatter, dateString), dateString);
}
- Date extraDate = dateTimeFormatter.parseDateTime("2003-04-24").toDate();
+ Date extraDate = parse(isoDateOptionalTimeFormatter, "2003-04-24");
mixedDates.put(extraDate, extraDate); // Date-typed field value
d = processAdd(chain, doc(f("id", "3395"), f(fieldName, mixedDates.values())));
assertNotNull(d);
@@ -912,4 +895,24 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
}
assertTrue(mixedDates.isEmpty());
}
+
+ private Date parse(DateTimeFormatter dateTimeFormatter, String dateString) {
+ final TemporalAccessor temporalAccessor = dateTimeFormatter.parseBest(dateString, OffsetDateTime::from,
+ ZonedDateTime::from, LocalDateTime::from, LocalDate::from, Instant::from);
+ return temporalToDate(temporalAccessor, dateTimeFormatter.getZone());
+ }
+
+ private Date temporalToDate(TemporalAccessor in, ZoneId timeZoneId) {
+ if (in instanceof OffsetDateTime) {
+ return Date.from(((OffsetDateTime) in).toInstant());
+ } else if (in instanceof ZonedDateTime) {
+ return Date.from(((ZonedDateTime) in).toInstant());
+ } else if (in instanceof LocalDateTime) {
+ return Date.from(((LocalDateTime) in).atZone(timeZoneId).toInstant());
+ } else if (in instanceof Instant) {
+ return Date.from((Instant) in);
+ } else {
+ return Date.from(((LocalDate) in).atStartOfDay(timeZoneId).toInstant());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/example/files/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/example/files/conf/solrconfig.xml b/solr/example/files/conf/solrconfig.xml
index bca0df8..7bc8d34 100644
--- a/solr/example/files/conf/solrconfig.xml
+++ b/solr/example/files/conf/solrconfig.xml
@@ -1161,22 +1161,10 @@
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory"/>
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/licenses/joda-time-2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/joda-time-2.2.jar.sha1 b/solr/licenses/joda-time-2.2.jar.sha1
deleted file mode 100644
index 5e68639..0000000
--- a/solr/licenses/joda-time-2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a5f29a7acaddea3f4af307e8cf2d0cc82645fd7d
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/licenses/joda-time-LICENSE-ASL.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/joda-time-LICENSE-ASL.txt b/solr/licenses/joda-time-LICENSE-ASL.txt
deleted file mode 100644
index d645695..0000000
--- a/solr/licenses/joda-time-LICENSE-ASL.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/licenses/joda-time-NOTICE.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/joda-time-NOTICE.txt b/solr/licenses/joda-time-NOTICE.txt
deleted file mode 100644
index dffbcf3..0000000
--- a/solr/licenses/joda-time-NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-=============================================================================
-= NOTICE file corresponding to section 4d of the Apache License Version 2.0 =
-=============================================================================
-This product includes software developed by
-Joda.org (http://www.joda.org/).
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/server/solr/configsets/_default/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml
index 51f2c63..d393316 100644
--- a/solr/server/solr/configsets/_default/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml
@@ -1141,22 +1141,10 @@
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/solr-ref-guide/src/schemaless-mode.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/schemaless-mode.adoc b/solr/solr-ref-guide/src/schemaless-mode.adoc
index b62a92e..0a2f555 100644
--- a/solr/solr-ref-guide/src/schemaless-mode.adoc
+++ b/solr/solr-ref-guide/src/schemaless-mode.adoc
@@ -103,22 +103,10 @@ To start, you should define it as follows (see the javadoc links below for updat
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
[16/31] lucene-solr:jira/http2: LUCENE-8204: Boolean queries with a
mix of required and optional clauses are now faster if the total hit count is
not required
Posted by da...@apache.org.
LUCENE-8204: Boolean queries with a mix of required and optional clauses are now faster if the total hit count is not required
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/49e3cca7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/49e3cca7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/49e3cca7
Branch: refs/heads/jira/http2
Commit: 49e3cca77f868dd8dcf766a528ab23c686a6edfc
Parents: 6845bbf
Author: Jim Ferenczi <ji...@apache.org>
Authored: Wed Aug 8 15:49:58 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed Aug 8 15:49:58 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +-
.../lucene/search/Boolean2ScorerSupplier.java | 2 +-
.../apache/lucene/search/ReqOptSumScorer.java | 175 +++++++++-----
.../lucene/search/TestReqOptSumScorer.java | 241 +++++++++++++++++--
.../lucene/search/RandomApproximationQuery.java | 10 +-
5 files changed, 352 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f0360a9..31c8988 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -135,7 +135,10 @@ Optimizations
to run faster. (Adrien Grand)
* LUCENE-8439: Disjunction max queries can skip blocks to select the top documents
- when the total hit count is not required. (Jim Ferenczi, Adrien Grand)
+ if the total hit count is not required. (Jim Ferenczi, Adrien Grand)
+
+* LUCENE-8204: Boolean queries with a mix of required and optional clauses are
+ now faster if the total hit count is not required. (Jim Ferenczi, Adrien Grand)
======================= Lucene 7.5.0 =======================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
index bae7941..a13547f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
@@ -111,7 +111,7 @@ final class Boolean2ScorerSupplier extends ScorerSupplier {
assert scoreMode.needsScores();
return new ReqOptSumScorer(
excl(req(subs.get(Occur.FILTER), subs.get(Occur.MUST), leadCost), subs.get(Occur.MUST_NOT), leadCost),
- opt(subs.get(Occur.SHOULD), minShouldMatch, scoreMode, leadCost));
+ opt(subs.get(Occur.SHOULD), minShouldMatch, scoreMode, leadCost), scoreMode);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
index 2463e6a..dc3ee02 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
@@ -18,108 +18,160 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/** A Scorer for queries with a required part and an optional part.
* Delays skipTo() on the optional part until a score() is needed.
*/
class ReqOptSumScorer extends Scorer {
- /** The scorers passed from the constructor.
- * These are set to null as soon as their next() or skipTo() returns false.
- */
private final Scorer reqScorer;
private final Scorer optScorer;
- private final float reqMaxScore;
+ private final DocIdSetIterator reqApproximation;
private final DocIdSetIterator optApproximation;
private final TwoPhaseIterator optTwoPhase;
- private boolean optIsRequired;
private final DocIdSetIterator approximation;
private final TwoPhaseIterator twoPhase;
- final MaxScoreSumPropagator maxScorePropagator;
- /** Construct a <code>ReqOptScorer</code>.
+ private float minScore = 0;
+ private float reqMaxScore;
+ private boolean optIsRequired;
+
+ /**
+ * Construct a <code>ReqOptScorer</code>.
+ *
* @param reqScorer The required scorer. This must match.
* @param optScorer The optional scorer. This is used for scoring only.
+ * @param scoreMode How the produced scorers will be consumed.
*/
- public ReqOptSumScorer(
- Scorer reqScorer,
- Scorer optScorer) throws IOException
- {
+ public ReqOptSumScorer(Scorer reqScorer, Scorer optScorer, ScoreMode scoreMode) throws IOException {
super(reqScorer.weight);
assert reqScorer != null;
assert optScorer != null;
this.reqScorer = reqScorer;
this.optScorer = optScorer;
- reqScorer.advanceShallow(0);
- this.reqMaxScore = reqScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
- this.maxScorePropagator = new MaxScoreSumPropagator(Arrays.asList(reqScorer, optScorer));
-
final TwoPhaseIterator reqTwoPhase = reqScorer.twoPhaseIterator();
this.optTwoPhase = optScorer.twoPhaseIterator();
- final DocIdSetIterator reqApproximation;
if (reqTwoPhase == null) {
reqApproximation = reqScorer.iterator();
} else {
- reqApproximation= reqTwoPhase.approximation();
+ reqApproximation = reqTwoPhase.approximation();
}
if (optTwoPhase == null) {
optApproximation = optScorer.iterator();
} else {
- optApproximation= optTwoPhase.approximation();
+ optApproximation = optTwoPhase.approximation();
}
+ if (scoreMode != ScoreMode.TOP_SCORES) {
+ approximation = reqApproximation;
+ this.reqMaxScore = Float.POSITIVE_INFINITY;
+ } else {
+ reqScorer.advanceShallow(0);
+ optScorer.advanceShallow(0);
+ this.reqMaxScore = reqScorer.getMaxScore(NO_MORE_DOCS);
+ this.approximation = new DocIdSetIterator() {
+ int upTo = -1;
+ float maxScore;
- approximation = new DocIdSetIterator() {
+ private void moveToNextBlock(int target) throws IOException {
+ upTo = advanceShallow(target);
+ float reqMaxScoreBlock = reqScorer.getMaxScore(upTo);
+ maxScore = getMaxScore(upTo);
- private int nextCommonDoc(int reqDoc) throws IOException {
- int optDoc = optApproximation.docID();
- if (optDoc > reqDoc) {
- reqDoc = reqApproximation.advance(optDoc);
+ // Potentially move to a conjunction
+ optIsRequired = reqMaxScoreBlock < minScore;
}
- while (true) { // invariant: reqDoc >= optDoc
- if (reqDoc == optDoc) {
- return reqDoc;
+ private int advanceImpacts(int target) throws IOException {
+ if (target > upTo) {
+ moveToNextBlock(target);
}
- optDoc = optApproximation.advance(reqDoc);
- if (optDoc == reqDoc) {
- return reqDoc;
+ while (true) {
+ if (maxScore >= minScore) {
+ return target;
+ }
+
+ if (upTo == NO_MORE_DOCS) {
+ return NO_MORE_DOCS;
+ }
+
+ target = upTo + 1;
+
+ moveToNextBlock(target);
}
- reqDoc = reqApproximation.advance(optDoc);
}
- }
- @Override
- public int nextDoc() throws IOException {
- int doc = reqApproximation.nextDoc();
- if (optIsRequired) {
- doc = nextCommonDoc(doc);
+ @Override
+ public int nextDoc() throws IOException {
+ return advanceInternal(reqApproximation.docID()+1);
}
- return doc;
- }
- @Override
- public int advance(int target) throws IOException {
- int doc = reqApproximation.advance(target);
- if (optIsRequired) {
- doc = nextCommonDoc(doc);
+ @Override
+ public int advance(int target) throws IOException {
+ return advanceInternal(target);
}
- return doc;
- }
- @Override
- public int docID() {
- return reqApproximation.docID();
- }
+ private int advanceInternal(int target) throws IOException {
+ if (target == NO_MORE_DOCS) {
+ reqApproximation.advance(target);
+ return NO_MORE_DOCS;
+ }
+ int reqDoc = target;
+ advanceHead: for (;;) {
+ if (minScore != 0) {
+ reqDoc = advanceImpacts(reqDoc);
+ }
+ if (reqApproximation.docID() < reqDoc) {
+ reqDoc = reqApproximation.advance(reqDoc);
+ }
+ if (reqDoc == NO_MORE_DOCS || optIsRequired == false) {
+ return reqDoc;
+ }
- @Override
- public long cost() {
- return reqApproximation.cost();
- }
+ int upperBound = reqMaxScore < minScore ? NO_MORE_DOCS : upTo;
+ if (reqDoc > upperBound) {
+ continue;
+ }
- };
+ // Find the next common doc within the current block
+ for (;;) { // invariant: reqDoc >= optDoc
+ int optDoc = optApproximation.docID();
+ if (optDoc < reqDoc) {
+ optDoc = optApproximation.advance(reqDoc);
+ }
+ if (optDoc > upperBound) {
+ reqDoc = upperBound + 1;
+ continue advanceHead;
+ }
+
+ if (optDoc != reqDoc) {
+ reqDoc = reqApproximation.advance(optDoc);
+ if (reqDoc > upperBound) {
+ continue advanceHead;
+ }
+ }
+
+ if (reqDoc == NO_MORE_DOCS || optDoc == reqDoc) {
+ return reqDoc;
+ }
+ }
+ }
+ }
+
+ @Override
+ public int docID() {
+ return reqApproximation.docID();
+ }
+
+ @Override
+ public long cost() {
+ return reqApproximation.cost();
+ }
+ };
+ }
if (reqTwoPhase == null && optTwoPhase == null) {
this.twoPhase = null;
@@ -212,10 +264,13 @@ class ReqOptSumScorer extends Scorer {
@Override
public int advanceShallow(int target) throws IOException {
- if (optScorer.docID() < target) {
- optScorer.advanceShallow(target);
+ int upTo = reqScorer.advanceShallow(target);
+ if (optScorer.docID() <= target) {
+ upTo = Math.min(upTo, optScorer.advanceShallow(target));
+ } else if (optScorer.docID() != NO_MORE_DOCS) {
+ upTo = Math.min(upTo, optScorer.docID() - 1);
}
- return reqScorer.advanceShallow(target);
+ return upTo;
}
@Override
@@ -229,8 +284,9 @@ class ReqOptSumScorer extends Scorer {
@Override
public void setMinCompetitiveScore(float minScore) {
+ this.minScore = minScore;
// Potentially move to a conjunction
- if (optIsRequired == false && minScore > reqMaxScore) {
+ if (reqMaxScore < minScore) {
optIsRequired = true;
}
}
@@ -242,5 +298,4 @@ class ReqOptSumScorer extends Scorer {
children.add(new ChildScorer(optScorer, "SHOULD"));
return children;
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
index f8fc6c2..0d4454d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
@@ -17,11 +17,20 @@
package org.apache.lucene.search;
import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -97,6 +106,120 @@ public class TestReqOptSumScorer extends LuceneTestCase {
dir.close();
}
+ public void testMaxBlock() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
+ FieldType ft = new FieldType();
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+ ft.setTokenized(true);
+ ft.freeze();
+
+ for (int i = 0; i < 1024; i++) {
+ // create documents with an increasing number of As and one B
+ Document doc = new Document();
+ doc.add(new Field("foo", new TermFreqTokenStream("a", i+1), ft));
+ if (random().nextFloat() < 0.5f) {
+ doc.add(new Field("foo", new TermFreqTokenStream("b", 1), ft));
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ w.close();
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new TestSimilarity.SimpleSimilarity());
+ // freq == score
+ // searcher.setSimilarity(new TestSimilarity.SimpleSimilarity());
+ final Query reqQ = new TermQuery(new Term("foo", "a"));
+ final Query optQ = new TermQuery(new Term("foo", "b"));
+ final Query boolQ = new BooleanQuery.Builder()
+ .add(reqQ, Occur.MUST)
+ .add(optQ, Occur.SHOULD)
+ .build();
+ Scorer actual = reqOptScorer(searcher, reqQ, optQ, true);
+ Scorer expected = searcher
+ .createWeight(boolQ, ScoreMode.COMPLETE, 1)
+ .scorer(searcher.getIndexReader().leaves().get(0));
+ actual.setMinCompetitiveScore(Math.nextUp(1));
+ // Checks that all blocks are fully visited
+ for (int i = 0; i < 1024; i++) {
+ assertEquals(i, actual.iterator().nextDoc());
+ assertEquals(i, expected.iterator().nextDoc());
+ assertEquals(actual.score(),expected.score(), 0);
+ }
+ reader.close();
+ dir.close();
+ }
+
+ public void testMaxScoreSegment() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
+ for (String[] values : Arrays.asList(
+ new String[]{ "A" }, // 0
+ new String[]{ "A" }, // 1
+ new String[]{ }, // 2
+ new String[]{ "A", "B" }, // 3
+ new String[]{ "A" }, // 4
+ new String[]{ "B" }, // 5
+ new String[]{ "A", "B" }, // 6
+ new String[]{ "B" } // 7
+ )) {
+ Document doc = new Document();
+ for (String value : values) {
+ doc.add(new StringField("foo", value, Store.NO));
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ w.close();
+
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = newSearcher(reader);
+ final Query reqQ = new ConstantScoreQuery(new TermQuery(new Term("foo", "A")));
+ final Query optQ = new ConstantScoreQuery(new TermQuery(new Term("foo", "B")));
+ Scorer scorer = reqOptScorer(searcher, reqQ, optQ, false);
+ assertEquals(0, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(1, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(3, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(4, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(6, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ scorer = reqOptScorer(searcher, reqQ, optQ, false);
+ scorer.setMinCompetitiveScore(Math.nextDown(1f));
+ assertEquals(0, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(1, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(3, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(4, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(6, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ scorer = reqOptScorer(searcher, reqQ, optQ, false);
+ scorer.setMinCompetitiveScore(Math.nextUp(1f));
+ assertEquals(3, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(6, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ scorer = reqOptScorer(searcher, reqQ, optQ, true);
+ scorer.setMinCompetitiveScore(Math.nextUp(2f));
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ reader.close();
+ dir.close();
+ }
+
public void testRandomFrequentOpt() throws IOException {
doTestRandom(0.5);
}
@@ -150,41 +273,125 @@ public class TestReqOptSumScorer extends LuceneTestCase {
searcher.search(query, coll);
ScoreDoc[] expectedFiltered = coll.topDocs().scoreDocs;
- for (int i = 0; i < 4; ++i) {
- Query must = mustTerm;
- if (i % 2 == 1) {
- must = new RandomApproximationQuery(must, random());
- }
- Query should = shouldTerm;
- if (i >= 2) {
- should = new RandomApproximationQuery(should, random());
- }
-
- query = new BooleanQuery.Builder()
- .add(must, Occur.MUST)
- .add(should, Occur.SHOULD)
+ CheckHits.checkTopScores(random(), query, searcher);
+
+ {
+ Query q = new BooleanQuery.Builder()
+ .add(new RandomApproximationQuery(mustTerm, random()), Occur.MUST)
+ .add(shouldTerm, Occur.SHOULD)
.build();
coll = TopScoreDocCollector.create(10, null, 1);
- searcher.search(query, coll);
+ searcher.search(q, coll);
ScoreDoc[] actual = coll.topDocs().scoreDocs;
-
CheckHits.checkEqual(query, expected, actual);
+ q = new BooleanQuery.Builder()
+ .add(mustTerm, Occur.MUST)
+ .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD)
+ .build();
+ coll = TopScoreDocCollector.create(10, null, 1);
+ searcher.search(q, coll);
+ actual = coll.topDocs().scoreDocs;
+ CheckHits.checkEqual(q, expected, actual);
+
+ q = new BooleanQuery.Builder()
+ .add(new RandomApproximationQuery(mustTerm, random()), Occur.MUST)
+ .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD)
+ .build();
+ coll = TopScoreDocCollector.create(10, null, 1);
+ searcher.search(q, coll);
+ actual = coll.topDocs().scoreDocs;
+ CheckHits.checkEqual(q, expected, actual);
+ }
+
+ {
+ Query nestedQ = new BooleanQuery.Builder()
+ .add(query, Occur.MUST)
+ .add(new TermQuery(new Term("f", "C")), Occur.FILTER)
+ .build();
+ CheckHits.checkTopScores(random(), nestedQ, searcher);
+
query = new BooleanQuery.Builder()
.add(query, Occur.MUST)
.add(new RandomApproximationQuery(new TermQuery(new Term("f", "C")), random()), Occur.FILTER)
.build();
coll = TopScoreDocCollector.create(10, null, 1);
- searcher.search(query, coll);
+ searcher.search(nestedQ, coll);
ScoreDoc[] actualFiltered = coll.topDocs().scoreDocs;
+ CheckHits.checkEqual(nestedQ, expectedFiltered, actualFiltered);
+ }
- CheckHits.checkEqual(query, expectedFiltered, actualFiltered);
+ {
+ query = new BooleanQuery.Builder()
+ .add(query, Occur.MUST)
+ .add(new TermQuery(new Term("f", "C")), Occur.SHOULD)
+ .build();
+
+ CheckHits.checkTopScores(random(), query, searcher);
+
+ query = new BooleanQuery.Builder()
+ .add(new TermQuery(new Term("f", "C")), Occur.MUST)
+ .add(query, Occur.SHOULD)
+ .build();
+
+ CheckHits.checkTopScores(random(), query, searcher);
}
r.close();
dir.close();
}
+ private static Scorer reqOptScorer(IndexSearcher searcher, Query reqQ, Query optQ, boolean withBlockScore) throws IOException {
+ Scorer reqScorer = searcher
+ .createWeight(reqQ, ScoreMode.TOP_SCORES, 1)
+ .scorer(searcher.getIndexReader().leaves().get(0));
+ Scorer optScorer = searcher
+ .createWeight(optQ, ScoreMode.TOP_SCORES, 1)
+ .scorer(searcher.getIndexReader().leaves().get(0));
+ if (withBlockScore) {
+ return new ReqOptSumScorer(reqScorer, optScorer, ScoreMode.TOP_SCORES);
+ } else {
+ return new ReqOptSumScorer(reqScorer, optScorer, ScoreMode.TOP_SCORES) {
+ @Override
+ public float getMaxScore(int upTo) {
+ return Float.POSITIVE_INFINITY;
+ }
+ };
+ }
+ }
+
+ private static class TermFreqTokenStream extends TokenStream {
+ private final String term;
+ private final int termFreq;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final TermFrequencyAttribute termFreqAtt = addAttribute(TermFrequencyAttribute.class);
+ private boolean finish;
+
+ public TermFreqTokenStream(String term, int termFreq) {
+ this.term = term;
+ this.termFreq = termFreq;
+ }
+
+ @Override
+ public boolean incrementToken() {
+ if (finish) {
+ return false;
+ }
+
+ clearAttributes();
+
+ termAtt.append(term);
+ termFreqAtt.setTermFrequency(termFreq);
+
+ finish = true;
+ return true;
+ }
+
+ @Override
+ public void reset() {
+ finish = false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java b/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
index a050b50..b43ccc4 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
@@ -110,6 +110,12 @@ public class RandomApproximationQuery extends Query {
@Override
public int advanceShallow(int target) throws IOException {
+ if (scorer.docID() > target && twoPhaseView.approximation.docID() != scorer.docID()) {
+ // The random approximation can return doc ids that are not present in the underlying
+ // scorer. These additional doc ids are always *before* the next matching doc so we
+ // cannot use them to shallow advance the main scorer which is already ahead.
+ target = scorer.docID();
+ }
return scorer.advanceShallow(target);
}
@@ -120,12 +126,12 @@ public class RandomApproximationQuery extends Query {
@Override
public int docID() {
- return scorer.docID();
+ return twoPhaseView.approximation().docID();
}
@Override
public DocIdSetIterator iterator() {
- return scorer.iterator();
+ return TwoPhaseIterator.asDocIdSetIterator(twoPhaseView);
}
}
[21/31] lucene-solr:jira/http2: SOLR-11008: Use a lighter config for
MetricsHandlerTest and ensure the core is up before the test starts
Posted by da...@apache.org.
SOLR-11008: Use a lighter config for MetricsHandlerTest and ensure the core is up before the test starts
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7c4584bd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7c4584bd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7c4584bd
Branch: refs/heads/jira/http2
Commit: 7c4584bd4f14917116b96a0931bab242d433e47a
Parents: 37e8fea
Author: Varun Thacker <va...@apache.org>
Authored: Tue Aug 7 13:19:44 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Aug 8 13:28:00 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../test/org/apache/solr/handler/admin/MetricsHandlerTest.java | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c4584bd/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7468451..9c80773 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -282,6 +282,8 @@ Other Changes
* SOLR-12617: Remove Commons BeanUtils as a dependency (Varun Thacker)
+* SOLR-11008: Use a lighter config for MetricsHandlerTest and ensure the core is up before the test starts (Varun Thacker)
+
================== 7.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c4584bd/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 0fe5ad7..1730355 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -35,7 +35,9 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml", "schema.xml");
+ initCore("solrconfig-minimal.xml", "schema.xml");
+ h.getCoreContainer().waitForLoadingCoresToFinish(30000);
+
// manually register some metrics in solr.jvm and solr.jetty - TestHarness doesn't init them
Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "foo");
c.inc();
[03/31] lucene-solr:jira/http2: SOLR-11881: Retry update requests
from leaders to followers
Posted by da...@apache.org.
SOLR-11881: Retry update requests from leaders to followers
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c338cf61
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c338cf61
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c338cf61
Branch: refs/heads/jira/http2
Commit: c338cf61e7baba4908c31e02beda47ae3e201752
Parents: ea2e564
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Mon Aug 6 15:56:49 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Mon Aug 6 15:56:49 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../apache/solr/update/SolrCmdDistributor.java | 230 ++++++++----
.../solr/update/StreamingSolrClients.java | 5 +-
.../processor/DistributedUpdateProcessor.java | 36 +-
.../TimeRoutedAliasUpdateProcessor.java | 4 +-
.../solr/update/MockStreamingSolrClients.java | 27 +-
.../solr/update/SolrCmdDistributorTest.java | 370 +++++++++++++++++--
7 files changed, 540 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c338cf61/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 605e837..5fb1f88 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -233,6 +233,8 @@ Optimizations
introduced (splitMethod=link) which uses hard-linking of index files when possible, resulting in
significant speedups and reduced CPU / IO load on shard leader. (ab)
+* SOLR-11881: Retry update requests sent by leaders to it's followers (Varun Thacker, Mark Miller, Tomás Fernández Löbbe)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c338cf61/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 80e2253..d5aafec 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -17,12 +17,28 @@
package org.apache.solr.update;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
import org.apache.http.HttpResponse;
+import org.apache.http.NoHttpResponseException;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; // jdoc
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrException;
@@ -32,37 +48,21 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.Diagnostics;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
-import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker;
import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker;
+import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.invoke.MethodHandles;
-import java.net.ConnectException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-
/**
* Used for distributing commands from a shard leader to its replicas.
*/
public class SolrCmdDistributor implements Closeable {
- private static final int MAX_RETRIES_ON_FORWARD = 25;
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private StreamingSolrClients clients;
private boolean finished = false; // see finish()
private int retryPause = 500;
- private int maxRetriesOnForward = MAX_RETRIES_ON_FORWARD;
private final List<Error> allErrors = new ArrayList<>();
private final List<Error> errors = Collections.synchronizedList(new ArrayList<Error>());
@@ -79,9 +79,8 @@ public class SolrCmdDistributor implements Closeable {
this.completionService = new ExecutorCompletionService<>(updateShardHandler.getUpdateExecutor());
}
- public SolrCmdDistributor(StreamingSolrClients clients, int maxRetriesOnForward, int retryPause) {
+ public SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
this.clients = clients;
- this.maxRetriesOnForward = maxRetriesOnForward;
this.retryPause = retryPause;
completionService = new ExecutorCompletionService<>(clients.getUpdateExecutor());
}
@@ -107,55 +106,63 @@ public class SolrCmdDistributor implements Closeable {
List<Error> errors = new ArrayList<>(this.errors);
errors.addAll(clients.getErrors());
List<Error> resubmitList = new ArrayList<>();
+
+ if (log.isInfoEnabled() && errors.size() > 0) {
+ log.info("SolrCmdDistributor found {} errors", errors.size());
+ }
+
+ if (log.isDebugEnabled() && errors.size() > 0) {
+ StringBuilder builder = new StringBuilder("SolrCmdDistributor found:");
+ int maxErrorsToShow = 10;
+ for (Error e:errors) {
+ if (maxErrorsToShow-- <= 0) break;
+ builder.append("\n" + e);
+ }
+ if (errors.size() > 10) {
+ builder.append("\n... and ");
+ builder.append(errors.size() - 10);
+ builder.append(" more");
+ }
+ log.debug(builder.toString());
+ }
for (Error err : errors) {
try {
String oldNodeUrl = err.req.node.getUrl();
- // if there is a retry url, we want to retry...
- boolean isRetry = err.req.node.checkRetry();
-
- boolean doRetry = false;
- int rspCode = err.statusCode;
+ /*
+ * if this is a retryable request we may want to retry, depending on the error we received and
+ * the number of times we have already retried
+ */
+ boolean isRetry = err.req.shouldRetry(err);
if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
err.e);
// this can happen in certain situations such as close
if (isRetry) {
- if (rspCode == 404 || rspCode == 403 || rspCode == 503) {
- doRetry = true;
- }
-
- // if it's a connect exception, lets try again
- if (err.e instanceof SolrServerException) {
- if (((SolrServerException) err.e).getRootCause() instanceof ConnectException) {
- doRetry = true;
- }
- }
-
- if (err.e instanceof ConnectException) {
- doRetry = true;
- }
-
- if (err.req.retries < maxRetriesOnForward && doRetry) {
- err.req.retries++;
-
+ err.req.retries++;
+
+ if (err.req.node instanceof ForwardNode) {
SolrException.log(SolrCmdDistributor.log, "forwarding update to "
+ oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + " " + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + rspCode, err.e);
- try {
- Thread.sleep(retryPause);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- log.warn(null, e);
- }
-
- resubmitList.add(err);
+ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
+ + err.req.cmd.toString() + " params:"
+ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
} else {
- allErrors.add(err);
+ SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
+ + oldNodeUrl + " failed - retrying ... retries: "
+ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
+ + err.req.cmd.toString() + " params:"
+ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
+ }
+ try {
+ Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.warn(null, e);
}
+ resubmitList.add(err);
} else {
allErrors.add(err);
}
@@ -184,6 +191,10 @@ public class SolrCmdDistributor implements Closeable {
RollupRequestReplicationTracker rollupTracker,
LeaderRequestReplicationTracker leaderTracker) throws IOException {
+ if (!cmd.isDeleteById()) {
+ blockAndDoRetries(); // For DBQ, flush all writes before submitting
+ }
+
for (Node node : nodes) {
UpdateRequest uReq = new UpdateRequest();
uReq.setParams(params);
@@ -193,7 +204,6 @@ public class SolrCmdDistributor implements Closeable {
} else {
uReq.deleteByQuery(cmd.query);
}
-
submit(new Req(cmd, node, uReq, sync, rollupTracker, leaderTracker), false);
}
}
@@ -346,6 +356,16 @@ public class SolrCmdDistributor implements Closeable {
this.leaderTracker = leaderTracker;
}
+ /**
+ * @return true if this request should be retried after receiving a particular error
+ * false otherwise
+ */
+ public boolean shouldRetry(Error err) {
+ boolean isRetry = node.checkRetry(err);
+ isRetry &= uReq.getDeleteQuery() == null || uReq.getDeleteQuery().isEmpty(); //Don't retry DBQs
+ return isRetry && retries < node.getMaxRetries();
+ }
+
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("SolrCmdDistributor$Req: cmd=").append(cmd.toString());
@@ -440,27 +460,36 @@ public class SolrCmdDistributor implements Closeable {
public static abstract class Node {
public abstract String getUrl();
- public abstract boolean checkRetry();
+ public abstract boolean checkRetry(Error e);
public abstract String getCoreName();
public abstract String getBaseUrl();
public abstract ZkCoreNodeProps getNodeProps();
public abstract String getCollection();
public abstract String getShardId();
+ public abstract int getMaxRetries();
}
public static class StdNode extends Node {
protected ZkCoreNodeProps nodeProps;
protected String collection;
protected String shardId;
+ private final boolean retry;
+ private final int maxRetries;
public StdNode(ZkCoreNodeProps nodeProps) {
- this(nodeProps, null, null);
+ this(nodeProps, null, null, 0);
}
- public StdNode(ZkCoreNodeProps nodeProps, String collection, String shardId) {
+ public StdNode(ZkCoreNodeProps nodeProps, String collection, String shardId) {
+ this(nodeProps, collection, shardId, 0);
+ }
+
+ public StdNode(ZkCoreNodeProps nodeProps, String collection, String shardId, int maxRetries) {
this.nodeProps = nodeProps;
this.collection = collection;
this.shardId = shardId;
+ this.retry = maxRetries > 0;
+ this.maxRetries = maxRetries;
}
public String getCollection() {
@@ -482,9 +511,33 @@ public class SolrCmdDistributor implements Closeable {
}
@Override
- public boolean checkRetry() {
+ public boolean checkRetry(Error err) {
+ if (!retry) return false;
+
+ if (err.statusCode == 404 || err.statusCode == 403 || err.statusCode == 503) {
+ return true;
+ }
+
+ // if it's a connect exception, lets try again
+ if (err.e instanceof SolrServerException) {
+ if (isRetriableException(((SolrServerException) err.e).getRootCause())) {
+ return true;
+ }
+ } else {
+ if (isRetriableException(err.e)) {
+ return true;
+ }
+ }
return false;
}
+
+ /**
+ * @return true if Solr should retry in case of hitting this exception
+ * false otherwise
+ */
+ private boolean isRetriableException(Throwable t) {
+ return t instanceof SocketException || t instanceof NoHttpResponseException || t instanceof SocketTimeoutException;
+ }
@Override
public String getBaseUrl() {
@@ -506,6 +559,8 @@ public class SolrCmdDistributor implements Closeable {
result = prime * result + ((baseUrl == null) ? 0 : baseUrl.hashCode());
result = prime * result + ((coreName == null) ? 0 : coreName.hashCode());
result = prime * result + ((url == null) ? 0 : url.hashCode());
+ result = prime * result + Boolean.hashCode(retry);
+ result = prime * result + Integer.hashCode(maxRetries);
return result;
}
@@ -515,6 +570,8 @@ public class SolrCmdDistributor implements Closeable {
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
StdNode other = (StdNode) obj;
+ if (this.retry != other.retry) return false;
+ if (this.maxRetries != other.maxRetries) return false;
String baseUrl = nodeProps.getBaseUrl();
String coreName = nodeProps.getCoreName();
String url = nodeProps.getCoreUrl();
@@ -534,39 +591,56 @@ public class SolrCmdDistributor implements Closeable {
public ZkCoreNodeProps getNodeProps() {
return nodeProps;
}
+
+ @Override
+ public int getMaxRetries() {
+ return this.maxRetries;
+ }
}
// RetryNodes are used in the case of 'forward to leader' where we want
// to try the latest leader on a fail in the case the leader just went down.
- public static class RetryNode extends StdNode {
+ public static class ForwardNode extends StdNode {
private ZkStateReader zkStateReader;
- public RetryNode(ZkCoreNodeProps nodeProps, ZkStateReader zkStateReader, String collection, String shardId) {
- super(nodeProps, collection, shardId);
+ public ForwardNode(ZkCoreNodeProps nodeProps, ZkStateReader zkStateReader, String collection, String shardId, int maxRetries) {
+ super(nodeProps, collection, shardId, maxRetries);
this.zkStateReader = zkStateReader;
this.collection = collection;
this.shardId = shardId;
}
@Override
- public boolean checkRetry() {
- ZkCoreNodeProps leaderProps;
- try {
- leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderRetry(
- collection, shardId));
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- return false;
- } catch (Exception e) {
- // we retry with same info
- log.warn(null, e);
- return true;
+ public boolean checkRetry(Error err) {
+ boolean doRetry = false;
+ if (err.statusCode == 404 || err.statusCode == 403 || err.statusCode == 503) {
+ doRetry = true;
}
-
- this.nodeProps = leaderProps;
- return true;
+ // if it's a connect exception, lets try again
+ if (err.e instanceof SolrServerException && ((SolrServerException) err.e).getRootCause() instanceof ConnectException) {
+ doRetry = true;
+ } else if (err.e instanceof ConnectException) {
+ doRetry = true;
+ }
+ if (doRetry) {
+ ZkCoreNodeProps leaderProps;
+ try {
+ leaderProps = new ZkCoreNodeProps(zkStateReader.getLeaderRetry(
+ collection, shardId));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ } catch (Exception e) {
+ // we retry with same info
+ log.warn(null, e);
+ return true;
+ }
+
+ this.nodeProps = leaderProps;
+ }
+ return doRetry;
}
@Override
@@ -584,7 +658,7 @@ public class SolrCmdDistributor implements Closeable {
if (this == obj) return true;
if (!super.equals(obj)) return false;
if (getClass() != obj.getClass()) return false;
- RetryNode other = (RetryNode) obj;
+ ForwardNode other = (ForwardNode) obj;
if (nodeProps.getCoreUrl() == null) {
if (other.nodeProps.getCoreUrl() != null) return false;
} else if (!nodeProps.getCoreUrl().equals(other.nodeProps.getCoreUrl())) return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c338cf61/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
index eb4caec..eb92540 100644
--- a/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
+++ b/solr/core/src/java/org/apache/solr/update/StreamingSolrClients.java
@@ -143,7 +143,6 @@ class ErrorReportingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClien
@Override
public void handleError(Throwable ex) {
- req.trackRequestResult(null, false);
log.error("error", ex);
Error error = new Error();
error.e = (Exception) ex;
@@ -152,6 +151,10 @@ class ErrorReportingConcurrentUpdateSolrClient extends ConcurrentUpdateSolrClien
}
error.req = req;
errors.add(error);
+ if (!req.shouldRetry(error)) {
+ // only track the error if we are not retrying the request
+ req.trackRequestResult(null, false);
+ }
}
@Override
public void onSuccess(HttpResponse resp) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c338cf61/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
index a21d906..e1e7968 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
@@ -83,7 +83,7 @@ import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.SolrCmdDistributor;
import org.apache.solr.update.SolrCmdDistributor.Error;
import org.apache.solr.update.SolrCmdDistributor.Node;
-import org.apache.solr.update.SolrCmdDistributor.RetryNode;
+import org.apache.solr.update.SolrCmdDistributor.ForwardNode;
import org.apache.solr.update.SolrCmdDistributor.StdNode;
import org.apache.solr.update.SolrIndexSplitter;
import org.apache.solr.update.UpdateCommand;
@@ -114,6 +114,16 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
/**
+ * Request forwarded to a leader of a different shard will be retried up to this amount of times by default
+ */
+ static final int MAX_RETRIES_ON_FORWARD_DEAULT = 25;
+
+ /**
+ * Requests from leader to it's followers will be retried this amount of times by default
+ */
+ static final int MAX_RETRIES_TO_FOLLOWERS_DEFAULT = 3;
+
+ /**
* Values this processor supports for the <code>DISTRIB_UPDATE_PARAM</code>.
* This is an implementation detail exposed solely for tests.
*
@@ -175,6 +185,15 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private List<Node> nodes;
private Set<String> skippedCoreNodeNames;
private boolean isIndexChanged = false;
+
+ /**
+ * Number of times requests forwarded to some other shard's leader can be retried
+ */
+ private final int maxRetriesOnForward = MAX_RETRIES_ON_FORWARD_DEAULT;
+ /**
+ * Number of times requests from leaders to followers can be retried
+ */
+ private final int maxRetriesToFollowers = MAX_RETRIES_TO_FOLLOWERS_DEFAULT;
private UpdateCommand updateCommand; // the current command this processor is working on.
@@ -368,7 +387,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
} else if (!clusterState.getLiveNodes().contains(replica.getNodeName()) || replica.getState() == Replica.State.DOWN) {
skippedCoreNodeNames.add(replica.getName());
} else {
- nodes.add(new StdNode(new ZkCoreNodeProps(replica), collection, shardId));
+ nodes.add(new StdNode(new ZkCoreNodeProps(replica), collection, shardId, maxRetriesToFollowers));
}
}
return nodes;
@@ -377,7 +396,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// I need to forward on to the leader...
forwardToLeader = true;
return Collections.singletonList(
- new RetryNode(new ZkCoreNodeProps(leaderReplica), zkController.getZkStateReader(), collection, shardId));
+ new ForwardNode(new ZkCoreNodeProps(leaderReplica), zkController.getZkStateReader(), collection, shardId, maxRetriesOnForward));
}
} catch (InterruptedException e) {
@@ -779,7 +798,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
Set<String> replicasShouldBeInLowerTerms = new HashSet<>();
for (final SolrCmdDistributor.Error error : errors) {
- if (error.req.node instanceof RetryNode) {
+ if (error.req.node instanceof ForwardNode) {
// if it's a forward, any fail is a problem -
// otherwise we assume things are fine if we got it locally
// until we start allowing min replication param
@@ -1538,7 +1557,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
// don't forward to ourself
leaderForAnyShard = true;
} else {
- leaders.add(new RetryNode(coreLeaderProps, zkController.getZkStateReader(), collection, sliceName));
+ leaders.add(new ForwardNode(coreLeaderProps, zkController.getZkStateReader(), collection, sliceName, maxRetriesOnForward));
}
}
@@ -2100,7 +2119,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
private int achievedRf = Integer.MAX_VALUE;
private final int requestedRf;
- RollupRequestReplicationTracker(String minRepFact) {
+ public RollupRequestReplicationTracker(String minRepFact) {
try {
this.requestedRf = Integer.parseInt(minRepFact);
} catch (NumberFormatException nfe) {
@@ -2153,7 +2172,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
return requestedRf;
}
- LeaderRequestReplicationTracker(String shardId, int requestedRf) {
+ public LeaderRequestReplicationTracker(String shardId, int requestedRf) {
this.requestedRf = requestedRf;
this.myShardId = shardId;
}
@@ -2165,8 +2184,7 @@ public class DistributedUpdateProcessor extends UpdateRequestProcessor {
public void trackRequestResult(Node node, boolean success) {
if (log.isDebugEnabled()) {
- log.debug("trackRequestResult(" + node + "): success? " + success +
- ", shardId=" + myShardId);
+ log.debug("trackRequestResult({}): success? {}, shardId={}", node, success, myShardId);
}
if (success) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c338cf61/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
index 1d2d730..cd4ed00 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
@@ -401,8 +401,8 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
"No 'leader' replica available for shard " + slice.getName() + " of collection " + collection);
}
- return new SolrCmdDistributor.RetryNode(new ZkCoreNodeProps(leader), zkController.getZkStateReader(),
- collection, slice.getName());
+ return new SolrCmdDistributor.ForwardNode(new ZkCoreNodeProps(leader), zkController.getZkStateReader(),
+ collection, slice.getName(), DistributedUpdateProcessor.MAX_RETRIES_ON_FORWARD_DEAULT);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c338cf61/solr/core/src/test/org/apache/solr/update/MockStreamingSolrClients.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/MockStreamingSolrClients.java b/solr/core/src/test/org/apache/solr/update/MockStreamingSolrClients.java
index 72d39ff..c269c9e 100644
--- a/solr/core/src/test/org/apache/solr/update/MockStreamingSolrClients.java
+++ b/solr/core/src/test/org/apache/solr/update/MockStreamingSolrClients.java
@@ -16,19 +16,19 @@
*/
package org.apache.solr.update;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketException;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
-import java.io.IOException;
-import java.net.ConnectException;
-import java.net.SocketException;
-
public class MockStreamingSolrClients extends StreamingSolrClients {
- public enum Exp {CONNECT_EXCEPTION, SOCKET_EXCEPTION};
+ public enum Exp {CONNECT_EXCEPTION, SOCKET_EXCEPTION, BAD_REQUEST};
private volatile Exp exp = null;
@@ -46,12 +46,14 @@ public class MockStreamingSolrClients extends StreamingSolrClients {
this.exp = exp;
}
- private IOException exception() {
+ private Exception exception() {
switch (exp) {
case CONNECT_EXCEPTION:
return new ConnectException();
case SOCKET_EXCEPTION:
return new SocketException();
+ case BAD_REQUEST:
+ return new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Bad Request");
default:
break;
}
@@ -70,10 +72,17 @@ public class MockStreamingSolrClients extends StreamingSolrClients {
public NamedList<Object> request(SolrRequest request, String collection)
throws SolrServerException, IOException {
if (exp != null) {
- if (LuceneTestCase.random().nextBoolean()) {
- throw exception();
+ Exception e = exception();
+ if (e instanceof IOException) {
+ if (LuceneTestCase.random().nextBoolean()) {
+ throw (IOException)e;
+ } else {
+ throw new SolrServerException(e);
+ }
+ } else if (e instanceof SolrServerException) {
+ throw (SolrServerException)e;
} else {
- throw new SolrServerException(exception());
+ throw new SolrServerException(e);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c338cf61/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
index 1699b0d..24cf717 100644
--- a/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
@@ -16,15 +16,15 @@
*/
package org.apache.solr.update;
-import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
+import java.net.SocketException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;
-
+import javax.xml.parsers.ParserConfigurationException;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
@@ -34,6 +34,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -47,10 +48,12 @@ import org.apache.solr.index.LogDocMergePolicyFactory;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.update.MockStreamingSolrClients.Exp;
import org.apache.solr.update.SolrCmdDistributor.Error;
+import org.apache.solr.update.SolrCmdDistributor.ForwardNode;
import org.apache.solr.update.SolrCmdDistributor.Node;
-import org.apache.solr.update.SolrCmdDistributor.RetryNode;
import org.apache.solr.update.SolrCmdDistributor.StdNode;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
+import org.apache.solr.update.processor.DistributedUpdateProcessor.LeaderRequestReplicationTracker;
+import org.apache.solr.update.processor.DistributedUpdateProcessor.RollupRequestReplicationTracker;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -59,6 +62,8 @@ import org.xml.sax.SAXException;
// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
+ private static enum NodeType {FORWARD, STANDARD};
+
private AtomicInteger id = new AtomicInteger();
@BeforeClass
@@ -127,6 +132,7 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
shards = sb.toString();
}
+ @SuppressWarnings("unchecked")
@Test
@ShardsFixed(num = 4)
public void test() throws Exception {
@@ -325,31 +331,177 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
}
}
- testMaxRetries();
- testOneRetry();
+ testMaxRetries(NodeType.FORWARD);
+ testMaxRetries(NodeType.STANDARD);
+ testOneRetry(NodeType.FORWARD);
+ testOneRetry(NodeType.STANDARD);
testRetryNodeAgainstBadAddress();
- testRetryNodeWontRetrySocketError();
-
+ testStdNodeRetriesSocketError();
+ testForwardNodeWontRetrySocketError();
+ testNodeWontRetryBadRequest(NodeType.FORWARD);
+ testNodeWontRetryBadRequest(NodeType.STANDARD);
+ testMinRfOnRetries(NodeType.FORWARD);
+ testMinRfOnRetries(NodeType.STANDARD);
testDistribOpenSearcher();
+ testReqShouldRetryNoRetries();
+ testReqShouldRetryMaxRetries();
+ testReqShouldRetryBadRequest();
+ testReqShouldRetryNotFound();
+ testReqShouldRetryDBQ();
+ testDeletes(false, true);
+ testDeletes(false, false);
+ testDeletes(true, true);
+ testDeletes(true, false);
}
-
- private void testMaxRetries() throws IOException {
+
+ private void testDeletes(boolean dbq, boolean withFailures) throws Exception {
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+ solrclient.commit(true, true);
+ long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0)) {
- streamingClients.setExp(Exp.CONNECT_EXCEPTION);
+ try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
+ if (withFailures) {
+ streamingClients.setExp(Exp.CONNECT_EXCEPTION);
+ }
ArrayList<Node> nodes = new ArrayList<>();
- final HttpSolrClient solrclient1 = (HttpSolrClient) clients.get(0);
+
+ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
+ ZkStateReader.CORE_NAME_PROP, "");
final AtomicInteger retries = new AtomicInteger();
- ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient1.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
+ nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ Node retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
@Override
- public boolean checkRetry() {
+ public boolean checkRetry(Error err) {
+ streamingClients.setExp(null);
retries.incrementAndGet();
- return true;
+ return super.checkRetry(err);
}
};
+
+ nodes.add(retryNode);
+
+ for (int i = 0 ; i < 5 ; i++) {
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ int currentId = id.incrementAndGet();
+ cmd.solrDoc = sdoc("id", currentId);
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ cmdDistrib.distribAdd(cmd, nodes, params);
+ DeleteUpdateCommand dcmd = new DeleteUpdateCommand(null);
+ if (dbq) {
+ dcmd.setQuery("id:" + currentId);
+ } else {
+ dcmd.setId(String.valueOf(currentId));
+ }
+ cmdDistrib.distribDelete(dcmd, nodes, params, false, null, null);
+ }
+
+
+ CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
+ cmdDistrib.distribCommit(ccmd, nodes, new ModifiableSolrParams());
+ cmdDistrib.finish();
+
+ int expectedRetryCount = 0;
+ if (withFailures) {
+ if (dbq) {
+ expectedRetryCount = 1; // just the first cmd would be retried
+ } else {
+ expectedRetryCount = 10;
+ }
+ }
+ assertEquals(expectedRetryCount, retries.get());
+
+
+ long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
+
+ // we will get java.net.ConnectException which we retry on
+ assertEquals(numFoundBefore, numFoundAfter);
+ assertEquals(0, cmdDistrib.getErrors().size());
+ }
+ }
+
+ private void testMinRfOnRetries(NodeType nodeType) throws Exception {
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+ final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
+ try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
+ streamingClients.setExp(Exp.CONNECT_EXCEPTION);
+ ArrayList<Node> nodes = new ArrayList<>();
+
+ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
+ ZkStateReader.CORE_NAME_PROP, "");
+
+ final AtomicInteger retries = new AtomicInteger();
+ nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ if (nodeType == NodeType.FORWARD) {
+ nodes.add(new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ if (retries.incrementAndGet() >= 3) {
+ streamingClients.setExp(null);
+ }
+ return super.checkRetry(err);
+ }
+ });
+ } else {
+ nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ if (retries.incrementAndGet() >= 3) {
+ streamingClients.setExp(null);
+ }
+ return super.checkRetry(err);
+ }
+ });
+ }
+
+
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ cmd.solrDoc = sdoc("id", id.incrementAndGet());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ RollupRequestReplicationTracker rollupReqTracker = new RollupRequestReplicationTracker("2");
+ LeaderRequestReplicationTracker leaderReqTracker = new LeaderRequestReplicationTracker("shard1", 2);
+
+ cmdDistrib.distribAdd(cmd, nodes, params, false, rollupReqTracker, leaderReqTracker);
+ cmdDistrib.finish();
+ assertEquals(3, retries.get());
+ assertEquals(2, leaderReqTracker.getAchievedRf());// "2" here is because one would be the leader, that creates the instance of LeaderRequestReplicationTracker, the second one is the node
+
+ assertEquals(0, cmdDistrib.getErrors().size());
+ }
+ }
+
+ private void testMaxRetries(NodeType nodeType) throws IOException {
+ final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
+ try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
+ streamingClients.setExp(Exp.CONNECT_EXCEPTION);
+ ArrayList<Node> nodes = new ArrayList<>();
+ final HttpSolrClient solrclient1 = (HttpSolrClient) clients.get(0);
+
+ final AtomicInteger retries = new AtomicInteger();
+ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient1.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ Node retryNode;
+ if (nodeType == NodeType.FORWARD) {
+ retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 6) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ } else {
+ retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 6) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ }
+
+
nodes.add(retryNode);
AddUpdateCommand cmd = new AddUpdateCommand(null);
@@ -359,18 +511,61 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
cmdDistrib.distribAdd(cmd, nodes, params);
cmdDistrib.finish();
- assertEquals(6, retries.get());
+ assertEquals(7, retries.get());
assertEquals(1, cmdDistrib.getErrors().size());
}
}
- private void testOneRetry() throws Exception {
+ private void testReqShouldRetryNoRetries() {
+ Error err = getError(new SocketException());
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 0), new UpdateRequest(), true);
+ assertFalse(req.shouldRetry(err));
+ }
+
+ private void testReqShouldRetryDBQ() {
+ Error err = getError(new SocketException());
+ UpdateRequest dbqReq = new UpdateRequest();
+ dbqReq.deleteByQuery("*:*");
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), dbqReq, true);
+ assertFalse(req.shouldRetry(err));
+ }
+
+ private void testReqShouldRetryMaxRetries() {
+ Error err = getError(new SocketException());
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+ assertTrue(req.shouldRetry(err));
+ req.retries++;
+ assertFalse(req.shouldRetry(err));
+ }
+
+ private void testReqShouldRetryBadRequest() {
+ Error err = getError(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "bad request"));
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+ assertFalse(req.shouldRetry(err));
+ }
+
+ private void testReqShouldRetryNotFound() {
+ Error err = getError(new SolrException(SolrException.ErrorCode.NOT_FOUND, "not found"));
+ SolrCmdDistributor.Req req = new SolrCmdDistributor.Req(null, new StdNode(null, "collection1", "shard1", 1), new UpdateRequest(), true);
+ assertTrue(req.shouldRetry(err));
+ }
+
+ private Error getError(Exception e) {
+ Error err = new Error();
+ err.e = e;
+ if (e instanceof SolrException) {
+ err.statusCode = ((SolrException)e).code();
+ }
+ return err;
+ }
+
+ private void testOneRetry(NodeType nodeType) throws Exception {
final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
.getNumFound();
final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0)) {
+ try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
streamingClients.setExp(Exp.CONNECT_EXCEPTION);
ArrayList<Node> nodes = new ArrayList<>();
@@ -379,14 +574,26 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
final AtomicInteger retries = new AtomicInteger();
nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
- @Override
- public boolean checkRetry() {
- streamingClients.setExp(null);
- retries.incrementAndGet();
- return true;
- }
- };
+ Node retryNode;
+ if (nodeType == NodeType.FORWARD) {
+ retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ streamingClients.setExp(null);
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ } else {
+ retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ streamingClients.setExp(null);
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ }
nodes.add(retryNode);
@@ -412,12 +619,70 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
}
}
- private void testRetryNodeWontRetrySocketError() throws Exception {
+ private void testNodeWontRetryBadRequest(NodeType nodeType) throws Exception {
+ ignoreException("Bad Request");
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+ long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
+ final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
+ try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
+ streamingClients.setExp(Exp.BAD_REQUEST);
+ ArrayList<Node> nodes = new ArrayList<>();
+ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
+ ZkStateReader.CORE_NAME_PROP, "");
+
+ final AtomicInteger retries = new AtomicInteger();
+ Node retryNode;
+ if (nodeType == NodeType.FORWARD) {
+ retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ } else {
+ retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+ }
+ nodes.add(retryNode);
+
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ cmd.solrDoc = sdoc("id", id.incrementAndGet());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ CommitUpdateCommand ccmd = new CommitUpdateCommand(null, false);
+ cmdDistrib.distribAdd(cmd, nodes, params);
+
+ streamingClients.setExp(null);
+ cmdDistrib.distribCommit(ccmd, nodes, params);
+ cmdDistrib.finish();
+
+ // it will checkRetry, but not actually do it...
+ assertEquals(1, retries.get());
+
+
+ long numFoundAfter = solrclient.query(new SolrQuery("*:*")).getResults()
+ .getNumFound();
+
+ // we will get java.net.SocketException: Network is unreachable, which we don't retry on
+ assertEquals(numFoundBefore, numFoundAfter);
+ assertEquals(1, cmdDistrib.getErrors().size());
+ unIgnoreException("Bad Request");
+ }
+ }
+
+ private void testForwardNodeWontRetrySocketError() throws Exception {
final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
long numFoundBefore = solrclient.query(new SolrQuery("*:*")).getResults()
.getNumFound();
final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
- try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 5, 0)) {
+ try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
streamingClients.setExp(Exp.SOCKET_EXCEPTION);
ArrayList<Node> nodes = new ArrayList<>();
@@ -426,11 +691,11 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
final AtomicInteger retries = new AtomicInteger();
nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
+ ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
@Override
- public boolean checkRetry() {
+ public boolean checkRetry(Error err) {
retries.incrementAndGet();
- return true;
+ return super.checkRetry(err);
}
};
@@ -460,6 +725,41 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
assertEquals(1, cmdDistrib.getErrors().size());
}
}
+
+ private void testStdNodeRetriesSocketError() throws Exception {
+ final HttpSolrClient solrclient = (HttpSolrClient) clients.get(0);
+ final MockStreamingSolrClients streamingClients = new MockStreamingSolrClients(updateShardHandler);
+ try (SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(streamingClients, 0)) {
+ streamingClients.setExp(Exp.SOCKET_EXCEPTION);
+ ArrayList<Node> nodes = new ArrayList<>();
+
+ ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
+ ZkStateReader.CORE_NAME_PROP, "");
+
+ final AtomicInteger retries = new AtomicInteger();
+ nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ Node retryNode = new StdNode(new ZkCoreNodeProps(nodeProps), "collection1", "shard1", 5) {
+ @Override
+ public boolean checkRetry(Error err) {
+ retries.incrementAndGet();
+ return super.checkRetry(err);
+ }
+ };
+
+
+ nodes.add(retryNode);
+
+ AddUpdateCommand cmd = new AddUpdateCommand(null);
+ cmd.solrDoc = sdoc("id", id.incrementAndGet());
+ ModifiableSolrParams params = new ModifiableSolrParams();
+
+ cmdDistrib.distribAdd(cmd, nodes, params);
+ cmdDistrib.finish();
+
+ // it will checkRetry, but not actually do it...
+ assertEquals(6, retries.get());
+ }
+ }
private void testRetryNodeAgainstBadAddress() throws SolrServerException, IOException {
// Test RetryNode
@@ -471,14 +771,14 @@ public class SolrCmdDistributorTest extends BaseDistributedSearchTestCase {
ArrayList<Node> nodes = new ArrayList<>();
ZkNodeProps nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, "[ff01::114]:33332" + context, ZkStateReader.CORE_NAME_PROP, "");
- RetryNode retryNode = new RetryNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1") {
+ ForwardNode retryNode = new ForwardNode(new ZkCoreNodeProps(nodeProps), null, "collection1", "shard1", 5) {
@Override
- public boolean checkRetry() {
+ public boolean checkRetry(Error err) {
ZkNodeProps leaderProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP, solrclient.getBaseURL(),
ZkStateReader.CORE_NAME_PROP, "");
this.nodeProps = new ZkCoreNodeProps(leaderProps);
- return true;
+ return super.checkRetry(err);
}
};
[26/31] lucene-solr:jira/http2: SOLR-12555: Use `expectThrows` for
expected exceptions
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/TestZkChroot.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestZkChroot.java b/solr/core/src/test/org/apache/solr/cloud/TestZkChroot.java
index 874c6ec..a89d3e3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestZkChroot.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestZkChroot.java
@@ -99,23 +99,20 @@ public class TestZkChroot extends SolrTestCaseJ4 {
System.setProperty("bootstrap_conf", "false");
System.setProperty("zkHost", zkServer.getZkHost() + chroot);
-
- SolrZkClient zkClient = null;
-
- try {
- zkClient = new SolrZkClient(zkServer.getZkHost(),
- AbstractZkTestCase.TIMEOUT);
- assertFalse("Path '" + chroot + "' should not exist before the test",
- zkClient.exists(chroot, true));
- cores = CoreContainer.createAndLoad(home);
- fail("There should be a zk exception, as the initial path doesn't exist");
- } catch (ZooKeeperException e) {
- // expected
+
+ try(SolrZkClient zkClient = new SolrZkClient(zkServer.getZkHost(),
+ AbstractZkTestCase.TIMEOUT)) {
+ expectThrows(ZooKeeperException.class,
+ "did not get a top level exception when more then 4 updates failed",
+ () -> {
+ assertFalse("Path '" + chroot + "' should not exist before the test",
+ zkClient.exists(chroot, true));
+ cores = CoreContainer.createAndLoad(home);
+ });
assertFalse("Path shouldn't have been created",
zkClient.exists(chroot, true));// check the path was not created
} finally {
if (cores != null) cores.shutdown();
- if (zkClient != null) zkClient.close();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
index d2b6b08..4363382 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
@@ -178,14 +178,9 @@ public class ZkCLITest extends SolrTestCaseJ4 {
// test put file
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"putfile", "/solr.xml", SOLR_HOME + File.separator + "not-there.xml"};
- try {
- ZkCLI.main(args);
- fail("Should have had a file not found exception");
- } catch (FileNotFoundException fne) {
- String msg = fne.getMessage();
- assertTrue("Didn't find expected error message containing 'not-there.xml' in " + msg,
- msg.indexOf("not-there.xml") != -1);
- }
+ FileNotFoundException e = expectThrows(FileNotFoundException.class, () -> ZkCLI.main(args));
+ assertTrue("Didn't find expected error message containing 'not-there.xml' in " + e.getMessage(),
+ e.getMessage().indexOf("not-there.xml") != -1);
}
@Test
@@ -332,11 +327,8 @@ public class ZkCLITest extends SolrTestCaseJ4 {
File file = createTempFile("newfile", null).toFile();
String[] args = new String[] {"-zkhost", zkServer.getZkAddress(), "-cmd",
"getfile", getNode, file.getAbsolutePath()};
- try {
- ZkCLI.main(args);
- fail("Expected NoNodeException");
- } catch (KeeperException.NoNodeException ex) {
- }
+ KeeperException e = expectThrows(KeeperException.class, () -> ZkCLI.main(args));
+ assertEquals(e.code(), KeeperException.Code.NONODE);
}
@Test(expected = SolrException.class)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00aeb64c/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
index 1376fe8..120457c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkSolrClientTest.java
@@ -21,7 +21,6 @@ import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import junit.framework.Assert;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
@@ -117,7 +116,8 @@ public class ZkSolrClientTest extends SolrTestCaseJ4 {
AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
- zkClient = new SolrZkClient(server.getZkAddress(), AbstractZkTestCase.TIMEOUT);
+ final SolrZkClient zkClientConLoss = new SolrZkClient(server.getZkAddress(), AbstractZkTestCase.TIMEOUT);
+ zkClient = zkClientConLoss;
String shardsPath = "/collections/collection1/shards";
zkClient.makePath(shardsPath, false, true);
@@ -129,12 +129,10 @@ public class ZkSolrClientTest extends SolrTestCaseJ4 {
Thread.sleep(80);
- try {
- zkClient.makePath("collections/collection2", false);
- Assert.fail("Server should be down here");
- } catch (KeeperException.ConnectionLossException e) {
-
- }
+ expectThrows(KeeperException.class,
+ "Server should be down",
+ () -> zkClientConLoss.makePath("collections/collection2", false)
+ );
// bring server back up
server = new ZkTestServer(zkDir, zkServerPort);
@@ -204,18 +202,14 @@ public class ZkSolrClientTest extends SolrTestCaseJ4 {
ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(timeout);
final long start = System.nanoTime();
- try {
- zkCmdExecutor.retryOperation(() -> {
- if (System.nanoTime() - start > TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {
- throw new KeeperException.SessionExpiredException();
- }
- throw new KeeperException.ConnectionLossException();
+ expectThrows(KeeperException.SessionExpiredException.class, () -> {
+ zkCmdExecutor.retryOperation(() -> {
+ if (System.nanoTime() - start > TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {
+ throw new KeeperException.SessionExpiredException();
+ }
+ throw new KeeperException.ConnectionLossException();
+ });
});
- } catch(KeeperException.SessionExpiredException e) {
-
- } catch (Exception e) {
- fail("Expected " + KeeperException.SessionExpiredException.class.getSimpleName() + " but got " + e.getClass().getSimpleName());
- }
} finally {
if (server != null) {
server.shutdown();
@@ -334,31 +328,22 @@ public class ZkSolrClientTest extends SolrTestCaseJ4 {
zkClient.clean("/");
// should not work
- try {
- zkClient.makePath("/test/path/here", (byte[]) null, CreateMode.PERSISTENT, (Watcher) null, true, true, 1);
- fail("We should not be able to create this path");
- } catch (Exception e) {
-
- }
+ KeeperException e =expectThrows(KeeperException.NoNodeException.class,
+ "We should not be able to create this path",
+ () -> zkClient.makePath("/test/path/here", (byte[]) null, CreateMode.PERSISTENT, (Watcher) null, true, true, 1));
zkClient.clean("/");
ZkCmdExecutor zkCmdExecutor = new ZkCmdExecutor(30000);
- try {
- zkCmdExecutor.ensureExists("/collection/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2);
- fail("We should not be able to create this path");
- } catch (Exception e) {
-
- }
+ expectThrows(KeeperException.NoNodeException.class,
+ "We should not be able to create this path",
+ () -> zkCmdExecutor.ensureExists("/collection/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2));
zkClient.makePath("/collection", true);
- try {
- zkCmdExecutor.ensureExists("/collections/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2);
- fail("We should not be able to create this path");
- } catch (Exception e) {
-
- }
+ expectThrows(KeeperException.NoNodeException.class,
+ "We should not be able to create this path",
+ () -> zkCmdExecutor.ensureExists("/collections/collection/leader", (byte[]) null, CreateMode.PERSISTENT, zkClient, 2));
zkClient.makePath("/collection/collection", true);
byte[] bytes = new byte[10];