You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/10/25 07:36:19 UTC
[01/12] lucene-solr:jira/http2: LUCENE-8538: Add a Simple WKT Shape
Parser for creating Lucene Geometries (Polygon, Line, Rectangle, points,
and multi-variants) from WKT format.
Repository: lucene-solr
Updated Branches:
refs/heads/jira/http2 19f2d9fa5 -> c3e246fb3
LUCENE-8538: Add a Simple WKT Shape Parser for creating Lucene Geometries (Polygon, Line, Rectangle, points, and multi-variants) from WKT format.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bd714ca1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bd714ca1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bd714ca1
Branch: refs/heads/jira/http2
Commit: bd714ca1edc681cd871e6d385bd98a9cb9d7ffe6
Parents: e083b15
Author: Nicholas Knize <nk...@gmail.com>
Authored: Mon Oct 22 14:25:22 2018 -0500
Committer: Nicholas Knize <nk...@gmail.com>
Committed: Wed Oct 24 09:27:47 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../apache/lucene/geo/SimpleWKTShapeParser.java | 406 +++++++++++++++++++
.../lucene/geo/TestSimpleWKTShapeParsing.java | 206 ++++++++++
3 files changed, 615 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd714ca1/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 23d807b..4097ce0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -220,6 +220,9 @@ New Features
may be used to determine how to split the inner nodes, and dimensions N+1 to D
are ignored and stored as data dimensions at the leaves. (Nick Knize)
+* LUCENE-8538: Add a Simple WKT Shape Parser for creating Lucene Geometries (Polygon, Line,
+ Rectangle) from WKT format. (Nick Knize)
+
Improvements:
* LUCENE-8521: Change LatLonShape encoding to 7 dimensions instead of 6; where the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd714ca1/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java b/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java
new file mode 100644
index 0000000..17b595f
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.geo;
+
+import java.io.IOException;
+import java.io.StreamTokenizer;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Parses shape geometry represented in WKT format
+ *
+ * complies with OGC® document: 12-063r5 and ISO/IEC 13249-3:2016 standard
+ * located at http://docs.opengeospatial.org/is/12-063r5/12-063r5.html
+ */
+public class SimpleWKTShapeParser {
+ public static final String EMPTY = "EMPTY";
+ public static final String SPACE = " ";
+ public static final String LPAREN = "(";
+ public static final String RPAREN = ")";
+ public static final String COMMA = ",";
+ public static final String NAN = "NaN";
+
+ private static final String NUMBER = "<NUMBER>";
+ private static final String EOF = "END-OF-STREAM";
+ private static final String EOL = "END-OF-LINE";
+
+ // no instance
+ private SimpleWKTShapeParser() {}
+
+ public static Object parse(String wkt) throws IOException, ParseException {
+ return parseExpectedType(wkt, null);
+ }
+
+ public static Object parseExpectedType(String wkt, final ShapeType shapeType) throws IOException, ParseException {
+ try (StringReader reader = new StringReader(wkt)) {
+ // setup the tokenizer; configured to read words w/o numbers
+ StreamTokenizer tokenizer = new StreamTokenizer(reader);
+ tokenizer.resetSyntax();
+ tokenizer.wordChars('a', 'z');
+ tokenizer.wordChars('A', 'Z');
+ tokenizer.wordChars(128 + 32, 255);
+ tokenizer.wordChars('0', '9');
+ tokenizer.wordChars('-', '-');
+ tokenizer.wordChars('+', '+');
+ tokenizer.wordChars('.', '.');
+ tokenizer.whitespaceChars(0, ' ');
+ tokenizer.commentChar('#');
+ Object geometry = parseGeometry(tokenizer, shapeType);
+ checkEOF(tokenizer);
+ return geometry;
+ }
+ }
+
+ /** parse geometry from the stream tokenizer */
+ private static Object parseGeometry(StreamTokenizer stream, ShapeType shapeType) throws IOException, ParseException {
+ final ShapeType type = ShapeType.forName(nextWord(stream));
+ if (shapeType != null && shapeType != ShapeType.GEOMETRYCOLLECTION) {
+ if (type.wktName().equals(shapeType.wktName()) == false) {
+ throw new ParseException("Expected geometry type: [" + shapeType + "], but found: [" + type + "]", stream.lineno());
+ }
+ }
+ switch (type) {
+ case POINT:
+ return parsePoint(stream);
+ case MULTIPOINT:
+ return parseMultiPoint(stream);
+ case LINESTRING:
+ return parseLine(stream);
+ case MULTILINESTRING:
+ return parseMultiLine(stream);
+ case POLYGON:
+ return parsePolygon(stream);
+ case MULTIPOLYGON:
+ return parseMultiPolygon(stream);
+ case ENVELOPE:
+ return parseBBox(stream);
+ case GEOMETRYCOLLECTION:
+ return parseGeometryCollection(stream);
+ default:
+ throw new IllegalArgumentException("Unknown geometry type: " + type);
+ }
+ }
+
+ /** Parses a point as a double array */
+ private static double[] parsePoint(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ double[] pt = new double[]{nextNumber(stream), nextNumber(stream)};
+ if (isNumberNext(stream) == true) {
+ nextNumber(stream);
+ }
+ nextCloser(stream);
+ return pt;
+ }
+
+ /** Parses a list of points into latitude and longitude arraylists */
+ private static void parseCoordinates(StreamTokenizer stream, ArrayList lats, ArrayList lons)
+ throws IOException, ParseException {
+ boolean isOpenParen = false;
+ if (isNumberNext(stream) || (isOpenParen = nextWord(stream).equals(LPAREN))) {
+ parseCoordinate(stream, lats, lons);
+ }
+
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ isOpenParen = false;
+ if (isNumberNext(stream) || (isOpenParen = nextWord(stream).equals(LPAREN))) {
+ parseCoordinate(stream, lats, lons);
+ }
+ if (isOpenParen && nextCloser(stream).equals(RPAREN) == false) {
+ throw new ParseException("expected: [" + RPAREN + "] but found: [" + tokenString(stream) + "]", stream.lineno());
+ }
+ }
+
+ if (isOpenParen && nextCloser(stream).equals(RPAREN) == false) {
+ throw new ParseException("expected: [" + RPAREN + "] but found: [" + tokenString(stream) + "]", stream.lineno());
+ }
+ }
+
+ /** parses a single coordinate, w/ optional 3rd dimension */
+ private static void parseCoordinate(StreamTokenizer stream, ArrayList lats, ArrayList lons)
+ throws IOException, ParseException {
+ lons.add(nextNumber(stream));
+ lats.add(nextNumber(stream));
+ if (isNumberNext(stream)) {
+ nextNumber(stream);
+ }
+ }
+
+ /** parses a MULTIPOINT type */
+ private static double[][] parseMultiPoint(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ double[][] result = new double[lats.size()][2];
+ for (int i = 0; i < lats.size(); ++i) {
+ result[i] = new double[] {lons.get(i), lats.get(i)};
+ }
+ return result;
+ }
+
+ /** parses a LINESTRING */
+ private static Line parseLine(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ return new Line(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray());
+ }
+
+ /** parses a MULTILINESTRING */
+ private static Line[] parseMultiLine(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Line> lines = new ArrayList();
+ lines.add(parseLine(stream));
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ lines.add(parseLine(stream));
+ }
+ return lines.toArray(new Line[lines.size()]);
+ }
+
+ /** parses the hole of a polygon */
+ private static Polygon parsePolygonHole(StreamTokenizer stream) throws IOException, ParseException {
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ return new Polygon(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray());
+ }
+
+ /** parses a POLYGON */
+ private static Polygon parsePolygon(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ nextOpener(stream);
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ ArrayList<Polygon> holes = new ArrayList<>();
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ holes.add(parsePolygonHole(stream));
+ }
+
+ if (holes.isEmpty() == false) {
+ return new Polygon(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray(), holes.toArray(new Polygon[holes.size()]));
+ }
+ return new Polygon(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray());
+ }
+
+ /** parses a MULTIPOLYGON */
+ private static Polygon[] parseMultiPolygon(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Polygon> polygons = new ArrayList();
+ polygons.add(parsePolygon(stream));
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ polygons.add(parsePolygon(stream));
+ }
+ return polygons.toArray(new Polygon[polygons.size()]);
+ }
+
+ /** parses an ENVELOPE */
+ private static Rectangle parseBBox(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ double minLon = nextNumber(stream);
+ nextComma(stream);
+ double maxLon = nextNumber(stream);
+ nextComma(stream);
+ double maxLat = nextNumber(stream);
+ nextComma(stream);
+ double minLat = nextNumber(stream);
+ nextCloser(stream);
+ return new Rectangle(minLat, maxLat, minLon, maxLon);
+ }
+
+ /** parses a GEOMETRYCOLLECTION */
+ private static Object[] parseGeometryCollection(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Object> geometries = new ArrayList<>();
+ geometries.add(parseGeometry(stream, ShapeType.GEOMETRYCOLLECTION));
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ geometries.add(parseGeometry(stream, null));
+ }
+ return geometries.toArray(new Object[geometries.size()]);
+ }
+
+ /** next word in the stream */
+ private static String nextWord(StreamTokenizer stream) throws ParseException, IOException {
+ switch (stream.nextToken()) {
+ case StreamTokenizer.TT_WORD:
+ final String word = stream.sval;
+ return word.equalsIgnoreCase(EMPTY) ? EMPTY : word;
+ case '(': return LPAREN;
+ case ')': return RPAREN;
+ case ',': return COMMA;
+ }
+ throw new ParseException("expected word but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** next number in the stream */
+ private static double nextNumber(StreamTokenizer stream) throws IOException, ParseException {
+ if (stream.nextToken() == StreamTokenizer.TT_WORD) {
+ if (stream.sval.equalsIgnoreCase(NAN)) {
+ return Double.NaN;
+ } else {
+ try {
+ return Double.parseDouble(stream.sval);
+ } catch (NumberFormatException e) {
+ throw new ParseException("invalid number found: " + stream.sval, stream.lineno());
+ }
+ }
+ }
+ throw new ParseException("expected number but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** next token in the stream */
+ private static String tokenString(StreamTokenizer stream) {
+ switch (stream.ttype) {
+ case StreamTokenizer.TT_WORD: return stream.sval;
+ case StreamTokenizer.TT_EOF: return EOF;
+ case StreamTokenizer.TT_EOL: return EOL;
+ case StreamTokenizer.TT_NUMBER: return NUMBER;
+ }
+ return "'" + (char)stream.ttype + "'";
+ }
+
+ /** checks if the next token is a number */
+ private static boolean isNumberNext(StreamTokenizer stream) throws IOException {
+ final int type = stream.nextToken();
+ stream.pushBack();
+ return type == StreamTokenizer.TT_WORD;
+ }
+
+ /** checks if next token is an EMPTY or open paren */
+ private static String nextEmptyOrOpen(StreamTokenizer stream) throws IOException, ParseException {
+ final String next = nextWord(stream);
+ if (next.equals(EMPTY) || next.equals(LPAREN)) {
+ return next;
+ }
+ throw new ParseException("expected " + EMPTY + " or " + LPAREN
+ + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** checks if next token is a closing paren */
+ private static String nextCloser(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextWord(stream).equals(RPAREN)) {
+ return RPAREN;
+ }
+ throw new ParseException("expected " + RPAREN + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** expects a comma as next token */
+ private static String nextComma(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextWord(stream).equals(COMMA) == true) {
+ return COMMA;
+ }
+ throw new ParseException("expected " + COMMA + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** expects an open RPAREN as the next toke */
+ private static String nextOpener(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextWord(stream).equals(LPAREN)) {
+ return LPAREN;
+ }
+ throw new ParseException("expected " + LPAREN + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** expects either a closing LPAREN or comma as the next token */
+ private static String nextCloserOrComma(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextWord(stream);
+ if (token.equals(COMMA) || token.equals(RPAREN)) {
+ return token;
+ }
+ throw new ParseException("expected " + COMMA + " or " + RPAREN
+ + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** next word in the stream */
+ private static void checkEOF(StreamTokenizer stream) throws ParseException, IOException {
+ if (stream.nextToken() != StreamTokenizer.TT_EOF) {
+ throw new ParseException("expected end of WKT string but found additional text: "
+ + tokenString(stream), stream.lineno());
+ }
+ }
+
+ /** Enumerated type for Shapes */
+ public enum ShapeType {
+ POINT("point"),
+ MULTIPOINT("multipoint"),
+ LINESTRING("linestring"),
+ MULTILINESTRING("multilinestring"),
+ POLYGON("polygon"),
+ MULTIPOLYGON("multipolygon"),
+ GEOMETRYCOLLECTION("geometrycollection"),
+ ENVELOPE("envelope"); // not part of the actual WKB spec
+
+ private final String shapeName;
+ private static Map<String, ShapeType> shapeTypeMap = new HashMap<>();
+ private static final String BBOX = "BBOX";
+
+ static {
+ for (ShapeType type : values()) {
+ shapeTypeMap.put(type.shapeName, type);
+ }
+ shapeTypeMap.put(ENVELOPE.wktName().toLowerCase(Locale.ROOT), ENVELOPE);
+ }
+
+ ShapeType(String shapeName) {
+ this.shapeName = shapeName;
+ }
+
+ protected String typename() {
+ return shapeName;
+ }
+
+ /** wkt shape name */
+ public String wktName() {
+ return this == ENVELOPE ? BBOX : this.shapeName;
+ }
+
+ public static ShapeType forName(String shapename) {
+ String typename = shapename.toLowerCase(Locale.ROOT);
+ for (ShapeType type : values()) {
+ if(type.shapeName.equals(typename)) {
+ return type;
+ }
+ }
+ throw new IllegalArgumentException("unknown geo_shape ["+shapename+"]");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd714ca1/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java b/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java
new file mode 100644
index 0000000..e941ef4
--- /dev/null
+++ b/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.geo;
+
+import org.apache.lucene.geo.SimpleWKTShapeParser.ShapeType;
+import org.apache.lucene.util.LuceneTestCase;
+
+/** simple WKT parsing tests */
+public class TestSimpleWKTShapeParsing extends LuceneTestCase {
+
+ /** test simple Point */
+ public void testPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POINT + "(101.0 10.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof double[]);
+ double[] point = (double[])shape;
+ assertEquals(101d, point[0], 0d); // lon
+ assertEquals(10d, point[1], 1d); // lat
+ }
+
+ /** test POINT EMPTY returns null */
+ public void testEmptyPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POINT + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple MULTIPOINT */
+ public void testMultiPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTIPOINT + "(101.0 10.0, 180.0 90.0, -180.0 -90.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof double[][]);
+ double[][] pts = (double[][])shape;
+ assertEquals(3, pts.length,0);
+ assertEquals(101d, pts[0][0], 0);
+ assertEquals(10d, pts[0][1], 0);
+ assertEquals(180d, pts[1][0], 0);
+ assertEquals(90d, pts[1][1], 0);
+ assertEquals(-180d, pts[2][0], 0);
+ assertEquals(-90d, pts[2][1], 0);
+ }
+
+ /** test MULTIPOINT EMPTY returns null */
+ public void testEmptyMultiPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTIPOINT + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple LINESTRING */
+ public void testLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.LINESTRING + "(101.0 10.0, 180.0 90.0, -180.0 -90.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Line);
+ Line line = (Line)shape;
+ assertEquals(3, line.numPoints(),0);
+ assertEquals(101d, line.getLon(0), 0);
+ assertEquals(10d, line.getLat(0), 0);
+ assertEquals(180d, line.getLon(1), 0);
+ assertEquals(90d, line.getLat(1), 0);
+ assertEquals(-180d, line.getLon(2), 0);
+ assertEquals(-90d, line.getLat(2), 0);
+ }
+
+ /** test empty LINESTRING */
+ public void testEmptyLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.LINESTRING + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple MULTILINESTRING */
+ public void testMultiLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTILINESTRING + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0),");
+ b.append("(10.0 2.0, 11.0 2.0, 11.0 3.0, 10.0 3.0, 10.0 2.0))");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Line[]);
+ Line[] lines = (Line[])shape;
+ assertEquals(2, lines.length, 0);
+ }
+
+ /** test empty MULTILINESTRING */
+ public void testEmptyMultiLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTILINESTRING + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple polygon: POLYGON((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) */
+ public void testPolygon() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POLYGON + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0))\n");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Polygon);
+ Polygon polygon = (Polygon)shape;
+ assertEquals(new Polygon(new double[] {0.0, 0.0, 1.0, 1.0, 0.0},
+ new double[] {100.0, 101.0, 101.0, 100.0, 100.0}), polygon);
+ }
+
+ /** test polygon with hole */
+ public void testPolygonWithHole() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POLYGON + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), ");
+ b.append("(100.5 0.5, 100.5 0.75, 100.75 0.75, 100.75 0.5, 100.5 0.5))");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Polygon);
+ Polygon hole = new Polygon(new double[] {0.5, 0.75, 0.75, 0.5, 0.5},
+ new double[] {100.5, 100.5, 100.75, 100.75, 100.5});
+ Polygon expected = new Polygon(new double[] {0.0, 0.0, 1.0, 1.0, 0.0},
+ new double[] {100.0, 101.0, 101.0, 100.0, 100.0}, hole);
+ Polygon polygon = (Polygon)shape;
+
+ assertEquals(expected, polygon);
+ }
+
+ /** test MultiPolygon returns Polygon array */
+ public void testMultiPolygon() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTIPOLYGON + "(((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)),");
+ b.append("((10.0 2.0, 11.0 2.0, 11.0 3.0, 10.0 3.0, 10.0 2.0)))");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Polygon[]);
+ Polygon[] polygons = (Polygon[])shape;
+ assertEquals(2, polygons.length);
+ assertEquals(new Polygon(new double[] {0.0, 0.0, 1.0, 1.0, 0.0},
+ new double[] {100.0, 101.0, 101.0, 100.0, 100.0}), polygons[0]);
+ assertEquals(new Polygon(new double[] {2.0, 2.0, 3.0, 3.0, 2.0},
+ new double[] {10.0, 11.0, 11.0, 10.0, 10.0}), polygons[1]);
+ }
+
+ /** polygon must be closed */
+ public void testPolygonNotClosed() {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POLYGON + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0))\n");
+
+ IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+ SimpleWKTShapeParser.parse(b.toString());
+ });
+ assertTrue(expected.getMessage(),
+ expected.getMessage().contains("first and last points of the polygon must be the same (it must close itself)"));
+ }
+
+ /** test simple ENVELOPE (minLon, maxLon, maxLat, minLat) */
+ public void testEnvelope() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.ENVELOPE + "(-180.0, 180.0, 90.0, -90.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Rectangle);
+ Rectangle bbox = (Rectangle)shape;
+ assertEquals(-180d, bbox.minLon, 0);
+ assertEquals(180d, bbox.maxLon, 0);
+ assertEquals(-90d, bbox.minLat, 0);
+ assertEquals(90d, bbox.maxLat, 0);
+ }
+
+ /** test simple geometry collection */
+ public void testGeometryCollection() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.GEOMETRYCOLLECTION + "(");
+ b.append(ShapeType.MULTIPOLYGON + "(((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)),");
+ b.append("((10.0 2.0, 11.0 2.0, 11.0 3.0, 10.0 3.0, 10.0 2.0))),");
+ b.append(ShapeType.POINT + "(101.0 10.0),");
+ b.append(ShapeType.LINESTRING + "(101.0 10.0, 180.0 90.0, -180.0 -90.0),");
+ b.append(ShapeType.ENVELOPE + "(-180.0, 180.0, 90.0, -90.0)");
+ b.append(")");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Object[]);
+ Object[] shapes = (Object[]) shape;
+ assertEquals(4, shapes.length);
+ assertTrue(shapes[0] instanceof Polygon[]);
+ assertTrue(shapes[1] instanceof double[]);
+ assertTrue(shapes[2] instanceof Line);
+ assertTrue(shapes[3] instanceof Rectangle);
+ }
+}
[12/12] lucene-solr:jira/http2: Merge branch 'master' into jira/http1
Posted by da...@apache.org.
Merge branch 'master' into jira/http1
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c3e246fb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c3e246fb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c3e246fb
Branch: refs/heads/jira/http2
Commit: c3e246fb3829cd7fd6b7519a450b28c598afe065
Parents: 19f2d9f 3e87499
Author: Cao Manh Dat <da...@apache.org>
Authored: Thu Oct 25 14:36:03 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Thu Oct 25 14:36:03 2018 +0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
lucene/common-build.xml | 2 +-
.../apache/lucene/geo/SimpleWKTShapeParser.java | 406 +++++++++
.../lucene/geo/TestSimpleWKTShapeParsing.java | 206 +++++
solr/CHANGES.txt | 2 +
...lrconfig-distrib-update-processor-chains.xml | 2 +-
.../conf/solrconfig-doctransformers.xml | 2 +-
.../solr/collection1/conf/solrconfig-hash.xml | 2 +-
.../conf/solrconfig-master-throttled.xml | 2 +-
.../collection1/conf/solrconfig-paramset.xml | 2 +-
.../solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../solr/collection1/conf/solrconfig-tagger.xml | 2 +-
.../cloud/TestCloudJSONFacetJoinDomain.java | 853 ------------------
.../solr/cloud/TestCloudJSONFacetSKG.java | 677 ---------------
.../TestSolrConfigHandlerConcurrent.java | 4 +-
.../org/apache/solr/search/facet/DebugAgg.java | 2 +-
.../facet/TestCloudJSONFacetJoinDomain.java | 855 +++++++++++++++++++
.../search/facet/TestCloudJSONFacetSKG.java | 678 +++++++++++++++
solr/solr-ref-guide/src/collections-api.adoc | 32 +-
.../src/stream-decorator-reference.adoc | 2 +-
.../src/stream-evaluator-reference.adoc | 5 +-
.../solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../solr/configsets/ml/conf/solrconfig.xml | 2 +-
.../configsets/streaming/conf/solrconfig.xml | 2 +-
.../solr/cloud/MultiSolrCloudTestCase.java | 1 +
25 files changed, 2186 insertions(+), 1562 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3e246fb/solr/CHANGES.txt
----------------------------------------------------------------------
[02/12] lucene-solr:jira/http2: Fix typo in
stream-decorator-reference.adoc file.
Posted by da...@apache.org.
Fix typo in stream-decorator-reference.adoc file.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cebf7039
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cebf7039
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cebf7039
Branch: refs/heads/jira/http2
Commit: cebf7039ee047302d4c52fdb282f43c58ed65595
Parents: bd714ca
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:22:51 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:22:51 2018 +0100
----------------------------------------------------------------------
solr/solr-ref-guide/src/stream-decorator-reference.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cebf7039/solr/solr-ref-guide/src/stream-decorator-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-decorator-reference.adoc b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
index c2d6160..5549fc2 100644
--- a/solr/solr-ref-guide/src/stream-decorator-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
@@ -612,7 +612,7 @@ eval(expr)
----
In the example above the `eval` expression reads the first tuple from the underlying expression. It then compiles and
-executes the string Streaming Expression in the epxr_s field.
+executes the string Streaming Expression in the expr_s field.
== executor
[09/12] lucene-solr:jira/http2: SOLR-12793: Move
TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test
package
Posted by da...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
new file mode 100644
index 0000000..91b912f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
@@ -0,0 +1,855 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.facet;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.cloud.TestCloudPivotFacet;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * Tests randomized JSON Facets, sometimes using query 'join' domain transfers and/or domain 'filter' options
+ * </p>
+ * <p>
+ * The results of each facet constraint count will be compared with a verification query using an equivalent filter
+ * </p>
+ *
+ * @see TestCloudPivotFacet
+ */
+public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
+ private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
+
+ private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
+ private static final int MAX_FIELD_NUM = 15;
+ private static final int UNIQUE_FIELD_VALS = 20;
+
+ // NOTE: set to 'true' to see if refinement testing is adequate (should get fails occasionally)
+ private static final boolean FORCE_DISABLE_REFINEMENT = false;
+
+ /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
+ /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
+
+ /** A basic client for operations at the cloud level, default collection will be set */
+ private static CloudSolrClient CLOUD_CLIENT;
+ /** One client per node */
+ private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
+
+ @BeforeClass
+ private static void createMiniSolrCloudCluster() throws Exception {
+ // sanity check constants
+ assertTrue("bad test constants: some suffixes will never be tested",
+ (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
+
+ // we need DVs on point fields to compute stats & facets
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
+
+ // multi replicas should not matter...
+ final int repFactor = usually() ? 1 : 2;
+ // ... but we definitely want to test multiple shards
+ final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
+ final int numNodes = (numShards * repFactor);
+
+ final String configName = DEBUG_LABEL + "_config-set";
+ final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
+
+ configureCluster(numNodes).addConfig(configName, configDir).configure();
+
+ Map<String, String> collectionProperties = new LinkedHashMap<>();
+ collectionProperties.put("config", "solrconfig-tlog.xml");
+ collectionProperties.put("schema", "schema_latest.xml");
+ CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
+ .setProperties(collectionProperties)
+ .process(cluster.getSolrClient());
+
+ CLOUD_CLIENT = cluster.getSolrClient();
+ CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
+
+ waitForRecoveriesToFinish(CLOUD_CLIENT);
+
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
+ }
+
+ final int numDocs = atLeast(100);
+ for (int id = 0; id < numDocs; id++) {
+ SolrInputDocument doc = sdoc("id", ""+id);
+ for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
+ // NOTE: some docs may have no value in a field
+ final int numValsThisDoc = TestUtil.nextInt(random(), 0, (usually() ? 3 : 6));
+ for (int v = 0; v < numValsThisDoc; v++) {
+ final String fieldValue = randFieldValue(fieldNum);
+
+ // for each fieldNum, there are actaully two fields: one string, and one integer
+ doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
+ doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
+ }
+ }
+ CLOUD_CLIENT.add(doc);
+ if (random().nextInt(100) < 1) {
+ CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
+ }
+ if (random().nextInt(100) < 5) {
+ CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
+ }
+ }
+ CLOUD_CLIENT.commit();
+ }
+
+ /**
+ * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
+ * uses that number and one of hte specified suffixes in it's name.
+ *
+ * @see #STR_FIELD_SUFFIXES
+ * @see #INT_FIELD_SUFFIXES
+ * @see #MAX_FIELD_NUM
+ * @see #randFieldValue
+ */
+ private static String field(final String[] suffixes, final int fieldNum) {
+ assert fieldNum < MAX_FIELD_NUM;
+
+ final String suffix = suffixes[fieldNum % suffixes.length];
+ return "field_" + fieldNum + suffix;
+ }
+ private static String strfield(final int fieldNum) {
+ return field(STR_FIELD_SUFFIXES, fieldNum);
+ }
+ private static String intfield(final int fieldNum) {
+ return field(INT_FIELD_SUFFIXES, fieldNum);
+ }
+
+ /**
+ * Given a (random) field number, returns a random (integer based) value for that field.
+ * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
+ * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
+ * will match fewer documents based on how far apart the field numbers are.
+ *
+ * @see #UNIQUE_FIELD_VALS
+ * @see #field
+ */
+ private static String randFieldValue(final int fieldNum) {
+ return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
+ }
+
+
+ @AfterClass
+ private static void afterClass() throws Exception {
+ CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+ for (HttpSolrClient client : CLIENTS) {
+ client.close();
+ }
+ CLIENTS = null;
+ }
+
+ /** Sanity check that malformed requests produce errors */
+ public void testMalformedGivesError() throws Exception {
+
+ ignoreException(".*'join' domain change.*");
+
+ for (String join : Arrays.asList("bogus",
+ "{ }",
+ "{ from:null, to:foo_s }",
+ "{ from:foo_s }",
+ "{ from:foo_s, to:foo_s, bogus:'what what?' }",
+ "{ to:foo_s, bogus:'what what?' }")) {
+ SolrException e = expectThrows(SolrException.class, () -> {
+ final SolrParams req = params("q", "*:*", "json.facet",
+ "{ x : { type:terms, field:x_s, domain: { join:"+join+" } } }");
+ final NamedList trash = getRandClient(random()).request(new QueryRequest(req));
+ });
+ assertEquals(join + " -> " + e, SolrException.ErrorCode.BAD_REQUEST.code, e.code());
+ assertTrue(join + " -> " + e, e.getMessage().contains("'join' domain change"));
+ }
+ }
+
+ public void testSanityCheckDomainMethods() throws Exception {
+ {
+ final JoinDomain empty = new JoinDomain(null, null, null);
+ assertEquals(null, empty.toJSONFacetParamValue());
+ final SolrParams out = empty.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals(null, out.get("safe_key"));
+ assertEquals("qqq", out.get("q"));
+ }
+ {
+ final JoinDomain join = new JoinDomain("xxx", "yyy", null);
+ assertEquals("domain:{join:{from:xxx,to:yyy}}", join.toJSONFacetParamValue().toString());
+ final SolrParams out = join.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals("qqq", out.get("safe_key"));
+ assertEquals("{!join from=xxx to=yyy v=$safe_key}", out.get("q"));
+
+ }
+ {
+ final JoinDomain filter = new JoinDomain(null, null, "zzz");
+ assertEquals("domain:{filter:'zzz'}", filter.toJSONFacetParamValue().toString());
+ final SolrParams out = filter.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals(null, out.get("safe_key"));
+ assertEquals("zzz AND qqq", out.get("q"));
+ }
+ {
+ final JoinDomain both = new JoinDomain("xxx", "yyy", "zzz");
+ assertEquals("domain:{join:{from:xxx,to:yyy},filter:'zzz'}", both.toJSONFacetParamValue().toString());
+ final SolrParams out = both.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals("qqq", out.get("safe_key"));
+ assertEquals("zzz AND {!join from=xxx to=yyy v=$safe_key}", out.get("q"));
+ }
+ }
+
+ /**
+ * Test some small, hand crafted, but non-trivial queries that are
+ * easier to trace/debug then a pure random monstrosity.
+ * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
+ */
+ public void testBespoke() throws Exception {
+
+ { // sanity check our test methods can handle a query matching no docs
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
+ top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
+ facets.put("empty_top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, strfield(7) + ":bogus");
+ assertEquals("Empty search result shouldn't have found a single bucket",
+ UNIQUE_FIELD_VALS, maxBuckets.get());
+ }
+
+ { // sanity check our test methods can handle a query where a facet filter prevents any doc from having terms
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, "-*:*"));
+ top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
+ facets.put("filtered_top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
+ assertEquals("Empty join filter shouldn't have found a single bucket",
+ UNIQUE_FIELD_VALS, maxBuckets.get());
+ }
+
+ { // sanity check our test methods can handle a query where a facet filter prevents any doc from having sub-terms
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(8), strfield(8), null));
+ top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(null, null, "-*:*")));
+ facets.put("filtered_top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+
+ { // strings
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
+ top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ }
+
+ { // ints
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(intfield(9), new JoinDomain(intfield(5), intfield(9), null));
+ facets.put("top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+intfield(7)+":16 OR "+intfield(3)+":13)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ }
+
+ { // some domains with filter only, no actual join
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, strfield(9)+":[* TO *]"));
+ top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(null, null, strfield(3)+":[* TO 5]")));
+ facets.put("top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+
+ }
+
+ { // low limits, explicit refinement
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9),
+ new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
+ 5, 0, true);
+ top.subFacets.put("facet_5", new TermFacet(strfield(11),
+ new JoinDomain(strfield(8), strfield(8), null),
+ 10, 0, true));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
+ }
+
+ { // low limit, high overrequest
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9),
+ new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
+ 5, UNIQUE_FIELD_VALS + 10, false);
+ top.subFacets.put("facet_5", new TermFacet(strfield(11),
+ new JoinDomain(strfield(8), strfield(8), null),
+ 10, UNIQUE_FIELD_VALS + 10, false));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
+ }
+
+ { // low limit, low overrequest, explicit refinement
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9),
+ new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
+ 5, 7, true);
+ top.subFacets.put("facet_5", new TermFacet(strfield(11),
+ new JoinDomain(strfield(8), strfield(8), null),
+ 10, 7, true));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
+ }
+
+ }
+
+ public void testTheTestRandomRefineParam() {
+ // sanity check that randomRefineParam never violates isRefinementNeeded
+ // (should be imposisble ... unless someone changes/breaks the randomization logic in the future)
+ final int numIters = atLeast(100);
+ for (int iter = 0; iter < numIters; iter++) {
+ final Integer limit = TermFacet.randomLimitParam(random());
+ final Integer overrequest = TermFacet.randomOverrequestParam(random());
+ final Boolean refine = TermFacet.randomRefineParam(random(), limit, overrequest);
+ if (TermFacet.isRefinementNeeded(limit, overrequest)) {
+ assertEquals("limit: " + limit + ", overrequest: " + overrequest + ", refine: " + refine,
+ Boolean.TRUE, refine);
+ }
+ }
+ }
+
+ public void testTheTestTermFacetShouldFreakOutOnBadRefineOptions() {
+ expectThrows(AssertionError.class, () -> {
+ final TermFacet bogus = new TermFacet("foo", null, 5, 0, false);
+ });
+ }
+
+ public void testRandom() throws Exception {
+
+ // we put a safety valve in place on the maximum number of buckets that we are willing to verify
+ // across *all* the queries that we do.
+ // that way if the randomized queries we build all have relatively small facets, so be it, but if
+ // we get a really big one early on, we can test as much as possible, skip other iterations.
+ //
+ // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
+ final int maxBucketsAllowed = atLeast(2000);
+ final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
+
+ final int numIters = atLeast(20);
+ for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
+ assertFacetCountsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(), buildRandomQuery());
+ }
+ assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
+ }
+
+ /**
+ * Generates a random query string across the randomized fields/values in the index
+ *
+ * @see #randFieldValue
+ * @see #field
+ */
+ private static String buildRandomQuery() {
+ if (0 == TestUtil.nextInt(random(), 0,10)) {
+ return "*:*";
+ }
+ final int numClauses = TestUtil.nextInt(random(), 3, 10);
+ List<String> clauses = new ArrayList<String>(numClauses);
+ for (int c = 0; c < numClauses; c++) {
+ final int fieldNum = random().nextInt(MAX_FIELD_NUM);
+ // keep queries simple, just use str fields - not point of test
+ clauses.add(strfield(fieldNum) + ":" + randFieldValue(fieldNum));
+ }
+ return "(" + StringUtils.join(clauses, " OR ") + ")";
+ }
+
+ /**
+ * Given a set of (potentially nested) term facets, and a base query string, asserts that
+ * the actual counts returned when executing that query with those facets match the expected results
+ * of filtering on the equivalent facet terms+domain
+ */
+ private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ Map<String,TermFacet> expected,
+ final String query) throws SolrServerException, IOException {
+
+ final SolrParams baseParams = params("q", query, "rows","0");
+ final SolrParams facetParams = params("json.facet", ""+TermFacet.toJSONFacetParamValue(expected));
+ final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
+
+ log.info("Doing full run: {}", initParams);
+
+ QueryResponse rsp = null;
+ // JSON Facets not (currently) available from QueryResponse...
+ NamedList topNamedList = null;
+ try {
+ rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
+ assertNotNull(initParams + " is null rsp?", rsp);
+ topNamedList = rsp.getResponse();
+ assertNotNull(initParams + " is null topNamedList?", topNamedList);
+ } catch (Exception e) {
+ throw new RuntimeException("init query failed: " + initParams + ": " +
+ e.getMessage(), e);
+ }
+ try {
+ final NamedList facetResponse = (NamedList) topNamedList.get("facets");
+ assertNotNull("null facet results?", facetResponse);
+ assertEquals("numFound mismatch with top count?",
+ rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
+ if (0 == rsp.getResults().getNumFound()) {
+ // when the query matches nothing, we should expect no top level facets
+ expected = Collections.emptyMap();
+ }
+ assertFacetCountsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
+ } catch (AssertionError e) {
+ throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
+ } finally {
+ log.info("Ending full run");
+ }
+ }
+
+ /**
+ * Recursive Helper method that walks the actual facet response, comparing the counts to the expected output
+ * based on the equivalent filters generated from the original TermFacet.
+ */
+ private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ final Map<String,TermFacet> expected,
+ final SolrParams baseParams,
+ final NamedList actualFacetResponse) throws SolrServerException, IOException {
+
+ for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
+ final String facetKey = entry.getKey();
+ final TermFacet facet = entry.getValue();
+ final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
+ assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
+ final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
+ assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
+
+ if (buckets.isEmpty()) {
+ // should only happen if the baseParams query does not match any docs with our field X
+ final long docsWithField = getRandClient(random()).query
+ (facet.applyValueConstraintAndDomain(baseParams, facetKey, "[* TO *]")).getResults().getNumFound();
+ assertEquals(facetKey + " has no buckets, but docs in query exist with field: " + facet.field,
+ 0, docsWithField);
+ }
+
+ for (NamedList bucket : buckets) {
+ final long count = ((Number) bucket.get("count")).longValue();
+ final String fieldVal = bucket.get("val").toString(); // int or stringified int
+
+ // change our query to filter on the fieldVal, and wrap in the facet domain (if any)
+ final SolrParams verifyParams = facet.applyValueConstraintAndDomain(baseParams, facetKey, fieldVal);
+
+ // check the count for this bucket
+ assertEquals(facetKey + ": " + verifyParams,
+ count, getRandClient(random()).query(verifyParams).getResults().getNumFound());
+
+ if (maxBucketsToCheck.decrementAndGet() <= 0) {
+ return;
+ }
+
+ // recursively check subFacets
+ if (! facet.subFacets.isEmpty()) {
+ assertFacetCountsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
+ }
+ }
+ }
+ assertTrue("facets have unexpected keys left over: " + actualFacetResponse,
+ // should alwasy be a count, maybe a 'val' if we're a subfacet
+ (actualFacetResponse.size() == expected.size() + 1) ||
+ (actualFacetResponse.size() == expected.size() + 2));
+ }
+
+
+ /**
+ * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
+ *
+ * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
+ */
+ private static final class TermFacet {
+ public final String field;
+ public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
+ public final JoinDomain domain; // may be null
+ public final Integer limit; // may be null
+ public final Integer overrequest; // may be null
+ public final Boolean refine; // may be null
+
+ /** Simplified constructor asks for limit = # unique vals */
+ public TermFacet(String field, JoinDomain domain) {
+ this(field, domain, UNIQUE_FIELD_VALS, 0, false);
+ }
+ public TermFacet(String field, JoinDomain domain, Integer limit, Integer overrequest, Boolean refine) {
+ assert null != field;
+ this.field = field;
+ this.domain = domain;
+ this.limit = limit;
+ this.overrequest = overrequest;
+ this.refine = refine;
+ if (isRefinementNeeded(limit, overrequest)) {
+ assertEquals("Invalid refine param based on limit & overrequest: " + this.toString(),
+ Boolean.TRUE, refine);
+ }
+ }
+
+ /**
+ * Returns new SolrParams that:
+ * <ul>
+ * <li>copy the original SolrParams</li>
+ * <li>modify/wrap the original "q" param to capture the domain change for this facet (if any)</li>
+ * <li>add a filter query against this field with the specified value</li>
+ * </ul>
+ *
+ * @see JoinDomain#applyDomainToQuery
+ */
+ public SolrParams applyValueConstraintAndDomain(SolrParams orig, String facetKey, String facetVal) {
+ // first wrap our original query in the domain if there is one...
+ if (null != domain) {
+ orig = domain.applyDomainToQuery(facetKey + "_q", orig);
+ }
+ // then filter by the facet value we need to test...
+ final ModifiableSolrParams out = new ModifiableSolrParams(orig);
+ out.set("q", field + ":" + facetVal + " AND " + orig.get("q"));
+
+ return out;
+ }
+
+ /**
+ * recursively generates the <code>json.facet</code> param value to use for testing this facet
+ */
+ private CharSequence toJSONFacetParamValue() {
+ final String limitStr = (null == limit) ? "" : (", limit:" + limit);
+ final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
+ final String refineStr = (null == refine) ? "" : ", refine:" + refine;
+ final StringBuilder sb = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + refineStr);
+ if (! subFacets.isEmpty()) {
+ sb.append(", facet:");
+ sb.append(toJSONFacetParamValue(subFacets));
+ }
+ if (null != domain) {
+ CharSequence ds = domain.toJSONFacetParamValue();
+ if (null != ds) {
+ sb.append(", ").append(ds);
+ }
+ }
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
+ * use for testing them against in a solr request.
+ */
+ public static CharSequence toJSONFacetParamValue(Map<String,TermFacet> facets) {
+ assert null != facets;
+ assert 0 < facets.size();
+ StringBuilder sb = new StringBuilder("{");
+ for (String key : facets.keySet()) {
+ sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
+ sb.append(" ,");
+ }
+ sb.setLength(sb.length() - 1);
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Factory method for generating some random (nested) facets.
+ *
+ * For simplicity, each facet will have a unique key name, regardless of it's depth under other facets
+ *
+ * @see JoinDomain
+ */
+ public static Map<String,TermFacet> buildRandomFacets() {
+ // for simplicity, use a unique facet key regardless of depth - simplifies verification
+ AtomicInteger keyCounter = new AtomicInteger(0);
+ final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
+ return buildRandomFacets(keyCounter, maxDepth);
+ }
+
+ /**
+ * picks a random value for the "limit" param, biased in favor of interesting test cases
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Integer randomLimitParam(Random r) {
+ final int limit = 1 + r.nextInt(UNIQUE_FIELD_VALS * 2);
+ if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
+ return -1; // unlimited
+ } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
+ return null; // sometimes, don't specify limit if it's the default
+ }
+ return limit;
+ }
+
+ /**
+ * picks a random value for the "overrequest" param, biased in favor of interesting test cases
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Integer randomOverrequestParam(Random r) {
+ switch(r.nextInt(10)) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ return 0; // 40% of the time, no overrequest to better stress refinement
+ case 4:
+ case 5:
+ return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
+ case 6:
+ return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
+ default: break;
+ }
+ // else.... either leave param unspecified (or redundently specify the -1 default)
+ return r.nextBoolean() ? null : -1;
+ }
+
+ /**
+ * picks a random value for the "refine" param, that is garunteed to be suitable for
+ * the specified limit & overrequest params.
+ *
+ * @return a value to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #randomLimitParam
+ * @see #randomOverrequestParam
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Boolean randomRefineParam(Random r, Integer limitParam, Integer overrequestParam) {
+ if (isRefinementNeeded(limitParam, overrequestParam)) {
+ return true;
+ }
+
+ // refinement is not required
+ if (0 == r.nextInt(10)) { // once in a while, turn on refinement even if it isn't needed.
+ return true;
+ }
+ // explicitly or implicitly indicate refinement is not needed
+ return r.nextBoolean() ? false : null;
+ }
+
+ /**
+ * Deterministicly identifies if the specified limit & overrequest params <b>require</b>
+ * a "refine:true" param be used in the the request, in order for the counts to be 100% accurate.
+ *
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static boolean isRefinementNeeded(Integer limitParam, Integer overrequestParam) {
+
+ if (FORCE_DISABLE_REFINEMENT) {
+ return false;
+ }
+
+ // use the "effective" values if the params are null
+ final int limit = null == limitParam ? DEFAULT_LIMIT : limitParam;
+ final int overrequest = null == overrequestParam ? 0 : overrequestParam;
+
+ return
+ // don't presume how much overrequest will be done by default, just check the limit
+ (overrequest < 0 && limit < UNIQUE_FIELD_VALS)
+ // if the user specified overrequest is not "enough" to get all unique values
+ || (overrequest >= 0 && (long)limit + overrequest < UNIQUE_FIELD_VALS);
+ }
+
+ /**
+ * recursive helper method for building random facets
+ *
+ * @param keyCounter used to ensure every generated facet has a unique key name
+ * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
+ */
+ private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
+ final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
+ Map<String,TermFacet> results = new LinkedHashMap<>();
+ for (int i = 0; i < numFacets; i++) {
+ final JoinDomain domain = JoinDomain.buildRandomDomain();
+ assert null != domain;
+ final Integer limit = randomLimitParam(random());
+ final Integer overrequest = randomOverrequestParam(random());
+ final TermFacet facet = new TermFacet(field(random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES,
+ random().nextInt(MAX_FIELD_NUM)),
+ domain, limit, overrequest,
+ randomRefineParam(random(), limit, overrequest));
+ results.put("facet_" + keyCounter.incrementAndGet(), facet);
+ if (0 < maxDepth) {
+ // if we're going wide, don't go deep
+ final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
+ facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
+ }
+ }
+ return results;
+ }
+ }
+
+
+ /**
+ * Models a Domain Change which includes either a 'join' or a 'filter' or both
+ */
+ private static final class JoinDomain {
+ public final String from;
+ public final String to;
+ public final String filter; // not bothering with more then 1 filter, not the point of the test
+
+ /**
+ * @param from left side of join field name, null if domain involves no joining
+ * @param to right side of join field name, null if domain involves no joining
+ * @param filter filter to apply to domain, null if domain involves no filtering
+ */
+ public JoinDomain(String from, String to, String filter) {
+ assert ! ((null == from) ^ (null == to)) : "if from is null, to must be null";
+ this.from = from;
+ this.to = to;
+ this.filter = filter;
+ }
+
+ /**
+ * @return the JSON string representing this domain for use in a facet param, or null if no domain should be used
+ * */
+ public CharSequence toJSONFacetParamValue() {
+ if (null == from && null == filter) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder("domain:{");
+ if (null != from) {
+ assert null != to;
+ sb. append("join:{from:").append(from).append(",to:").append(to).append("}");
+ if (null != filter){
+ sb.append(",");
+ }
+
+ }
+ if (null != filter) {
+ sb.append("filter:'").append(filter).append("'");
+ }
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Given some original SolrParams, returns new SolrParams where the original "q" param is wrapped
+ * as needed to apply the equivalent transformation to a query as this domain would to a facet
+ */
+ public SolrParams applyDomainToQuery(String safeKey, SolrParams in) {
+ assert null == in.get(safeKey); // shouldn't be possible if every facet uses a unique key string
+
+ String q = in.get("q");
+ final ModifiableSolrParams out = new ModifiableSolrParams(in);
+ if (null != from) {
+ out.set(safeKey, in.get("q"));
+ q = "{!join from="+from+" to="+to+" v=$"+safeKey+"}";
+ }
+ if (null != filter) {
+ q = filter + " AND " + q;
+ }
+ out.set("q", q);
+ return out;
+ }
+
+ /**
+ * Factory method for creating a random domain change to use with a facet - may return an 'noop' JoinDomain,
+ * but will never return null.
+ */
+ public static JoinDomain buildRandomDomain() {
+
+ // use consistent type on both sides of join
+ final String[] suffixes = random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES;
+
+ final boolean noJoin = random().nextBoolean();
+
+ String from = null;
+ String to = null;
+ for (;;) {
+ if (noJoin) break;
+ from = field(suffixes, random().nextInt(MAX_FIELD_NUM));
+ to = field(suffixes, random().nextInt(MAX_FIELD_NUM));
+ // HACK: joined numeric point fields need docValues.. for now just skip _is fields if we are dealing with points.
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP) && (from.endsWith("_is") || to.endsWith("_is")))
+ {
+ continue;
+ }
+ break;
+ }
+
+ // keep it simple, only filter on string fields - not point of test
+ final String filterField = strfield(random().nextInt(MAX_FIELD_NUM));
+
+ final String filter = random().nextBoolean() ? null : filterField+":[* TO *]";
+ return new JoinDomain(from, to, filter);
+ }
+ }
+
+ /**
+ * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
+ * at a node in our cluster
+ */
+ public static SolrClient getRandClient(Random rand) {
+ int numClients = CLIENTS.size();
+ int idx = TestUtil.nextInt(rand, 0, numClients);
+
+ return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
+ }
+
+ public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
+ assert null != client.getDefaultCollection();
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
+ client.getZkStateReader(),
+ true, true, 330);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java
new file mode 100644
index 0000000..e212993
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java
@@ -0,0 +1,678 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.facet;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang.StringUtils;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import static org.apache.solr.search.facet.RelatednessAgg.computeRelatedness;
+import static org.apache.solr.search.facet.RelatednessAgg.roundTo5Digits;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * A randomized test of nested facets using the <code>relatedness()</code> function, that asserts the
+ * accuracy the results for all the buckets returned using verification queries of the (expected)
+ * foreground & background queries based on the nested facet terms.
+ * <p>
+ * Note that unlike normal facet "count" verification, using a high limit + overrequest isn't a substitute
+ * for refinement in order to ensure accurate "skg" computation across shards. For that reason, this
+ * tests forces <code>refine: true</code> (unlike {@link TestCloudJSONFacetJoinDomain}) and specifices a
+ * <code>domain: { 'query':'*:*' }</code> for every facet, in order to garuntee that all shards
+ * participate in all facets, so that the popularity & relatedness values returned can be proven
+ * with validation requests.
+ * </p>
+ * <p>
+ * (Refinement alone is not enough. Using the '*:*' query as the facet domain is neccessary to
+ * prevent situations where a single shardX may return candidate bucket with no child-buckets due to
+ * the normal facet intersections, but when refined on other shardY(s), can produce "high scoring"
+ * SKG child-buckets, which would then be missing the foreground/background "size" contributions from
+ * shardX.
+ * </p>
+ *
+ * @see TestCloudJSONFacetJoinDomain
+ */
+@Slow
+public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
+ private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
+
+ private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
+ private static final int MAX_FIELD_NUM = 15;
+ private static final int UNIQUE_FIELD_VALS = 50;
+
+ /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
+ /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
+
+ /** A basic client for operations at the cloud level, default collection will be set */
+ private static CloudSolrClient CLOUD_CLIENT;
+ /** One client per node */
+ private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
+
+ @BeforeClass
+ private static void createMiniSolrCloudCluster() throws Exception {
+ // sanity check constants
+ assertTrue("bad test constants: some suffixes will never be tested",
+ (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
+
+ // we need DVs on point fields to compute stats & facets
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
+
+ // multi replicas should not matter...
+ final int repFactor = usually() ? 1 : 2;
+ // ... but we definitely want to test multiple shards
+ final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
+ final int numNodes = (numShards * repFactor);
+
+ final String configName = DEBUG_LABEL + "_config-set";
+ final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
+
+ configureCluster(numNodes).addConfig(configName, configDir).configure();
+
+ Map<String, String> collectionProperties = new LinkedHashMap<>();
+ collectionProperties.put("config", "solrconfig-tlog.xml");
+ collectionProperties.put("schema", "schema_latest.xml");
+ CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
+ .setProperties(collectionProperties)
+ .process(cluster.getSolrClient());
+
+ CLOUD_CLIENT = cluster.getSolrClient();
+ CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
+
+ waitForRecoveriesToFinish(CLOUD_CLIENT);
+
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
+ }
+
+ final int numDocs = atLeast(100);
+ for (int id = 0; id < numDocs; id++) {
+ SolrInputDocument doc = sdoc("id", ""+id);
+ for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
+ // NOTE: we ensure every doc has at least one value in each field
+ // that way, if a term is returned for a parent there there is garunteed to be at least one
+ // one term in the child facet as well.
+ //
+ // otherwise, we'd face the risk of a single shardX returning parentTermX as a top term for
+ // the parent facet, but having no child terms -- meanwhile on refinement another shardY that
+ // did *not* returned parentTermX in phase#1, could return some *new* child terms under
+ // parentTermX, but their stats would not include the bgCount from shardX.
+ //
+ // in normal operation, this is an edge case that isn't a big deal because the ratios &
+ // relatedness scores are statistically approximate, but for the purpose of this test where
+ // we verify correctness via exactness we need all shards to contribute to the SKG statistics
+ final int numValsThisDoc = TestUtil.nextInt(random(), 1, (usually() ? 5 : 10));
+ for (int v = 0; v < numValsThisDoc; v++) {
+ final String fieldValue = randFieldValue(fieldNum);
+
+ // for each fieldNum, there are actaully two fields: one string, and one integer
+ doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
+ doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
+ }
+ }
+ CLOUD_CLIENT.add(doc);
+ if (random().nextInt(100) < 1) {
+ CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
+ }
+ if (random().nextInt(100) < 5) {
+ CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
+ }
+ }
+ CLOUD_CLIENT.commit();
+ }
+
+ /**
+ * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
+ * uses that number and one of hte specified suffixes in it's name.
+ *
+ * @see #STR_FIELD_SUFFIXES
+ * @see #INT_FIELD_SUFFIXES
+ * @see #MAX_FIELD_NUM
+ * @see #randFieldValue
+ */
+ private static String field(final String[] suffixes, final int fieldNum) {
+ assert fieldNum < MAX_FIELD_NUM;
+
+ final String suffix = suffixes[fieldNum % suffixes.length];
+ return "field_" + fieldNum + suffix;
+ }
+ private static String strfield(final int fieldNum) {
+ return field(STR_FIELD_SUFFIXES, fieldNum);
+ }
+ private static String intfield(final int fieldNum) {
+ return field(INT_FIELD_SUFFIXES, fieldNum);
+ }
+
+ /**
+ * Given a (random) field number, returns a random (integer based) value for that field.
+ * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
+ * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
+ * will match fewer documents based on how far apart the field numbers are.
+ *
+ * @see #UNIQUE_FIELD_VALS
+ * @see #field
+ */
+ private static String randFieldValue(final int fieldNum) {
+ return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
+ }
+
+
+ @AfterClass
+ private static void afterClass() throws Exception {
+ CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+ for (HttpSolrClient client : CLIENTS) {
+ client.close();
+ }
+ CLIENTS = null;
+ }
+
+ /**
+ * Test some small, hand crafted, but non-trivial queries that are
+ * easier to trace/debug then a pure random monstrosity.
+ * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
+ */
+ public void testBespoke() throws Exception {
+ { // trivial single level facet
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, null);
+ facets.put("top1", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+
+ { // trivial single level facet w/sorting on skg
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, "skg desc");
+ facets.put("top2", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+
+ { // trivial single level facet w/ 2 diff ways to request "limit = (effectively) Infinite"
+ // to sanity check refinement of buckets missing from other shard in both cases
+
+ // NOTE that these two queries & facets *should* effectively identical given that the
+ // very large limit value is big enough no shard will ever return that may terms,
+ // but the "limit=-1" case it actaully triggers slightly different code paths
+ // because it causes FacetField.returnsPartial() to be "true"
+ for (int limit : new int[] { 999999999, -1 }) {
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ facets.put("top_facet_limit__" + limit, new TermFacet(strfield(9), limit, 0, "skg desc"));
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+ }
+ }
+
+ public void testRandom() throws Exception {
+
+ // since the "cost" of verifying the stats for each bucket is so high (see TODO in verifySKGResults())
+ // we put a safety valve in place on the maximum number of buckets that we are willing to verify
+ // across *all* the queries that we do.
+ // that way if the randomized queries we build all have relatively small facets, so be it, but if
+ // we get a really big one early on, we can test as much as possible, skip other iterations.
+ //
+ // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
+ final int maxBucketsAllowed = atLeast(2000);
+ final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
+
+ final int numIters = atLeast(10);
+ for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
+ assertFacetSKGsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(),
+ buildRandomQuery(), buildRandomQuery(), buildRandomQuery());
+ }
+ assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
+
+
+ }
+
+ /**
+ * Generates a random query string across the randomized fields/values in the index
+ *
+ * @see #randFieldValue
+ * @see #field
+ */
+ private static String buildRandomQuery() {
+ if (0 == TestUtil.nextInt(random(), 0,10)) {
+ return "*:*";
+ }
+ final int numClauses = TestUtil.nextInt(random(), 3, 10);
+ final String[] clauses = new String[numClauses];
+ for (int c = 0; c < numClauses; c++) {
+ final int fieldNum = random().nextInt(MAX_FIELD_NUM);
+ // keep queries simple, just use str fields - not point of test
+ clauses[c] = strfield(fieldNum) + ":" + randFieldValue(fieldNum);
+ }
+ return buildORQuery(clauses);
+ }
+
+ private static String buildORQuery(String... clauses) {
+ assert 0 < clauses.length;
+ return "(" + StringUtils.join(clauses, " OR ") + ")";
+ }
+
+ /**
+ * Given a set of term facets, and top level query strings, asserts that
+ * the SKG stats for each facet term returned when executing that query with those foreground/background
+ * queries match the expected results of executing the equivalent queries in isolation.
+ *
+ * @see #verifySKGResults
+ */
+ private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ Map<String,TermFacet> expected,
+ final String query,
+ final String foreQ,
+ final String backQ) throws SolrServerException, IOException {
+ final SolrParams baseParams = params("rows","0", "fore", foreQ, "back", backQ);
+
+ final SolrParams facetParams = params("q", query,
+ "json.facet", ""+TermFacet.toJSONFacetParamValue(expected,null));
+ final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
+
+ log.info("Doing full run: {}", initParams);
+
+ QueryResponse rsp = null;
+ // JSON Facets not (currently) available from QueryResponse...
+ NamedList topNamedList = null;
+ try {
+ rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
+ assertNotNull(initParams + " is null rsp?", rsp);
+ topNamedList = rsp.getResponse();
+ assertNotNull(initParams + " is null topNamedList?", topNamedList);
+ } catch (Exception e) {
+ throw new RuntimeException("init query failed: " + initParams + ": " +
+ e.getMessage(), e);
+ }
+ try {
+ final NamedList facetResponse = (NamedList) topNamedList.get("facets");
+ assertNotNull("null facet results?", facetResponse);
+ assertEquals("numFound mismatch with top count?",
+ rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
+
+ // Note: even if the query has numFound=0, our explicit background query domain should
+ // still force facet results
+ // (even if the background query matches nothing, that just means there will be no
+ // buckets in those facets)
+ assertFacetSKGsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
+
+ } catch (AssertionError e) {
+ throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
+ } finally {
+ log.info("Ending full run");
+ }
+ }
+
+ /**
+ * Recursive helper method that walks the actual facet response, comparing the SKG results to
+ * the expected output based on the equivalent filters generated from the original TermFacet.
+ */
+ private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ final Map<String,TermFacet> expected,
+ final SolrParams baseParams,
+ final NamedList actualFacetResponse) throws SolrServerException, IOException {
+
+ for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
+ final String facetKey = entry.getKey();
+ final TermFacet facet = entry.getValue();
+ final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
+ assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
+ final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
+ assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
+
+ if (buckets.isEmpty()) {
+ // should only happen if the background query does not match any docs with field X
+ final long docsWithField = getNumFound(params("_trace", "noBuckets",
+ "rows", "0",
+ "q", facet.field+":[* TO *]",
+ "fq", baseParams.get("back")));
+
+ assertEquals(facetKey + " has no buckets, but docs in background exist with field: " + facet.field,
+ 0, docsWithField);
+ }
+
+ // NOTE: it's important that we do this depth first -- not just because it's the easiest way to do it,
+ // but because it means that our maxBucketsToCheck will ensure we do a lot of deep sub-bucket checking,
+ // not just all the buckets of the top level(s) facet(s)
+ for (NamedList bucket : buckets) {
+ final String fieldVal = bucket.get("val").toString(); // int or stringified int
+
+ verifySKGResults(facetKey, facet, baseParams, fieldVal, bucket);
+ if (maxBucketsToCheck.decrementAndGet() <= 0) {
+ return;
+ }
+
+ final SolrParams verifyParams = SolrParams.wrapAppended(baseParams,
+ params("fq", facet.field + ":" + fieldVal));
+
+ // recursively check subFacets
+ if (! facet.subFacets.isEmpty()) {
+ assertFacetSKGsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
+ }
+ }
+ }
+
+ { // make sure we don't have any facet keys we don't expect
+ // a little hackish because subfacets have extra keys...
+ final LinkedHashSet expectedKeys = new LinkedHashSet(expected.keySet());
+ expectedKeys.add("count");
+ if (0 <= actualFacetResponse.indexOf("val",0)) {
+ expectedKeys.add("val");
+ expectedKeys.add("skg");
+ }
+ assertEquals("Unexpected keys in facet response",
+ expectedKeys, actualFacetResponse.asShallowMap().keySet());
+ }
+ }
+
+ /**
+ * Verifies that the popularity & relatedness values containined in a single SKG bucket
+ * match the expected values based on the facet field & bucket value, as well the existing
+ * filterParams.
+ *
+ * @see #assertFacetSKGsAreCorrect
+ */
+ private void verifySKGResults(String facetKey, TermFacet facet, SolrParams filterParams,
+ String fieldVal, NamedList<Object> bucket)
+ throws SolrServerException, IOException {
+
+ final String bucketQ = facet.field+":"+fieldVal;
+ final NamedList<Object> skgBucket = (NamedList<Object>) bucket.get("skg");
+ assertNotNull(facetKey + "/bucket:" + bucket.toString(), skgBucket);
+
+ // TODO: make this more efficient?
+ // ideally we'd do a single query w/4 facet.queries, one for each count
+ // but formatting the queries is a pain, currently we leverage the accumulated fq's
+ final long fgSize = getNumFound(SolrParams.wrapAppended(params("_trace", "fgSize",
+ "rows","0",
+ "q","{!query v=$fore}"),
+ filterParams));
+ final long bgSize = getNumFound(params("_trace", "bgSize",
+ "rows","0",
+ "q", filterParams.get("back")));
+
+ final long fgCount = getNumFound(SolrParams.wrapAppended(params("_trace", "fgCount",
+ "rows","0",
+ "q","{!query v=$fore}",
+ "fq", bucketQ),
+ filterParams));
+ final long bgCount = getNumFound(params("_trace", "bgCount",
+ "rows","0",
+ "q", bucketQ,
+ "fq", filterParams.get("back")));
+
+ assertEquals(facetKey + "/bucket:" + bucket + " => fgPop should be: " + fgCount + " / " + bgSize,
+ roundTo5Digits((double) fgCount / bgSize),
+ skgBucket.get("foreground_popularity"));
+ assertEquals(facetKey + "/bucket:" + bucket + " => bgPop should be: " + bgCount + " / " + bgSize,
+ roundTo5Digits((double) bgCount / bgSize),
+ skgBucket.get("background_popularity"));
+ assertEquals(facetKey + "/bucket:" + bucket + " => relatedness is wrong",
+ roundTo5Digits(computeRelatedness(fgCount, fgSize, bgCount, bgSize)),
+ skgBucket.get("relatedness"));
+
+ }
+
+
+ /**
+ * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
+ *
+ * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
+ */
+ private static final class TermFacet {
+ public final String field;
+ public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
+ public final Integer limit; // may be null
+ public final Integer overrequest; // may be null
+ public final String sort; // may be null
+ /** Simplified constructor asks for limit = # unique vals */
+ public TermFacet(String field) {
+ this(field, UNIQUE_FIELD_VALS, 0, "skg desc");
+
+ }
+ public TermFacet(String field, Integer limit, Integer overrequest, String sort) {
+ assert null != field;
+ this.field = field;
+ this.limit = limit;
+ this.overrequest = overrequest;
+ this.sort = sort;
+ }
+
+ /**
+ * recursively generates the <code>json.facet</code> param value to use for testing this facet
+ */
+ private CharSequence toJSONFacetParamValue() {
+ final String limitStr = (null == limit) ? "" : (", limit:" + limit);
+ final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
+ final String sortStr = (null == sort) ? "" : (", sort: '" + sort + "'");
+ final StringBuilder sb
+ = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + sortStr);
+
+ // see class javadocs for why we always use refine:true & the query:'*:*' domain for this test.
+ sb.append(", refine: true, domain: { query: '*:*' }, facet:");
+ sb.append(toJSONFacetParamValue(subFacets, "skg : 'relatedness($fore,$back)'"));
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
+ * use for testing them against in a solr request.
+ */
+ public static CharSequence toJSONFacetParamValue(final Map<String,TermFacet> facets,
+ final String extraJson) {
+ assert null != facets;
+ if (0 == facets.size() && null == extraJson) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder("{ processEmpty: true, ");
+ for (String key : facets.keySet()) {
+ sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
+ sb.append(" ,");
+ }
+ if (null == extraJson) {
+ sb.setLength(sb.length() - 1);
+ } else {
+ sb.append(extraJson);
+ }
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Factory method for generating some random facets.
+ *
+ * For simplicity, each facet will have a unique key name.
+ */
+ public static Map<String,TermFacet> buildRandomFacets() {
+ // for simplicity, use a unique facet key regardless of depth - simplifies verification
+ // and le's us enforce a hard limit on the total number of facets in a request
+ AtomicInteger keyCounter = new AtomicInteger(0);
+
+ final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
+ return buildRandomFacets(keyCounter, maxDepth);
+ }
+
+ /**
+ * picks a random value for the "sort" param, biased in favor of interesting test cases
+ *
+ * @return a sort string (w/direction), or null to specify nothing (trigger default behavior)
+ * @see #randomLimitParam
+ */
+ public static String randomSortParam(Random r) {
+
+ // IMPORTANT!!!
+ // if this method is modified to produce new sorts, make sure to update
+ // randomLimitParam to account for them if they are impacted by SOLR-12556
+ final String dir = random().nextBoolean() ? "asc" : "desc";
+ switch(r.nextInt(4)) {
+ case 0: return null;
+ case 1: return "count " + dir;
+ case 2: return "skg " + dir;
+ case 3: return "index " + dir;
+ default: throw new RuntimeException("Broken case statement");
+ }
+ }
+ /**
+ * picks a random value for the "limit" param, biased in favor of interesting test cases
+ *
+ * <p>
+ * <b>NOTE:</b> Due to SOLR-12556, we have to force an overrequest of "all" possible terms for
+ * some sort values.
+ * </p>
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ * @see #randomSortParam
+ */
+ public static Integer randomLimitParam(Random r, final String sort) {
+ if (null != sort) {
+ if (sort.equals("count asc") || sort.startsWith("skg")) {
+ // of the known types of sorts produced, these are at risk of SOLR-12556
+ // so request (effectively) unlimited num buckets
+ return r.nextBoolean() ? UNIQUE_FIELD_VALS : -1;
+ }
+ }
+ final int limit = 1 + r.nextInt((int) (UNIQUE_FIELD_VALS * 1.5F));
+ if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
+ return -1; // unlimited
+ } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
+ return null; // sometimes, don't specify limit if it's the default
+ }
+ return limit;
+ }
+
+ /**
+ * picks a random value for the "overrequest" param, biased in favor of interesting test cases.
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Integer randomOverrequestParam(Random r) {
+ switch(r.nextInt(10)) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ return 0; // 40% of the time, disable overrequest to better stress refinement
+ case 4:
+ case 5:
+ return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
+ case 6:
+ return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
+ default: break;
+ }
+ // else.... either leave param unspecified (or redundently specify the -1 default)
+ return r.nextBoolean() ? null : -1;
+ }
+
+ /**
+ * recursive helper method for building random facets
+ *
+ * @param keyCounter used to ensure every generated facet has a unique key name
+ * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
+ */
+ private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
+ final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
+ Map<String,TermFacet> results = new LinkedHashMap<>();
+ for (int i = 0; i < numFacets; i++) {
+ if (keyCounter.get() < 3) { // a hard limit on the total number of facets (regardless of depth) to reduce OOM risk
+
+ final String sort = randomSortParam(random());
+ final Integer limit = randomLimitParam(random(), sort);
+ final Integer overrequest = randomOverrequestParam(random());
+ final TermFacet facet = new TermFacet(field((random().nextBoolean()
+ ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES),
+ random().nextInt(MAX_FIELD_NUM)),
+ limit, overrequest, sort);
+ results.put("facet_" + keyCounter.incrementAndGet(), facet);
+ if (0 < maxDepth) {
+ // if we're going wide, don't go deep
+ final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
+ facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
+ }
+ }
+ }
+ return results;
+ }
+ }
+
+ /**
+ * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
+ * at a node in our cluster
+ */
+ public static SolrClient getRandClient(Random rand) {
+ int numClients = CLIENTS.size();
+ int idx = TestUtil.nextInt(rand, 0, numClients);
+
+ return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
+ }
+
+ /**
+ * Uses a random SolrClient to execture a request and returns only the numFound
+ * @see #getRandClient
+ */
+ public static long getNumFound(final SolrParams req) throws SolrServerException, IOException {
+ return getRandClient(random()).query(req).getResults().getNumFound();
+ }
+
+ public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
+ assert null != client.getDefaultCollection();
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
+ client.getZkStateReader(),
+ true, true, 330);
+ }
+
+}
[11/12] lucene-solr:jira/http2: Tweak test-help example for running
tests within a package
Posted by da...@apache.org.
Tweak test-help example for running tests within a package
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3e87499d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3e87499d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3e87499d
Branch: refs/heads/jira/http2
Commit: 3e87499d72131f2ee54b03fa83ed177413fd3cbc
Parents: 71988c7
Author: Varun Thacker <va...@apache.org>
Authored: Wed Oct 24 15:33:23 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Oct 24 15:33:23 2018 -0700
----------------------------------------------------------------------
lucene/common-build.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e87499d/lucene/common-build.xml
----------------------------------------------------------------------
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index 1a6839b..c9463ea 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -1349,7 +1349,7 @@ ant test "-Dtests.class=*.ClassName"
ant test -Dtestcase=ClassName
# Run all tests in a package and sub-packages
-ant test "-Dtests.class=org.apache.lucene.package.*"
+ant test "-Dtests.class=org.apache.lucene.package.Test*|org.apache.lucene.package.*Test"
# Run any test methods that contain 'esi' (like: ...r*esi*ze...).
ant test "-Dtests.method=*esi*"
[05/12] lucene-solr:jira/http2: SOLR-9425: fix NullPointerException
in TestSolrConfigHandlerConcurrent
Posted by da...@apache.org.
SOLR-9425: fix NullPointerException in TestSolrConfigHandlerConcurrent
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab14cc95
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab14cc95
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab14cc95
Branch: refs/heads/jira/http2
Commit: ab14cc9566e15743eb168b36ca63d2b3197ba0a1
Parents: c277674
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:30:44 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:30:44 2018 +0100
----------------------------------------------------------------------
.../org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab14cc95/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
index 6aa4e1f..c2599a0 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
@@ -66,11 +66,11 @@ public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBa
for (Object o : caches.entrySet()) {
final Map.Entry e = (Map.Entry) o;
if (e.getValue() instanceof Map) {
+ List<String> errs = new ArrayList<>();
+ collectErrors.add(errs);
Map value = (Map) e.getValue();
Thread t = new Thread(() -> {
try {
- List<String> errs = new ArrayList<>();
- collectErrors.add(errs);
invokeBulkCall((String)e.getKey() , errs, value);
} catch (Exception e1) {
e1.printStackTrace();
[06/12] lucene-solr:jira/http2: SOLR-12772: Correct dotProduct
parameters+syntax documentation
Posted by da...@apache.org.
SOLR-12772: Correct dotProduct parameters+syntax documentation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e62fe459
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e62fe459
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e62fe459
Branch: refs/heads/jira/http2
Commit: e62fe459834a287cccf01f25ec03b8764f7a5af7
Parents: ab14cc9
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:49:35 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:49:35 2018 +0100
----------------------------------------------------------------------
solr/solr-ref-guide/src/stream-evaluator-reference.adoc | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e62fe459/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-evaluator-reference.adoc b/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
index a88915f..bb13e77 100644
--- a/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
@@ -626,11 +626,12 @@ div(fieldA,add(fieldA,fieldB)) // fieldA / (fieldA + fieldB)
== dotProduct
-The `dotProduct` function returns the https://en.wikipedia.org/wiki/Dot_product[dotproduct] of a numeric array.
+The `dotProduct` function returns the https://en.wikipedia.org/wiki/Dot_product[dotproduct] of two numeric arrays.
=== dotProduct Parameters
* `numeric array`
+* `numeric array`
=== dotProduct Returns
@@ -639,7 +640,7 @@ A number.
=== dotProduct Syntax
[source,text]
-dotProduct(numericArray)
+dotProduct(numericArray, numericArray)
== ebeAdd
[08/12] lucene-solr:jira/http2: SOLR-12827: remove monospace in
headings; correct formatting for heading level indicator;
remove redundant "Example #" in the example titles
Posted by da...@apache.org.
SOLR-12827: remove monospace in headings; correct formatting for heading level indicator; remove redundant "Example #" in the example titles
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/16ee8476
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/16ee8476
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/16ee8476
Branch: refs/heads/jira/http2
Commit: 16ee847663d67d1b003228d809dac6d5e30f2682
Parents: b72f05d
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Oct 24 15:04:33 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Oct 24 15:04:33 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/collections-api.adoc | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/16ee8476/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index 47a09d0..673cb1b 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -1208,13 +1208,11 @@ http://localhost:8983/solr/admin/collections?action=CLUSTERPROP&name=urlScheme&v
</response>
----
-=== Deeply Nested Cluster Properties ===
+=== Setting Cluster-Wide Defaults
-==== `defaults` ====
-It is possible to set cluster-wide default values for certain attributes of a collection.
+It is possible to set cluster-wide default values for certain attributes of a collection, using the `defaults` parameter.
-
-*Example 1: Set/update default values*
+*Set/update default values*
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
@@ -1232,7 +1230,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
}' http://localhost:8983/api/cluster
----
-*Example 2: Unset the value of `nrtReplicas` alone*
+*Unset the only value of `nrtReplicas`*
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
@@ -1247,7 +1245,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
}' http://localhost:8983/api/cluster
----
-*Example 2: Unset all values in `defaults`*
+*Unset all values in `defaults`*
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
@@ -1256,10 +1254,10 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
}' http://localhost:8983/api/cluster
----
-NOTE: Until Solr 7.5, cluster properties supported a "collectionDefaults" key which is now deprecated. Using the API
-structure for Solr 7.4 or Solr 7.5 with "collectionDefaults" will continue to work but the format of the properties
-will automatically be converted to the new nested structure. Support for the "collectionDefaults" key will be
-removed in Solr 9.
+NOTE: Until Solr 7.5, cluster properties supported a `collectionDefaults` key which is now deprecated and
+replaced with `defaults`. Using the `collectionDefaults` parameter in Solr 7.4 or 7.5 will continue to work
+ but the format of the properties will automatically be converted to the new nested structure.
+Support for the "collectionDefaults" key will be removed in Solr 9.
[[collectionprop]]
== COLLECTIONPROP: Collection Properties
[03/12] lucene-solr:jira/http2: SOLR-12905: MultiSolrCloudTestCase
now clears static clusterId2cluster in @AfterClass
Posted by da...@apache.org.
SOLR-12905: MultiSolrCloudTestCase now clears static clusterId2cluster in @AfterClass
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7fc91dea
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7fc91dea
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7fc91dea
Branch: refs/heads/jira/http2
Commit: 7fc91deaba25ae91bc9b2c4ae2875fc74c2c19aa
Parents: cebf703
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:25:10 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:25:10 2018 +0100
----------------------------------------------------------------------
.../src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fc91dea/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
index 4562c3e..eb0a677 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
@@ -102,6 +102,7 @@ public abstract class MultiSolrCloudTestCase extends SolrTestCaseJ4 {
for (MiniSolrCloudCluster cluster : clusterId2cluster.values()) {
cluster.shutdown();
}
+ clusterId2cluster.clear();
}
}
[10/12] lucene-solr:jira/http2: SOLR-12793: Move
TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test
package
Posted by da...@apache.org.
SOLR-12793: Move TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test package
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/71988c75
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/71988c75
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/71988c75
Branch: refs/heads/jira/http2
Commit: 71988c756b76a96cb96fc8f86183219a4a008389
Parents: 16ee847
Author: Varun Thacker <va...@apache.org>
Authored: Wed Oct 24 13:57:28 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Oct 24 13:57:35 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../cloud/TestCloudJSONFacetJoinDomain.java | 853 ------------------
.../solr/cloud/TestCloudJSONFacetSKG.java | 677 ---------------
.../org/apache/solr/search/facet/DebugAgg.java | 2 +-
.../facet/TestCloudJSONFacetJoinDomain.java | 855 +++++++++++++++++++
.../search/facet/TestCloudJSONFacetSKG.java | 678 +++++++++++++++
6 files changed, 1536 insertions(+), 1531 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 32c8ae2..bb8f86d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -178,6 +178,8 @@ Other Changes
* SOLR-12423: Upgrade to Tika 1.19.1 when available (Tim Allison via Erick Erickson)
+* SOLR-12793: Move TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test package (Varun Thacker)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
deleted file mode 100644
index 150a08d..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
+++ /dev/null
@@ -1,853 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.lucene.util.TestUtil;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.search.facet.FacetField;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * Tests randomized JSON Facets, sometimes using query 'join' domain transfers and/or domain 'filter' options
- * </p>
- * <p>
- * The results of each facet constraint count will be compared with a verification query using an equivalent filter
- * </p>
- *
- * @see TestCloudPivotFacet
- */
-public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
- private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
-
- private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
- private static final int MAX_FIELD_NUM = 15;
- private static final int UNIQUE_FIELD_VALS = 20;
-
- // NOTE: set to 'true' to see if refinement testing is adequate (should get fails occasionally)
- private static final boolean FORCE_DISABLE_REFINEMENT = false;
-
- /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
- /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
-
- /** A basic client for operations at the cloud level, default collection will be set */
- private static CloudSolrClient CLOUD_CLIENT;
- /** One client per node */
- private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
-
- @BeforeClass
- private static void createMiniSolrCloudCluster() throws Exception {
- // sanity check constants
- assertTrue("bad test constants: some suffixes will never be tested",
- (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
-
- // we need DVs on point fields to compute stats & facets
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
-
- // multi replicas should not matter...
- final int repFactor = usually() ? 1 : 2;
- // ... but we definitely want to test multiple shards
- final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
- final int numNodes = (numShards * repFactor);
-
- final String configName = DEBUG_LABEL + "_config-set";
- final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
-
- configureCluster(numNodes).addConfig(configName, configDir).configure();
-
- Map<String, String> collectionProperties = new LinkedHashMap<>();
- collectionProperties.put("config", "solrconfig-tlog.xml");
- collectionProperties.put("schema", "schema_latest.xml");
- CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
- .setProperties(collectionProperties)
- .process(cluster.getSolrClient());
-
- CLOUD_CLIENT = cluster.getSolrClient();
- CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
-
- waitForRecoveriesToFinish(CLOUD_CLIENT);
-
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
- }
-
- final int numDocs = atLeast(100);
- for (int id = 0; id < numDocs; id++) {
- SolrInputDocument doc = sdoc("id", ""+id);
- for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
- // NOTE: some docs may have no value in a field
- final int numValsThisDoc = TestUtil.nextInt(random(), 0, (usually() ? 3 : 6));
- for (int v = 0; v < numValsThisDoc; v++) {
- final String fieldValue = randFieldValue(fieldNum);
-
- // for each fieldNum, there are actaully two fields: one string, and one integer
- doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
- doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
- }
- }
- CLOUD_CLIENT.add(doc);
- if (random().nextInt(100) < 1) {
- CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
- }
- if (random().nextInt(100) < 5) {
- CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
- }
- }
- CLOUD_CLIENT.commit();
- }
-
- /**
- * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
- * uses that number and one of hte specified suffixes in it's name.
- *
- * @see #STR_FIELD_SUFFIXES
- * @see #INT_FIELD_SUFFIXES
- * @see #MAX_FIELD_NUM
- * @see #randFieldValue
- */
- private static String field(final String[] suffixes, final int fieldNum) {
- assert fieldNum < MAX_FIELD_NUM;
-
- final String suffix = suffixes[fieldNum % suffixes.length];
- return "field_" + fieldNum + suffix;
- }
- private static String strfield(final int fieldNum) {
- return field(STR_FIELD_SUFFIXES, fieldNum);
- }
- private static String intfield(final int fieldNum) {
- return field(INT_FIELD_SUFFIXES, fieldNum);
- }
-
- /**
- * Given a (random) field number, returns a random (integer based) value for that field.
- * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
- * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
- * will match fewer documents based on how far apart the field numbers are.
- *
- * @see #UNIQUE_FIELD_VALS
- * @see #field
- */
- private static String randFieldValue(final int fieldNum) {
- return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
- }
-
-
- @AfterClass
- private static void afterClass() throws Exception {
- CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
- for (HttpSolrClient client : CLIENTS) {
- client.close();
- }
- CLIENTS = null;
- }
-
- /** Sanity check that malformed requests produce errors */
- public void testMalformedGivesError() throws Exception {
-
- ignoreException(".*'join' domain change.*");
-
- for (String join : Arrays.asList("bogus",
- "{ }",
- "{ from:null, to:foo_s }",
- "{ from:foo_s }",
- "{ from:foo_s, to:foo_s, bogus:'what what?' }",
- "{ to:foo_s, bogus:'what what?' }")) {
- SolrException e = expectThrows(SolrException.class, () -> {
- final SolrParams req = params("q", "*:*", "json.facet",
- "{ x : { type:terms, field:x_s, domain: { join:"+join+" } } }");
- final NamedList trash = getRandClient(random()).request(new QueryRequest(req));
- });
- assertEquals(join + " -> " + e, SolrException.ErrorCode.BAD_REQUEST.code, e.code());
- assertTrue(join + " -> " + e, e.getMessage().contains("'join' domain change"));
- }
- }
-
- public void testSanityCheckDomainMethods() throws Exception {
- {
- final JoinDomain empty = new JoinDomain(null, null, null);
- assertEquals(null, empty.toJSONFacetParamValue());
- final SolrParams out = empty.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals(null, out.get("safe_key"));
- assertEquals("qqq", out.get("q"));
- }
- {
- final JoinDomain join = new JoinDomain("xxx", "yyy", null);
- assertEquals("domain:{join:{from:xxx,to:yyy}}", join.toJSONFacetParamValue().toString());
- final SolrParams out = join.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals("qqq", out.get("safe_key"));
- assertEquals("{!join from=xxx to=yyy v=$safe_key}", out.get("q"));
-
- }
- {
- final JoinDomain filter = new JoinDomain(null, null, "zzz");
- assertEquals("domain:{filter:'zzz'}", filter.toJSONFacetParamValue().toString());
- final SolrParams out = filter.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals(null, out.get("safe_key"));
- assertEquals("zzz AND qqq", out.get("q"));
- }
- {
- final JoinDomain both = new JoinDomain("xxx", "yyy", "zzz");
- assertEquals("domain:{join:{from:xxx,to:yyy},filter:'zzz'}", both.toJSONFacetParamValue().toString());
- final SolrParams out = both.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals("qqq", out.get("safe_key"));
- assertEquals("zzz AND {!join from=xxx to=yyy v=$safe_key}", out.get("q"));
- }
- }
-
- /**
- * Test some small, hand crafted, but non-trivial queries that are
- * easier to trace/debug then a pure random monstrosity.
- * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
- */
- public void testBespoke() throws Exception {
-
- { // sanity check our test methods can handle a query matching no docs
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
- top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
- facets.put("empty_top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, strfield(7) + ":bogus");
- assertEquals("Empty search result shouldn't have found a single bucket",
- UNIQUE_FIELD_VALS, maxBuckets.get());
- }
-
- { // sanity check our test methods can handle a query where a facet filter prevents any doc from having terms
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, "-*:*"));
- top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
- facets.put("filtered_top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
- assertEquals("Empty join filter shouldn't have found a single bucket",
- UNIQUE_FIELD_VALS, maxBuckets.get());
- }
-
- { // sanity check our test methods can handle a query where a facet filter prevents any doc from having sub-terms
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(8), strfield(8), null));
- top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(null, null, "-*:*")));
- facets.put("filtered_top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
-
- { // strings
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
- top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- }
-
- { // ints
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(intfield(9), new JoinDomain(intfield(5), intfield(9), null));
- facets.put("top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+intfield(7)+":16 OR "+intfield(3)+":13)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- }
-
- { // some domains with filter only, no actual join
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, strfield(9)+":[* TO *]"));
- top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(null, null, strfield(3)+":[* TO 5]")));
- facets.put("top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
-
- }
-
- { // low limits, explicit refinement
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9),
- new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
- 5, 0, true);
- top.subFacets.put("facet_5", new TermFacet(strfield(11),
- new JoinDomain(strfield(8), strfield(8), null),
- 10, 0, true));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
- }
-
- { // low limit, high overrequest
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9),
- new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
- 5, UNIQUE_FIELD_VALS + 10, false);
- top.subFacets.put("facet_5", new TermFacet(strfield(11),
- new JoinDomain(strfield(8), strfield(8), null),
- 10, UNIQUE_FIELD_VALS + 10, false));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
- }
-
- { // low limit, low overrequest, explicit refinement
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9),
- new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
- 5, 7, true);
- top.subFacets.put("facet_5", new TermFacet(strfield(11),
- new JoinDomain(strfield(8), strfield(8), null),
- 10, 7, true));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
- }
-
- }
-
- public void testTheTestRandomRefineParam() {
- // sanity check that randomRefineParam never violates isRefinementNeeded
- // (should be imposisble ... unless someone changes/breaks the randomization logic in the future)
- final int numIters = atLeast(100);
- for (int iter = 0; iter < numIters; iter++) {
- final Integer limit = TermFacet.randomLimitParam(random());
- final Integer overrequest = TermFacet.randomOverrequestParam(random());
- final Boolean refine = TermFacet.randomRefineParam(random(), limit, overrequest);
- if (TermFacet.isRefinementNeeded(limit, overrequest)) {
- assertEquals("limit: " + limit + ", overrequest: " + overrequest + ", refine: " + refine,
- Boolean.TRUE, refine);
- }
- }
- }
-
- public void testTheTestTermFacetShouldFreakOutOnBadRefineOptions() {
- expectThrows(AssertionError.class, () -> {
- final TermFacet bogus = new TermFacet("foo", null, 5, 0, false);
- });
- }
-
- public void testRandom() throws Exception {
-
- // we put a safety valve in place on the maximum number of buckets that we are willing to verify
- // across *all* the queries that we do.
- // that way if the randomized queries we build all have relatively small facets, so be it, but if
- // we get a really big one early on, we can test as much as possible, skip other iterations.
- //
- // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
- final int maxBucketsAllowed = atLeast(2000);
- final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
-
- final int numIters = atLeast(20);
- for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
- assertFacetCountsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(), buildRandomQuery());
- }
- assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
- }
-
- /**
- * Generates a random query string across the randomized fields/values in the index
- *
- * @see #randFieldValue
- * @see #field
- */
- private static String buildRandomQuery() {
- if (0 == TestUtil.nextInt(random(), 0,10)) {
- return "*:*";
- }
- final int numClauses = TestUtil.nextInt(random(), 3, 10);
- List<String> clauses = new ArrayList<String>(numClauses);
- for (int c = 0; c < numClauses; c++) {
- final int fieldNum = random().nextInt(MAX_FIELD_NUM);
- // keep queries simple, just use str fields - not point of test
- clauses.add(strfield(fieldNum) + ":" + randFieldValue(fieldNum));
- }
- return "(" + StringUtils.join(clauses, " OR ") + ")";
- }
-
- /**
- * Given a set of (potentially nested) term facets, and a base query string, asserts that
- * the actual counts returned when executing that query with those facets match the expected results
- * of filtering on the equivalent facet terms+domain
- */
- private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
- Map<String,TermFacet> expected,
- final String query) throws SolrServerException, IOException {
-
- final SolrParams baseParams = params("q", query, "rows","0");
- final SolrParams facetParams = params("json.facet", ""+TermFacet.toJSONFacetParamValue(expected));
- final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
-
- log.info("Doing full run: {}", initParams);
-
- QueryResponse rsp = null;
- // JSON Facets not (currently) available from QueryResponse...
- NamedList topNamedList = null;
- try {
- rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
- assertNotNull(initParams + " is null rsp?", rsp);
- topNamedList = rsp.getResponse();
- assertNotNull(initParams + " is null topNamedList?", topNamedList);
- } catch (Exception e) {
- throw new RuntimeException("init query failed: " + initParams + ": " +
- e.getMessage(), e);
- }
- try {
- final NamedList facetResponse = (NamedList) topNamedList.get("facets");
- assertNotNull("null facet results?", facetResponse);
- assertEquals("numFound mismatch with top count?",
- rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
- if (0 == rsp.getResults().getNumFound()) {
- // when the query matches nothing, we should expect no top level facets
- expected = Collections.emptyMap();
- }
- assertFacetCountsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
- } catch (AssertionError e) {
- throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
- } finally {
- log.info("Ending full run");
- }
- }
-
- /**
- * Recursive Helper method that walks the actual facet response, comparing the counts to the expected output
- * based on the equivalent filters generated from the original TermFacet.
- */
- private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
- final Map<String,TermFacet> expected,
- final SolrParams baseParams,
- final NamedList actualFacetResponse) throws SolrServerException, IOException {
-
- for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
- final String facetKey = entry.getKey();
- final TermFacet facet = entry.getValue();
- final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
- assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
- final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
- assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
-
- if (buckets.isEmpty()) {
- // should only happen if the baseParams query does not match any docs with our field X
- final long docsWithField = getRandClient(random()).query
- (facet.applyValueConstraintAndDomain(baseParams, facetKey, "[* TO *]")).getResults().getNumFound();
- assertEquals(facetKey + " has no buckets, but docs in query exist with field: " + facet.field,
- 0, docsWithField);
- }
-
- for (NamedList bucket : buckets) {
- final long count = ((Number) bucket.get("count")).longValue();
- final String fieldVal = bucket.get("val").toString(); // int or stringified int
-
- // change our query to filter on the fieldVal, and wrap in the facet domain (if any)
- final SolrParams verifyParams = facet.applyValueConstraintAndDomain(baseParams, facetKey, fieldVal);
-
- // check the count for this bucket
- assertEquals(facetKey + ": " + verifyParams,
- count, getRandClient(random()).query(verifyParams).getResults().getNumFound());
-
- if (maxBucketsToCheck.decrementAndGet() <= 0) {
- return;
- }
-
- // recursively check subFacets
- if (! facet.subFacets.isEmpty()) {
- assertFacetCountsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
- }
- }
- }
- assertTrue("facets have unexpected keys left over: " + actualFacetResponse,
- // should alwasy be a count, maybe a 'val' if we're a subfacet
- (actualFacetResponse.size() == expected.size() + 1) ||
- (actualFacetResponse.size() == expected.size() + 2));
- }
-
-
- /**
- * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
- *
- * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
- */
- private static final class TermFacet {
- public final String field;
- public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
- public final JoinDomain domain; // may be null
- public final Integer limit; // may be null
- public final Integer overrequest; // may be null
- public final Boolean refine; // may be null
-
- /** Simplified constructor asks for limit = # unique vals */
- public TermFacet(String field, JoinDomain domain) {
- this(field, domain, UNIQUE_FIELD_VALS, 0, false);
- }
- public TermFacet(String field, JoinDomain domain, Integer limit, Integer overrequest, Boolean refine) {
- assert null != field;
- this.field = field;
- this.domain = domain;
- this.limit = limit;
- this.overrequest = overrequest;
- this.refine = refine;
- if (isRefinementNeeded(limit, overrequest)) {
- assertEquals("Invalid refine param based on limit & overrequest: " + this.toString(),
- Boolean.TRUE, refine);
- }
- }
-
- /**
- * Returns new SolrParams that:
- * <ul>
- * <li>copy the original SolrParams</li>
- * <li>modify/wrap the original "q" param to capture the domain change for this facet (if any)</li>
- * <li>add a filter query against this field with the specified value</li>
- * </ul>
- *
- * @see JoinDomain#applyDomainToQuery
- */
- public SolrParams applyValueConstraintAndDomain(SolrParams orig, String facetKey, String facetVal) {
- // first wrap our original query in the domain if there is one...
- if (null != domain) {
- orig = domain.applyDomainToQuery(facetKey + "_q", orig);
- }
- // then filter by the facet value we need to test...
- final ModifiableSolrParams out = new ModifiableSolrParams(orig);
- out.set("q", field + ":" + facetVal + " AND " + orig.get("q"));
-
- return out;
- }
-
- /**
- * recursively generates the <code>json.facet</code> param value to use for testing this facet
- */
- private CharSequence toJSONFacetParamValue() {
- final String limitStr = (null == limit) ? "" : (", limit:" + limit);
- final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
- final String refineStr = (null == refine) ? "" : ", refine:" + refine;
- final StringBuilder sb = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + refineStr);
- if (! subFacets.isEmpty()) {
- sb.append(", facet:");
- sb.append(toJSONFacetParamValue(subFacets));
- }
- if (null != domain) {
- CharSequence ds = domain.toJSONFacetParamValue();
- if (null != ds) {
- sb.append(", ").append(ds);
- }
- }
- sb.append("}");
- return sb;
- }
-
- /**
- * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
- * use for testing them against in a solr request.
- */
- public static CharSequence toJSONFacetParamValue(Map<String,TermFacet> facets) {
- assert null != facets;
- assert 0 < facets.size();
- StringBuilder sb = new StringBuilder("{");
- for (String key : facets.keySet()) {
- sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
- sb.append(" ,");
- }
- sb.setLength(sb.length() - 1);
- sb.append("}");
- return sb;
- }
-
- /**
- * Factory method for generating some random (nested) facets.
- *
- * For simplicity, each facet will have a unique key name, regardless of it's depth under other facets
- *
- * @see JoinDomain
- */
- public static Map<String,TermFacet> buildRandomFacets() {
- // for simplicity, use a unique facet key regardless of depth - simplifies verification
- AtomicInteger keyCounter = new AtomicInteger(0);
- final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
- return buildRandomFacets(keyCounter, maxDepth);
- }
-
- /**
- * picks a random value for the "limit" param, biased in favor of interesting test cases
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- */
- public static Integer randomLimitParam(Random r) {
- final int limit = 1 + r.nextInt(UNIQUE_FIELD_VALS * 2);
- if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
- return -1; // unlimited
- } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
- return null; // sometimes, don't specify limit if it's the default
- }
- return limit;
- }
-
- /**
- * picks a random value for the "overrequest" param, biased in favor of interesting test cases
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- */
- public static Integer randomOverrequestParam(Random r) {
- switch(r.nextInt(10)) {
- case 0:
- case 1:
- case 2:
- case 3:
- return 0; // 40% of the time, no overrequest to better stress refinement
- case 4:
- case 5:
- return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
- case 6:
- return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
- default: break;
- }
- // else.... either leave param unspecified (or redundently specify the -1 default)
- return r.nextBoolean() ? null : -1;
- }
-
- /**
- * picks a random value for the "refine" param, that is garunteed to be suitable for
- * the specified limit & overrequest params.
- *
- * @return a value to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #randomLimitParam
- * @see #randomOverrequestParam
- * @see #UNIQUE_FIELD_VALS
- */
- public static Boolean randomRefineParam(Random r, Integer limitParam, Integer overrequestParam) {
- if (isRefinementNeeded(limitParam, overrequestParam)) {
- return true;
- }
-
- // refinement is not required
- if (0 == r.nextInt(10)) { // once in a while, turn on refinement even if it isn't needed.
- return true;
- }
- // explicitly or implicitly indicate refinement is not needed
- return r.nextBoolean() ? false : null;
- }
-
- /**
- * Deterministicly identifies if the specified limit & overrequest params <b>require</b>
- * a "refine:true" param be used in the the request, in order for the counts to be 100% accurate.
- *
- * @see #UNIQUE_FIELD_VALS
- */
- public static boolean isRefinementNeeded(Integer limitParam, Integer overrequestParam) {
-
- if (FORCE_DISABLE_REFINEMENT) {
- return false;
- }
-
- // use the "effective" values if the params are null
- final int limit = null == limitParam ? DEFAULT_LIMIT : limitParam;
- final int overrequest = null == overrequestParam ? 0 : overrequestParam;
-
- return
- // don't presume how much overrequest will be done by default, just check the limit
- (overrequest < 0 && limit < UNIQUE_FIELD_VALS)
- // if the user specified overrequest is not "enough" to get all unique values
- || (overrequest >= 0 && (long)limit + overrequest < UNIQUE_FIELD_VALS);
- }
-
- /**
- * recursive helper method for building random facets
- *
- * @param keyCounter used to ensure every generated facet has a unique key name
- * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
- */
- private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
- final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
- Map<String,TermFacet> results = new LinkedHashMap<>();
- for (int i = 0; i < numFacets; i++) {
- final JoinDomain domain = JoinDomain.buildRandomDomain();
- assert null != domain;
- final Integer limit = randomLimitParam(random());
- final Integer overrequest = randomOverrequestParam(random());
- final TermFacet facet = new TermFacet(field(random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES,
- random().nextInt(MAX_FIELD_NUM)),
- domain, limit, overrequest,
- randomRefineParam(random(), limit, overrequest));
- results.put("facet_" + keyCounter.incrementAndGet(), facet);
- if (0 < maxDepth) {
- // if we're going wide, don't go deep
- final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
- facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
- }
- }
- return results;
- }
- }
-
-
- /**
- * Models a Domain Change which includes either a 'join' or a 'filter' or both
- */
- private static final class JoinDomain {
- public final String from;
- public final String to;
- public final String filter; // not bothering with more then 1 filter, not the point of the test
-
- /**
- * @param from left side of join field name, null if domain involves no joining
- * @param to right side of join field name, null if domain involves no joining
- * @param filter filter to apply to domain, null if domain involves no filtering
- */
- public JoinDomain(String from, String to, String filter) {
- assert ! ((null == from) ^ (null == to)) : "if from is null, to must be null";
- this.from = from;
- this.to = to;
- this.filter = filter;
- }
-
- /**
- * @return the JSON string representing this domain for use in a facet param, or null if no domain should be used
- * */
- public CharSequence toJSONFacetParamValue() {
- if (null == from && null == filter) {
- return null;
- }
- StringBuilder sb = new StringBuilder("domain:{");
- if (null != from) {
- assert null != to;
- sb. append("join:{from:").append(from).append(",to:").append(to).append("}");
- if (null != filter){
- sb.append(",");
- }
-
- }
- if (null != filter) {
- sb.append("filter:'").append(filter).append("'");
- }
- sb.append("}");
- return sb;
- }
-
- /**
- * Given some original SolrParams, returns new SolrParams where the original "q" param is wrapped
- * as needed to apply the equivalent transformation to a query as this domain would to a facet
- */
- public SolrParams applyDomainToQuery(String safeKey, SolrParams in) {
- assert null == in.get(safeKey); // shouldn't be possible if every facet uses a unique key string
-
- String q = in.get("q");
- final ModifiableSolrParams out = new ModifiableSolrParams(in);
- if (null != from) {
- out.set(safeKey, in.get("q"));
- q = "{!join from="+from+" to="+to+" v=$"+safeKey+"}";
- }
- if (null != filter) {
- q = filter + " AND " + q;
- }
- out.set("q", q);
- return out;
- }
-
- /**
- * Factory method for creating a random domain change to use with a facet - may return an 'noop' JoinDomain,
- * but will never return null.
- */
- public static JoinDomain buildRandomDomain() {
-
- // use consistent type on both sides of join
- final String[] suffixes = random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES;
-
- final boolean noJoin = random().nextBoolean();
-
- String from = null;
- String to = null;
- for (;;) {
- if (noJoin) break;
- from = field(suffixes, random().nextInt(MAX_FIELD_NUM));
- to = field(suffixes, random().nextInt(MAX_FIELD_NUM));
- // HACK: joined numeric point fields need docValues.. for now just skip _is fields if we are dealing with points.
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP) && (from.endsWith("_is") || to.endsWith("_is")))
- {
- continue;
- }
- break;
- }
-
- // keep it simple, only filter on string fields - not point of test
- final String filterField = strfield(random().nextInt(MAX_FIELD_NUM));
-
- final String filter = random().nextBoolean() ? null : filterField+":[* TO *]";
- return new JoinDomain(from, to, filter);
- }
- }
-
- /**
- * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
- * at a node in our cluster
- */
- public static SolrClient getRandClient(Random rand) {
- int numClients = CLIENTS.size();
- int idx = TestUtil.nextInt(rand, 0, numClients);
-
- return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
- }
-
- public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
- assert null != client.getDefaultCollection();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
- client.getZkStateReader(),
- true, true, 330);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
deleted file mode 100644
index 24384cc..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
+++ /dev/null
@@ -1,677 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.lang.StringUtils;
-
-import org.apache.lucene.util.LuceneTestCase.Slow;
-import org.apache.lucene.util.TestUtil;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.search.facet.FacetField;
-import static org.apache.solr.search.facet.RelatednessAgg.computeRelatedness;
-import static org.apache.solr.search.facet.RelatednessAgg.roundTo5Digits;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * A randomized test of nested facets using the <code>relatedness()</code> function, that asserts the
- * accuracy the results for all the buckets returned using verification queries of the (expected)
- * foreground & background queries based on the nested facet terms.
- * <p>
- * Note that unlike normal facet "count" verification, using a high limit + overrequest isn't a substitute
- * for refinement in order to ensure accurate "skg" computation across shards. For that reason, this
- * tests forces <code>refine: true</code> (unlike {@link TestCloudJSONFacetJoinDomain}) and specifices a
- * <code>domain: { 'query':'*:*' }</code> for every facet, in order to garuntee that all shards
- * participate in all facets, so that the popularity & relatedness values returned can be proven
- * with validation requests.
- * </p>
- * <p>
- * (Refinement alone is not enough. Using the '*:*' query as the facet domain is neccessary to
- * prevent situations where a single shardX may return candidate bucket with no child-buckets due to
- * the normal facet intersections, but when refined on other shardY(s), can produce "high scoring"
- * SKG child-buckets, which would then be missing the foreground/background "size" contributions from
- * shardX.
- * </p>
- *
- * @see TestCloudJSONFacetJoinDomain
- */
-@Slow
-public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
- private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
-
- private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
- private static final int MAX_FIELD_NUM = 15;
- private static final int UNIQUE_FIELD_VALS = 50;
-
- /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
- /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
-
- /** A basic client for operations at the cloud level, default collection will be set */
- private static CloudSolrClient CLOUD_CLIENT;
- /** One client per node */
- private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
-
- @BeforeClass
- private static void createMiniSolrCloudCluster() throws Exception {
- // sanity check constants
- assertTrue("bad test constants: some suffixes will never be tested",
- (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
-
- // we need DVs on point fields to compute stats & facets
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
-
- // multi replicas should not matter...
- final int repFactor = usually() ? 1 : 2;
- // ... but we definitely want to test multiple shards
- final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
- final int numNodes = (numShards * repFactor);
-
- final String configName = DEBUG_LABEL + "_config-set";
- final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
-
- configureCluster(numNodes).addConfig(configName, configDir).configure();
-
- Map<String, String> collectionProperties = new LinkedHashMap<>();
- collectionProperties.put("config", "solrconfig-tlog.xml");
- collectionProperties.put("schema", "schema_latest.xml");
- CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
- .setProperties(collectionProperties)
- .process(cluster.getSolrClient());
-
- CLOUD_CLIENT = cluster.getSolrClient();
- CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
-
- waitForRecoveriesToFinish(CLOUD_CLIENT);
-
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
- }
-
- final int numDocs = atLeast(100);
- for (int id = 0; id < numDocs; id++) {
- SolrInputDocument doc = sdoc("id", ""+id);
- for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
- // NOTE: we ensure every doc has at least one value in each field
- // that way, if a term is returned for a parent there there is garunteed to be at least one
- // one term in the child facet as well.
- //
- // otherwise, we'd face the risk of a single shardX returning parentTermX as a top term for
- // the parent facet, but having no child terms -- meanwhile on refinement another shardY that
- // did *not* returned parentTermX in phase#1, could return some *new* child terms under
- // parentTermX, but their stats would not include the bgCount from shardX.
- //
- // in normal operation, this is an edge case that isn't a big deal because the ratios &
- // relatedness scores are statistically approximate, but for the purpose of this test where
- // we verify correctness via exactness we need all shards to contribute to the SKG statistics
- final int numValsThisDoc = TestUtil.nextInt(random(), 1, (usually() ? 5 : 10));
- for (int v = 0; v < numValsThisDoc; v++) {
- final String fieldValue = randFieldValue(fieldNum);
-
- // for each fieldNum, there are actaully two fields: one string, and one integer
- doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
- doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
- }
- }
- CLOUD_CLIENT.add(doc);
- if (random().nextInt(100) < 1) {
- CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
- }
- if (random().nextInt(100) < 5) {
- CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
- }
- }
- CLOUD_CLIENT.commit();
- }
-
- /**
- * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
- * uses that number and one of hte specified suffixes in it's name.
- *
- * @see #STR_FIELD_SUFFIXES
- * @see #INT_FIELD_SUFFIXES
- * @see #MAX_FIELD_NUM
- * @see #randFieldValue
- */
- private static String field(final String[] suffixes, final int fieldNum) {
- assert fieldNum < MAX_FIELD_NUM;
-
- final String suffix = suffixes[fieldNum % suffixes.length];
- return "field_" + fieldNum + suffix;
- }
- private static String strfield(final int fieldNum) {
- return field(STR_FIELD_SUFFIXES, fieldNum);
- }
- private static String intfield(final int fieldNum) {
- return field(INT_FIELD_SUFFIXES, fieldNum);
- }
-
- /**
- * Given a (random) field number, returns a random (integer based) value for that field.
- * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
- * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
- * will match fewer documents based on how far apart the field numbers are.
- *
- * @see #UNIQUE_FIELD_VALS
- * @see #field
- */
- private static String randFieldValue(final int fieldNum) {
- return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
- }
-
-
- @AfterClass
- private static void afterClass() throws Exception {
- CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
- for (HttpSolrClient client : CLIENTS) {
- client.close();
- }
- CLIENTS = null;
- }
-
- /**
- * Test some small, hand crafted, but non-trivial queries that are
- * easier to trace/debug then a pure random monstrosity.
- * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
- */
- public void testBespoke() throws Exception {
- { // trivial single level facet
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, null);
- facets.put("top1", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
-
- { // trivial single level facet w/sorting on skg
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, "skg desc");
- facets.put("top2", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
-
- { // trivial single level facet w/ 2 diff ways to request "limit = (effectively) Infinite"
- // to sanity check refinement of buckets missing from other shard in both cases
-
- // NOTE that these two queries & facets *should* effectively identical given that the
- // very large limit value is big enough no shard will ever return that may terms,
- // but the "limit=-1" case it actaully triggers slightly different code paths
- // because it causes FacetField.returnsPartial() to be "true"
- for (int limit : new int[] { 999999999, -1 }) {
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- facets.put("top_facet_limit__" + limit, new TermFacet(strfield(9), limit, 0, "skg desc"));
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
- }
- }
-
- public void testRandom() throws Exception {
-
- // since the "cost" of verifying the stats for each bucket is so high (see TODO in verifySKGResults())
- // we put a safety valve in place on the maximum number of buckets that we are willing to verify
- // across *all* the queries that we do.
- // that way if the randomized queries we build all have relatively small facets, so be it, but if
- // we get a really big one early on, we can test as much as possible, skip other iterations.
- //
- // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
- final int maxBucketsAllowed = atLeast(2000);
- final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
-
- final int numIters = atLeast(10);
- for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
- assertFacetSKGsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(),
- buildRandomQuery(), buildRandomQuery(), buildRandomQuery());
- }
- assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
-
-
- }
-
- /**
- * Generates a random query string across the randomized fields/values in the index
- *
- * @see #randFieldValue
- * @see #field
- */
- private static String buildRandomQuery() {
- if (0 == TestUtil.nextInt(random(), 0,10)) {
- return "*:*";
- }
- final int numClauses = TestUtil.nextInt(random(), 3, 10);
- final String[] clauses = new String[numClauses];
- for (int c = 0; c < numClauses; c++) {
- final int fieldNum = random().nextInt(MAX_FIELD_NUM);
- // keep queries simple, just use str fields - not point of test
- clauses[c] = strfield(fieldNum) + ":" + randFieldValue(fieldNum);
- }
- return buildORQuery(clauses);
- }
-
- private static String buildORQuery(String... clauses) {
- assert 0 < clauses.length;
- return "(" + StringUtils.join(clauses, " OR ") + ")";
- }
-
- /**
- * Given a set of term facets, and top level query strings, asserts that
- * the SKG stats for each facet term returned when executing that query with those foreground/background
- * queries match the expected results of executing the equivalent queries in isolation.
- *
- * @see #verifySKGResults
- */
- private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
- Map<String,TermFacet> expected,
- final String query,
- final String foreQ,
- final String backQ) throws SolrServerException, IOException {
- final SolrParams baseParams = params("rows","0", "fore", foreQ, "back", backQ);
-
- final SolrParams facetParams = params("q", query,
- "json.facet", ""+TermFacet.toJSONFacetParamValue(expected,null));
- final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
-
- log.info("Doing full run: {}", initParams);
-
- QueryResponse rsp = null;
- // JSON Facets not (currently) available from QueryResponse...
- NamedList topNamedList = null;
- try {
- rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
- assertNotNull(initParams + " is null rsp?", rsp);
- topNamedList = rsp.getResponse();
- assertNotNull(initParams + " is null topNamedList?", topNamedList);
- } catch (Exception e) {
- throw new RuntimeException("init query failed: " + initParams + ": " +
- e.getMessage(), e);
- }
- try {
- final NamedList facetResponse = (NamedList) topNamedList.get("facets");
- assertNotNull("null facet results?", facetResponse);
- assertEquals("numFound mismatch with top count?",
- rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
-
- // Note: even if the query has numFound=0, our explicit background query domain should
- // still force facet results
- // (even if the background query matches nothing, that just means there will be no
- // buckets in those facets)
- assertFacetSKGsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
-
- } catch (AssertionError e) {
- throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
- } finally {
- log.info("Ending full run");
- }
- }
-
- /**
- * Recursive helper method that walks the actual facet response, comparing the SKG results to
- * the expected output based on the equivalent filters generated from the original TermFacet.
- */
- private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
- final Map<String,TermFacet> expected,
- final SolrParams baseParams,
- final NamedList actualFacetResponse) throws SolrServerException, IOException {
-
- for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
- final String facetKey = entry.getKey();
- final TermFacet facet = entry.getValue();
- final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
- assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
- final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
- assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
-
- if (buckets.isEmpty()) {
- // should only happen if the background query does not match any docs with field X
- final long docsWithField = getNumFound(params("_trace", "noBuckets",
- "rows", "0",
- "q", facet.field+":[* TO *]",
- "fq", baseParams.get("back")));
-
- assertEquals(facetKey + " has no buckets, but docs in background exist with field: " + facet.field,
- 0, docsWithField);
- }
-
- // NOTE: it's important that we do this depth first -- not just because it's the easiest way to do it,
- // but because it means that our maxBucketsToCheck will ensure we do a lot of deep sub-bucket checking,
- // not just all the buckets of the top level(s) facet(s)
- for (NamedList bucket : buckets) {
- final String fieldVal = bucket.get("val").toString(); // int or stringified int
-
- verifySKGResults(facetKey, facet, baseParams, fieldVal, bucket);
- if (maxBucketsToCheck.decrementAndGet() <= 0) {
- return;
- }
-
- final SolrParams verifyParams = SolrParams.wrapAppended(baseParams,
- params("fq", facet.field + ":" + fieldVal));
-
- // recursively check subFacets
- if (! facet.subFacets.isEmpty()) {
- assertFacetSKGsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
- }
- }
- }
-
- { // make sure we don't have any facet keys we don't expect
- // a little hackish because subfacets have extra keys...
- final LinkedHashSet expectedKeys = new LinkedHashSet(expected.keySet());
- expectedKeys.add("count");
- if (0 <= actualFacetResponse.indexOf("val",0)) {
- expectedKeys.add("val");
- expectedKeys.add("skg");
- }
- assertEquals("Unexpected keys in facet response",
- expectedKeys, actualFacetResponse.asShallowMap().keySet());
- }
- }
-
- /**
- * Verifies that the popularity & relatedness values containined in a single SKG bucket
- * match the expected values based on the facet field & bucket value, as well the existing
- * filterParams.
- *
- * @see #assertFacetSKGsAreCorrect
- */
- private void verifySKGResults(String facetKey, TermFacet facet, SolrParams filterParams,
- String fieldVal, NamedList<Object> bucket)
- throws SolrServerException, IOException {
-
- final String bucketQ = facet.field+":"+fieldVal;
- final NamedList<Object> skgBucket = (NamedList<Object>) bucket.get("skg");
- assertNotNull(facetKey + "/bucket:" + bucket.toString(), skgBucket);
-
- // TODO: make this more efficient?
- // ideally we'd do a single query w/4 facet.queries, one for each count
- // but formatting the queries is a pain, currently we leverage the accumulated fq's
- final long fgSize = getNumFound(SolrParams.wrapAppended(params("_trace", "fgSize",
- "rows","0",
- "q","{!query v=$fore}"),
- filterParams));
- final long bgSize = getNumFound(params("_trace", "bgSize",
- "rows","0",
- "q", filterParams.get("back")));
-
- final long fgCount = getNumFound(SolrParams.wrapAppended(params("_trace", "fgCount",
- "rows","0",
- "q","{!query v=$fore}",
- "fq", bucketQ),
- filterParams));
- final long bgCount = getNumFound(params("_trace", "bgCount",
- "rows","0",
- "q", bucketQ,
- "fq", filterParams.get("back")));
-
- assertEquals(facetKey + "/bucket:" + bucket + " => fgPop should be: " + fgCount + " / " + bgSize,
- roundTo5Digits((double) fgCount / bgSize),
- skgBucket.get("foreground_popularity"));
- assertEquals(facetKey + "/bucket:" + bucket + " => bgPop should be: " + bgCount + " / " + bgSize,
- roundTo5Digits((double) bgCount / bgSize),
- skgBucket.get("background_popularity"));
- assertEquals(facetKey + "/bucket:" + bucket + " => relatedness is wrong",
- roundTo5Digits(computeRelatedness(fgCount, fgSize, bgCount, bgSize)),
- skgBucket.get("relatedness"));
-
- }
-
-
- /**
- * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
- *
- * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
- */
- private static final class TermFacet {
- public final String field;
- public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
- public final Integer limit; // may be null
- public final Integer overrequest; // may be null
- public final String sort; // may be null
- /** Simplified constructor asks for limit = # unique vals */
- public TermFacet(String field) {
- this(field, UNIQUE_FIELD_VALS, 0, "skg desc");
-
- }
- public TermFacet(String field, Integer limit, Integer overrequest, String sort) {
- assert null != field;
- this.field = field;
- this.limit = limit;
- this.overrequest = overrequest;
- this.sort = sort;
- }
-
- /**
- * recursively generates the <code>json.facet</code> param value to use for testing this facet
- */
- private CharSequence toJSONFacetParamValue() {
- final String limitStr = (null == limit) ? "" : (", limit:" + limit);
- final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
- final String sortStr = (null == sort) ? "" : (", sort: '" + sort + "'");
- final StringBuilder sb
- = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + sortStr);
-
- // see class javadocs for why we always use refine:true & the query:'*:*' domain for this test.
- sb.append(", refine: true, domain: { query: '*:*' }, facet:");
- sb.append(toJSONFacetParamValue(subFacets, "skg : 'relatedness($fore,$back)'"));
- sb.append("}");
- return sb;
- }
-
- /**
- * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
- * use for testing them against in a solr request.
- */
- public static CharSequence toJSONFacetParamValue(final Map<String,TermFacet> facets,
- final String extraJson) {
- assert null != facets;
- if (0 == facets.size() && null == extraJson) {
- return "";
- }
-
- StringBuilder sb = new StringBuilder("{ processEmpty: true, ");
- for (String key : facets.keySet()) {
- sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
- sb.append(" ,");
- }
- if (null == extraJson) {
- sb.setLength(sb.length() - 1);
- } else {
- sb.append(extraJson);
- }
- sb.append("}");
- return sb;
- }
-
- /**
- * Factory method for generating some random facets.
- *
- * For simplicity, each facet will have a unique key name.
- */
- public static Map<String,TermFacet> buildRandomFacets() {
- // for simplicity, use a unique facet key regardless of depth - simplifies verification
- // and le's us enforce a hard limit on the total number of facets in a request
- AtomicInteger keyCounter = new AtomicInteger(0);
-
- final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
- return buildRandomFacets(keyCounter, maxDepth);
- }
-
- /**
- * picks a random value for the "sort" param, biased in favor of interesting test cases
- *
- * @return a sort string (w/direction), or null to specify nothing (trigger default behavior)
- * @see #randomLimitParam
- */
- public static String randomSortParam(Random r) {
-
- // IMPORTANT!!!
- // if this method is modified to produce new sorts, make sure to update
- // randomLimitParam to account for them if they are impacted by SOLR-12556
- final String dir = random().nextBoolean() ? "asc" : "desc";
- switch(r.nextInt(4)) {
- case 0: return null;
- case 1: return "count " + dir;
- case 2: return "skg " + dir;
- case 3: return "index " + dir;
- default: throw new RuntimeException("Broken case statement");
- }
- }
- /**
- * picks a random value for the "limit" param, biased in favor of interesting test cases
- *
- * <p>
- * <b>NOTE:</b> Due to SOLR-12556, we have to force an overrequest of "all" possible terms for
- * some sort values.
- * </p>
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- * @see #randomSortParam
- */
- public static Integer randomLimitParam(Random r, final String sort) {
- if (null != sort) {
- if (sort.equals("count asc") || sort.startsWith("skg")) {
- // of the known types of sorts produced, these are at risk of SOLR-12556
- // so request (effectively) unlimited num buckets
- return r.nextBoolean() ? UNIQUE_FIELD_VALS : -1;
- }
- }
- final int limit = 1 + r.nextInt((int) (UNIQUE_FIELD_VALS * 1.5F));
- if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
- return -1; // unlimited
- } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
- return null; // sometimes, don't specify limit if it's the default
- }
- return limit;
- }
-
- /**
- * picks a random value for the "overrequest" param, biased in favor of interesting test cases.
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- */
- public static Integer randomOverrequestParam(Random r) {
- switch(r.nextInt(10)) {
- case 0:
- case 1:
- case 2:
- case 3:
- return 0; // 40% of the time, disable overrequest to better stress refinement
- case 4:
- case 5:
- return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
- case 6:
- return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
- default: break;
- }
- // else.... either leave param unspecified (or redundently specify the -1 default)
- return r.nextBoolean() ? null : -1;
- }
-
- /**
- * recursive helper method for building random facets
- *
- * @param keyCounter used to ensure every generated facet has a unique key name
- * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
- */
- private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
- final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
- Map<String,TermFacet> results = new LinkedHashMap<>();
- for (int i = 0; i < numFacets; i++) {
- if (keyCounter.get() < 3) { // a hard limit on the total number of facets (regardless of depth) to reduce OOM risk
-
- final String sort = randomSortParam(random());
- final Integer limit = randomLimitParam(random(), sort);
- final Integer overrequest = randomOverrequestParam(random());
- final TermFacet facet = new TermFacet(field((random().nextBoolean()
- ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES),
- random().nextInt(MAX_FIELD_NUM)),
- limit, overrequest, sort);
- results.put("facet_" + keyCounter.incrementAndGet(), facet);
- if (0 < maxDepth) {
- // if we're going wide, don't go deep
- final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
- facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
- }
- }
- }
- return results;
- }
- }
-
- /**
- * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
- * at a node in our cluster
- */
- public static SolrClient getRandClient(Random rand) {
- int numClients = CLIENTS.size();
- int idx = TestUtil.nextInt(rand, 0, numClients);
-
- return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
- }
-
- /**
- * Uses a random SolrClient to execture a request and returns only the numFound
- * @see #getRandClient
- */
- public static long getNumFound(final SolrParams req) throws SolrServerException, IOException {
- return getRandClient(random()).query(req).getResults().getNumFound();
- }
-
- public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
- assert null != client.getDefaultCollection();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
- client.getZkStateReader(),
- true, true, 330);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
index 9de0d12..a661198 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
@@ -32,7 +32,7 @@ import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.ValueSourceParser;
-public class DebugAgg extends AggValueSource {
+class DebugAgg extends AggValueSource {
public static AtomicLong parses = new AtomicLong(0);
public static class Parser extends ValueSourceParser {
[07/12] lucene-solr:jira/http2: SOLR-9317: cleanup a couple of typos;
add some description to addreplica examples
Posted by da...@apache.org.
SOLR-9317: cleanup a couple of typos; add some description to addreplica examples
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b72f05da
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b72f05da
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b72f05da
Branch: refs/heads/jira/http2
Commit: b72f05dac63f88929056627525857f45b303154b
Parents: e62fe45
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Oct 24 14:21:35 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Oct 24 14:41:36 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/collections-api.adoc | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b72f05da/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index a5021da..47a09d0 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -1056,14 +1056,14 @@ If the exact shard name is not known, users may pass the `\_route_` value and th
Ignored if the `shard` parameter is also specified.
`node`::
-The name of the node where the replica should be created (optional)
+The name of the node where the replica should be created (optional).
`createNodeSet`::
A comma-separated list of nodes among which the best ones will be chosen to place the replicas (optional)
+
The format is a comma-separated list of node_names, such as `localhost:8983_solr,localhost:8984_solr,localhost:8985_solr`.
-If neither `node`, nor `createNodeSet` is specified then the best node(s) from among all the live nodes in the cluster are chosen.
+NOTE: If neither `node` nor `createNodeSet` are specified then the best node(s) from among all the live nodes in the cluster are chosen.
`instanceDir`::
The instanceDir for the core that will be created.
@@ -1074,11 +1074,9 @@ The directory in which the core should be created.
`type`::
The type of replica to create. These possible values are allowed:
+
---
* `nrt`: The NRT type maintains a transaction log and updates its index locally. This is the default and the most commonly used.
* `tlog`: The TLOG type maintains a transaction log but only updates its index via replication.
* `pull`: The PULL type does not maintain a transaction log and only updates its index via replication. This type is not eligible to become a leader.
---
+
See the section <<shards-and-indexing-data-in-solrcloud.adoc#types-of-replicas,Types of Replicas>> for more information about replica type options.
@@ -1104,6 +1102,8 @@ Request ID to track this action which will be <<Asynchronous Calls,processed asy
*Input*
+Create a replica for the "test" collection on the node "192.167.1.2:8983_solr".
+
[source,text]
----
http://localhost:8983/solr/admin/collections?action=ADDREPLICA&collection=test2&shard=shard2&node=192.167.1.2:8983_solr&wt=xml
@@ -1130,6 +1130,10 @@ http://localhost:8983/solr/admin/collections?action=ADDREPLICA&collection=test2&
</response>
----
+*Input*
+
+Create a replica for the "gettingstarted" collection with one PULL replica and one TLOG replica.
+
[source,text]
----
http://localhost:8983/solr/admin/collections?action=addreplica&collection=gettingstarted&shard=shard1&tlogReplicas=1&pullReplicas=1
[04/12] lucene-solr:jira/http2: SOLR-12873: Replace few remaining
occurrences of LUCENE_CURRENT with LATEST for luceneMatchVersion.
Posted by da...@apache.org.
SOLR-12873: Replace few remaining occurrences of LUCENE_CURRENT with LATEST for luceneMatchVersion.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c277674f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c277674f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c277674f
Branch: refs/heads/jira/http2
Commit: c277674f0ed5ab810432a1ad18c174f75dd4a9be
Parents: 7fc91de
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:28:12 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:28:12 2018 +0100
----------------------------------------------------------------------
.../conf/solrconfig-distrib-update-processor-chains.xml | 2 +-
.../solr/collection1/conf/solrconfig-doctransformers.xml | 2 +-
solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml | 2 +-
.../solr/collection1/conf/solrconfig-master-throttled.xml | 2 +-
.../src/test-files/solr/collection1/conf/solrconfig-paramset.xml | 2 +-
solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../src/test-files/solr/collection1/conf/solrconfig-tagger.xml | 2 +-
.../src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml | 2 +-
.../test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml | 2 +-
10 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
index f91be75..c9c351f 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
@@ -20,7 +20,7 @@
<config>
<jmx />
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
<!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
index 5a7d65e..f3a0bd5 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
index cad70d8..6600f7c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
index 4a53776..1a37f3f 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
@@ -18,7 +18,7 @@
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
<schemaFactory class="ClassicIndexSchemaFactory"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
index 7eefc9d..a9e71f6 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
@@ -20,7 +20,7 @@
<!-- a basic solrconfig that tests can use when they want simple minimal solrconfig/schema
DO NOT ADD THINGS TO THIS CONFIG! -->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
index 8ec6489..ac8ea62 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
index e0d3677..c97ce08 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
@@ -24,7 +24,7 @@
<!-- a basic solrconfig that tests can use when they want simple minimal solrconfig/schema
DO NOT ADD THINGS TO THIS CONFIG! -->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
index af659ad..129cc48 100644
--- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
+++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
index b002a30..0120f35 100644
--- a/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
+++ b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
index 70e9403..0e13a5a 100644
--- a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
+++ b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>