You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/08/09 19:27:03 UTC
[01/17] lucene-solr:jira/solr-12470: SOLR-12629: The predict
evaluator should work with the polyfit function
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-12470 d8fb69166 -> e1a5ee6fa
SOLR-12629: The predict evaluator should work with the polyfit function
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2ca5a6d1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2ca5a6d1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2ca5a6d1
Branch: refs/heads/jira/solr-12470
Commit: 2ca5a6d12e48e5d58811d7991da028cb2d70e86f
Parents: 3898653
Author: Joel Bernstein <jb...@apache.org>
Authored: Tue Aug 7 10:12:07 2018 -0400
Committer: Joel <jo...@alfresco.com>
Committed: Tue Aug 7 10:31:53 2018 -0400
----------------------------------------------------------------------
.../solr/client/solrj/io/eval/PolyFitEvaluator.java | 7 ++++++-
.../client/solrj/io/stream/MathExpressionTest.java | 15 ++++++++++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2ca5a6d1/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
index 5320f18..5292c94 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PolyFitEvaluator.java
@@ -96,6 +96,11 @@ public class PolyFitEvaluator extends RecursiveNumericEvaluator implements ManyV
list.add(yvalue);
}
- return list;
+
+ VectorFunction vec = new VectorFunction(pf, list);
+ vec.addToContext("x", x);
+ vec.addToContext("y", y);
+
+ return vec;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2ca5a6d1/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 3806b22..98a52a6 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -2856,7 +2856,8 @@ public class MathExpressionTest extends SolrCloudTestCase {
public void testPolyfit() throws Exception {
String cexpr = "let(echo=true," +
" a=array(0,1,2,3,4,5,6,7)," +
- " fit=polyfit(a, 1))";
+ " fit=polyfit(a, 1)," +
+ " predictions=predict(fit, a))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
paramsLoc.set("expr", cexpr);
paramsLoc.set("qt", "/stream");
@@ -2876,6 +2877,18 @@ public class MathExpressionTest extends SolrCloudTestCase {
assertTrue(out.get(5).intValue() == 5);
assertTrue(out.get(6).intValue() == 6);
assertTrue(out.get(7).intValue() == 7);
+
+
+ out = (List<Number>)tuples.get(0).get("predictions");
+ assertTrue(out.size() == 8);
+ assertTrue(out.get(0).intValue() == 0);
+ assertTrue(out.get(1).intValue() == 1);
+ assertTrue(out.get(2).intValue() == 2);
+ assertTrue(out.get(3).intValue() == 3);
+ assertTrue(out.get(4).intValue() == 4);
+ assertTrue(out.get(5).intValue() == 5);
+ assertTrue(out.get(6).intValue() == 6);
+ assertTrue(out.get(7).intValue() == 7);
}
@Test
[04/17] lucene-solr:jira/solr-12470: LUCENE-8439: Disjunction max
queries can skip blocks to select the top documents when the total hit count
is not required
Posted by ab...@apache.org.
LUCENE-8439: Disjunction max queries can skip blocks to select the top documents when the total hit count is not required
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ba9b18f3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ba9b18f3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ba9b18f3
Branch: refs/heads/jira/solr-12470
Commit: ba9b18f36743dc9674478dce5bbf2da509ef41c0
Parents: 38bf976
Author: Jim Ferenczi <ji...@apache.org>
Authored: Wed Aug 8 12:34:42 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed Aug 8 12:34:42 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../org/apache/lucene/search/BlockMaxDISI.java | 94 ++++++++++++++++++++
.../lucene/search/Boolean2ScorerSupplier.java | 2 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../lucene/search/DisjunctionMaxQuery.java | 2 +-
.../lucene/search/DisjunctionMaxScorer.java | 80 ++++++++++-------
.../apache/lucene/search/DisjunctionScorer.java | 22 ++++-
.../lucene/search/DisjunctionSumScorer.java | 4 +-
.../lucene/search/TestDisjunctionMaxQuery.java | 77 +++++++++++++++-
9 files changed, 244 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 865f5e2..f0360a9 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -134,6 +134,9 @@ Optimizations
or phrase queries as sub queries, which know how to leverage this information
to run faster. (Adrien Grand)
+* LUCENE-8439: Disjunction max queries can skip blocks to select the top documents
+ when the total hit count is not required. (Jim Ferenczi, Adrien Grand)
+
======================= Lucene 7.5.0 =======================
API Changes:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java b/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java
new file mode 100644
index 0000000..804f96f
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/BlockMaxDISI.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.search;
+
+import java.io.IOException;
+
+/**
+ * {@link DocIdSetIterator} that skips non-competitive docs by checking
+ * the max score of the provided {@link Scorer} for the current block.
+ * Call {@link #setMinCompetitiveScore(float)} in order to give this iterator the ability
+ * to skip low-scoring documents.
+ * @lucene.internal
+ */
+public class BlockMaxDISI extends DocIdSetIterator {
+ protected final Scorer scorer;
+ private final DocIdSetIterator in;
+ private float minScore;
+ private float maxScore;
+ private int upTo = -1;
+
+ public BlockMaxDISI(DocIdSetIterator iterator, Scorer scorer) {
+ this.in = iterator;
+ this.scorer = scorer;
+ }
+
+ @Override
+ public int docID() {
+ return in.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return advance(docID()+1);
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ int doc = advanceImpacts(target);
+ return in.advance(doc);
+ }
+
+ @Override
+ public long cost() {
+ return in.cost();
+ }
+
+ public void setMinCompetitiveScore(float minScore) {
+ this.minScore = minScore;
+ }
+
+ private void moveToNextBlock(int target) throws IOException {
+ upTo = scorer.advanceShallow(target);
+ maxScore = scorer.getMaxScore(upTo);
+ }
+
+ private int advanceImpacts(int target) throws IOException {
+ if (minScore == -1 || target == NO_MORE_DOCS) {
+ return target;
+ }
+
+ if (target > upTo) {
+ moveToNextBlock(target);
+ }
+
+ while (true) {
+ if (maxScore >= minScore) {
+ return target;
+ }
+
+ if (upTo == NO_MORE_DOCS) {
+ return NO_MORE_DOCS;
+ }
+
+ target = upTo + 1;
+
+ moveToNextBlock(target);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
index 3401215..bae7941 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
@@ -187,7 +187,7 @@ final class Boolean2ScorerSupplier extends ScorerSupplier {
} else if (scoreMode == ScoreMode.TOP_SCORES) {
return new WANDScorer(weight, optionalScorers);
} else {
- return new DisjunctionSumScorer(weight, optionalScorers, scoreMode.needsScores());
+ return new DisjunctionSumScorer(weight, optionalScorers, scoreMode);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
index 1fddd7c..5f20a6b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
@@ -309,7 +309,7 @@ final class BooleanWeight extends Weight {
} else {
Scorer prohibitedScorer = prohibited.size() == 1
? prohibited.get(0)
- : new DisjunctionSumScorer(this, prohibited, false);
+ : new DisjunctionSumScorer(this, prohibited, ScoreMode.COMPLETE_NO_SCORES);
if (prohibitedScorer.twoPhaseIterator() != null) {
// ReqExclBulkScorer can't deal efficiently with two-phased prohibited clauses
return null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
index 5b99c3c..3cdc0bc 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
@@ -148,7 +148,7 @@ public final class DisjunctionMaxQuery extends Query implements Iterable<Query>
// only one sub-scorer in this segment
return scorers.get(0);
} else {
- return new DisjunctionMaxScorer(this, tieBreakerMultiplier, scorers, scoreMode.needsScores());
+ return new DisjunctionMaxScorer(this, tieBreakerMultiplier, scorers, scoreMode);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
index 0434e0f..dd04d4c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
@@ -21,6 +21,8 @@ import java.util.List;
import org.apache.lucene.util.MathUtil;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/**
* The Scorer for DisjunctionMaxQuery. The union of all documents generated by the the subquery scorers
* is generated in document number order. The score for each document is the maximum of the scores computed
@@ -28,9 +30,9 @@ import org.apache.lucene.util.MathUtil;
* for the other subqueries that generate the document.
*/
final class DisjunctionMaxScorer extends DisjunctionScorer {
+ private final List<Scorer> subScorers;
/* Multiplier applied to non-maximum-scoring subqueries for a document as they are summed into the result. */
private final float tieBreakerMultiplier;
- private final float maxScore;
/**
* Creates a new instance of DisjunctionMaxScorer
@@ -43,40 +45,13 @@ final class DisjunctionMaxScorer extends DisjunctionScorer {
* @param subScorers
* The sub scorers this Scorer should iterate on
*/
- DisjunctionMaxScorer(Weight weight, float tieBreakerMultiplier, List<Scorer> subScorers, boolean needsScores) throws IOException {
- super(weight, subScorers, needsScores);
+ DisjunctionMaxScorer(Weight weight, float tieBreakerMultiplier, List<Scorer> subScorers, ScoreMode scoreMode) throws IOException {
+ super(weight, subScorers, scoreMode);
+ this.subScorers = subScorers;
this.tieBreakerMultiplier = tieBreakerMultiplier;
if (tieBreakerMultiplier < 0 || tieBreakerMultiplier > 1) {
throw new IllegalArgumentException("tieBreakerMultiplier must be in [0, 1]");
}
-
- if (needsScores == false) {
- this.maxScore = Float.MAX_VALUE;
- } else {
- float scoreMax = 0;
- double otherScoreSum = 0;
- for (Scorer scorer : subScorers) {
- scorer.advanceShallow(0);
- float subScore = scorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
- if (subScore >= scoreMax) {
- otherScoreSum += scoreMax;
- scoreMax = subScore;
- } else {
- otherScoreSum += subScore;
- }
- }
-
- if (tieBreakerMultiplier == 0) {
- this.maxScore = scoreMax;
- } else {
- // The error of sums depends on the order in which values are summed up. In
- // order to avoid this issue, we compute an upper bound of the value that
- // the sum may take. If the max relative error is b, then it means that two
- // sums are always within 2*b of each other.
- otherScoreSum *= (1 + 2 * MathUtil.sumRelativeErrorBound(subScorers.size() - 1));
- this.maxScore = (float) (scoreMax + otherScoreSum * tieBreakerMultiplier);
- }
- }
}
@Override
@@ -96,7 +71,48 @@ final class DisjunctionMaxScorer extends DisjunctionScorer {
}
@Override
+ public int advanceShallow(int target) throws IOException {
+ int upTo = NO_MORE_DOCS;
+ for (Scorer scorer : subScorers) {
+ if (scorer.docID() <= target) {
+ upTo = Math.min(scorer.advanceShallow(target), upTo);
+ } else if (scorer.docID() < NO_MORE_DOCS) {
+ upTo = Math.min(scorer.docID()-1, upTo);
+ }
+ }
+ return upTo;
+ }
+
+ @Override
public float getMaxScore(int upTo) throws IOException {
- return maxScore;
+ float scoreMax = 0;
+ double otherScoreSum = 0;
+ for (Scorer scorer : subScorers) {
+ if (scorer.docID() <= upTo) {
+ float subScore = scorer.getMaxScore(upTo);
+ if (subScore >= scoreMax) {
+ otherScoreSum += scoreMax;
+ scoreMax = subScore;
+ } else {
+ otherScoreSum += subScore;
+ }
+ }
+ }
+
+ if (tieBreakerMultiplier == 0) {
+ return scoreMax;
+ } else {
+ // The error of sums depends on the order in which values are summed up. In
+ // order to avoid this issue, we compute an upper bound of the value that
+ // the sum may take. If the max relative error is b, then it means that two
+ // sums are always within 2*b of each other.
+ otherScoreSum *= (1 + 2 * MathUtil.sumRelativeErrorBound(subScorers.size() - 1));
+ return (float) (scoreMax + otherScoreSum * tieBreakerMultiplier);
+ }
+ }
+
+ @Override
+ public void setMinCompetitiveScore(float minScore) {
+ getBlockMaxApprox().setMinCompetitiveScore(minScore);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
index 147b993..249521e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java
@@ -32,10 +32,11 @@ abstract class DisjunctionScorer extends Scorer {
private final boolean needsScores;
private final DisiPriorityQueue subScorers;
- private final DisjunctionDISIApproximation approximation;
+ private final DocIdSetIterator approximation;
+ private final BlockMaxDISI blockMaxApprox;
private final TwoPhase twoPhase;
- protected DisjunctionScorer(Weight weight, List<Scorer> subScorers, boolean needsScores) {
+ protected DisjunctionScorer(Weight weight, List<Scorer> subScorers, ScoreMode scoreMode) throws IOException {
super(weight);
if (subScorers.size() <= 1) {
throw new IllegalArgumentException("There must be at least 2 subScorers");
@@ -45,8 +46,17 @@ abstract class DisjunctionScorer extends Scorer {
final DisiWrapper w = new DisiWrapper(scorer);
this.subScorers.add(w);
}
- this.needsScores = needsScores;
- this.approximation = new DisjunctionDISIApproximation(this.subScorers);
+ this.needsScores = scoreMode != ScoreMode.COMPLETE_NO_SCORES;
+ if (scoreMode == ScoreMode.TOP_SCORES) {
+ for (Scorer scorer : subScorers) {
+ scorer.advanceShallow(0);
+ }
+ this.blockMaxApprox = new BlockMaxDISI(new DisjunctionDISIApproximation(this.subScorers), this);
+ this.approximation = blockMaxApprox;
+ } else {
+ this.approximation = new DisjunctionDISIApproximation(this.subScorers);
+ this.blockMaxApprox = null;
+ }
boolean hasApproximation = false;
float sumMatchCost = 0;
@@ -167,6 +177,10 @@ abstract class DisjunctionScorer extends Scorer {
return subScorers.top().doc;
}
+ BlockMaxDISI getBlockMaxApprox() {
+ return blockMaxApprox;
+ }
+
DisiWrapper getSubMatches() throws IOException {
if (twoPhase == null) {
return subScorers.topList();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
index 06e8c4a..18835ed 100644
--- a/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
@@ -28,8 +28,8 @@ final class DisjunctionSumScorer extends DisjunctionScorer {
* @param weight The weight to be used.
* @param subScorers Array of at least two subscorers.
*/
- DisjunctionSumScorer(Weight weight, List<Scorer> subScorers, boolean needsScores) throws IOException {
- super(weight, subScorers, needsScores);
+ DisjunctionSumScorer(Weight weight, List<Scorer> subScorers, ScoreMode scoreMode) throws IOException {
+ super(weight, subScorers, scoreMode);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ba9b18f3/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
index 2d72fc1..7fa1ed3 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
@@ -16,18 +16,22 @@
*/
package org.apache.lucene.search;
-
import java.io.IOException;
+import java.io.StringReader;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -522,6 +526,77 @@ public class TestDisjunctionMaxQuery extends LuceneTestCase {
assertEquals(bq.clauses().get(0), new BooleanClause(sub1, BooleanClause.Occur.SHOULD));
assertEquals(bq.clauses().get(1), new BooleanClause(sub2, BooleanClause.Occur.SHOULD));
}
+
+ public void testRandomTopDocs() throws Exception {
+ doTestRandomTopDocs(2, 0.05f, 0.05f);
+ doTestRandomTopDocs(2, 1.0f, 0.05f);
+ doTestRandomTopDocs(3, 1.0f, 0.5f, 0.05f);
+ doTestRandomTopDocs(4, 1.0f, 0.5f, 0.05f, 0f);
+ doTestRandomTopDocs(4, 1.0f, 0.5f, 0.05f, 0f);
+ }
+
+ private void doTestRandomTopDocs(int numFields, double... freqs) throws IOException {
+ assert numFields == freqs.length;
+ Directory dir = newDirectory();
+ IndexWriterConfig config = new IndexWriterConfig(new StandardAnalyzer());
+ IndexWriter w = new IndexWriter(dir, config);
+
+ int numDocs = atLeast(1000); // make sure some terms have skip data
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ for (int j = 0; j < numFields; j++) {
+ StringBuilder builder = new StringBuilder();
+ int numAs = random().nextDouble() < freqs[j] ? 0 : 1 + random().nextInt(5);
+ for (int k = 0; k < numAs; k++) {
+ if (builder.length() > 0) {
+ builder.append(' ');
+ }
+ builder.append('a');
+ }
+ if (random().nextBoolean()) {
+ doc.add(new StringField("field", "c", Field.Store.NO));
+ }
+ int numOthers = random().nextBoolean() ? 0 : 1 + random().nextInt(5);
+ for (int k = 0; k < numOthers; k++) {
+ if (builder.length() > 0) {
+ builder.append(' ');
+ }
+ builder.append(Integer.toString(random().nextInt()));
+ }
+ doc.add(new TextField(Integer.toString(j), new StringReader(builder.toString())));
+ }
+ w.addDocument(doc);
+ }
+ IndexReader reader = DirectoryReader.open(w);
+ w.close();
+ IndexSearcher searcher = newSearcher(reader);
+ for (int i = 0; i < 4; i++) {
+ List<Query> clauses = new ArrayList<>();
+ for (int j = 0; j < numFields; j++) {
+ if (i % 2 == 1) {
+ clauses.add(tq(Integer.toString(j), "a"));
+ } else {
+ float boost = random().nextBoolean() ? 0 : random().nextFloat();
+ if (boost > 0) {
+ clauses.add(tq(Integer.toString(j), "a", boost));
+ } else {
+ clauses.add(tq(Integer.toString(j), "a"));
+ }
+ }
+ }
+ float tieBreaker = random().nextFloat();
+ Query query = new DisjunctionMaxQuery(clauses, tieBreaker);
+ CheckHits.checkTopScores(random(), query, searcher);
+
+ query = new BooleanQuery.Builder()
+ .add(new DisjunctionMaxQuery(clauses, tieBreaker), BooleanClause.Occur.MUST)
+ .add(tq("field", "c"), BooleanClause.Occur.FILTER)
+ .build();
+ CheckHits.checkTopScores(random(), query, searcher);
+ }
+ reader.close();
+ dir.close();
+ }
/** macro */
protected Query tq(String f, String t) {
[15/17] lucene-solr:jira/solr-12470: SOLR-12636: escape arrow glyph
to fix precommit
Posted by ab...@apache.org.
SOLR-12636: escape arrow glyph to fix precommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6c11284e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6c11284e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6c11284e
Branch: refs/heads/jira/solr-12470
Commit: 6c11284e7c46bac110b1915719907d647d490091
Parents: cb1db48
Author: Steve Rowe <sa...@apache.org>
Authored: Thu Aug 9 09:56:17 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Thu Aug 9 09:56:17 2018 -0400
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-policy-preferences.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c11284e/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
index 82af94c..8714b38 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
@@ -246,7 +246,7 @@ This supports values calculated at the time of execution.
** `freedisk`
* `#ALL` : This is applied to the `replica` attribute only. This means all replicas that meet the rule condition.
* `#EQUAL`: This is applied to the `replica` and `cores` attributes only. This means equal number of replicas/cores in each bucket. The buckets can be defined using an array operator (`[]`) or `#EACH`. The buckets can be defined using the following properties:
-** `node` <- <<Rule Types,global rules>>, i.e. with the `cores` attribute, may only specify this attribute
+** `node` \<- <<Rule Types,global rules>>, i.e. with the `cores` attribute, may only specify this attribute
** `sysprop.*`
** `port`
** `diskType`
[14/17] lucene-solr:jira/solr-12470: SOLR-11863: Add knnRegress
Stream Evaluator to support nearest neighbor regression
Posted by ab...@apache.org.
SOLR-11863: Add knnRegress Stream Evaluator to support nearest neighbor regression
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cb1db482
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cb1db482
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cb1db482
Branch: refs/heads/jira/solr-12470
Commit: cb1db482523cf33b7927b5155d506202d8ddbd89
Parents: e9f3a3c
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Aug 8 21:05:02 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Aug 8 21:05:21 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/Lang.java | 1 +
.../solr/client/solrj/io/eval/KnnEvaluator.java | 18 +-
.../solrj/io/eval/KnnRegressionEvaluator.java | 194 +++++++++++++++++++
.../solrj/io/eval/MinMaxScaleEvaluator.java | 2 +-
.../client/solrj/io/eval/PredictEvaluator.java | 30 ++-
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 71 +++++++
7 files changed, 311 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index a01a841..6f170c4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -246,6 +246,7 @@ public class Lang {
.withFunctionName("zeros", ZerosEvaluator.class)
.withFunctionName("getValue", GetValueEvaluator.class)
.withFunctionName("setValue", SetValueEvaluator.class)
+ .withFunctionName("knnRegress", KnnRegressionEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
index 251e092..81607cf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnEvaluator.java
@@ -67,8 +67,6 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
throw new IOException("The third parameter for knn should be k.");
}
- double[][] data = matrix.getData();
-
DistanceMeasure distanceMeasure = null;
if(values.length == 4) {
@@ -77,6 +75,15 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
distanceMeasure = new EuclideanDistance();
}
+ return search(matrix, vec, k, distanceMeasure);
+ }
+
+ public static Matrix search(Matrix observations,
+ double[] vec,
+ int k,
+ DistanceMeasure distanceMeasure) {
+
+ double[][] data = observations.getData();
TreeSet<Neighbor> neighbors = new TreeSet();
for(int i=0; i<data.length; i++) {
double distance = distanceMeasure.compute(vec, data[i]);
@@ -87,8 +94,9 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
}
double[][] out = new double[neighbors.size()][];
- List<String> rowLabels = matrix.getRowLabels();
+ List<String> rowLabels = observations.getRowLabels();
List<String> newRowLabels = new ArrayList();
+ List<Number> indexes = new ArrayList();
List<Number> distances = new ArrayList();
int i=-1;
@@ -102,6 +110,7 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
out[++i] = data[rowIndex];
distances.add(neighbor.getDistance());
+ indexes.add(rowIndex);
}
Matrix knn = new Matrix(out);
@@ -110,8 +119,9 @@ public class KnnEvaluator extends RecursiveObjectEvaluator implements ManyValueW
knn.setRowLabels(newRowLabels);
}
- knn.setColumnLabels(matrix.getColumnLabels());
+ knn.setColumnLabels(observations.getColumnLabels());
knn.setAttribute("distances", distances);
+ knn.setAttribute("indexes", indexes);
return knn;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
new file mode 100644
index 0000000..957936e
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.commons.math3.linear.Array2DRowRealMatrix;
+import org.apache.commons.math3.ml.distance.DistanceMeasure;
+import org.apache.commons.math3.ml.distance.EuclideanDistance;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class KnnRegressionEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public KnnRegressionEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object ... values) throws IOException {
+
+ if(values.length < 3) {
+ throw new IOException("knnRegress expects atleast three parameters: an observation matrix, an outcomes vector and k.");
+ }
+
+ Matrix observations = null;
+ List<Number> outcomes = null;
+ int k = 5;
+ DistanceMeasure distanceMeasure = new EuclideanDistance();
+
+ if(values[0] instanceof Matrix) {
+ observations = (Matrix)values[0];
+ } else {
+ throw new IOException("The first parameter for knnRegress should be the observation matrix.");
+ }
+
+ if(values[1] instanceof List) {
+ outcomes = (List) values[1];
+ } else {
+ throw new IOException("The second parameter for knnRegress should be outcome array. ");
+ }
+
+ if(values[2] instanceof Number) {
+ k = ((Number) values[2]).intValue();
+ } else {
+ throw new IOException("The third parameter for knnRegress should be k. ");
+ }
+
+ if(values.length == 4) {
+ if(values[3] instanceof DistanceMeasure) {
+ distanceMeasure = (DistanceMeasure) values[3];
+ throw new IOException("The fourth parameter for knnRegress should be a distance measure. ");
+ }
+ }
+
+ double[] outcomeData = new double[outcomes.size()];
+ for(int i=0; i<outcomeData.length; i++) {
+ outcomeData[i] = outcomes.get(i).doubleValue();
+ }
+
+ Map map = new HashMap();
+ map.put("k", k);
+ map.put("observations", observations.getRowCount());
+ map.put("features", observations.getColumnCount());
+ map.put("distance", distanceMeasure.getClass().getSimpleName());
+
+ return new KnnRegressionTuple(observations, outcomeData, k, distanceMeasure, map);
+ }
+
+
+ public static class KnnRegressionTuple extends Tuple {
+
+ private Matrix observations;
+ private Matrix scaledObservations;
+ private double[] outcomes;
+ private int k;
+ private DistanceMeasure distanceMeasure;
+
+ public KnnRegressionTuple(Matrix observations,
+ double[] outcomes,
+ int k,
+ DistanceMeasure distanceMeasure,
+ Map<?,?> map) {
+ super(map);
+ this.observations = observations;
+ this.outcomes = outcomes;
+ this.k = k;
+ this.distanceMeasure = distanceMeasure;
+ }
+
+ //MinMax Scale both the observations and the predictors
+
+ public double[] scale(double[] predictors) {
+ double[][] data = observations.getData();
+ //We need to scale the columns of the data matrix with along with the predictors
+ Array2DRowRealMatrix matrix = new Array2DRowRealMatrix(data);
+ Array2DRowRealMatrix transposed = (Array2DRowRealMatrix) matrix.transpose();
+ double[][] featureRows = transposed.getDataRef();
+
+ double[] scaledPredictors = new double[predictors.length];
+
+ for(int i=0; i<featureRows.length; i++) {
+ double[] featureRow = featureRows[i];
+ double[] combinedFeatureRow = new double[featureRow.length+1];
+ System.arraycopy(featureRow, 0, combinedFeatureRow, 0, featureRow.length);
+ combinedFeatureRow[featureRow.length] = predictors[i]; // Add the last feature from the predictor
+ double[] scaledFeatures = MinMaxScaleEvaluator.scale(combinedFeatureRow, 0, 1);
+ scaledPredictors[i] = scaledFeatures[featureRow.length];
+ System.arraycopy(scaledFeatures, 0, featureRow, 0, featureRow.length);
+ }
+
+ Array2DRowRealMatrix scaledFeatureMatrix = new Array2DRowRealMatrix(featureRows);
+
+
+ Array2DRowRealMatrix scaledObservationsMatrix= (Array2DRowRealMatrix)scaledFeatureMatrix.transpose();
+ this.scaledObservations = new Matrix(scaledObservationsMatrix.getDataRef());
+ return scaledPredictors;
+ }
+
+
+ public Matrix scale(Matrix predictors) {
+ double[][] observationData = observations.getData();
+ //We need to scale the columns of the data matrix with along with the predictors
+ Array2DRowRealMatrix observationMatrix = new Array2DRowRealMatrix(observationData);
+ Array2DRowRealMatrix observationTransposed = (Array2DRowRealMatrix) observationMatrix.transpose();
+ double[][] observationFeatureRows = observationTransposed.getDataRef();
+
+ double[][] predictorsData = predictors.getData();
+ //We need to scale the columns of the data matrix with along with the predictors
+ Array2DRowRealMatrix predictorMatrix = new Array2DRowRealMatrix(predictorsData);
+ Array2DRowRealMatrix predictorTransposed = (Array2DRowRealMatrix) predictorMatrix.transpose();
+ double[][] predictorFeatureRows = predictorTransposed.getDataRef();
+
+ for(int i=0; i<observationFeatureRows.length; i++) {
+ double[] observationFeatureRow = observationFeatureRows[i];
+ double[] predictorFeatureRow = predictorFeatureRows[i];
+ double[] combinedFeatureRow = new double[observationFeatureRow.length+predictorFeatureRow.length];
+ System.arraycopy(observationFeatureRow, 0, combinedFeatureRow, 0, observationFeatureRow.length);
+ System.arraycopy(predictorFeatureRow, 0, combinedFeatureRow, observationFeatureRow.length, predictorFeatureRow.length);
+
+ double[] scaledFeatures = MinMaxScaleEvaluator.scale(combinedFeatureRow, 0, 1);
+ System.arraycopy(scaledFeatures, 0, observationFeatureRow, 0, observationFeatureRow.length);
+ System.arraycopy(scaledFeatures, observationFeatureRow.length, predictorFeatureRow, 0, predictorFeatureRow.length);
+ }
+
+ Array2DRowRealMatrix scaledFeatureMatrix = new Array2DRowRealMatrix(observationFeatureRows);
+ Array2DRowRealMatrix scaledObservationsMatrix= (Array2DRowRealMatrix)scaledFeatureMatrix.transpose();
+ this.scaledObservations = new Matrix(scaledObservationsMatrix.getDataRef());
+
+ Array2DRowRealMatrix scaledPredictorMatrix = new Array2DRowRealMatrix(predictorFeatureRows);
+ Array2DRowRealMatrix scaledTransposedPredictorMatrix= (Array2DRowRealMatrix)scaledPredictorMatrix.transpose();
+ return new Matrix(scaledTransposedPredictorMatrix.getDataRef());
+ }
+
+
+ public double predict(double[] values) {
+
+ Matrix knn = KnnEvaluator.search(scaledObservations, values, k, distanceMeasure);
+ List<Number> indexes = (List<Number>)knn.getAttribute("indexes");
+
+ double sum = 0;
+
+ //Collect the outcomes for the nearest neighbors
+ for(Number n : indexes) {
+ sum += outcomes[n.intValue()];
+ }
+
+ //Return the average of the outcomes as the prediction.
+
+ return sum/((double)indexes.size());
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
index 60c6377..3996910 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MinMaxScaleEvaluator.java
@@ -76,7 +76,7 @@ public class MinMaxScaleEvaluator extends RecursiveObjectEvaluator implements Ma
}
}
- private double[] scale(double[] values, double min, double max) {
+ public static double[] scale(double[] values, double min, double max) {
double localMin = Double.MAX_VALUE;
double localMax = Double.MIN_VALUE;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
index 2444370..9385928 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/PredictEvaluator.java
@@ -43,7 +43,11 @@ public class PredictEvaluator extends RecursiveObjectEvaluator implements ManyVa
Object first = objects[0];
Object second = objects[1];
- if (!(first instanceof BivariateFunction) && !(first instanceof VectorFunction) && !(first instanceof RegressionEvaluator.RegressionTuple) && !(first instanceof OLSRegressionEvaluator.MultipleRegressionTuple)) {
+ if (!(first instanceof BivariateFunction) &&
+ !(first instanceof VectorFunction) &&
+ !(first instanceof RegressionEvaluator.RegressionTuple) &&
+ !(first instanceof OLSRegressionEvaluator.MultipleRegressionTuple) &&
+ !(first instanceof KnnRegressionEvaluator.KnnRegressionTuple)) {
throw new IOException(String.format(Locale.ROOT, "Invalid expression %s - found type %s for the first value, expecting a RegressionTuple", toExpression(constructingFactory), first.getClass().getSimpleName()));
}
@@ -83,6 +87,30 @@ public class PredictEvaluator extends RecursiveObjectEvaluator implements ManyVa
return predictions;
}
+ } else if (first instanceof KnnRegressionEvaluator.KnnRegressionTuple) {
+ KnnRegressionEvaluator.KnnRegressionTuple regressedTuple = (KnnRegressionEvaluator.KnnRegressionTuple) first;
+ if (second instanceof List) {
+ List<Number> list = (List<Number>) second;
+ double[] predictors = new double[list.size()];
+
+ for (int i = 0; i < list.size(); i++) {
+ predictors[i] = list.get(i).doubleValue();
+ }
+
+ predictors = regressedTuple.scale(predictors);
+
+ return regressedTuple.predict(predictors);
+ } else if (second instanceof Matrix) {
+
+ Matrix m = (Matrix) second;
+ m = regressedTuple.scale(m);
+ double[][] data = m.getData();
+ List<Number> predictions = new ArrayList();
+ for (double[] predictors : data) {
+ predictions.add(regressedTuple.predict(predictors));
+ }
+ return predictions;
+ }
} else if (first instanceof VectorFunction) {
VectorFunction vectorFunction = (VectorFunction) first;
UnivariateFunction univariateFunction = (UnivariateFunction)vectorFunction.getFunction();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 22b432f..df56844 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -69,7 +69,7 @@ public class TestLang extends LuceneTestCase {
TemporalEvaluatorDayOfQuarter.FUNCTION_NAME, "abs", "add", "div", "mult", "sub", "log", "pow",
"mod", "ceil", "floor", "sin", "asin", "sinh", "cos", "acos", "cosh", "tan", "atan", "tanh", "round", "sqrt",
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft", "euclidean","manhattan",
- "earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue"};
+ "earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue", "knnRegress"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cb1db482/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 98a52a6..a9be57e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -3395,6 +3395,77 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
@Test
+ public void testKnnRegress() throws Exception {
+ String cexpr = "let(echo=true, a=array(8.5, 12.89999962, 5.199999809, 10.69999981, 3.099999905, 3.5, 9.199999809, 9, 15.10000038, 10.19999981), " +
+ "b=array(5.099999905, 5.800000191, 2.099999905, 8.399998665, 2.900000095, 1.200000048, 3.700000048, 7.599999905, 7.699999809, 4.5)," +
+ "c=array(4.699999809, 8.800000191, 15.10000038, 12.19999981, 10.60000038, 3.5, 9.699999809, 5.900000095, 20.79999924, 7.900000095)," +
+ "d=array(85.09999847, 106.3000031, 50.20000076, 130.6000061, 54.79999924, 30.29999924, 79.40000153, 91, 135.3999939, 89.30000305)," +
+ "e=transpose(matrix(a, b, c))," +
+ "f=knnRegress(e, d, 1)," +
+ "g=predict(f, e))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<Number> predictions = (List<Number>)tuples.get(0).get("g");
+ assertEquals(predictions.size(), 10);
+ //k=1 should bring back only one prediction for the exact match in the training set
+ assertEquals(predictions.get(0).doubleValue(), 85.09999847, 0);
+ assertEquals(predictions.get(1).doubleValue(), 106.3000031, 0);
+ assertEquals(predictions.get(2).doubleValue(), 50.20000076, 0);
+ assertEquals(predictions.get(3).doubleValue(), 130.6000061, 0);
+ assertEquals(predictions.get(4).doubleValue(), 54.79999924, 0);
+ assertEquals(predictions.get(5).doubleValue(), 30.29999924, 0);
+ assertEquals(predictions.get(6).doubleValue(), 79.40000153, 0);
+ assertEquals(predictions.get(7).doubleValue(), 91, 0);
+ assertEquals(predictions.get(8).doubleValue(), 135.3999939, 0);
+ assertEquals(predictions.get(9).doubleValue(), 89.30000305, 0);
+
+ cexpr = "let(echo=true, a=array(8.5, 12.89999962, 5.199999809, 10.69999981, 3.099999905, 3.5, 9.199999809, 9, 15.10000038, 10.19999981), " +
+ "b=array(5.099999905, 5.800000191, 2.099999905, 8.399998665, 2.900000095, 1.200000048, 3.700000048, 7.599999905, 7.699999809, 4.5)," +
+ "c=array(4.699999809, 8.800000191, 15.10000038, 12.19999981, 10.60000038, 3.5, 9.699999809, 5.900000095, 20.79999924, 7.900000095)," +
+ "d=array(85.09999847, 106.3000031, 50.20000076, 130.6000061, 54.79999924, 30.29999924, 79.40000153, 91, 135.3999939, 89.30000305)," +
+ "e=transpose(matrix(a, b, c))," +
+ "f=knnRegress(e, d, 1)," +
+ "g=predict(f, array(8, 5, 4)))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Number prediction = (Number)tuples.get(0).get("g");
+ assertEquals(prediction.doubleValue(), 85.09999847, 0);
+
+ cexpr = "let(echo=true, a=array(8.5, 12.89999962, 5.199999809, 10.69999981, 3.099999905, 3.5, 9.199999809, 9, 15.10000038, 8.19999981), " +
+ "b=array(5.099999905, 5.800000191, 2.099999905, 8.399998665, 2.900000095, 1.200000048, 3.700000048, 7.599999905, 5.699999809, 4.5)," +
+ "c=array(4.699999809, 8.800000191, 15.10000038, 12.19999981, 10.60000038, 3.5, 9.699999809, 5.900000095, 20.79999924, 4.900000095)," +
+ "d=array(85.09999847, 106.3000031, 50.20000076, 130.6000061, 54.79999924, 30.29999924, 79.40000153, 91, 135.3999939, 89.30000305)," +
+ "e=transpose(matrix(a, b, c))," +
+ "f=knnRegress(e, d, 2)," +
+ "g=predict(f, array(8, 5, 4)))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ solrStream = new SolrStream(url, paramsLoc);
+ context = new StreamContext();
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ prediction = (Number)tuples.get(0).get("g");
+ assertEquals(prediction.doubleValue(), 87.20000076, 0);
+ }
+
+ @Test
public void testPlot() throws Exception {
String cexpr = "let(a=array(3,2,3), plot(type=scatter, x=a, y=array(5,6,3)))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
[05/17] lucene-solr:jira/solr-12470: SOLR-7767: "ZK Status" sub menu
under "Cloud" tab to see status of zookeeper ensemble
Posted by ab...@apache.org.
SOLR-7767: "ZK Status" sub menu under "Cloud" tab to see status of zookeeper ensemble
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9306922d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9306922d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9306922d
Branch: refs/heads/jira/solr-12470
Commit: 9306922d65a3995d5816ffd61ab59fedb5202e06
Parents: ba9b18f
Author: Jan Høydahl <ja...@apache.org>
Authored: Wed Aug 8 12:43:19 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Wed Aug 8 12:43:19 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/core/CoreContainer.java | 3 +
.../handler/admin/ZookeeperStatusHandler.java | 222 +++++++++++++++++++
.../admin/ZookeeperStatusHandlerTest.java | 85 +++++++
solr/solr-ref-guide/src/cloud-screens.adoc | 5 +
.../src/images/cloud-screens/cloud-zkstatus.png | Bin 0 -> 175359 bytes
.../apache/solr/common/params/CommonParams.java | 1 +
solr/webapp/web/css/angular/cloud.css | 71 +++++-
solr/webapp/web/css/angular/menu.css | 1 +
solr/webapp/web/index.html | 1 +
solr/webapp/web/js/angular/controllers/cloud.js | 38 +++-
solr/webapp/web/js/angular/services.js | 6 +
solr/webapp/web/partials/cloud.html | 59 +++++
13 files changed, 491 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3ede98b..7468451 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -140,6 +140,8 @@ New Features
* SOLR-8207: Add "Nodes" view to the Admin UI "Cloud" tab, listing nodes and key metrics (janhoy)
+* SOLR-7767: "ZK Status" sub menu under "Cloud" tab to see status of zookeeper ensemble (janhoy)
+
* SOLR-11990: Make it possible to co-locate replicas of multiple collections together in a node. A collection may be
co-located with another collection during collection creation time by specifying a 'withCollection' parameter. It can
also be co-located afterwards by using the modify collection API. The co-location guarantee is enforced regardless of
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/core/src/java/org/apache/solr/core/CoreContainer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 0583101..83384fb 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -85,6 +85,7 @@ import org.apache.solr.handler.admin.SecurityConfHandler;
import org.apache.solr.handler.admin.SecurityConfHandlerLocal;
import org.apache.solr.handler.admin.SecurityConfHandlerZk;
import org.apache.solr.handler.admin.ZookeeperInfoHandler;
+import org.apache.solr.handler.admin.ZookeeperStatusHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.logging.LogWatcher;
import org.apache.solr.logging.MDCLoggingContext;
@@ -120,6 +121,7 @@ import static org.apache.solr.common.params.CommonParams.INFO_HANDLER_PATH;
import static org.apache.solr.common.params.CommonParams.METRICS_HISTORY_PATH;
import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
import static org.apache.solr.common.params.CommonParams.ZK_PATH;
+import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
import static org.apache.solr.core.CorePropertiesLocator.PROPERTIES_FILENAME;
import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
@@ -564,6 +566,7 @@ public class CoreContainer {
this.backupRepoFactory = new BackupRepositoryFactory(cfg.getBackupRepositoryPlugins());
createHandler(ZK_PATH, ZookeeperInfoHandler.class.getName(), ZookeeperInfoHandler.class);
+ createHandler(ZK_STATUS_PATH, ZookeeperStatusHandler.class.getName(), ZookeeperStatusHandler.class);
collectionsHandler = createHandler(COLLECTIONS_HANDLER_PATH, cfg.getCollectionsHandlerClass(), CollectionsHandler.class);
healthCheckHandler = createHandler(HEALTH_CHECK_HANDLER_PATH, cfg.getHealthCheckHandlerClass(), HealthCheckHandler.class);
infoHandler = createHandler(INFO_HANDLER_PATH, cfg.getInfoHandlerClass(), InfoHandler.class);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
new file mode 100644
index 0000000..8842437
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ZookeeperStatusHandler.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.lang.invoke.MethodHandles;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Zookeeper Status handler, talks to ZK using sockets and four-letter words
+ *
+ * @since solr 7.5
+ */
+public final class ZookeeperStatusHandler extends RequestHandlerBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final int ZOOKEEPER_DEFAULT_PORT = 2181;
+ private static final String STATUS_RED = "red";
+ private static final String STATUS_GREEN = "green";
+ private static final String STATUS_YELLOW = "yellow";
+ private static final String STATUS_NA = "N/A";
+ private CoreContainer cores;
+
+ public ZookeeperStatusHandler(CoreContainer cc) {
+ this.cores = cc;
+ }
+
+ @Override
+ public String getDescription() {
+ return "Fetch Zookeeper status";
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.ADMIN;
+ }
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ final SolrParams params = req.getParams();
+ Map<String, String> map = new HashMap<>(1);
+ NamedList values = rsp.getValues();
+ values.add("zkStatus", getZkStatus(cores.getZkController().getZkServerAddress()));
+ }
+
+ /*
+ Gets all info from ZK API and returns as a map
+ */
+ protected Map<String, Object> getZkStatus(String zkHost) {
+ Map<String, Object> zkStatus = new HashMap<>();
+ List<String> zookeepers = Arrays.asList(zkHost.split("/")[0].split(","));
+ List<Object> details = new ArrayList<>();
+ int numOk = 0;
+ String status = STATUS_NA;
+ int standalone = 0;
+ int followers = 0;
+ int reportedFollowers = 0;
+ int leaders = 0;
+ List<String> errors = new ArrayList<>();
+ for (String zk : zookeepers) {
+ try {
+ Map<String, Object> stat = monitorZookeeper(zk);
+ details.add(stat);
+ if ("true".equals(String.valueOf(stat.get("ok")))) {
+ numOk++;
+ }
+ String state = String.valueOf(stat.get("zk_server_state"));
+ if ("follower".equals(state)) {
+ followers++;
+ } else if ("leader".equals(state)) {
+ leaders++;
+ reportedFollowers = Integer.parseInt(String.valueOf(stat.get("zk_followers")));
+ } else if ("standalone".equals(state)) {
+ standalone++;
+ }
+ } catch (SolrException se) {
+ log.warn("Failed talking to zookeeper" + zk, se);
+ errors.add(se.getMessage());
+ Map<String, Object> stat = new HashMap<>();
+ stat.put("host", zk);
+ stat.put("ok", false);
+ details.add(stat);
+ }
+ }
+ zkStatus.put("ensembleSize", zookeepers.size());
+ zkStatus.put("zkHost", zkHost);
+ zkStatus.put("details", details);
+ if (followers+leaders > 0 && standalone > 0) {
+ status = STATUS_RED;
+ errors.add("The zk nodes do not agree on their mode, check details");
+ }
+ if (standalone > 1) {
+ status = STATUS_RED;
+ errors.add("Only one zk allowed in standalone mode");
+ }
+ if (leaders > 1) {
+ zkStatus.put("mode", "ensemble");
+ status = STATUS_RED;
+ errors.add("Only one leader allowed, got " + leaders);
+ }
+ if (followers > 0 && leaders == 0) {
+ zkStatus.put("mode", "ensemble");
+ status = STATUS_RED;
+ errors.add("We do not have a leader");
+ }
+ if (leaders > 0 && followers != reportedFollowers) {
+ zkStatus.put("mode", "ensemble");
+ status = STATUS_RED;
+ errors.add("Leader reports " + reportedFollowers + " followers, but we only found " + followers +
+ ". Please check zkHost configuration");
+ }
+ if (followers+leaders == 0 && standalone == 1) {
+ zkStatus.put("mode", "standalone");
+ }
+ if (followers+leaders > 0 && (zookeepers.size())%2 == 0) {
+ if (!STATUS_RED.equals(status)) {
+ status = STATUS_YELLOW;
+ }
+ errors.add("We have an even number of zookeepers which is not recommended");
+ }
+ if (followers+leaders > 0 && standalone == 0) {
+ zkStatus.put("mode", "ensemble");
+ }
+ if (status.equals(STATUS_NA)) {
+ if (numOk == zookeepers.size()) {
+ status = STATUS_GREEN;
+ } else if (numOk < zookeepers.size() && numOk > zookeepers.size() / 2) {
+ status = STATUS_YELLOW;
+ errors.add("Some zookeepers are down: " + numOk + "/" + zookeepers.size());
+ } else {
+ status = STATUS_RED;
+ errors.add("Mismatch in number of zookeeper nodes live. numOK=" + numOk + ", expected " + zookeepers.size());
+ }
+ }
+ zkStatus.put("status", status);
+ if (!errors.isEmpty()) {
+ zkStatus.put("errors", errors);
+ }
+ return zkStatus;
+ }
+
+ private Map<String, Object> monitorZookeeper(String zkHostPort) {
+ List<String> lines = getZkRawResponse(zkHostPort, "mntr");
+ Map<String, Object> obj = new HashMap<>();
+ obj.put("host", zkHostPort);
+ obj.put("ok", "imok".equals(getZkRawResponse(zkHostPort, "ruok").get(0)));
+ for (String line : lines) {
+ obj.put(line.split("\t")[0], line.split("\t")[1]);
+ }
+ lines = getZkRawResponse(zkHostPort, "conf");
+ for (String line : lines) {
+ obj.put(line.split("=")[0], line.split("=")[1]);
+ }
+ return obj;
+ }
+
+ /**
+ * Sends a four-letter-word command to one particular Zookeeper server and returns the response as list of strings
+ * @param zkHostPort the host:port for one zookeeper server to access
+ * @param fourLetterWordCommand the custom 4-letter command to send to Zookeeper
+ * @return a list of lines returned from Zookeeper
+ */
+ private List<String> getZkRawResponse(String zkHostPort, String fourLetterWordCommand) {
+ String[] hostPort = zkHostPort.split(":");
+ String host = hostPort[0];
+ int port = ZOOKEEPER_DEFAULT_PORT;
+ if (hostPort.length > 1) {
+ port = Integer.parseInt(hostPort[1]);
+ }
+ try (
+ Socket socket = new Socket(host, port);
+ Writer writer = new OutputStreamWriter(socket.getOutputStream(), "utf-8");
+ PrintWriter out = new PrintWriter(writer, true);
+ BufferedReader in = new BufferedReader(new InputStreamReader(socket.getInputStream(), "utf-8"));) {
+ out.println(fourLetterWordCommand);
+ List<String> response = in.lines().collect(Collectors.toList());
+ log.debug("Got response from ZK on host {} and port {}: {}", host, port, response);
+ if (response == null || response.isEmpty()) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Empty response from Zookeeper " + zkHostPort);
+ }
+ return response;
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Failed talking to Zookeeper " + zkHostPort, e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
new file mode 100644
index 0000000..7ec8bf2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.admin;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.response.DelegationTokenResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ZookeeperStatusHandlerTest extends SolrCloudTestCase {
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(1)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+
+ @After
+ @Override
+ public void tearDown() throws Exception {
+ super.tearDown();
+ }
+
+ /*
+ Test the monitoring endpoint, used in the Cloud => ZkStatus Admin UI screen
+ NOTE: We do not currently test with multiple zookeepers, but the only difference is that there are multiple "details" objects and mode is "ensemble"...
+ */
+ @Test
+ public void monitorZookeeper() throws IOException, SolrServerException, InterruptedException, ExecutionException, TimeoutException {
+ URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl();
+ HttpSolrClient solr = new HttpSolrClient.Builder(baseUrl.toString()).build();
+ GenericSolrRequest mntrReq = new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/zookeeper/status", new ModifiableSolrParams());
+ mntrReq.setResponseParser(new DelegationTokenResponse.JsonMapResponseParser());
+ NamedList<Object> nl = solr.httpUriRequest(mntrReq).future.get(1000, TimeUnit.MILLISECONDS);
+
+ assertEquals("zkStatus", nl.getName(1));
+ Map<String,Object> zkStatus = (Map<String,Object>) nl.get("zkStatus");
+ assertEquals("green", zkStatus.get("status"));
+ assertEquals("standalone", zkStatus.get("mode"));
+ assertEquals(1L, zkStatus.get("ensembleSize"));
+ List<Object> detailsList = (List<Object>)zkStatus.get("details");
+ assertEquals(1, detailsList.size());
+ Map<String,Object> details = (Map<String,Object>) detailsList.get(0);
+ assertEquals(true, details.get("ok"));
+ assertTrue(Integer.parseInt((String) details.get("zk_znode_count")) > 50);
+ solr.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/solr-ref-guide/src/cloud-screens.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/cloud-screens.adoc b/solr/solr-ref-guide/src/cloud-screens.adoc
index 34982ee..cc5a5eb 100644
--- a/solr/solr-ref-guide/src/cloud-screens.adoc
+++ b/solr/solr-ref-guide/src/cloud-screens.adoc
@@ -42,6 +42,11 @@ image::images/cloud-screens/cloud-tree.png[image,width=487,height=250]
As an aid to debugging, the data shown in the "Tree" view can be exported locally using the following command `bin/solr zk ls -r /`
+== ZK Status view
+The "ZK Status" view gives an overview over the Zookeepers used by Solr. It lists whether running in `standalone` or `ensemble` mode, shows how many zookeepers are configured, and then displays a table listing detailed monitoring status for each of the zookeepers, inlcuding who is the leader, configuration parameters and more.
+
+image::images/cloud-screens/cloud-zkstatus.png[image,width=512,height=509]
+
== Graph views
The "Graph" view shows a graph of each collection, the shards that make up those collections, and the addresses and type ("NRT", "TLOG" or "PULL") of each replica for each shard.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png b/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png
new file mode 100644
index 0000000..b1b7452
Binary files /dev/null and b/solr/solr-ref-guide/src/images/cloud-screens/cloud-zkstatus.png differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index 63a763c..798410d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -182,6 +182,7 @@ public interface CommonParams {
String AUTHZ_PATH = "/admin/authorization";
String AUTHC_PATH = "/admin/authentication";
String ZK_PATH = "/admin/zookeeper";
+ String ZK_STATUS_PATH = "/admin/zookeeper/status";
String SYSTEM_INFO_PATH = "/admin/info/system";
String METRICS_PATH = "/admin/metrics";
String METRICS_HISTORY_PATH = "/admin/metrics/history";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/css/angular/cloud.css
----------------------------------------------------------------------
diff --git a/solr/webapp/web/css/angular/cloud.css b/solr/webapp/web/css/angular/cloud.css
index c3d54a5..5c8ce45 100644
--- a/solr/webapp/web/css/angular/cloud.css
+++ b/solr/webapp/web/css/angular/cloud.css
@@ -536,7 +536,8 @@ limitations under the License.
}
/* Styling of reload and details buttons */
-#content #cloud #controls
+#content #cloud #controls,
+#content #cloud #frame #zk-status-content #zk-controls
{
display: block;
height: 30px;
@@ -641,4 +642,70 @@ limitations under the License.
border-radius: 4px;
background-color: rgba(0,0,0,.5);
-webkit-box-shadow: 0 0 1px rgba(255,255,255,.5);
-}
\ No newline at end of file
+}
+#content #cloud #zk-table td,
+#content #cloud #zk-table th
+{
+ border: 0px solid #ddd;
+ border-bottom: 0.50px solid #eee;
+ padding-right: 5px;
+ padding-left: 5px;
+}
+
+#content #cloud #zk-table th
+{
+ border-bottom: 1px solid #ddd;
+ border-top: 1px solid #ddd;
+ font-weight: bolder;
+ font-stretch: extra-expanded;
+ background: #F8F8F8;
+}
+
+#content #cloud #zk-table
+{
+ border-top: 1px solid #c0c0c0;
+ margin-top: 10px;
+ border-collapse: collapse;
+
+ font-weight: bold;
+}
+
+#content #cloud #zk-table #detail-divider
+{
+ background-color: #f8f8f8;
+ height: 10px;
+}
+
+.zookeeper-status
+{
+ font-size: large;
+}
+
+.zookeeper-errors
+{
+ background-color: lightpink;
+ padding: 10px;
+ border: 1px;
+ margin-top: 10px;
+ margin-bottom: 10px;
+}
+
+.zookeeper-errors li::before
+{
+ content: "- ";
+}
+
+.zkstatus-green
+{
+ color: darkgreen;
+}
+
+.zkstatus-yellow
+{
+ color: orange;
+}
+
+.zkstatus-red
+{
+ color: red;
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/css/angular/menu.css
----------------------------------------------------------------------
diff --git a/solr/webapp/web/css/angular/menu.css b/solr/webapp/web/css/angular/menu.css
index ba5e0b6..f4e04c1 100644
--- a/solr/webapp/web/css/angular/menu.css
+++ b/solr/webapp/web/css/angular/menu.css
@@ -261,6 +261,7 @@ limitations under the License.
#menu #cloud.global p a { background-image: url( ../../img/ico/network-cloud.png ); }
#menu #cloud.global .tree a { background-image: url( ../../img/ico/folder-tree.png ); }
#menu #cloud.global .nodes a { background-image: url( ../../img/solr-ico.png ); }
+#menu #cloud.global .zkstatus a { background-image: url( ../../img/ico/node-master.png ); }
#menu #cloud.global .graph a { background-image: url( ../../img/ico/molecule.png ); }
#menu #cloud.global .rgraph a { background-image: url( ../../img/ico/asterisk.png ); }
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/index.html
----------------------------------------------------------------------
diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html
index 256af89..0663805 100644
--- a/solr/webapp/web/index.html
+++ b/solr/webapp/web/index.html
@@ -152,6 +152,7 @@ limitations under the License.
<ul ng-show="showingCloud">
<li class="nodes" ng-class="{active:page=='cloud-nodes'}"><a href="#/~cloud?view=nodes">Nodes</a></li>
<li class="tree" ng-class="{active:page=='cloud-tree'}"><a href="#/~cloud?view=tree">Tree</a></li>
+ <li class="zkstatus" ng-class="{active:page=='cloud-zkstatus'}"><a href="#/~cloud?view=zkstatus">ZK Status</a></li>
<li class="graph" ng-class="{active:page=='cloud-graph'}"><a href="#/~cloud?view=graph">Graph</a></li>
<li class="rgraph" ng-class="{active:page=='cloud-rgraph'}"><a href="#/~cloud?view=rgraph">Graph (Radial)</a></li>
</ul>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/js/angular/controllers/cloud.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js
index a033ce5..8cbd4db 100644
--- a/solr/webapp/web/js/angular/controllers/cloud.js
+++ b/solr/webapp/web/js/angular/controllers/cloud.js
@@ -16,7 +16,7 @@
*/
solrAdminApp.controller('CloudController',
- function($scope, $location, Zookeeper, Constants, Collections, System, Metrics) {
+ function($scope, $location, Zookeeper, Constants, Collections, System, Metrics, ZookeeperStatus) {
$scope.showDebug = false;
@@ -41,6 +41,9 @@ solrAdminApp.controller('CloudController',
} else if (view === "nodes") {
$scope.resetMenu("cloud-nodes", Constants.IS_ROOT_PAGE);
nodesSubController($scope, Collections, System, Metrics);
+ } else if (view === "zkstatus") {
+ $scope.resetMenu("cloud-zkstatus", Constants.IS_ROOT_PAGE);
+ zkStatusSubController($scope, ZookeeperStatus, false);
}
}
);
@@ -486,7 +489,39 @@ var nodesSubController = function($scope, Collections, System, Metrics) {
$scope.initClusterState();
};
+var zkStatusSubController = function($scope, ZookeeperStatus) {
+ $scope.showZkStatus = true;
+ $scope.showNodes = false;
+ $scope.showTree = false;
+ $scope.showGraph = false;
+ $scope.tree = {};
+ $scope.showData = false;
+ $scope.showDetails = false;
+
+ $scope.toggleDetails = function() {
+ $scope.showDetails = !$scope.showDetails === true;
+ };
+
+ $scope.initZookeeper = function() {
+ ZookeeperStatus.monitor({}, function(data) {
+ $scope.zkState = data.zkStatus;
+ $scope.mainKeys = ["ok", "clientPort", "zk_server_state", "zk_version",
+ "zk_approximate_data_size", "zk_znode_count", "zk_num_alive_connections"];
+ $scope.detailKeys = ["dataDir", "dataLogDir",
+ "zk_avg_latency", "zk_max_file_descriptor_count", "zk_watch_count",
+ "zk_packets_sent", "zk_packets_received",
+ "tickTime", "maxClientCnxns", "minSessionTimeout", "maxSessionTimeout"];
+ $scope.ensembleMainKeys = ["serverId", "electionPort", "quorumPort"];
+ $scope.ensembleDetailKeys = ["peerType", "electionAlg", "initLimit", "syncLimit",
+ "zk_followers", "zk_synced_followers", "zk_pending_syncs"];
+ });
+ };
+
+ $scope.initZookeeper();
+};
+
var treeSubController = function($scope, Zookeeper) {
+ $scope.showZkStatus = false;
$scope.showTree = true;
$scope.showGraph = false;
$scope.tree = {};
@@ -545,6 +580,7 @@ function secondsForHumans ( seconds ) {
}
var graphSubController = function ($scope, Zookeeper, isRadial) {
+ $scope.showZkStatus = false;
$scope.showTree = false;
$scope.showGraph = true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/js/angular/services.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js
index 66f2654..e3dcd3f 100644
--- a/solr/webapp/web/js/angular/services.js
+++ b/solr/webapp/web/js/angular/services.js
@@ -81,6 +81,12 @@ solrAdminServices.factory('System',
}}
});
}])
+.factory('ZookeeperStatus',
+ ['$resource', function($resource) {
+ return $resource('admin/zookeeper/status', {wt:'json', _:Date.now()}, {
+ "monitor": {}
+ });
+ }])
.factory('Properties',
['$resource', function($resource) {
return $resource('admin/info/properties', {'wt':'json', '_':Date.now()});
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9306922d/solr/webapp/web/partials/cloud.html
----------------------------------------------------------------------
diff --git a/solr/webapp/web/partials/cloud.html b/solr/webapp/web/partials/cloud.html
index 078c9af..825e195 100644
--- a/solr/webapp/web/partials/cloud.html
+++ b/solr/webapp/web/partials/cloud.html
@@ -18,6 +18,65 @@ limitations under the License.
<div id="frame">
+ <div id="zk-status-content" class="content clearfix" ng-show="showZkStatus">
+ <div id="zk-controls">
+ <a class="reload" ng-click="initZookeeper()"><span>Refresh</span></a>
+ <a class="details-button" ng-click="toggleDetails()" ng-class="{on:showDetails}">
+ <span>Toggle details</span>
+ </a>
+ </div>
+
+ <div class="zookeeper-status">Status: <span class="zkstatus-{{zkState.status}}">{{zkState.status}}</span></div>
+ <div class="zookeeper-errors" ng-show="zkState.errors">
+ Errors:
+ <ul>
+ <li ng-repeat="error in zkState.errors">{{error}}</li>
+ </ul>
+ </div>
+ <div>ZK connection string: {{zkState.zkHost}}</div>
+ <div>Ensemble size: {{zkState.ensembleSize}}</div>
+ <div>Ensemble mode: {{zkState.mode}}</div>
+
+ <table id="zk-table">
+ <thead>
+ <tr>
+ <th></th>
+ <th ng-repeat="host in zkState.details">{{host.host}}</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr ng-repeat="key in mainKeys">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details" ng-style="key === 'zk_server_state' && host[key] === 'leader' ? {'font-weight': 'bold'} : {'font-weight': 'normal'}">
+ {{key === 'zk_version' ? host[key].split("-")[0] : host[key]}}
+ </td>
+ </tr>
+ <tr ng-repeat="key in ensembleMainKeys" ng-show="zkState.mode === 'ensemble'">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details">
+ {{host[key]}}
+ </td>
+ </tr>
+ <tr id="detail-divider" ng-show="showDetails" >
+ <td ng-class="details"></td>
+ <td ng-repeat="host in zkState.details" ng-class="details"></td>
+ </tr>
+ <tr ng-repeat="key in detailKeys" ng-show="showDetails">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details">
+ {{host[key]}}
+ </td>
+ </tr>
+ <tr ng-repeat="key in ensembleDetailKeys" ng-show="showDetails && zkState.mode === 'ensemble'">
+ <td>{{key}}</td>
+ <td ng-repeat="host in zkState.details">
+ {{host[key]}}
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+
<div id="tree-content" class="content clearfix" ng-show="showTree">
<jstree class="tree" on-select="showTreeLink(url)" id="tree" data="tree"></jstree>
[11/17] lucene-solr:jira/solr-12470: SOLR-12636: Improve autoscaling
policy rules documentation
Posted by ab...@apache.org.
SOLR-12636: Improve autoscaling policy rules documentation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/37e8fea2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/37e8fea2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/37e8fea2
Branch: refs/heads/jira/solr-12470
Commit: 37e8fea22e532f9b9deb9ce040427cb8abc4a455
Parents: cbaedb4
Author: Steve Rowe <sa...@apache.org>
Authored: Wed Aug 8 14:23:08 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Wed Aug 8 14:23:08 2018 -0400
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
...olrcloud-autoscaling-policy-preferences.adoc | 167 +++++++++++++------
2 files changed, 117 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37e8fea2/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
index 2c4c433..2215c86 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
@@ -61,7 +61,7 @@ You can learn more about preferences in the <<solrcloud-autoscaling-policy-prefe
A cluster policy is a set of conditions that a node, shard, or collection must satisfy before it can be chosen as the target of a cluster management operation. These conditions are applied across the cluster regardless of the collection being managed. For example, the condition `{"cores":"<10", "node":"#ANY"}` means that any node must have less than 10 Solr cores in total, regardless of which collection they belong to.
-There are many metrics on which the condition can be based, e.g., system load average, heap usage, free disk space, etc. The full list of supported metrics can be found in the section describing <<solrcloud-autoscaling-policy-preferences.adoc#policy-attributes,Autoscaling Policy Attributes>>.
+There are many metrics on which the condition can be based, e.g., system load average, heap usage, free disk space, etc. The full list of supported metrics can be found in the section describing <<solrcloud-autoscaling-policy-preferences.adoc#policy-rule-attributes,Autoscaling Policy Rule Attributes>>.
When a node, shard, or collection does not satisfy the policy, we call it a *violation*. Solr ensures that cluster management operations minimize the number of violations. Cluster management operations are currently invoked manually. In the future, these cluster management operations may be invoked automatically in response to cluster events such as a node being added or lost.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37e8fea2/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
index 9641424..82af94c 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
@@ -20,9 +20,11 @@
The autoscaling policy and preferences are a set of rules and sorting preferences that help Solr select the target of cluster management operations so the overall load on the cluster remains balanced.
+Solr consults the configured policy and preferences when performing <<Commands That Use Autoscaling Policy and Preferences,Collections API commands>> in all contexts: manual, e.g. using `bin/solr`; semi-automatic, via the <<solrcloud-autoscaling-api.adoc#suggestions-api,Suggestions API>> or the Admin UI's <<suggestions-screen.adoc#suggestions-screen,Suggestions Screen>>; or fully automatic, via configured <<solrcloud-autoscaling-triggers.adoc#solrcloud-autoscaling-triggers,Triggers>>.
+
== Cluster Preferences Specification
-A preference is a hint to Solr on how to sort nodes based on their utilization. The default cluster preference is to sort by the total number of Solr cores (or replicas) hosted by a node. Therefore, by default, when selecting a node to add a replica, Solr can apply the preferences and choose the node with the least number of cores.
+A preference is a hint to Solr on how to sort nodes based on their utilization. The default cluster preference is to sort by the total number of Solr cores (or replicas) hosted by a node. Therefore, by default, when selecting a node to which to add a replica, Solr can apply the preferences and choose the node with the fewest cores.
More than one preference can be added to break ties. For example, we may choose to use free disk space to break ties if the number of cores on two nodes are the same. The node with the higher free disk space can be chosen as the target of the cluster operation.
@@ -86,16 +88,85 @@ In this example, we add a precision to the `freedisk` parameter so that nodes wi
A policy is a hard rule to be satisfied by each node. If a node does not satisfy the rule then it is called a *violation*. Solr ensures that the number of violations are minimized while invoking any cluster management operations.
-=== Policy Attributes
-A policy can have the following attributes:
+=== Policy Rule Structure
+
+==== Rule Types
+
+Policy rules can be either global or per-collection:
+
+* *Global rules* constrain the number of cores per node or node group. This type of rule applies to cores from all collections hosted on the specified node(s). As a result, <<Defining Collection-Specific Policies,collection-specific policies>>, which are associated with individual collections, may not contain global rules.
+* *Per-collection rules* constrain the number of replicas per node or node group.
+
+Global rules have three parts:
+
+* <<Node Selector>>
+* <<Core Count Constraint>>
+* <<Rule Strictness>>
+
+Per-collection rules have four parts:
+
+* <<Node Selector>>
+* <<Replica Selector and Rule Evaluation Context>>
+* <<Replica Count Constraint>>
+* <<Rule Strictness>>
+
+==== Node Selector
+
+Rule evaluation is restricted to node(s) matching the value of one of the following attributes: `node`, `port`, `ip_\*`, `sysprop.*`, or `diskType`. For replica/core count constraints other than `#EQUAL`, a condition specified in one of the following attributes may instead be used to select nodes: `freedisk`, `host`, `sysLoadAvg`, `heapUsage`, `nodeRole`, or `metrics.*`.
+
+Except for `node`, the attributes above cause selected nodes to be partitioned into node groups. A node group is referred to as a "bucket". Those attributes usable with the `#EQUAL` directive may define buckets either via the value `#EACH` or an array `["value1", ...]` (a subset of all possible values); in both cases, each node is placed in the bucket corresponding to the matching attribute value.
+
+The `node` attribute always places each selected node into its own bucket, regardless of the attribute value's form (`#ANY`, `node-name`, or `["node1-name", ...]`).
+
+Replica and core count constraints, described below, are evaluated against the total number in each bucket.
+
+==== Core Count Constraint
+
+The `cores` attribute value can be specified in one of the following forms:
+
+* the `#EQUAL` directive, which will cause cores to be distributed equally among the nodes specified via the rule's <<Node Selector>>.
+* a constraint on the core count on each <<Node Selector,selected node>>, specified as one of:
+** an integer value (e.g. `2`), a lower bound (e.g. `>0`), or an upper bound (e.g. `<3`)
+** a decimal value, interpreted as an acceptable range of core counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g. `1.6`: `1` or `2` is acceptable, and `2` is preferred)
+** a range of acceptable core counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g. `3-5`)
+** a percentage (e.g. `33%`), which is multiplied by the number of cores in the cluster at runtime. This value is then interpreted as described above for literal decimal values.
+
+==== Replica Selector and Rule Evaluation Context
+
+Rule evaluation can be restricted to replicas that meet any combination of the following conditions:
+
+* The replica is of a shard belonging to the collection specified in the `collection` attribute value. (Not usable with per-collection policies.)
+* The replica is of a shard specified in the `shard` attribute value.
+* The replica has the replica type specified in the `type` attribute value (`NRT`, `TLOG`, or `PULL`).
+
+If none of the above attributes is specified, then the rule is evaluated separately for each collection against all types of replicas of all shards.
+
+Specifying `#EACH` as the `shard` attribute value causes the rule to be evaluated separately for each shard of each collection.
+
+==== Replica Count Constraint
+
+The `replica` attribute value can be specified in one of the following forms:
+
+* `#ALL`: All <<Replica Selector and Rule Evaluation Context,selected replicas>> will be placed on the <<Node Selector,selected nodes>>.
+* `#EQUAL`: Distribute <<Replica Selector and Rule Evaluation Context,selected replicas>> evenly among all the <<Node Selector,selected nodes>>.
+* a constraint on the replica count on each <<Node Selector,selected node>>, specified as one of:
+** an integer value (e.g. `2`), a lower bound (e.g. `>0`), or an upper bound (e.g. `<3`)
+** a decimal value, interpreted as an acceptable range of replica counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g. `1.6`: `1` or `2` is acceptable, and `2` is preferred)
+** a range of acceptable replica counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g. `3-5`)
+** a percentage (e.g. `33%`), which is multiplied by the number of <<Replica Selector and Rule Evaluation Context,selected replicas>> at runtime. This value is then interpreted as described above for literal decimal values.
+
+==== Rule Strictness
+
+By default, the rule must be satisfied, and if it can't, then no action will be taken.
+
+If the `strict` attribute value is specified as `false`, Solr tries to satisfy the rule on a best effort basis, but if no node can satisfy the rule then any node may be chosen.
+
+=== Policy Rule Attributes
+
+A policy rule can have the following attributes:
`cores`::
-This is a special attribute that applies to the entire cluster. It can only be used along with the `node` attribute and no other. The value of this attribute can be
-* a positive integer . e.g : "`3`"
-* a number with a decimal value . e.g: "`1.66`" . This means both 1 and 2 are acceptable values but the system would prefer `2`
-* a number range. Such as `"3-5"` . This means `3,4,5` are acceptable values
-* a percentage value . e.g: `33%` . This is computed to a decimal value at runtime
-* `#EQUAL` : Divide the no:of cores equally among all the nodes or a subset of nodes
+This is a required attribute for <<Rule Types,global rules>>. It can only be used along with the `node` attribute and no other. See <<Core Count Constraint>> for possible attribute values.
`collection`::
The name of the collection to which the policy rule should apply. If omitted, the rule applies to all collections. This attribute is optional.
@@ -107,15 +178,7 @@ The name of the shard to which the policy rule should apply. If omitted, the rul
The type of the replica to which the policy rule should apply. If omitted, the rule is applied for all replica types of this collection/shard. The allowed values are `NRT`, `TLOG` and `PULL`
`replica`::
-This is a required attribute. The number of replicas that must exist to satisfy the rule. This must be one of
-
-* a positive integer . e.g : "`3`"
-* a number with a decimal value . e.g: "`1.66`" . This means both 1 and 2 are acceptable values but the system would prefer `2`
-* a number range. Such as `"3-5"` . This means `3,4,5` are acceptable values
-* `#ALL` : All replicas of a given collection or shard
-* a percentage value . e.g: `33%` . This is computed to a decimal value at runtime
-* `#EQUAL` : Divide the no:of replicas equally among all the nodes qualifying a certain property and place equal no:of them in each node
-
+This is a required attribute for <<Rule Types,per-collection rules>>. The number of replicas that must exist to satisfy the rule. See <<Replica Count Constraint>> for possible attribute values.
`strict`::
An optional boolean value. The default is `true`. If true, the rule must be satisfied. If false, Solr tries to satisfy the rule on a best effort basis but if no node can satisfy the rule then any node may be chosen.
@@ -123,7 +186,7 @@ An optional boolean value. The default is `true`. If true, the rule must be sati
One and only one of the following attributes can be specified in addition to the above attributes:
`node`::
-The name of the node to which the rule should apply. The default value is `#ANY` which means that any node in the cluster may satisfy the rule.
+The name of the node to which the rule should apply.
`port`::
The port of the node to which the rule should apply.
@@ -155,7 +218,7 @@ Any arbitrary metric. For example, `metrics:solr.node:CONTAINER.fs.totalSpace`.
`diskType`::
The type of disk drive being used for Solr's `coreRootDirectory`. The only two supported values are `rotational` and `ssd`. Refer to `coreRootDirectory` parameter in the <<format-of-solr-xml.adoc#solr-xml-parameters, Solr.xml Parameters>> section.
+
-It's value is fetched from the Metrics API with the key named `solr.node:CONTAINER.fs.coreRoot.spins`. The disk type is auto-detected by Lucene using various heuristics and it is not guaranteed to be correct across all platforms or operating systems. Refer to the <<taking-solr-to-production.adoc#dynamic-defaults-for-concurrentmergescheduler, Dynamic defaults for ConcurrentMergeScheduler>> section for more details.
+Its value is fetched from the Metrics API with the key named `solr.node:CONTAINER.fs.coreRoot.spins`. The disk type is auto-detected by Lucene using various heuristics and it is not guaranteed to be correct across all platforms or operating systems. Refer to the <<taking-solr-to-production.adoc#dynamic-defaults-for-concurrentmergescheduler, Dynamic defaults for ConcurrentMergeScheduler>> section for more details.
=== Policy Operators
@@ -164,9 +227,8 @@ Each attribute in the policy may specify one of the following operators along wi
* `<`: Less than
* `>`: Greater than
* `!`: Not
-* Range operator `(-)` : a value such as `"3-5"` means a value between 3 to 5 (inclusive). This is only supported in the following attributes
-** `replica`
-* array operator . e.g: `sysprop.zone = ["east", "west","apac"]`. This is equivalent to having multiple rules with each of these values. This can be used in the following attributes
+* Range operator `(-)`: a value such as `"3-5"` means a value between 3 to 5 (inclusive). This is only supported in the `replica` and `cores` attributes.
+* Array operator `[]`. e.g: `sysprop.zone = ["east", "west","apac"]`. This is equivalent to having multiple rules with each of these values. This can be used in the following attributes
** `sysprop.*`
** `port`
** `ip_*`
@@ -174,46 +236,47 @@ Each attribute in the policy may specify one of the following operators along wi
** `diskType`
* None means equal
-==== Special functions
+==== Special Functions
+
This supports values calculated at the time of execution.
* `%` : A certain percentage of the value. This is supported by the following attributes
** `replica`
+** `cores`
** `freedisk`
-* `#ALL` : This is applied to the `replica` attribute only. This means all replicas qualifying a certain clause
-* `#EQUAL`: This is applied to the `replica` attribute only. This means equal no:of replicas in each bucket.The buckets can be defined using an array operator (`[]`) or `#EACH` .The buckets can be defined on the following properties
-** `node`
+* `#ALL` : This is applied to the `replica` attribute only. This means all replicas that meet the rule condition.
+* `#EQUAL`: This is applied to the `replica` and `cores` attributes only. This means equal number of replicas/cores in each bucket. The buckets can be defined using an array operator (`[]`) or `#EACH`. The buckets can be defined using the following properties:
+** `node` <- <<Rule Types,global rules>>, i.e. with the `cores` attribute, may only specify this attribute
** `sysprop.*`
** `port`
** `diskType`
** `ip_*`
-****
-Some content here
-****
-
-
=== Examples of Policy Rules
==== Limit Replica Placement
+
Do not place more than one replica of the same shard on the same node:
[source,json]
{"replica": "<2", "shard": "#EACH", "node": "#ANY"}
==== Limit Cores per Node
+
Do not place more than 10 cores in any node. This rule can only be added to the cluster policy because it mentions the `cores` attribute that is only applicable cluster-wide.
[source,json]
{"cores": "<10", "node": "#ANY"}
==== Place Replicas Based on Port
+
Place exactly 1 replica of each shard of collection `xyz` on a node running on port `8983`
[source,json]
{"replica": 1, "shard": "#EACH", "collection": "xyz", "port": "8983"}
==== Place Replicas Based on a System Property
+
Place all replicas on a node with system property `availability_zone=us-east-1a`.
[source,json]
@@ -221,25 +284,28 @@ Place all replicas on a node with system property `availability_zone=us-east-1a`
===== Use Percentage
-====== example 1
-Place roughly a maximum of a 3rd of the replicas of a shard in a node. In the following example, the value of replica is computed in real time.
+====== Example 1
+
+Place roughly a maximum of a 3rd of the replicas of a shard in a node. In the following example, the value of `replica` is computed in real time:
+
[source,json]
{"replica": "33%", "shard": "#EACH", "node": "#ANY"}
-If the no:of of replicas in a shard is `2` , `33% of 2 = 0.66` . This means a node may have a maximum of `1` and a minimum of `0` replicas of each shard.
+If the number of replicas in a shard is `2`, `33% of 2 = 0.66`. This means a node may have a maximum of `1` and a minimum of `0` replicas of each shard.
-It is possible to get the same effect by hard coding the value of replica as follows
+It is possible to get the same effect by hard coding the value of `replica` as follows:
[source,json]
{"replica": 0.66, "shard": "#EACH", "node": "#ANY"}
-or using the range operator
+or using the range operator:
[source,json]
{"replica": "0-1", "shard": "#EACH", "node": "#ANY"}
-====== example 2
-Distribute replicas across datacenters east and west at a `1:2` ratio
+====== Example 2
+
+Distribute replicas across datacenters `east` and `west` at a `1:2` ratio:
[source,json]
{"replica": "33%", "shard": "#EACH", "sysprop.zone": "east"}
@@ -247,9 +313,7 @@ Distribute replicas across datacenters east and west at a `1:2` ratio
For the above rule to work, all nodes must the started with a system property called `"zone"`
-== example 3
-
-Distribute replicas equally in each zone
+==== Distribute Replicas Equally in Each Zone
[source,json]
{"replica": "#EQUAL", "shard": "#EACH", "sysprop.zone": ["east", "west"]}
@@ -259,15 +323,16 @@ or simply as follows
[source,json]
{"replica": "#EQUAL", "shard": "#EACH", "sysprop.zone": "#EACH"}
-
==== Place Replicas Based on Node Role
+
Do not place any replica on a node which has the overseer role. Note that the role is added by the `addRole` collection API. It is *not* automatically the node which is currently the overseer.
[source,json]
{"replica": 0, "nodeRole": "overseer"}
==== Place Replicas Based on Free Disk
-Place all replicas in nodes with freedisk more than 500GB. Here again, we have to write the rule in the negative sense.
+
+Place all replicas in nodes with freedisk more than 500GB.
[source,json]
{"replica": "#ALL", "freedisk": ">500"}
@@ -276,19 +341,19 @@ Keep all replicas in nodes with over `50%` freedisk
[source,json]
{"replica": "#ALL", "freedisk": ">50%"}
-
==== Try to Place Replicas Based on Free Disk
+
Place all replicas in nodes with freedisk more than 500GB when possible. Here we use the strict keyword to signal that this rule is to be honored on a best effort basis.
[source,json]
{"replica": "#ALL", "freedisk": ">500", "strict" : false}
-==== Try to Place all Replicas of type TLOG on Nodes with SSD Drives
+==== Try to Place All Replicas of Type TLOG on Nodes with SSD Drives
[source,json]
{ "replica": "#ALL","type" : "TLOG", "diskType" : "ssd" }
-==== Try to Place all Replicas of type PULL on Nodes with Rotational Disk Drives
+==== Try to Place All Replicas of Type PULL on Nodes with Rotational Disk Drives
[source,json]
{ "replica": "#ALL", "type" : "PULL" , "diskType" : "rotational"}
@@ -304,9 +369,11 @@ It is possible to override conditions specified in the cluster policy using coll
Also, if `maxShardsPerNode` is specified during the time of collection creation, then both `maxShardsPerNode` and the policy rules must be satisfied.
-Some attributes such as `cores` can only be used in the cluster policy. See the section above on policy attributes for details.
+Some attributes such as `cores` can only be used in the cluster policy. See the section <<Policy Rule Attributes>> for details.
+
+== Commands That Use Autoscaling Policy and Preferences
-The policy is used by these <<collections-api.adoc#collections-api,Collections API>> commands:
+The configured autoscaling policy and preferences are used by these <<collections-api.adoc#collections-api,Collections API>> commands:
* CREATE
* CREATESHARD
@@ -315,5 +382,3 @@ The policy is used by these <<collections-api.adoc#collections-api,Collections A
* SPLITSHARD
* UTILIZENODE
* MOVEREPLICA
-
-In the future, the policy and preferences will be used by the Autoscaling framework to automatically change the cluster in response to events such as a node being added or lost.
[09/17] lucene-solr:jira/solr-12470: SOLR-12618:
AutoScalingHandlerTest failing in jenkins
Posted by ab...@apache.org.
SOLR-12618: AutoScalingHandlerTest failing in jenkins
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9b418a45
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9b418a45
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9b418a45
Branch: refs/heads/jira/solr-12470
Commit: 9b418a4593c2cd4f49aab2485c5614d11e1bb5eb
Parents: 2674c53
Author: Noble Paul <no...@apache.org>
Authored: Thu Aug 9 01:24:03 2018 +1000
Committer: Noble Paul <no...@apache.org>
Committed: Thu Aug 9 01:24:03 2018 +1000
----------------------------------------------------------------------
.../autoscaling/AutoScalingHandlerTest.java | 43 ++--
.../cloud/autoscaling/AddReplicaSuggester.java | 10 +-
.../client/solrj/cloud/autoscaling/Clause.java | 18 +-
.../cloud/autoscaling/MoveReplicaSuggester.java | 3 +-
.../solrj/cloud/autoscaling/NodeVariable.java | 5 +-
.../client/solrj/cloud/autoscaling/Policy.java | 2 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 38 ++-
.../solrj/cloud/autoscaling/RangeVal.java | 10 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 6 +-
.../solrj/cloud/autoscaling/ReplicaInfo.java | 17 +-
.../solrj/cloud/autoscaling/Suggester.java | 14 +-
.../solrj/cloud/autoscaling/VariableBase.java | 1 +
.../solrj/cloud/autoscaling/Violation.java | 10 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 2 +-
.../solr/common/ConditionalMapWriter.java | 75 ++++++
.../java/org/apache/solr/common/MapWriter.java | 6 +
.../solrj/cloud/autoscaling/TestPolicy.java | 16 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 240 +++++++++++++++++++
.../apache/solr/cloud/SolrCloudTestCase.java | 19 ++
19 files changed, 462 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 5b8ec21..b9e8e66 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -23,6 +23,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
@@ -675,7 +676,6 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
public void testReadApi() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
// first trigger
@@ -693,9 +693,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
String setClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
+ " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}]" +
"}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
response = solrClient.request(req);
@@ -713,17 +711,9 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertEquals(response.get("result").toString(), "success");
String setPolicyCommand = "{'set-policy': {" +
- " 'xyz':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]," +
- " 'policy1':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]," +
- " 'policy2':[" +
- " {'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]" +
- "}}";
+ " 'xyz':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
+ " 'policy1':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
+ " 'policy2':[{'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}]}}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setPolicyCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -747,7 +737,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
List<Map> clusterPolicy = (List<Map>) response.get("cluster-policy");
assertNotNull(clusterPolicy);
- assertEquals(3, clusterPolicy.size());
+ assertEquals(2, clusterPolicy.size());
Map policies = (Map) response.get("policies");
assertNotNull(policies);
@@ -766,11 +756,11 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
for (int i = 0; i < 2; i++) {
Map node = (Map) sortedNodes.get(i);
assertNotNull(node);
- assertEquals(6, node.size());
assertNotNull(node.get("node"));
assertNotNull(node.get("cores"));
assertEquals(0d, node.get("cores"));
assertNotNull(node.get("freedisk"));
+ assertNotNull(node.get("replicas"));
assertTrue(node.get("freedisk") instanceof Double);
assertNotNull(node.get("sysLoadAvg"));
assertTrue(node.get("sysLoadAvg") instanceof Double);
@@ -800,8 +790,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
String tempClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
+ " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}"+
" ]" +
"}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, tempClusterPolicyCommand);
@@ -831,12 +820,26 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
for (Map<String, Object> violation : violations) {
assertEquals("readApiTestViolations", violation.get("collection"));
assertEquals("shard1", violation.get("shard"));
- assertEquals(2d, getObjectByPath(violation, true, "violation/delta"));
+ assertEquals(1.0d, getObjectByPath(violation, true, "violation/delta"));
assertEquals(3l, getObjectByPath(violation, true, "violation/replica/NRT"));
assertNotNull(violation.get("clause"));
}
+ log.info("Before starting new jetty ,{}", cluster.getJettySolrRunners()
+ .stream()
+ .map(jettySolrRunner -> jettySolrRunner.getNodeName()).collect(Collectors.toList()));
JettySolrRunner runner1 = cluster.startJettySolrRunner();
cluster.waitForAllNodes(30);
+ log.info("started new jetty {}", runner1.getNodeName());
+
+ response = waitForResponse(namedList -> {
+ List l = (List) Utils.getObjectByPath(namedList, false, "diagnostics/liveNodes");
+ if (l != null && l.contains(runner1.getNodeName())) return true;
+ return false;
+ },
+ createAutoScalingRequest(SolrRequest.METHOD.GET, "/diagnostics", null),
+ 200,
+ 20,
+ runner1.getNodeName() + " could not come up ");
req = createAutoScalingRequest(SolrRequest.METHOD.GET, "/suggestions", null);
response = solrClient.request(req);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
index 2a31e71..a498932 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/AddReplicaSuggester.java
@@ -47,16 +47,19 @@ class AddReplicaSuggester extends Suggester {
//iterate through nodes and identify the least loaded
List<Violation> leastSeriousViolation = null;
Row bestNode = null;
+ double[] bestDeviation = null;
for (int i = getMatrix().size() - 1; i >= 0; i--) {
Row row = getMatrix().get(i);
if (!isNodeSuitableForReplicaAddition(row)) continue;
Row tmpRow = row.addReplica(shard.first(), shard.second(), type, strict);
- List<Violation> errs = testChangedMatrix(strict, tmpRow.session);
-
+ double[] deviation = new double[1];
+ List<Violation> errs = testChangedMatrix(strict, tmpRow.session, deviation);
if (!containsNewErrors(errs)) {
- if (isLessSerious(errs, leastSeriousViolation)) {
+ if ((errs.isEmpty() && isLessDeviant(bestDeviation, deviation)) ||//there are no violations but this is deviating less
+ isLessSerious(errs, leastSeriousViolation)) {//there are errors , but this has less serious violation
leastSeriousViolation = errs;
bestNode = tmpRow;
+ bestDeviation = deviation;
}
}
}
@@ -73,6 +76,7 @@ class AddReplicaSuggester extends Suggester {
return null;
}
+
@Override
public CollectionParams.CollectionAction getAction() {
return ADDREPLICA;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
index cd9212b..e325d10 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Clause.java
@@ -42,6 +42,7 @@ import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.EQUAL;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.GREATER_THAN;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.LESS_THAN;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.NOT_EQUAL;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.RANGE_EQUAL;
import static org.apache.solr.client.solrj.cloud.autoscaling.Operand.WILDCARD;
import static org.apache.solr.client.solrj.cloud.autoscaling.Policy.ANY;
import static org.apache.solr.common.params.CoreAdminParams.COLLECTION;
@@ -215,7 +216,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
if (this.isPerCollectiontag() && that.isPerCollectiontag()) {
v = Integer.compare(this.replica.op.priority, that.replica.op.priority);
if (v == 0) {// higher the number of replicas , harder to satisfy
- v = Preference.compareWithTolerance((Double) this.replica.val, (Double) that.replica.val, 1);
+ Double thisVal = this.replica.val instanceof RangeVal ? ((RangeVal) this.replica.val).max.doubleValue() : (Double) this.replica.val;
+ Double thatVal = that.replica.val instanceof RangeVal ? ((RangeVal) that.replica.val).max.doubleValue() : (Double) that.replica.val;
+ v = Preference.compareWithTolerance(thisVal, thatVal, 1);
v = this.replica.op == LESS_THAN ? v : v * -1;
}
if (v == 0) v = compareTypes(this.type, that.type);
@@ -240,7 +243,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
//replica value is zero
boolean isReplicaZero() {
- return replica != null && replica.getOperand() == Operand.EQUAL &&
+ return replica != null && replica.getOperand() == Operand.EQUAL && replica.val instanceof Double &&
Preference.compareWithTolerance(0d, (Double) replica.val, 1) == 0;
}
@@ -353,7 +356,7 @@ public class Clause implements MapWriter, Comparable<Clause> {
return operand;
}
- public List<Violation> test(Policy.Session session) {
+ public List<Violation> test(Policy.Session session, double[] deviations) {
ComputedValueEvaluator computedValueEvaluator = new ComputedValueEvaluator(session);
Violation.Ctx ctx = new Violation.Ctx(this, session.matrix, computedValueEvaluator);
if (isPerCollectiontag()) {
@@ -377,6 +380,13 @@ public class Clause implements MapWriter, Comparable<Clause> {
sealedClause.getReplica().delta(replicas),
tag.varType.meta.isNodeSpecificVal() ? null : counts.getKey());
tag.varType.addViolatingReplicas(ctx.reset(counts.getKey(), replicas, violation));
+ } else {
+ if (deviations != null && sealedClause.replica.op == RANGE_EQUAL) {
+ Number actualCount = replicas.getVal(type);
+ Double realDelta = ((RangeVal) sealedClause.replica.val).realDelta(actualCount.doubleValue());
+ realDelta = this.isReplicaZero() ? -1 * realDelta : realDelta;
+ deviations[0] += Math.abs(realDelta);
+ }
}
}
}
@@ -422,7 +432,9 @@ public class Clause implements MapWriter, Comparable<Clause> {
tagVsCount.get(row.node).increment(shards.getValue());
}
} else {
+ tagVsCount.computeIfAbsent(String.valueOf(t.getValue()), s -> new ReplicaCount());
boolean pass = sealedClause.getTag().isPass(tagVal);
+ if(!pass && !isReplicaZero()) continue;
tagVsCount.computeIfAbsent(pass ? String.valueOf(tagVal) : "", s -> new ReplicaCount());
if (pass) {
tagVsCount.get(String.valueOf(tagVal)).increment(shards.getValue());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
index fffaee2..9e778c0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/MoveReplicaSuggester.java
@@ -62,7 +62,8 @@ public class MoveReplicaSuggester extends Suggester {
if (!isNodeSuitableForReplicaAddition(targetRow)) continue;
targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first
Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node
- List<Violation> errs = testChangedMatrix(strict, srcRowModified.session);
+ double[] deviation = new double[1];
+ List<Violation> errs = testChangedMatrix(strict, srcRowModified.session, deviation);
srcRowModified.session.applyRules(); // now resort the nodes with the new values
Policy.Session tmpSession = srcRowModified.session;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
index 3292800..48dcd5e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NodeVariable.java
@@ -33,13 +33,12 @@ public class NodeVariable extends VariableBase {
if (ctx.violation.replicaCountDelta > 0) {//there are more replicas than necessary
for (int i = 0; i < Math.abs(ctx.violation.replicaCountDelta); i++) {
Suggester suggester = ctx.session.getSuggester(MOVEREPLICA)
+ .forceOperation(true)
.hint(Suggester.Hint.SRC_NODE, ctx.violation.node)
.hint(ctx.violation.shard.equals(ANY) ? Suggester.Hint.COLL : Suggester.Hint.COLL_SHARD,
ctx.violation.shard.equals(ANY) ? ctx.violation.coll : new Pair<>(ctx.violation.coll, ctx.violation.shard));
- ctx.addSuggestion(suggester);
+ if(ctx.addSuggestion(suggester) == null) break;
}
}
-
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index b398caf..8b0a956 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -572,7 +572,7 @@ public class Policy implements MapWriter {
setApproxValuesAndSortNodes(clusterPreferences, matrix);
for (Clause clause : expandedClauses) {
- List<Violation> errs = clause.test(this);
+ List<Violation> errs = clause.test(this, null);
violations.addAll(errs);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index f662eff..a88617c 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -28,12 +28,14 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
@@ -183,30 +185,20 @@ public class PolicyHelper {
public static MapWriter getDiagnostics(Policy policy, SolrCloudManager cloudManager) {
Policy.Session session = policy.createSession(cloudManager);
List<Row> sorted = session.getSortedNodes();
- List<Violation> violations = session.getViolations();
-
- List<Preference> clusterPreferences = policy.getClusterPreferences();
-
- List<Map<String, Object>> sortedNodes = new ArrayList<>(sorted.size());
- for (Row row : sorted) {
- Map<String, Object> map = Utils.makeMap("node", row.node);
- map.put("isLive", row.isLive);
- for (Cell cell : row.getCells()) {
- for (Preference clusterPreference : clusterPreferences) {
- Policy.SortParam name = clusterPreference.getName();
- if (cell.getName().equalsIgnoreCase(name.name())) {
- map.put(name.name(), cell.getValue());
- break;
- }
- }
+ return ew -> ew.put("sortedNodes", (IteratorWriter) iw -> {
+ for (Row row : sorted) {
+ iw.add((MapWriter) ew1 -> {
+ ew1.put("node", row.node).
+ put("isLive", row.isLive);
+ for (Cell cell : row.getCells())
+ ew1.put(cell.name, cell.val,
+ (Predicate) o -> o != null && (!(o instanceof Map) || !((Map) o).isEmpty()));
+ ew1.put("replicas", row.collectionVsShardVsReplicas);
+ });
}
- sortedNodes.add(map);
- }
-
- return ew -> {
- ew.put("sortedNodes", sortedNodes);
- ew.put("violations", violations);
- };
+ }).put("liveNodes", cloudManager.getClusterStateProvider().getLiveNodes())
+ .put("violations", session.getViolations())
+ .put("config", session.getPolicy());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
index 16cfe6a..b9f6b8d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/RangeVal.java
@@ -35,8 +35,12 @@ class RangeVal implements MapWriter {
Double.compare(testVal.doubleValue(), max.doubleValue()) <= 0;
}
+ public Double realDelta(double v) {
+ if (actual != null) return v - actual.doubleValue();
+ else return delta(v);
+ }
+
public Double delta(double v) {
-// if (actual != null) return v - actual.doubleValue();
if (v >= max.doubleValue()) return v - max.doubleValue();
if (v <= min.doubleValue()) return v - min.doubleValue();
return 0d;
@@ -49,6 +53,8 @@ class RangeVal implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put("min", min).put("max", max).putIfNotNull("actual", actual);
+ ew.put("min", min)
+ .put("max", max)
+ .putIfNotNull("actual", actual);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
index 588508f..7e17df1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaCount.java
@@ -33,9 +33,9 @@ class ReplicaCount implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put(Replica.Type.NRT.name(), nrt);
- ew.put(Replica.Type.PULL.name(), pull);
- ew.put(Replica.Type.TLOG.name(), tlog);
+ if (nrt > 0) ew.put(Replica.Type.NRT.name(), nrt);
+ if (pull > 0) ew.put(Replica.Type.PULL.name(), pull);
+ if (tlog > 0) ew.put(Replica.Type.TLOG.name(), tlog);
ew.put("count", total());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
index edc7269..25f12bd 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
@@ -53,7 +53,7 @@ public class ReplicaInfo implements MapWriter {
}
public ReplicaInfo(String name, String core, String coll, String shard, Replica.Type type, String node, Map<String, Object> vals) {
- if(vals==null) vals = Collections.emptyMap();
+ if (vals == null) vals = Collections.emptyMap();
this.name = name;
if (vals != null) {
this.variables.putAll(vals);
@@ -66,6 +66,21 @@ public class ReplicaInfo implements MapWriter {
this.node = node;
}
+ ReplicaInfo(Map<String, Object> map) {
+ this.name = map.keySet().iterator().next();
+ Map details = (Map) map.get(name);
+ details = Utils.getDeepCopy(details, 4);
+ this.collection = (String) details.remove("collection");
+ this.shard = (String) details.remove("shard");
+ this.core = (String) details.remove("core");
+ this.node = (String) details.remove("node_name");
+ this.isLeader = Boolean.parseBoolean((String) details.getOrDefault("leader", "false"));
+ details.remove("leader");
+ type = Replica.Type.valueOf((String) details.getOrDefault("type", "NRT"));
+ details.remove("type");
+ this.variables.putAll(details);
+ }
+
public Object clone() {
return new ReplicaInfo(name, core, collection, shard, type, node, variables);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index 9b9b60e..db0aab4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -35,6 +35,7 @@ import java.util.function.Predicate;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.common.ConditionalMapWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.DocCollection;
@@ -70,6 +71,10 @@ public abstract class Suggester implements MapWriter {
this.session = session.copy();
}
+ boolean isLessDeviant(double[] previousBest, double[] newDeviation) {
+ if (previousBest == null) return true;
+ return newDeviation[0] < previousBest[0];
+ }
public Suggester hint(Hint hint, Object value) {
hint.validator.accept(value);
if (hint.multiValued) {
@@ -203,7 +208,9 @@ public abstract class Suggester implements MapWriter {
@Override
public void writeMap(EntryWriter ew) throws IOException {
ew.put("type", violation == null ? "improvement" : "violation");
- ew.putIfNotNull("violation", violation);
+ ew.putIfNotNull("violation",
+ new ConditionalMapWriter(violation,
+ (k, v) -> !"violatingReplicas".equals(k)));
ew.put("operation", operation);
}
@@ -237,6 +244,7 @@ public abstract class Suggester implements MapWriter {
boolean containsNewErrors(List<Violation> violations) {
boolean isTxOpen = session.transaction != null && session.transaction.isOpen();
+ if (violations.size() > originalViolations.size()) return true;
for (Violation v : violations) {
//the computed value can change over time. So it's better to evaluate it in the end
if (isTxOpen && v.getClause().hasComputedValue) continue;
@@ -272,12 +280,12 @@ public abstract class Suggester implements MapWriter {
}
}
- List<Violation> testChangedMatrix(boolean strict, Policy.Session session) {
+ List<Violation> testChangedMatrix(boolean strict, Policy.Session session, double[] deviation) {
Policy.setApproxValuesAndSortNodes(session.getPolicy().clusterPreferences, session.matrix);
List<Violation> errors = new ArrayList<>();
for (Clause clause : session.expandedClauses) {
if (strict || clause.strict) {
- List<Violation> errs = clause.test(session);
+ List<Violation> errs = clause.test(session, deviation);
if (!errs.isEmpty()) {
errors.addAll(errs);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
index 8b3f1e1..4387eb1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/VariableBase.java
@@ -107,6 +107,7 @@ public class VariableBase implements Variable {
val = condition.op.readRuleValue(condition);
if (val != condition.val) return val;
}
+ if (!isRuleVal && "".equals(val) && this.varType.type != String.class) val = -1;
if (name == null) name = this.varType.tagName;
if (varType.type == Double.class) {
Double num = Clause.parseDouble(name, val);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
index 39b3c0b..53f7924 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Violation.java
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Objects;
import java.util.function.Function;
+import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.Utils;
@@ -160,13 +161,20 @@ public class Violation implements MapWriter {
ew.putIfNotNull("collection", coll);
ew.putIfNotNull("shard", shard);
ew.putIfNotNull("node", node);
- ew.putIfNotNull("tagKey", String.valueOf(tagKey));
+ ew.putStringIfNotNull("tagKey", tagKey);
ew.putIfNotNull("violation", (MapWriter) ew1 -> {
if (getClause().isPerCollectiontag()) ew1.put("replica", actualVal);
else ew1.put(clause.tag.name, String.valueOf(actualVal));
ew1.putIfNotNull("delta", replicaCountDelta);
});
ew.put("clause", getClause());
+ if (!replicaInfoAndErrs.isEmpty()) {
+ ew.put("violatingReplicas", (IteratorWriter) iw -> {
+ for (ReplicaInfoAndErr replicaInfoAndErr : replicaInfoAndErrs) {
+ iw.add(replicaInfoAndErr.replicaInfo);
+ }
+ });
+ }
}
static class Ctx {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
index 83fb25a..d148e21 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientNodeStateProvider.java
@@ -94,7 +94,7 @@ public class SolrClientNodeStateProvider implements NodeStateProvider, MapWriter
return solrClient.getClusterStateProvider();
}
- private void readReplicaDetails() throws IOException {
+ protected void readReplicaDetails() throws IOException {
ClusterStateProvider clusterStateProvider = getClusterStateProvider();
ClusterState clusterState = clusterStateProvider.getClusterState();
if (clusterState == null) { // zkStateReader still initializing
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
new file mode 100644
index 0000000..9a1106f
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/common/ConditionalMapWriter.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.common;
+
+import java.io.IOException;
+import java.util.function.BiPredicate;
+
+public class ConditionalMapWriter implements MapWriter {
+ private final MapWriter delegate;
+ private final BiPredicate<String, Object> predicate;
+
+ public ConditionalMapWriter(MapWriter delegate, BiPredicate<String, Object> predicate) {
+ this.delegate = delegate;
+ this.predicate = predicate;
+ }
+
+ private class EntryWriterWrapper implements EntryWriter {
+ private final EntryWriter delegate;
+
+ EntryWriterWrapper(EntryWriter delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public EntryWriter put(String k, Object v) throws IOException {
+ if (predicate.test(k, v)) delegate.put(k, v);
+ return this;
+ }
+
+ @Override
+ public EntryWriter put(String k, int v) throws IOException {
+ return put(k, Integer.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, long v) throws IOException {
+ return put(k, Long.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, float v) throws IOException {
+ return put(k, Float.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, double v) throws IOException {
+ return put(k, Double.valueOf(v));
+ }
+
+ @Override
+ public EntryWriter put(String k, boolean v) throws IOException {
+ return put(k, Boolean.valueOf(v));
+ }
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ delegate.writeMap(new EntryWriterWrapper(ew));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
index fc57e3d..56f3b8d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/MapWriter.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.Predicate;
import org.apache.solr.common.util.Utils;
@@ -102,6 +103,11 @@ public interface MapWriter extends MapSerializable {
return this;
}
+ default EntryWriter put(String k, Object v, Predicate<Object> p) throws IOException {
+ if (p.test(v)) put(k, v);
+ return this;
+ }
+
default EntryWriter putIfNotNull(String k, Object v) throws IOException {
if(v != null) put(k,v);
return this;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 3bd58aa..19db8fc 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -2467,8 +2467,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
" }}}";
String autoScalingjson = " { cluster-policy:[" +
- " { replica :'<34%', shard: '#EACH', sysprop.az : east}," +
- " { replica :'<67%', shard: '#EACH', sysprop.az : west}" +
+ " { replica :'33%', shard: '#EACH', sysprop.az : east}," +
+ " { replica :'67%', shard: '#EACH', sysprop.az : west}" +
" ]," +
" cluster-preferences :[{ minimize : cores }]}";
@@ -2480,6 +2480,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
List<String> nodes = new ArrayList<>();
+ int westCount = 0, eastCount = 0;
for (int i = 0; i < 12; i++) {
SolrRequest suggestion = txn.getCurrentSession()
.getSuggester(ADDREPLICA)
@@ -2488,9 +2489,13 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertNotNull(suggestion);
String node = suggestion.getParams().get("node");
nodes.add(node);
+ if ("10.0.0.6:8983_solr".equals(node)) eastCount++;
+ if ("10.0.0.6:7574_solr".equals(node)) westCount++;
if (i % 3 == 1) assertEquals("10.0.0.6:8983_solr", node);
else assertEquals("10.0.0.6:7574_solr", node);
}
+ assertEquals(8, westCount);
+ assertEquals(4, eastCount);
List<Violation> violations = txn.close();
assertTrue(violations.isEmpty());
@@ -2655,7 +2660,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
"}";
AutoScalingConfig cfg = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
List<Violation> violations = cfg.getPolicy().createSession(cloudManagerWithData(dataproviderdata)).getViolations();
- assertEquals(2, violations.size());
+ assertEquals("expected 2 violations", 2, violations.size());
List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(cfg, cloudManagerWithData(dataproviderdata));
assertEquals(2, suggestions.size());
for (Suggester.SuggestionInfo suggestion : suggestions) {
@@ -3562,11 +3567,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
Object root = Utils.fromJSONString(writer.toString());
assertEquals(2l,
Utils.getObjectByPath(root, true, "violations[0]/violation/replica/NRT"));
- assertEquals(0l,
- Utils.getObjectByPath(root, true, "violations[0]/violation/replica/PULL"));
- assertEquals(0l,
- Utils.getObjectByPath(root, true, "violations[0]/violation/replica/TLOG"));
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
index c1d69b9..d6a5d51 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy2.java
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -288,4 +289,243 @@ public class TestPolicy2 extends SolrTestCaseJ4 {
};
}
+ public void testAutoScalingHandlerFailure() {
+ String diagnostics = "{" +
+ " 'diagnostics': {" +
+ " 'sortedNodes': [" +
+ " {" +
+ " 'node': '127.0.0.1:63191_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.97510064011647," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'readApiTestViolations': {" +
+ " 'shard1': [" +
+ " {" +
+ " 'core_node5': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n2'," +
+ " 'leader': 'true'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node7': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n4'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node12': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n10'," +
+ " 'base_url': 'https://127.0.0.1:63191/solr'," +
+ " 'node_name': '127.0.0.1:63191_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63192_solr'," +
+ " 'isLive': true," +
+ " 'cores': 3.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {" +
+ " 'readApiTestViolations': {" +
+ " 'shard1': [" +
+ " {" +
+ " 'core_node3': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n1'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node9': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n6'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'core_node11': {" +
+ " 'core': 'readApiTestViolations_shard1_replica_n8'," +
+ " 'base_url': 'https://127.0.0.1:63192/solr'," +
+ " 'node_name': '127.0.0.1:63192_solr'," +
+ " 'state': 'active'," +
+ " 'type': 'NRT'," +
+ " 'force_set_state': 'false'," +
+ " 'INDEX.sizeInGB': 6.426125764846802E-8," +
+ " 'shard': 'shard1'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]" +
+ " }" +
+ " }" +
+ " }," +
+ " {" +
+ " 'node': '127.0.0.1:63219_solr'," +
+ " 'isLive': true," +
+ " 'cores': 0.0," +
+ " 'freedisk': 680.908073425293," +
+ " 'heapUsage': 24.98878807983566," +
+ " 'sysLoadAvg': 272.75390625," +
+ " 'totaldisk': 1037.938980102539," +
+ " 'replicas': {}" +
+ " }" +
+ " ]," +
+ " 'liveNodes': [" +
+ " '127.0.0.1:63191_solr'," +
+ " '127.0.0.1:63192_solr'," +
+ " '127.0.0.1:63219_solr'" +
+ " ]," +
+ " 'violations': [" +
+ " {" +
+ " 'collection': 'readApiTestViolations'," +
+ " 'shard': 'shard1'," +
+ " 'node': '127.0.0.1:63191_solr'," +
+ " 'violation': {" +
+ " 'replica': {'NRT': 3, 'count': 3}," +
+ " 'delta': 2.0" +
+ " }," +
+ " 'clause': {" +
+ " 'replica': '<3'," +
+ " 'shard': '#EACH'," +
+ " 'node': '#ANY'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }," +
+ " {" +
+ " 'collection': 'readApiTestViolations'," +
+ " 'shard': 'shard1'," +
+ " 'node': '127.0.0.1:63192_solr'," +
+ " 'violation': {" +
+ " 'replica': {'NRT': 3, 'count': 3}," +
+ " 'delta': 2.0" +
+ " }," +
+ " 'clause': {" +
+ " 'replica': '<2'," +
+ " 'shard': '#EACH'," +
+ " 'node': '#ANY'," +
+ " 'collection': 'readApiTestViolations'" +
+ " }" +
+ " }" +
+ " ]," +
+ " 'config': {" +
+ " 'cluster-preferences': [" +
+ " {'minimize': 'cores', 'precision': 3}," +
+ " {'maximize': 'freedisk', 'precision': 100}," +
+ " {'minimize': 'sysLoadAvg', 'precision': 10}," +
+ " {'minimize': 'heapUsage', 'precision': 10}" +
+ " ]," +
+ " 'cluster-policy': [" +
+ " {'cores': '<10', 'node': '#ANY'}," +
+ " {'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole': 'overseer', 'replica': 0}" +
+ " ]" +
+ " }" +
+ " }}";
+ Map<String, Object> m = (Map<String, Object>) Utils.fromJSONString(diagnostics);
+
+ Policy policy = new Policy((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config"));
+ SolrCloudManager cloudManagerFromDiagnostics = createCloudManagerFromDiagnostics(m);
+ Policy.Session session = policy.createSession(cloudManagerFromDiagnostics);
+ List<Suggester.SuggestionInfo> suggestions = PolicyHelper.getSuggestions(new AutoScalingConfig((Map<String, Object>) Utils.getObjectByPath(m, false, "diagnostics/config")), cloudManagerFromDiagnostics);
+ assertEquals(2, suggestions.size());
+
+ }
+
+ static SolrCloudManager createCloudManagerFromDiagnostics(Map<String, Object> m) {
+ List<Map> sortedNodes = (List<Map>) Utils.getObjectByPath(m, false, "diagnostics/sortedNodes");
+ Set<String> liveNodes = new HashSet<>();
+ SolrClientNodeStateProvider nodeStateProvider = new SolrClientNodeStateProvider(null) {
+ @Override
+ protected void readReplicaDetails() {
+ for (Object o : sortedNodes) {
+ String node = (String) ((Map) o).get("node");
+ liveNodes.add(node);
+ Map nodeDetails = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(node, s -> new LinkedHashMap<>());
+ Map<String, Map<String, List<Map>>> replicas = (Map<String, Map<String, List<Map>>>) ((Map) o).get("replicas");
+ replicas.forEach((coll, shardVsReplicas) -> shardVsReplicas
+ .forEach((shard, repDetails) -> {
+ List<ReplicaInfo> reps = (List) ((Map) nodeDetails
+ .computeIfAbsent(coll, o1 -> new LinkedHashMap<>()))
+ .computeIfAbsent(shard, o12 -> new ArrayList<ReplicaInfo>());
+ for (Map map : repDetails) reps.add(new ReplicaInfo(map));
+ }));
+ }
+
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return nodeVsCollectionVsShardVsReplicaInfo.get(node) == null ?
+ Collections.emptyMap() :
+ nodeVsCollectionVsShardVsReplicaInfo.get(node);
+ }
+
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ for (Map n : sortedNodes) if (n.get("node").equals(node)) return n;
+ return Collections.emptyMap();
+ }
+ };
+ return new DelegatingCloudManager(null) {
+ @Override
+ public NodeStateProvider getNodeStateProvider() {
+ return nodeStateProvider;
+ }
+
+ @Override
+ public ClusterStateProvider getClusterStateProvider() {
+ return new DelegatingClusterStateProvider(null) {
+ @Override
+ public Set<String> getLiveNodes() {
+ return liveNodes;
+ }
+ };
+ }
+ };
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9b418a45/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index a4c3b6d..d0cc4c5 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -49,6 +50,7 @@ import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.NamedList;
import org.junit.AfterClass;
import org.junit.Before;
@@ -352,4 +354,21 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
}
}
+ protected NamedList waitForResponse(Predicate<NamedList> predicate, SolrRequest request, int intervalInMillis, int numRetries, String messageOnFail) {
+ int i = 0;
+ for (; i < numRetries; i++) {
+ try {
+ NamedList<Object> response = cluster.getSolrClient().request(request);
+ if (predicate.test(response)) return response;
+ Thread.sleep(intervalInMillis);
+ } catch (RuntimeException rte) {
+ throw rte;
+ } catch (Exception e) {
+ throw new RuntimeException("error executing request", e);
+ }
+ }
+ fail("Tried " + i + " times , could not succeed. " + messageOnFail);
+ return null;
+ }
+
}
[17/17] lucene-solr:jira/solr-12470: Merge branch 'master' into
jira/solr-12470
Posted by ab...@apache.org.
Merge branch 'master' into jira/solr-12470
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e1a5ee6f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e1a5ee6f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e1a5ee6f
Branch: refs/heads/jira/solr-12470
Commit: e1a5ee6fa1e2346fb37727328cc4634e670f341e
Parents: d8fb691 63fc124
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Aug 9 21:26:51 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Aug 9 21:26:51 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 8 +
.../org/apache/lucene/search/BlockMaxDISI.java | 94 +++++
.../lucene/search/Boolean2ScorerSupplier.java | 4 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../lucene/search/DisjunctionMaxQuery.java | 2 +-
.../lucene/search/DisjunctionMaxScorer.java | 80 ++--
.../apache/lucene/search/DisjunctionScorer.java | 22 +-
.../lucene/search/DisjunctionSumScorer.java | 4 +-
.../apache/lucene/search/ReqOptSumScorer.java | 175 +++++---
.../lucene/search/TestDisjunctionMaxQuery.java | 77 +++-
.../lucene/search/TestReqOptSumScorer.java | 241 ++++++++++-
lucene/ivy-versions.properties | 1 -
.../org/apache/lucene/document/LatLonShape.java | 13 +-
.../document/LatLonShapeBoundingBoxQuery.java | 420 +++++--------------
.../document/LatLonShapePolygonQuery.java | 301 ++-----------
.../lucene/document/LatLonShapeQuery.java | 394 +++++++++++++++++
.../document/BaseLatLonShapeTestCase.java | 46 +-
.../document/TestLatLonLineShapeQueries.java | 40 +-
.../document/TestLatLonPointShapeQueries.java | 20 +-
.../document/TestLatLonPolygonShapeQueries.java | 36 +-
.../apache/lucene/document/TestLatLonShape.java | 3 +-
.../lucene/search/RandomApproximationQuery.java | 10 +-
solr/CHANGES.txt | 20 +
.../configsets/collection1/conf/solrconfig.xml | 20 +-
solr/core/ivy.xml | 1 -
.../org/apache/solr/core/ConfigSetService.java | 7 +-
.../org/apache/solr/core/CoreContainer.java | 3 +
.../handler/admin/ZookeeperStatusHandler.java | 222 ++++++++++
.../solr/handler/export/DoubleValueSortDoc.java | 93 ++++
.../solr/handler/export/ExportWriter.java | 13 +-
.../solr/handler/export/QuadValueSortDoc.java | 126 ++++++
.../solr/handler/export/SingleValueSortDoc.java | 82 ++++
.../org/apache/solr/handler/export/SortDoc.java | 7 +-
.../solr/handler/export/TripleValueSortDoc.java | 110 +++++
.../apache/solr/handler/loader/XMLLoader.java | 18 +-
.../ParseDateFieldUpdateProcessorFactory.java | 82 +++-
...dd-schema-fields-update-processor-chains.xml | 20 +-
...lrconfig-parsing-update-processor-chains.xml | 49 +--
.../collection1/conf/solrconfig-schemaless.xml | 20 +-
.../configsets/_default/conf/solrconfig.xml | 20 +-
.../autoscaling/AutoScalingHandlerTest.java | 43 +-
.../solr/handler/admin/MetricsHandlerTest.java | 4 +-
.../admin/ZookeeperStatusHandlerTest.java | 85 ++++
.../apache/solr/update/AddBlockUpdateTest.java | 148 +++++++
...dSchemaFieldsUpdateProcessorFactoryTest.java | 23 +-
.../ParsingFieldUpdateProcessorsTest.java | 157 +++----
solr/example/files/conf/solrconfig.xml | 20 +-
solr/licenses/joda-time-2.2.jar.sha1 | 1 -
solr/licenses/joda-time-LICENSE-ASL.txt | 202 ---------
solr/licenses/joda-time-NOTICE.txt | 5 -
.../configsets/_default/conf/solrconfig.xml | 20 +-
solr/solr-ref-guide/src/cloud-screens.adoc | 5 +
.../src/images/cloud-screens/cloud-zkstatus.png | Bin 0 -> 175359 bytes
solr/solr-ref-guide/src/schemaless-mode.adoc | 20 +-
...solrcloud-autoscaling-auto-add-replicas.adoc | 6 +-
.../src/solrcloud-autoscaling-overview.adoc | 2 +-
...olrcloud-autoscaling-policy-preferences.adoc | 167 +++++---
.../src/transforming-result-documents.adoc | 9 +-
.../cloud/autoscaling/AddReplicaSuggester.java | 10 +-
.../client/solrj/cloud/autoscaling/Clause.java | 18 +-
.../cloud/autoscaling/MoveReplicaSuggester.java | 3 +-
.../solrj/cloud/autoscaling/NodeVariable.java | 5 +-
.../client/solrj/cloud/autoscaling/Policy.java | 2 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 38 +-
.../solrj/cloud/autoscaling/RangeVal.java | 10 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 6 +-
.../solrj/cloud/autoscaling/ReplicaInfo.java | 17 +-
.../solrj/cloud/autoscaling/Suggester.java | 14 +-
.../solrj/cloud/autoscaling/VariableBase.java | 1 +
.../solrj/cloud/autoscaling/Violation.java | 10 +-
.../solrj/impl/SolrClientNodeStateProvider.java | 2 +-
.../org/apache/solr/client/solrj/io/Lang.java | 1 +
.../solr/client/solrj/io/eval/KnnEvaluator.java | 18 +-
.../solrj/io/eval/KnnRegressionEvaluator.java | 194 +++++++++
.../solrj/io/eval/MinMaxScaleEvaluator.java | 2 +-
.../client/solrj/io/eval/PolyFitEvaluator.java | 7 +-
.../client/solrj/io/eval/PredictEvaluator.java | 30 +-
.../solr/client/solrj/util/ClientUtils.java | 20 +-
.../solr/common/ConditionalMapWriter.java | 75 ++++
.../java/org/apache/solr/common/MapWriter.java | 6 +
.../apache/solr/common/params/CommonParams.java | 1 +
.../java/org/apache/solr/common/util/XML.java | 98 ++---
.../solrj/cloud/autoscaling/TestPolicy.java | 16 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 240 +++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 86 +++-
.../apache/solr/cloud/SolrCloudTestCase.java | 19 +
.../org/apache/solr/util/BaseTestHarness.java | 23 +-
solr/webapp/web/css/angular/cloud.css | 71 +++-
solr/webapp/web/css/angular/menu.css | 1 +
solr/webapp/web/index.html | 1 +
solr/webapp/web/js/angular/controllers/cloud.js | 42 +-
solr/webapp/web/js/angular/services.js | 6 +
solr/webapp/web/partials/cloud.html | 59 +++
94 files changed, 3535 insertions(+), 1442 deletions(-)
----------------------------------------------------------------------
[10/17] lucene-solr:jira/solr-12470: LUCENE-8447: Add DISJOINT and
WITHIN support to LatLonShape queries
Posted by ab...@apache.org.
LUCENE-8447: Add DISJOINT and WITHIN support to LatLonShape queries
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cbaedb47
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cbaedb47
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cbaedb47
Branch: refs/heads/jira/solr-12470
Commit: cbaedb470c8ddc7b21e8ff0b2729c6cf97fbd3d0
Parents: 9b418a4
Author: Nicholas Knize <nk...@gmail.com>
Authored: Mon Aug 6 16:31:21 2018 -0500
Committer: Nicholas Knize <nk...@gmail.com>
Committed: Wed Aug 8 12:50:05 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
.../org/apache/lucene/document/LatLonShape.java | 13 +-
.../document/LatLonShapeBoundingBoxQuery.java | 420 +++++--------------
.../document/LatLonShapePolygonQuery.java | 301 ++-----------
.../lucene/document/LatLonShapeQuery.java | 394 +++++++++++++++++
.../document/BaseLatLonShapeTestCase.java | 46 +-
.../document/TestLatLonLineShapeQueries.java | 40 +-
.../document/TestLatLonPointShapeQueries.java | 20 +-
.../document/TestLatLonPolygonShapeQueries.java | 36 +-
.../apache/lucene/document/TestLatLonShape.java | 3 +-
10 files changed, 659 insertions(+), 616 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 31c8988..f1e6ebb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -225,6 +225,8 @@ Changes in Runtime Behavior:
Improvements
+* LUCENE-8447: Add DISJOINT and WITHIN support to LatLonShape queries. (Nick Knize)
+
* LUCENE-8440: Add support for indexing and searching Line and Point shapes using LatLonShape encoding (Nick Knize)
* LUCENE-8435: Add new LatLonShapePolygonQuery for querying indexed LatLonShape fields by arbitrary polygons (Nick Knize)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
index 01a31ad..3ac171f9 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShape.java
@@ -131,12 +131,12 @@ public class LatLonShape {
* note: does not currently support dateline crossing boxes
* todo split dateline crossing boxes into two queries like {@link LatLonPoint#newBoxQuery}
**/
- public static Query newBoxQuery(String field, double minLatitude, double maxLatitude, double minLongitude, double maxLongitude) {
- return new LatLonShapeBoundingBoxQuery(field, minLatitude, maxLatitude, minLongitude, maxLongitude);
+ public static Query newBoxQuery(String field, QueryRelation queryRelation, double minLatitude, double maxLatitude, double minLongitude, double maxLongitude) {
+ return new LatLonShapeBoundingBoxQuery(field, queryRelation, minLatitude, maxLatitude, minLongitude, maxLongitude);
}
- public static Query newPolygonQuery(String field, Polygon... polygons) {
- return new LatLonShapePolygonQuery(field, polygons);
+ public static Query newPolygonQuery(String field, QueryRelation queryRelation, Polygon... polygons) {
+ return new LatLonShapePolygonQuery(field, queryRelation, polygons);
}
/** polygons are decomposed into tessellated triangles using {@link org.apache.lucene.geo.Tessellator}
@@ -167,4 +167,9 @@ public class LatLonShape {
NumericUtils.intToSortableBytes(cX, bytes, BYTES * 5);
}
}
+
+ /** Query Relation Types **/
+ public enum QueryRelation {
+ INTERSECTS, WITHIN, DISJOINT
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
index 9779210..e797043 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeBoundingBoxQuery.java
@@ -16,29 +16,11 @@
*/
package org.apache.lucene.document;
-import java.io.IOException;
import java.util.Arrays;
-import java.util.Objects;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Tessellator;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.ScorerSupplier;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.DocIdSetBuilder;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.FutureArrays;
import org.apache.lucene.util.NumericUtils;
@@ -58,19 +40,18 @@ import static org.apache.lucene.geo.GeoUtils.orient;
*
* @lucene.experimental
**/
-class LatLonShapeBoundingBoxQuery extends Query {
- final String field;
+final class LatLonShapeBoundingBoxQuery extends LatLonShapeQuery {
final byte[] bbox;
final int minX;
final int maxX;
final int minY;
final int maxY;
- public LatLonShapeBoundingBoxQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
+ public LatLonShapeBoundingBoxQuery(String field, LatLonShape.QueryRelation queryRelation, double minLat, double maxLat, double minLon, double maxLon) {
+ super(field, queryRelation);
if (minLon > maxLon) {
throw new IllegalArgumentException("dateline crossing bounding box queries are not supported for [" + field + "]");
}
- this.field = field;
this.bbox = new byte[4 * LatLonPoint.BYTES];
this.minX = encodeLongitudeCeil(minLon);
this.maxX = encodeLongitude(maxLon);
@@ -83,301 +64,129 @@ class LatLonShapeBoundingBoxQuery extends Query {
}
@Override
- public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- return new ConstantScoreWeight(this, boost) {
-
- private boolean edgeIntersectsQuery(double ax, double ay, double bx, double by) {
- // top
- if (orient(ax, ay, bx, by, minX, maxY) * orient(ax, ay, bx, by, maxX, maxY) <= 0 &&
- orient(minX, maxY, maxX, maxY, ax, ay) * orient(minX, maxY, maxX, maxY, bx, by) <= 0) {
- return true;
- }
-
- // right
- if (orient(ax, ay, bx, by, maxX, maxY) * orient(ax, ay, bx, by, maxX, minY) <= 0 &&
- orient(maxX, maxY, maxX, minY, ax, ay) * orient(maxX, maxY, maxX, minY, bx, by) <= 0) {
- return true;
- }
-
- // bottom
- if (orient(ax, ay, bx, by, maxX, minY) * orient(ax, ay, bx, by, minX, minY) <= 0 &&
- orient(maxX, minY, minX, minY, ax, ay) * orient(maxX, minY, minX, minY, bx, by) <= 0) {
- return true;
- }
-
- // left
- if (orient(ax, ay, bx, by, minX, minY) * orient(ax, ay, bx, by, minX, maxY) <= 0 &&
- orient(minX, minY, minX, maxY, ax, ay) * orient(minX, minY, minX, maxY, bx, by) <= 0) {
- return true;
- }
- return false;
- }
-
- private boolean queryContains(byte[] t, int point) {
- final int yIdx = 2 * LatLonPoint.BYTES * point;
- final int xIdx = yIdx + LatLonPoint.BYTES;
-
- if (FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 0, LatLonPoint.BYTES) < 0 || //minY
- FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 || //maxY
- FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 || // minX
- FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, bbox.length) > 0) {
- return false;
- }
- return true;
- }
-
- private boolean queryIntersects(int ax, int ay, int bx, int by, int cx, int cy) {
- // check each edge of the triangle against the query
- if (edgeIntersectsQuery(ax, ay, bx, by) ||
- edgeIntersectsQuery(bx, by, cx, cy) ||
- edgeIntersectsQuery(cx, cy, ax, ay)) {
- return true;
- }
- return false;
- }
-
- private boolean queryCrossesTriangle(byte[] t) {
- // 1. query contains any triangle points
- if (queryContains(t, 0) || queryContains(t, 1) || queryContains(t, 2)) {
- return true;
- }
-
- int aY = NumericUtils.sortableBytesToInt(t, 0);
- int aX = NumericUtils.sortableBytesToInt(t, LatLonPoint.BYTES);
- int bY = NumericUtils.sortableBytesToInt(t, 2 * LatLonPoint.BYTES);
- int bX = NumericUtils.sortableBytesToInt(t, 3 * LatLonPoint.BYTES);
- int cY = NumericUtils.sortableBytesToInt(t, 4 * LatLonPoint.BYTES);
- int cX = NumericUtils.sortableBytesToInt(t, 5 * LatLonPoint.BYTES);
-
- int tMinX = StrictMath.min(StrictMath.min(aX, bX), cX);
- int tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX);
- int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY);
- int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY);
-
- // 2. check bounding boxes are disjoint
- if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) {
- return false;
- }
-
- // 3. check triangle contains any query points
- if (Tessellator.pointInTriangle(minX, minY, aX, aY, bX, bY, cX, cY)) {
- return true;
- } else if (Tessellator.pointInTriangle(maxX, minY, aX, aY, bX, bY, cX, cY)) {
- return true;
- } else if (Tessellator.pointInTriangle(maxX, maxY, aX, aY, bX, bY, cX, cY)) {
- return true;
- } else if (Tessellator.pointInTriangle(minX, maxY, aX, aY, bX, bY, cX, cY)) {
- return true;
- }
-
-
- // 4. last ditch effort: check crossings
- if (queryIntersects(aX, aY, bX, bY, cX, cY)) {
- return true;
- }
- return false;
- }
-
- private Relation relateRangeToQuery(byte[] minTriangle, byte[] maxTriangle) {
- // compute bounding box
- int minXOfs = 0;
- int minYOfs = 0;
- int maxXOfs = 0;
- int maxYOfs = 0;
- for (int d = 1; d < 3; ++d) {
- // check minX
- int aOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- int bOfs = (d * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minXOfs = d;
- }
- // check maxX
- aOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxXOfs = d;
- }
- // check minY
- aOfs = minYOfs * 2 * LatLonPoint.BYTES;
- bOfs = d * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minYOfs = d;
- }
- // check maxY
- aOfs = maxYOfs * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxYOfs = d;
- }
- }
- minXOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- maxXOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- minYOfs *= 2 * LatLonPoint.BYTES;
- maxYOfs *= 2 * LatLonPoint.BYTES;
-
- // check bounding box (DISJOINT)
- if (FutureArrays.compareUnsigned(minTriangle, minXOfs, minXOfs + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) > 0 ||
- FutureArrays.compareUnsigned(maxTriangle, maxXOfs, maxXOfs + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 ||
- FutureArrays.compareUnsigned(minTriangle, minYOfs, minYOfs + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 ||
- FutureArrays.compareUnsigned(maxTriangle, maxYOfs, maxYOfs + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) < 0) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
-
- if (FutureArrays.compareUnsigned(minTriangle, minXOfs, minXOfs + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) > 0 &&
- FutureArrays.compareUnsigned(maxTriangle, maxXOfs, maxXOfs + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) < 0 &&
- FutureArrays.compareUnsigned(minTriangle, minYOfs, minYOfs + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) > 0 &&
- FutureArrays.compareUnsigned(maxTriangle, maxYOfs, maxYOfs + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0) {
- return Relation.CELL_INSIDE_QUERY;
- }
- return Relation.CELL_CROSSES_QUERY;
- }
-
- private IntersectVisitor getIntersectVisitor(DocIdSetBuilder result) {
- return new IntersectVisitor() {
-
- DocIdSetBuilder.BulkAdder adder;
-
- @Override
- public void grow(int count) {
- adder = result.grow(count);
- }
-
- @Override
- public void visit(int docID) throws IOException {
- adder.add(docID);
- }
-
- @Override
- public void visit(int docID, byte[] t) throws IOException {
- if (queryCrossesTriangle(t)) {
- adder.add(docID);
- }
- }
+ protected Relation relateRangeBBoxToQuery(int minXOffset, int minYOffset, byte[] minTriangle,
+ int maxXOffset, int maxYOffset, byte[] maxTriangle) {
+ // check bounding box (DISJOINT)
+ if (FutureArrays.compareUnsigned(minTriangle, minXOffset, minXOffset + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(maxTriangle, maxXOffset, maxXOffset + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 ||
+ FutureArrays.compareUnsigned(minTriangle, minYOffset, minYOffset + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(maxTriangle, maxYOffset, maxYOffset + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
- @Override
- public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
- return relateRangeToQuery(minTriangle, maxTriangle);
- }
- };
- }
+ if (FutureArrays.compareUnsigned(minTriangle, minXOffset, minXOffset + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) > 0 &&
+ FutureArrays.compareUnsigned(maxTriangle, maxXOffset, maxXOffset + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, 4 * LatLonPoint.BYTES) < 0 &&
+ FutureArrays.compareUnsigned(minTriangle, minYOffset, minYOffset + LatLonPoint.BYTES, bbox, 0, LatLonPoint.BYTES) > 0 &&
+ FutureArrays.compareUnsigned(maxTriangle, maxYOffset, maxYOffset + LatLonPoint.BYTES, bbox, 2 * LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0) {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ return Relation.CELL_CROSSES_QUERY;
+ }
- /**
- * Create a visitor that clears documents that do NOT match the bounding box query.
- */
- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {
- return new IntersectVisitor() {
+ /** returns true if the query matches the encoded triangle */
+ @Override
+ protected boolean queryMatches(byte[] t) {
+ if (queryRelation == LatLonShape.QueryRelation.WITHIN) {
+ return queryContains(t, 0) && queryContains(t, 1) && queryContains(t, 2);
+ }
+ return queryIntersects(t);
+ }
- @Override
- public void visit(int docID) {
- result.clear(docID);
- cost[0]--;
- }
+ /** returns true if the query intersects the encoded triangle */
+ protected boolean queryIntersects(byte[] t) {
- @Override
- public void visit(int docID, byte[] packedTriangle) {
- if (queryCrossesTriangle(packedTriangle) == false) {
- result.clear(docID);
- cost[0]--;
- }
- }
+ // 1. query contains any triangle points
+ if (queryContains(t, 0) || queryContains(t, 1) || queryContains(t, 2)) {
+ return true;
+ }
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- Relation r = relateRangeToQuery(minPackedValue, maxPackedValue);
- if (r == Relation.CELL_OUTSIDE_QUERY) {
- return Relation.CELL_INSIDE_QUERY;
- } else if (r == Relation.CELL_INSIDE_QUERY) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- return r;
- }
- };
- }
+ int aY = NumericUtils.sortableBytesToInt(t, 0);
+ int aX = NumericUtils.sortableBytesToInt(t, LatLonPoint.BYTES);
+ int bY = NumericUtils.sortableBytesToInt(t, 2 * LatLonPoint.BYTES);
+ int bX = NumericUtils.sortableBytesToInt(t, 3 * LatLonPoint.BYTES);
+ int cY = NumericUtils.sortableBytesToInt(t, 4 * LatLonPoint.BYTES);
+ int cX = NumericUtils.sortableBytesToInt(t, 5 * LatLonPoint.BYTES);
+
+ int tMinX = StrictMath.min(StrictMath.min(aX, bX), cX);
+ int tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX);
+ int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY);
+ int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY);
+
+ // 2. check bounding boxes are disjoint
+ if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) {
+ return false;
+ }
- @Override
- public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
- LeafReader reader = context.reader();
- PointValues values = reader.getPointValues(field);
- if (values == null) {
- return null;
- }
+ // 3. check triangle contains any query points
+ if (Tessellator.pointInTriangle(minX, minY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ } else if (Tessellator.pointInTriangle(maxX, minY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ } else if (Tessellator.pointInTriangle(maxX, maxY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ } else if (Tessellator.pointInTriangle(minX, maxY, aX, aY, bX, bY, cX, cY)) {
+ return true;
+ }
- boolean allDocsMatch = true;
- if (values.getDocCount() != reader.maxDoc() ||
- relateRangeToQuery(values.getMinPackedValue(), values.getMaxPackedValue()) != Relation.CELL_INSIDE_QUERY) {
- allDocsMatch = false;
- }
- final Weight weight = this;
- if (allDocsMatch) {
- return new ScorerSupplier() {
- @Override
- public Scorer get(long leadCost) throws IOException {
- return new ConstantScoreScorer(weight, score(),
- DocIdSetIterator.all(reader.maxDoc()));
- }
+ // 4. last ditch effort: check crossings
+ if (queryIntersects(aX, aY, bX, bY, cX, cY)) {
+ return true;
+ }
+ return false;
+ }
- @Override
- public long cost() {
- return reader.maxDoc();
- }
- };
- } else {
- return new ScorerSupplier() {
- final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
- final IntersectVisitor visitor = getIntersectVisitor(result);
- long cost = -1;
+ /** returns true if the edge (defined by (ax, ay) (bx, by)) intersects the query */
+ private boolean edgeIntersectsQuery(double ax, double ay, double bx, double by) {
+ // top
+ if (orient(ax, ay, bx, by, minX, maxY) * orient(ax, ay, bx, by, maxX, maxY) <= 0 &&
+ orient(minX, maxY, maxX, maxY, ax, ay) * orient(minX, maxY, maxX, maxY, bx, by) <= 0) {
+ return true;
+ }
- @Override
- public Scorer get(long leadCost) throws IOException {
- if (values.getDocCount() == reader.maxDoc()
- && values.getDocCount() == values.size()
- && cost() > reader.maxDoc() / 2) {
- // If all docs have exactly one value and the cost is greater
- // than half the leaf size then maybe we can make things faster
- // by computing the set of documents that do NOT match the query
- final FixedBitSet result = new FixedBitSet(reader.maxDoc());
- result.set(0, reader.maxDoc());
- int[] cost = new int[]{reader.maxDoc()};
- values.intersect(getInverseIntersectVisitor(result, cost));
- final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
- return new ConstantScoreScorer(weight, score(), iterator);
- }
+ // right
+ if (orient(ax, ay, bx, by, maxX, maxY) * orient(ax, ay, bx, by, maxX, minY) <= 0 &&
+ orient(maxX, maxY, maxX, minY, ax, ay) * orient(maxX, maxY, maxX, minY, bx, by) <= 0) {
+ return true;
+ }
- values.intersect(visitor);
- DocIdSetIterator iterator = result.build().iterator();
- return new ConstantScoreScorer(weight, score(), iterator);
- }
+ // bottom
+ if (orient(ax, ay, bx, by, maxX, minY) * orient(ax, ay, bx, by, minX, minY) <= 0 &&
+ orient(maxX, minY, minX, minY, ax, ay) * orient(maxX, minY, minX, minY, bx, by) <= 0) {
+ return true;
+ }
- @Override
- public long cost() {
- if (cost == -1) {
- // Computing the cost may be expensive, so only do it if necessary
- cost = values.estimatePointCount(visitor);
- assert cost >= 0;
- }
- return cost;
- }
- };
- }
- }
+ // left
+ if (orient(ax, ay, bx, by, minX, minY) * orient(ax, ay, bx, by, minX, maxY) <= 0 &&
+ orient(minX, minY, minX, maxY, ax, ay) * orient(minX, minY, minX, maxY, bx, by) <= 0) {
+ return true;
+ }
+ return false;
+ }
- @Override
- public Scorer scorer(LeafReaderContext context) throws IOException {
- ScorerSupplier scorerSupplier = scorerSupplier(context);
- if (scorerSupplier == null) {
- return null;
- }
- return scorerSupplier.get(Long.MAX_VALUE);
- }
+ /** returns true if the query contains the triangle vertex */
+ private boolean queryContains(byte[] t, int point) {
+ final int yIdx = 2 * LatLonPoint.BYTES * point;
+ final int xIdx = yIdx + LatLonPoint.BYTES;
- @Override
- public boolean isCacheable(LeafReaderContext ctx) {
- return true;
- }
- };
+ if (FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 0, LatLonPoint.BYTES) < 0 || //minY
+ FutureArrays.compareUnsigned(t, yIdx, xIdx, bbox, 2 * LatLonPoint.BYTES, 3 * LatLonPoint.BYTES) > 0 || //maxY
+ FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES) < 0 || // minX
+ FutureArrays.compareUnsigned(t, xIdx, xIdx + LatLonPoint.BYTES, bbox, 3 * LatLonPoint.BYTES, bbox.length) > 0) {
+ return false;
+ }
+ return true;
}
- public String getField() {
- return field;
+ /** returns true if the query intersects the provided triangle (in encoded space) */
+ private boolean queryIntersects(int ax, int ay, int bx, int by, int cx, int cy) {
+ // check each edge of the triangle against the query
+ if (edgeIntersectsQuery(ax, ay, bx, by) ||
+ edgeIntersectsQuery(bx, by, cx, cy) ||
+ edgeIntersectsQuery(cx, cy, ax, ay)) {
+ return true;
+ }
+ return false;
}
@Override
@@ -385,15 +194,14 @@ class LatLonShapeBoundingBoxQuery extends Query {
return sameClassAs(o) && equalsTo(getClass().cast(o));
}
- private boolean equalsTo(LatLonShapeBoundingBoxQuery o) {
- return Objects.equals(field, o.field) &&
- Arrays.equals(bbox, o.bbox);
+ @Override
+ protected boolean equalsTo(Object o) {
+ return super.equalsTo(o) && Arrays.equals(bbox, ((LatLonShapeBoundingBoxQuery)o).bbox);
}
@Override
public int hashCode() {
- int hash = classHash();
- hash = 31 * hash + field.hashCode();
+ int hash = super.hashCode();
hash = 31 * hash + Arrays.hashCode(bbox);
return hash;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
index be47971..755946e 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapePolygonQuery.java
@@ -16,39 +16,13 @@
*/
package org.apache.lucene.document;
-import java.io.IOException;
import java.util.Arrays;
-import java.util.Objects;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.GeoEncodingUtils;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
-import org.apache.lucene.geo.Rectangle;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.PointValues;
-import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
-import org.apache.lucene.search.ConstantScoreScorer;
-import org.apache.lucene.search.ConstantScoreWeight;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.ScorerSupplier;
-import org.apache.lucene.search.Weight;
-import org.apache.lucene.util.BitSetIterator;
-import org.apache.lucene.util.DocIdSetBuilder;
-import org.apache.lucene.util.FixedBitSet;
-import org.apache.lucene.util.FutureArrays;
-import org.apache.lucene.util.NumericUtils;
-
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitudeCeil;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil;
/**
* Finds all previously indexed shapes that intersect the specified arbitrary.
@@ -58,15 +32,15 @@ import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitudeCeil;
*
* @lucene.experimental
**/
-public class LatLonShapePolygonQuery extends Query {
- final String field;
+final class LatLonShapePolygonQuery extends LatLonShapeQuery {
final Polygon[] polygons;
+ final private Polygon2D poly2D;
-
- public LatLonShapePolygonQuery(String field, Polygon... polygons) {
- if (field == null) {
- throw new IllegalArgumentException("field must not be null");
- }
+ /**
+ * Creates a query that matches all indexed shapes to the provided polygons
+ */
+ public LatLonShapePolygonQuery(String field, QueryRelation queryRelation, Polygon... polygons) {
+ super(field, queryRelation);
if (polygons == null) {
throw new IllegalArgumentException("polygons must not be null");
}
@@ -76,236 +50,40 @@ public class LatLonShapePolygonQuery extends Query {
for (int i = 0; i < polygons.length; i++) {
if (polygons[i] == null) {
throw new IllegalArgumentException("polygon[" + i + "] must not be null");
+ } else if (polygons[i].minLon > polygons[i].maxLon) {
+ throw new IllegalArgumentException("LatLonShapePolygonQuery does not currently support querying across dateline.");
}
}
- this.field = field;
this.polygons = polygons.clone();
+ this.poly2D = Polygon2D.create(polygons);
}
@Override
- public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- final Rectangle box = Rectangle.fromPolygon(polygons);
- final byte minLat[] = new byte[Integer.BYTES];
- final byte maxLat[] = new byte[Integer.BYTES];
- final byte minLon[] = new byte[Integer.BYTES];
- final byte maxLon[] = new byte[Integer.BYTES];
- NumericUtils.intToSortableBytes(encodeLatitudeCeil(box.minLat), minLat, 0);
- NumericUtils.intToSortableBytes(encodeLatitude(box.maxLat), maxLat, 0);
- NumericUtils.intToSortableBytes(encodeLongitudeCeil(box.minLon), minLon, 0);
- NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0);
-
- final Polygon2D polygon = Polygon2D.create(polygons);
-
- return new ConstantScoreWeight(this, boost) {
-
- private Relation relateRangeToQuery(byte[] minTriangle, byte[] maxTriangle) {
- // compute bounding box
- int minXOfs = 0;
- int minYOfs = 0;
- int maxXOfs = 0;
- int maxYOfs = 0;
- for (int d = 1; d < 3; ++d) {
- // check minX
- int aOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- int bOfs = (d * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minXOfs = d;
- }
- // check maxX
- aOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxXOfs = d;
- }
- // check minY
- aOfs = minYOfs * 2 * LatLonPoint.BYTES;
- bOfs = d * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
- minYOfs = d;
- }
- // check maxY
- aOfs = maxYOfs * 2 * LatLonPoint.BYTES;
- if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
- maxYOfs = d;
- }
- }
- minXOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- maxXOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
- minYOfs *= 2 * LatLonPoint.BYTES;
- maxYOfs *= 2 * LatLonPoint.BYTES;
-
- double minLat = GeoEncodingUtils.decodeLatitude(minTriangle, minYOfs);
- double minLon = GeoEncodingUtils.decodeLongitude(minTriangle, minXOfs);
- double maxLat = GeoEncodingUtils.decodeLatitude(maxTriangle, maxYOfs);
- double maxLon = GeoEncodingUtils.decodeLongitude(maxTriangle, maxXOfs);
-
- // check internal node against query
- return polygon.relate(minLat, maxLat, minLon, maxLon);
- }
-
- private boolean queryCrossesTriangle(byte[] t) {
- double ay = GeoEncodingUtils.decodeLatitude(t, 0);
- double ax = GeoEncodingUtils.decodeLongitude(t, LatLonPoint.BYTES);
- double by = GeoEncodingUtils.decodeLatitude(t, 2 * LatLonPoint.BYTES);
- double bx = GeoEncodingUtils.decodeLongitude(t, 3 * LatLonPoint.BYTES);
- double cy = GeoEncodingUtils.decodeLatitude(t, 4 * LatLonPoint.BYTES);
- double cx = GeoEncodingUtils.decodeLongitude(t, 5 * LatLonPoint.BYTES);
- return polygon.relateTriangle(ax, ay, bx, by, cx, cy) != Relation.CELL_OUTSIDE_QUERY;
- }
-
- private IntersectVisitor getIntersectVisitor(DocIdSetBuilder result) {
- return new IntersectVisitor() {
-
- DocIdSetBuilder.BulkAdder adder;
-
- @Override
- public void grow(int count) {
- adder = result.grow(count);
- }
-
- @Override
- public void visit(int docID) throws IOException {
- adder.add(docID);
- }
-
- @Override
- public void visit(int docID, byte[] t) throws IOException {
- if (queryCrossesTriangle(t)) {
- adder.add(docID);
- }
- }
-
- @Override
- public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
- return relateRangeToQuery(minTriangle, maxTriangle);
- }
- };
- }
-
- /**
- * Create a visitor that clears documents that do NOT match the polygon query.
- */
- private IntersectVisitor getInverseIntersectVisitor(FixedBitSet result, int[] cost) {
- return new IntersectVisitor() {
-
- @Override
- public void visit(int docID) {
- result.clear(docID);
- cost[0]--;
- }
-
- @Override
- public void visit(int docID, byte[] packedTriangle) {
- if (queryCrossesTriangle(packedTriangle) == false) {
- result.clear(docID);
- cost[0]--;
- }
- }
-
- @Override
- public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
- Relation r = relateRangeToQuery(minPackedValue, maxPackedValue);
- if (r == Relation.CELL_OUTSIDE_QUERY) {
- return Relation.CELL_INSIDE_QUERY;
- } else if (r == Relation.CELL_INSIDE_QUERY) {
- return Relation.CELL_OUTSIDE_QUERY;
- }
- return r;
- }
- };
- }
-
- @Override
- public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
- LeafReader reader = context.reader();
- PointValues values = reader.getPointValues(field);
- if (values == null) {
- // No docs in this segment had any points fields
- return null;
- }
- FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
- if (fieldInfo == null) {
- // No docs in this segment indexed this field at all
- return null;
- }
-
- boolean allDocsMatch = true;
- if (values.getDocCount() != reader.maxDoc() ||
- relateRangeToQuery(values.getMinPackedValue(), values.getMaxPackedValue()) != Relation.CELL_INSIDE_QUERY) {
- allDocsMatch = false;
- }
-
- final Weight weight = this;
- if (allDocsMatch) {
- return new ScorerSupplier() {
- @Override
- public Scorer get(long leadCost) throws IOException {
- return new ConstantScoreScorer(weight, score(),
- DocIdSetIterator.all(reader.maxDoc()));
- }
-
- @Override
- public long cost() {
- return reader.maxDoc();
- }
- };
- } else {
- return new ScorerSupplier() {
- final DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
- final IntersectVisitor visitor = getIntersectVisitor(result);
- long cost = -1;
-
- @Override
- public Scorer get(long leadCost) throws IOException {
- if (values.getDocCount() == reader.maxDoc()
- && values.getDocCount() == values.size()
- && cost() > reader.maxDoc() / 2) {
- // If all docs have exactly one value and the cost is greater
- // than half the leaf size then maybe we can make things faster
- // by computing the set of documents that do NOT match the query
- final FixedBitSet result = new FixedBitSet(reader.maxDoc());
- result.set(0, reader.maxDoc());
- int[] cost = new int[]{reader.maxDoc()};
- values.intersect(getInverseIntersectVisitor(result, cost));
- final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
- return new ConstantScoreScorer(weight, score(), iterator);
- }
-
- values.intersect(visitor);
- DocIdSetIterator iterator = result.build().iterator();
- return new ConstantScoreScorer(weight, score(), iterator);
- }
-
- @Override
- public long cost() {
- if (cost == -1) {
- // Computing the cost may be expensive, so only do it if necessary
- cost = values.estimatePointCount(visitor);
- assert cost >= 0;
- }
- return cost;
- }
- };
- }
- }
-
- @Override
- public Scorer scorer(LeafReaderContext context) throws IOException {
- ScorerSupplier scorerSupplier = scorerSupplier(context);
- if (scorerSupplier == null) {
- return null;
- }
- return scorerSupplier.get(Long.MAX_VALUE);
- }
-
- @Override
- public boolean isCacheable(LeafReaderContext ctx) {
- return true;
- }
- };
+ protected Relation relateRangeBBoxToQuery(int minXOffset, int minYOffset, byte[] minTriangle,
+ int maxXOffset, int maxYOffset, byte[] maxTriangle) {
+ double minLat = GeoEncodingUtils.decodeLatitude(minTriangle, minYOffset);
+ double minLon = GeoEncodingUtils.decodeLongitude(minTriangle, minXOffset);
+ double maxLat = GeoEncodingUtils.decodeLatitude(maxTriangle, maxYOffset);
+ double maxLon = GeoEncodingUtils.decodeLongitude(maxTriangle, maxXOffset);
+
+ // check internal node against query
+ return poly2D.relate(minLat, maxLat, minLon, maxLon);
}
- public String getField() {
- return field;
+ @Override
+ protected boolean queryMatches(byte[] triangle) {
+ double ay = GeoEncodingUtils.decodeLatitude(triangle, 0);
+ double ax = GeoEncodingUtils.decodeLongitude(triangle, LatLonPoint.BYTES);
+ double by = GeoEncodingUtils.decodeLatitude(triangle, 2 * LatLonPoint.BYTES);
+ double bx = GeoEncodingUtils.decodeLongitude(triangle, 3 * LatLonPoint.BYTES);
+ double cy = GeoEncodingUtils.decodeLatitude(triangle, 4 * LatLonPoint.BYTES);
+ double cx = GeoEncodingUtils.decodeLongitude(triangle, 5 * LatLonPoint.BYTES);
+
+ if (queryRelation == QueryRelation.WITHIN) {
+ return poly2D.relateTriangle(ax, ay, bx, by, cx, cy) == Relation.CELL_INSIDE_QUERY;
+ }
+ // INTERSECTS
+ return poly2D.relateTriangle(ax, ay, bx, by, cx, cy) != Relation.CELL_OUTSIDE_QUERY;
}
@Override
@@ -323,18 +101,13 @@ public class LatLonShapePolygonQuery extends Query {
}
@Override
- public boolean equals(Object o) {
- return sameClassAs(o) && equalsTo(getClass().cast(o));
- }
-
- private boolean equalsTo(LatLonShapePolygonQuery o) {
- return Objects.equals(field, o.field) && Arrays.equals(polygons, o.polygons);
+ protected boolean equalsTo(Object o) {
+ return super.equalsTo(o) && Arrays.equals(polygons, ((LatLonShapePolygonQuery)o).polygons);
}
@Override
public int hashCode() {
- int hash = classHash();
- hash = 31 * hash + field.hashCode();
+ int hash = super.hashCode();
hash = 31 * hash + Arrays.hashCode(polygons);
return hash;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java
new file mode 100644
index 0000000..c678941
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/document/LatLonShapeQuery.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.io.IOException;
+import java.util.Objects;
+
+import org.apache.lucene.document.LatLonShape.QueryRelation;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.PointValues.IntersectVisitor;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.BitSetIterator;
+import org.apache.lucene.util.DocIdSetBuilder;
+import org.apache.lucene.util.FixedBitSet;
+import org.apache.lucene.util.FutureArrays;
+
+/**
+ * Base LatLonShape Query class providing common query logic for
+ * {@link LatLonShapeBoundingBoxQuery} and {@link LatLonShapePolygonQuery}
+ *
+ * Note: this class implements the majority of the INTERSECTS, WITHIN, DISJOINT relation logic
+ *
+ * @lucene.experimental
+ **/
+abstract class LatLonShapeQuery extends Query {
+ /** field name */
+ final String field;
+ /** query relation
+ * disjoint: {@code CELL_OUTSIDE_QUERY}
+ * intersects: {@code CELL_CROSSES_QUERY},
+ * within: {@code CELL_WITHIN_QUERY} */
+ final LatLonShape.QueryRelation queryRelation;
+
+ protected LatLonShapeQuery(String field, final QueryRelation queryType) {
+ if (field == null) {
+ throw new IllegalArgumentException("field must not be null");
+ }
+ this.field = field;
+ this.queryRelation = queryType;
+ }
+
+ /**
+ * relates an internal node (bounding box of a range of triangles) to the target query
+ * Note: logic is specific to query type
+ * see {@link LatLonShapeBoundingBoxQuery#relateRangeToQuery} and {@link LatLonShapePolygonQuery#relateRangeToQuery}
+ */
+ protected abstract Relation relateRangeBBoxToQuery(int minXOffset, int minYOffset, byte[] minTriangle,
+ int maxXOffset, int maxYOffset, byte[] maxTriangle);
+
+ /** returns true if the provided triangle matches the query */
+ protected abstract boolean queryMatches(byte[] triangle);
+
+ /** relates a range of triangles (internal node) to the query */
+ protected Relation relateRangeToQuery(byte[] minTriangle, byte[] maxTriangle) {
+ // compute bounding box of internal node
+ int minXOfs = 0;
+ int minYOfs = 0;
+ int maxXOfs = 0;
+ int maxYOfs = 0;
+ for (int d = 1; d < 3; ++d) {
+ // check minX
+ int aOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ int bOfs = (d * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
+ minXOfs = d;
+ }
+ // check maxX
+ aOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
+ maxXOfs = d;
+ }
+ // check minY
+ aOfs = minYOfs * 2 * LatLonPoint.BYTES;
+ bOfs = d * 2 * LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(minTriangle, bOfs, bOfs + LatLonPoint.BYTES, minTriangle, aOfs, aOfs + LatLonPoint.BYTES) < 0) {
+ minYOfs = d;
+ }
+ // check maxY
+ aOfs = maxYOfs * 2 * LatLonPoint.BYTES;
+ if (FutureArrays.compareUnsigned(maxTriangle, bOfs, bOfs + LatLonPoint.BYTES, maxTriangle, aOfs, aOfs + LatLonPoint.BYTES) > 0) {
+ maxYOfs = d;
+ }
+ }
+ minXOfs = (minXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ maxXOfs = (maxXOfs * 2 * LatLonPoint.BYTES) + LatLonPoint.BYTES;
+ minYOfs *= 2 * LatLonPoint.BYTES;
+ maxYOfs *= 2 * LatLonPoint.BYTES;
+
+ Relation r = relateRangeBBoxToQuery(minXOfs, minYOfs, minTriangle, maxXOfs, maxYOfs, maxTriangle);
+
+ if (queryRelation == QueryRelation.DISJOINT) {
+ return transposeRelation(r);
+ }
+ return r;
+ }
+
+ @Override
+ public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+
+ return new ConstantScoreWeight(this, boost) {
+
+ /** create a visitor that adds documents that match the query using a sparse bitset. (Used by INTERSECT) */
+ protected IntersectVisitor getSparseIntersectVisitor(DocIdSetBuilder result) {
+ return new IntersectVisitor() {
+ DocIdSetBuilder.BulkAdder adder;
+
+ @Override
+ public void grow(int count) {
+ adder = result.grow(count);
+ }
+
+ @Override
+ public void visit(int docID) throws IOException {
+ adder.add(docID);
+ }
+
+ @Override
+ public void visit(int docID, byte[] t) throws IOException {
+ if (queryMatches(t)) {
+ adder.add(docID);
+ }
+ }
+
+ @Override
+ public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
+ return relateRangeToQuery(minTriangle, maxTriangle);
+ }
+ };
+ }
+
+ /** create a visitor that adds documents that match the query using a dense bitset. (Used by WITHIN, DISJOINT) */
+ protected IntersectVisitor getDenseIntersectVisitor(FixedBitSet intersect, FixedBitSet disjoint) {
+ return new IntersectVisitor() {
+
+ @Override
+ public void visit(int docID) throws IOException {
+ if (queryRelation == QueryRelation.DISJOINT) {
+ // if DISJOINT query set the doc in the disjoint bitset
+ disjoint.set(docID);
+ } else {
+ // for INTERSECT, and WITHIN queries we set the intersect bitset
+ intersect.set(docID);
+ }
+ }
+
+ @Override
+ public void visit(int docID, byte[] t) throws IOException {
+ if (queryMatches(t)) {
+ intersect.set(docID);
+ } else {
+ disjoint.set(docID);
+ }
+ }
+
+ @Override
+ public Relation compare(byte[] minTriangle, byte[] maxTriangle) {
+ return relateRangeToQuery(minTriangle, maxTriangle);
+ }
+ };
+ }
+
+ /** get a scorer supplier for INTERSECT queries */
+ protected ScorerSupplier getIntersectScorerSupplier(LeafReader reader, PointValues values, Weight weight) throws IOException {
+ DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
+ IntersectVisitor visitor = getSparseIntersectVisitor(result);
+ return new RelationScorerSupplier(values, visitor) {
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ return getIntersectsScorer(LatLonShapeQuery.this, reader, weight, result, score());
+ }
+ };
+ }
+
+ /** get a scorer supplier for all other queries (DISJOINT, WITHIN) */
+ protected ScorerSupplier getScorerSupplier(LeafReader reader, PointValues values, Weight weight) throws IOException {
+ if (queryRelation == QueryRelation.INTERSECTS) {
+ return getIntersectScorerSupplier(reader, values, weight);
+ }
+
+ FixedBitSet intersect = new FixedBitSet(reader.maxDoc());
+ FixedBitSet disjoint = new FixedBitSet(reader.maxDoc());
+ IntersectVisitor visitor = getDenseIntersectVisitor(intersect, disjoint);
+ return new RelationScorerSupplier(values, visitor) {
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ return getScorer(LatLonShapeQuery.this, weight, intersect, disjoint, score());
+ }
+ };
+ }
+
+ @Override
+ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+ LeafReader reader = context.reader();
+ PointValues values = reader.getPointValues(field);
+ if (values == null) {
+ // No docs in this segment had any points fields
+ return null;
+ }
+ FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+ if (fieldInfo == null) {
+ // No docs in this segment indexed this field at all
+ return null;
+ }
+
+ boolean allDocsMatch = true;
+ if (values.getDocCount() != reader.maxDoc() ||
+ relateRangeToQuery(values.getMinPackedValue(), values.getMaxPackedValue()) != Relation.CELL_INSIDE_QUERY) {
+ allDocsMatch = false;
+ }
+
+ final Weight weight = this;
+ if (allDocsMatch) {
+ return new ScorerSupplier() {
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ return new ConstantScoreScorer(weight, score(),
+ DocIdSetIterator.all(reader.maxDoc()));
+ }
+
+ @Override
+ public long cost() {
+ return reader.maxDoc();
+ }
+ };
+ } else {
+ return getScorerSupplier(reader, values, weight);
+ }
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ ScorerSupplier scorerSupplier = scorerSupplier(context);
+ if (scorerSupplier == null) {
+ return null;
+ }
+ return scorerSupplier.get(Long.MAX_VALUE);
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return true;
+ }
+ };
+ }
+
+ /** returns the field name */
+ public String getField() {
+ return field;
+ }
+
+ /** returns the query relation */
+ public QueryRelation getQueryRelation() {
+ return queryRelation;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = classHash();
+ hash = 31 * hash + field.hashCode();
+ hash = 31 * hash + queryRelation.hashCode();
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ return sameClassAs(o) && equalsTo(o);
+ }
+
+ protected boolean equalsTo(Object o) {
+ return Objects.equals(field, ((LatLonShapeQuery)o).field) && this.queryRelation == ((LatLonShapeQuery)o).queryRelation;
+ }
+
+ /** transpose the relation; INSIDE becomes OUTSIDE, OUTSIDE becomes INSIDE, CROSSES remains unchanged */
+ private static Relation transposeRelation(Relation r) {
+ if (r == Relation.CELL_INSIDE_QUERY) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ } else if (r == Relation.CELL_OUTSIDE_QUERY) {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ return Relation.CELL_CROSSES_QUERY;
+ }
+
+ /** utility class for implementing constant score logic specifig to INTERSECT, WITHIN, and DISJOINT */
+ protected static abstract class RelationScorerSupplier extends ScorerSupplier {
+ PointValues values;
+ IntersectVisitor visitor;
+ long cost = -1;
+
+ RelationScorerSupplier(PointValues values, IntersectVisitor visitor) {
+ this.values = values;
+ this.visitor = visitor;
+ }
+
+ /** create a visitor that clears documents that do NOT match the polygon query; used with INTERSECTS */
+ private IntersectVisitor getInverseIntersectVisitor(LatLonShapeQuery query, FixedBitSet result, int[] cost) {
+ return new IntersectVisitor() {
+
+ @Override
+ public void visit(int docID) {
+ result.clear(docID);
+ cost[0]--;
+ }
+
+ @Override
+ public void visit(int docID, byte[] packedTriangle) {
+ if (query.queryMatches(packedTriangle) == false) {
+ result.clear(docID);
+ cost[0]--;
+ }
+ }
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+ return transposeRelation(query.relateRangeToQuery(minPackedValue, maxPackedValue));
+ }
+ };
+ }
+
+ /** returns a Scorer for INTERSECT queries that uses a sparse bitset */
+ protected Scorer getIntersectsScorer(LatLonShapeQuery query, LeafReader reader, Weight weight,
+ DocIdSetBuilder docIdSetBuilder, final float boost) throws IOException {
+ if (values.getDocCount() == reader.maxDoc()
+ && values.getDocCount() == values.size()
+ && cost() > reader.maxDoc() / 2) {
+ // If all docs have exactly one value and the cost is greater
+ // than half the leaf size then maybe we can make things faster
+ // by computing the set of documents that do NOT match the query
+ final FixedBitSet result = new FixedBitSet(reader.maxDoc());
+ result.set(0, reader.maxDoc());
+ int[] cost = new int[]{reader.maxDoc()};
+ values.intersect(getInverseIntersectVisitor(query, result, cost));
+ final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
+ return new ConstantScoreScorer(weight, boost, iterator);
+ }
+
+ values.intersect(visitor);
+ DocIdSetIterator iterator = docIdSetBuilder.build().iterator();
+ return new ConstantScoreScorer(weight, boost, iterator);
+ }
+
+ /** returns a Scorer for all other (non INTERSECT) queries */
+ protected Scorer getScorer(LatLonShapeQuery query, Weight weight,
+ FixedBitSet intersect, FixedBitSet disjoint, final float boost) throws IOException {
+ values.intersect(visitor);
+ DocIdSetIterator iterator;
+ if (query.queryRelation == QueryRelation.DISJOINT) {
+ disjoint.andNot(intersect);
+ iterator = new BitSetIterator(disjoint, cost());
+ } else if (query.queryRelation == QueryRelation.WITHIN) {
+ intersect.andNot(disjoint);
+ iterator = new BitSetIterator(intersect, cost());
+ } else {
+ iterator = new BitSetIterator(intersect, cost());
+ }
+ return new ConstantScoreScorer(weight, boost, iterator);
+ }
+
+ @Override
+ public long cost() {
+ if (cost == -1) {
+ // Computing the cost may be expensive, so only do it if necessary
+ cost = values.estimatePointCount(visitor);
+ assert cost >= 0;
+ }
+ return cost;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java b/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
index a7560ee..191e2cb 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/BaseLatLonShapeTestCase.java
@@ -22,6 +22,7 @@ import java.util.HashSet;
import java.util.Set;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
@@ -94,6 +95,16 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
return new Polygon(lats, lons);
}
+ protected Line quantizeLine(Line line) {
+ double[] lats = new double[line.numPoints()];
+ double[] lons = new double[line.numPoints()];
+ for (int i = 0; i < lats.length; ++i) {
+ lats[i] = quantizeLat(line.getLat(i));
+ lons[i] = quantizeLon(line.getLon(i));
+ }
+ return new Line(lats, lons);
+ }
+
protected abstract Field[] createIndexableFields(String field, Object shape);
private void addShapeToDoc(String field, Document doc, Object shape) {
@@ -103,12 +114,12 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
}
}
- protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
- return LatLonShape.newBoxQuery(field, minLat, maxLat, minLon, maxLon);
+ protected Query newRectQuery(String field, QueryRelation queryRelation, double minLat, double maxLat, double minLon, double maxLon) {
+ return LatLonShape.newBoxQuery(field, queryRelation, minLat, maxLat, minLon, maxLon);
}
- protected Query newPolygonQuery(String field, Polygon... polygons) {
- return LatLonShape.newPolygonQuery(field, polygons);
+ protected Query newPolygonQuery(String field, QueryRelation queryRelation, Polygon... polygons) {
+ return LatLonShape.newPolygonQuery(field, queryRelation, polygons);
}
// A particularly tricky adversary for BKD tree:
@@ -240,7 +251,8 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
break;
}
}
- Query query = newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon);
+ QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
+ Query query = newRectQuery(FIELD_NAME, queryRelation, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon);
if (VERBOSE) {
System.out.println(" query=" + query);
@@ -280,7 +292,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
expected = false;
} else {
// check quantized poly against quantized query
- expected = getValidator().testBBoxQuery(quantizeLatCeil(rect.minLat), quantizeLat(rect.maxLat),
+ expected = getValidator(queryRelation).testBBoxQuery(quantizeLatCeil(rect.minLat), quantizeLat(rect.maxLat),
quantizeLonCeil(rect.minLon), quantizeLon(rect.maxLon), shapes[id]);
}
@@ -292,10 +304,11 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
+ b.append(" relation=" + queryRelation + "\n");
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" shape=" + shapes[id] + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
- b.append(" rect=Rectangle(" + quantizeLatCeil(rect.minLat) + " TO " + quantizeLat(rect.maxLat) + " lon=" + quantizeLonCeil(rect.minLon) + " TO " + quantizeLon(rect.maxLon) + ")");
+ b.append(" rect=Rectangle(" + quantizeLatCeil(rect.minLat) + " TO " + quantizeLat(rect.maxLat) + " lon=" + quantizeLonCeil(rect.minLon) + " TO " + quantizeLon(rect.maxLon) + ")\n");
if (true) {
fail("wrong hit (first of possibly more):\n\n" + b);
} else {
@@ -326,7 +339,8 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
// Polygon
Polygon queryPolygon = GeoTestUtil.nextPolygon();
Polygon2D queryPoly2D = Polygon2D.create(queryPolygon);
- Query query = newPolygonQuery(FIELD_NAME, queryPolygon);
+ QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
+ Query query = newPolygonQuery(FIELD_NAME, queryRelation, queryPolygon);
if (VERBOSE) {
System.out.println(" query=" + query);
@@ -365,7 +379,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
} else if (shapes[id] == null) {
expected = false;
} else {
- expected = getValidator().testPolygonQuery(queryPoly2D, shapes[id]);
+ expected = getValidator(queryRelation).testPolygonQuery(queryPoly2D, shapes[id]);
}
if (hits.get(docID) != expected) {
@@ -376,6 +390,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
} else {
b.append("FAIL: id=" + id + " should not match but did\n");
}
+ b.append(" relation=" + queryRelation + "\n");
b.append(" query=" + query + " docID=" + docID + "\n");
b.append(" shape=" + shapes[id] + "\n");
b.append(" deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
@@ -394,7 +409,7 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
}
}
- protected abstract Validator getValidator();
+ protected abstract Validator getValidator(QueryRelation relation);
/** internal point class for testing point shapes */
protected static class Point {
@@ -466,8 +481,13 @@ public abstract class BaseLatLonShapeTestCase extends LuceneTestCase {
}
}
- protected interface Validator {
- boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape);
- boolean testPolygonQuery(Polygon2D poly2d, Object shape);
+ protected abstract class Validator {
+ protected QueryRelation queryRelation = QueryRelation.INTERSECTS;
+ public abstract boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape);
+ public abstract boolean testPolygonQuery(Polygon2D poly2d, Object shape);
+
+ public void setRelation(QueryRelation relation) {
+ this.queryRelation = relation;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
index 21367dc..9a91232 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonLineShapeQueries.java
@@ -16,16 +16,12 @@
*/
package org.apache.lucene.document;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.index.PointValues.Relation;
-import static org.apache.lucene.geo.GeoEncodingUtils.decodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.decodeLongitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLatitude;
-import static org.apache.lucene.geo.GeoEncodingUtils.encodeLongitude;
-
/** random bounding box and polygon query tests for random generated {@link Line} types */
public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
@@ -42,17 +38,25 @@ public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
}
@Override
- protected Validator getValidator() {
+ protected Validator getValidator(QueryRelation queryRelation) {
+ VALIDATOR.setRelation(queryRelation);
return VALIDATOR;
}
- protected class LineValidator implements Validator {
+ protected class LineValidator extends Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
+ Line l = (Line)shape;
+ if (queryRelation == QueryRelation.WITHIN) {
+ // within: bounding box of shape should be within query box
+ return minLat <= quantizeLat(l.minLat) && maxLat >= quantizeLat(l.maxLat)
+ && minLon <= quantizeLon(l.minLon) && maxLon >= quantizeLon(l.maxLon);
+ }
+
// to keep it simple we convert the bbox into a polygon and use poly2d
Polygon2D p = Polygon2D.create(new Polygon[] {new Polygon(new double[] {minLat, minLat, maxLat, maxLat, minLat},
new double[] {minLon, maxLon, maxLon, minLon, minLon})});
- return testLine(p, (Line)shape);
+ return testLine(p, l);
}
@Override
@@ -62,11 +66,12 @@ public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
private boolean testLine(Polygon2D queryPoly, Line line) {
double ax, ay, bx, by, temp;
+ Relation r;
for (int i = 0, j = 1; j < line.numPoints(); ++i, ++j) {
- ay = decodeLatitude(encodeLatitude(line.getLat(i)));
- ax = decodeLongitude(encodeLongitude(line.getLon(i)));
- by = decodeLatitude(encodeLatitude(line.getLat(j)));
- bx = decodeLongitude(encodeLongitude(line.getLon(j)));
+ ay = quantizeLat(line.getLat(i));
+ ax = quantizeLon(line.getLon(i));
+ by = quantizeLat(line.getLat(j));
+ bx = quantizeLon(line.getLon(j));
if (ay > by) {
temp = ay;
ay = by;
@@ -84,11 +89,16 @@ public class TestLatLonLineShapeQueries extends BaseLatLonShapeTestCase {
bx = temp;
}
}
- if (queryPoly.relateTriangle(ax, ay, bx, by, ax, ay) != Relation.CELL_OUTSIDE_QUERY) {
- return true;
+ r = queryPoly.relateTriangle(ax, ay, bx, by, ax, ay);
+ if (queryRelation == QueryRelation.DISJOINT) {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return false;
+ } else if (queryRelation == QueryRelation.WITHIN) {
+ if (r != Relation.CELL_INSIDE_QUERY) return false;
+ } else {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return true;
}
}
- return false;
+ return queryRelation == QueryRelation.INTERSECTS ? false : true;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
index 3adb26b..df924fe 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPointShapeQueries.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.document;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.index.PointValues.Relation;
@@ -41,17 +42,22 @@ public class TestLatLonPointShapeQueries extends BaseLatLonShapeTestCase {
}
@Override
- protected Validator getValidator() {
+ protected Validator getValidator(QueryRelation relation) {
+ VALIDATOR.setRelation(relation);
return VALIDATOR;
}
- protected class PointValidator implements Validator {
+ protected class PointValidator extends Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
Point p = (Point)shape;
double lat = decodeLatitude(encodeLatitude(p.lat));
double lon = decodeLongitude(encodeLongitude(p.lon));
- return (lat < minLat || lat > maxLat || lon < minLon || lon > maxLon) == false;
+ boolean isDisjoint = lat < minLat || lat > maxLat || lon < minLon || lon > maxLon;
+ if (queryRelation == QueryRelation.DISJOINT) {
+ return isDisjoint;
+ }
+ return isDisjoint == false;
}
@Override
@@ -60,7 +66,13 @@ public class TestLatLonPointShapeQueries extends BaseLatLonShapeTestCase {
double lat = decodeLatitude(encodeLatitude(p.lat));
double lon = decodeLongitude(encodeLongitude(p.lon));
// for consistency w/ the query we test the point as a triangle
- return poly2d.relateTriangle(lon, lat, lon, lat, lon, lat) != Relation.CELL_OUTSIDE_QUERY;
+ Relation r = poly2d.relateTriangle(lon, lat, lon, lat, lon, lat);
+ if (queryRelation == QueryRelation.WITHIN) {
+ return r == Relation.CELL_INSIDE_QUERY;
+ } else if (queryRelation == QueryRelation.DISJOINT) {
+ return r == Relation.CELL_OUTSIDE_QUERY;
+ }
+ return r != Relation.CELL_OUTSIDE_QUERY;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
index 17eb6e8..9b844ab 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
@@ -18,11 +18,13 @@ package org.apache.lucene.document;
import java.util.List;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.geo.Tessellator;
import org.apache.lucene.index.PointValues.Relation;
+/** random bounding box and polygon query tests for random indexed {@link Polygon} types */
public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
protected final PolygonValidator VALIDATOR = new PolygonValidator();
@@ -53,30 +55,46 @@ public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
}
@Override
- protected Validator getValidator() {
+ protected Validator getValidator(QueryRelation relation) {
+ VALIDATOR.setRelation(relation);
return VALIDATOR;
}
- protected class PolygonValidator implements Validator {
+ protected class PolygonValidator extends Validator {
@Override
public boolean testBBoxQuery(double minLat, double maxLat, double minLon, double maxLon, Object shape) {
- Polygon2D poly = Polygon2D.create(quantizePolygon((Polygon)shape));
- return poly.relate(minLat, maxLat, minLon, maxLon) != Relation.CELL_OUTSIDE_QUERY;
+ Polygon p = (Polygon)shape;
+ if (queryRelation == QueryRelation.WITHIN) {
+ // within: bounding box of shape should be within query box
+ return minLat <= quantizeLat(p.minLat) && maxLat >= quantizeLat(p.maxLat)
+ && minLon <= quantizeLon(p.minLon) && maxLon >= quantizeLon(p.maxLon);
+ }
+
+ Polygon2D poly = Polygon2D.create(quantizePolygon(p));
+ Relation r = poly.relate(minLat, maxLat, minLon, maxLon);
+ if (queryRelation == QueryRelation.DISJOINT) {
+ return r == Relation.CELL_OUTSIDE_QUERY;
+ }
+ return r != Relation.CELL_OUTSIDE_QUERY;
}
@Override
public boolean testPolygonQuery(Polygon2D query, Object shape) {
-
List<Tessellator.Triangle> tessellation = Tessellator.tessellate((Polygon) shape);
for (Tessellator.Triangle t : tessellation) {
// we quantize the triangle for consistency with the index
- if (query.relateTriangle(quantizeLon(t.getLon(0)), quantizeLat(t.getLat(0)),
+ Relation r = query.relateTriangle(quantizeLon(t.getLon(0)), quantizeLat(t.getLat(0)),
quantizeLon(t.getLon(1)), quantizeLat(t.getLat(1)),
- quantizeLon(t.getLon(2)), quantizeLat(t.getLat(2))) != Relation.CELL_OUTSIDE_QUERY) {
- return true;
+ quantizeLon(t.getLon(2)), quantizeLat(t.getLat(2)));
+ if (queryRelation == QueryRelation.DISJOINT) {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return false;
+ } else if (queryRelation == QueryRelation.WITHIN) {
+ if (r != Relation.CELL_INSIDE_QUERY) return false;
+ } else {
+ if (r != Relation.CELL_OUTSIDE_QUERY) return true;
}
}
- return false;
+ return queryRelation == QueryRelation.INTERSECTS ? false : true;
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cbaedb47/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
index 3aa5ace..2f6e5e1 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
@@ -17,6 +17,7 @@
package org.apache.lucene.document;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+import org.apache.lucene.document.LatLonShape.QueryRelation;
import org.apache.lucene.geo.GeoTestUtil;
import org.apache.lucene.geo.Line;
import org.apache.lucene.geo.Polygon;
@@ -52,7 +53,7 @@ public class TestLatLonShape extends LuceneTestCase {
}
protected Query newRectQuery(String field, double minLat, double maxLat, double minLon, double maxLon) {
- return LatLonShape.newBoxQuery(field, minLat, maxLat, minLon, maxLon);
+ return LatLonShape.newBoxQuery(field, QueryRelation.INTERSECTS, minLat, maxLat, minLon, maxLon);
}
@Ignore
[02/17] lucene-solr:jira/solr-12470: SOLR-12586: Change
ParseDateFieldUpdateProcessorFactory to use java.time.DateTimeFormatter,
not Joda Time. Note: slightly different pattern language! Remove Joda Time.
Closes #428
Posted by ab...@apache.org.
SOLR-12586: Change ParseDateFieldUpdateProcessorFactory to use java.time.DateTimeFormatter, not Joda Time.
Note: slightly different pattern language!
Remove Joda Time.
Closes #428
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c3887b35
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c3887b35
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c3887b35
Branch: refs/heads/jira/solr-12470
Commit: c3887b351e77ab6d23ddd662564b3d0d977895e4
Parents: 2ca5a6d
Author: David Smiley <ds...@apache.org>
Authored: Tue Aug 7 11:51:16 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Aug 7 11:51:16 2018 -0400
----------------------------------------------------------------------
lucene/ivy-versions.properties | 1 -
solr/CHANGES.txt | 10 +
.../configsets/collection1/conf/solrconfig.xml | 20 +-
solr/core/ivy.xml | 1 -
.../org/apache/solr/core/ConfigSetService.java | 7 +-
.../ParseDateFieldUpdateProcessorFactory.java | 82 ++++++--
...dd-schema-fields-update-processor-chains.xml | 20 +-
...lrconfig-parsing-update-processor-chains.xml | 49 ++---
.../collection1/conf/solrconfig-schemaless.xml | 20 +-
.../configsets/_default/conf/solrconfig.xml | 20 +-
...dSchemaFieldsUpdateProcessorFactoryTest.java | 23 +--
.../ParsingFieldUpdateProcessorsTest.java | 157 +++++++-------
solr/example/files/conf/solrconfig.xml | 20 +-
solr/licenses/joda-time-2.2.jar.sha1 | 1 -
solr/licenses/joda-time-LICENSE-ASL.txt | 202 -------------------
solr/licenses/joda-time-NOTICE.txt | 5 -
.../configsets/_default/conf/solrconfig.xml | 20 +-
solr/solr-ref-guide/src/schemaless-mode.adoc | 20 +-
18 files changed, 218 insertions(+), 460 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 3a1ef2b..d7141bd 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -77,7 +77,6 @@ io.prometheus.version = 0.2.0
/javax.activation/activation = 1.1.1
/javax.servlet/javax.servlet-api = 3.1.0
-/joda-time/joda-time = 2.2
/junit/junit = 4.10
/mecab/mecab-ipadic = 2.7.0-20070801
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d5edef3..94960bb 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -44,11 +44,21 @@ Upgrade Notes
MemoryDocValues). If you used postingsFormat="Memory" or docValuesFormat="Memory"
switch to "Direct" instead. (Dawid Weiss)
+* SOLR-12586: The date format patterns used by ParseDateFieldUpdateProcessorFactory (present in "schemaless mode")
+ are now interpreted by Java 8's java.time.DateTimeFormatter instead of Joda Time. The pattern language is very
+ similar but not the same. Typically, simply update the pattern by changing an uppercase 'Z' to lowercase 'z' and
+ that's it. For the current recommended set of patterns in schemaless mode, see "Schemaless Mode" in the ref guide.
+ (David Smiley, Bar Rotstein)
+
Other Changes
----------------------
* SOLR-12614: Make "Nodes" view the default in AdminUI "Cloud" tab (janhoy)
+* SOLR-12586: Remove Joda Time dependency. Upgrade ParseDateFieldUpdateProcessorFactory (present in "schemaless mode")
+ to use Java 8's java.time.DateTimeFormatter instead (see upgrade notes).
+ (David Smiley, Bar Rotstein)
+
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml b/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
index 72c5430..37c41d8 100644
--- a/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
+++ b/solr/contrib/prometheus-exporter/src/test-files/configsets/collection1/conf/solrconfig.xml
@@ -164,22 +164,10 @@
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index ee6fe80..5a0fd09 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -44,7 +44,6 @@
<dependency org="org.ow2.asm" name="asm-commons" rev="${/org.ow2.asm/asm-commons}" conf="compile"/>
<dependency org="org.restlet.jee" name="org.restlet" rev="${/org.restlet.jee/org.restlet}" conf="compile"/>
<dependency org="org.restlet.jee" name="org.restlet.ext.servlet" rev="${/org.restlet.jee/org.restlet.ext.servlet}" conf="compile"/>
- <dependency org="joda-time" name="joda-time" rev="${/joda-time/joda-time}" conf="compile"/>
<dependency org="dom4j" name="dom4j" rev="${/dom4j/dom4j}" conf="compile"/>
<dependency org="com.carrotsearch" name="hppc" rev="${/com.carrotsearch/hppc}" conf="compile"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
index 69e160b..7ce1a52 100644
--- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java
@@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.time.Instant;
import java.util.Locale;
import java.util.concurrent.ExecutionException;
@@ -33,8 +34,6 @@ import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.IndexSchemaFactory;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -212,12 +211,10 @@ public abstract class ConfigSetService {
super(loader, configSetBase);
}
- public static final DateTimeFormatter cacheKeyFormatter = DateTimeFormat.forPattern("yyyyMMddHHmmss");
-
public static String cacheName(Path schemaFile) throws IOException {
long lastModified = Files.getLastModifiedTime(schemaFile).toMillis();
return String.format(Locale.ROOT, "%s:%s",
- schemaFile.toString(), cacheKeyFormatter.print(lastModified));
+ schemaFile.toString(), Instant.ofEpochMilli(lastModified).toString());
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
index 234b48b..f0ea5d2 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
@@ -17,6 +17,17 @@
package org.apache.solr.update.processor;
import java.lang.invoke.MethodHandles;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
+import java.time.temporal.TemporalQueries;
import java.util.Collection;
import java.util.Date;
import java.util.LinkedHashMap;
@@ -24,6 +35,7 @@ import java.util.Locale;
import java.util.Map;
import org.apache.commons.lang.LocaleUtils;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
@@ -31,10 +43,6 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.DateValueFieldType;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,8 +65,8 @@ import org.slf4j.LoggerFactory;
* </p>
* <p>
* One or more date "format" specifiers must be specified. See
- * <a href="http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html"
- * >Joda-time's DateTimeFormat javadocs</a> for a description of format strings.
+ * <a href="https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html"
+ * >Java 8's DateTimeFormatter javadocs</a> for a description of format strings.
* </p>
* <p>
* A default time zone name or offset may optionally be specified for those dates
@@ -115,9 +123,8 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
for (Map.Entry<String,DateTimeFormatter> format : formats.entrySet()) {
DateTimeFormatter parser = format.getValue();
try {
- DateTime dateTime = parser.parseDateTime(srcStringVal);
- return dateTime.withZone(DateTimeZone.UTC).toDate();
- } catch (IllegalArgumentException e) {
+ return Date.from(parseInstant(parser, srcStringVal));
+ } catch (DateTimeParseException e) {
log.debug("value '{}' is not parseable with format '{}'",
new Object[] { srcStringVal, format.getKey() });
}
@@ -144,15 +151,18 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
}
Object defaultTimeZoneParam = args.remove(DEFAULT_TIME_ZONE_PARAM);
- DateTimeZone defaultTimeZone = DateTimeZone.UTC;
+ ZoneId defaultTimeZone = ZoneOffset.UTC;
if (null != defaultTimeZoneParam) {
- defaultTimeZone = DateTimeZone.forID(defaultTimeZoneParam.toString());
+ defaultTimeZone = ZoneId.of(defaultTimeZoneParam.toString());
}
Collection<String> formatsParam = args.removeConfigArgs(FORMATS_PARAM);
if (null != formatsParam) {
for (String value : formatsParam) {
- formats.put(value, DateTimeFormat.forPattern(value).withZone(defaultTimeZone).withLocale(locale));
+ DateTimeFormatter formatter = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .appendPattern(value).toFormatter(locale).withZone(defaultTimeZone);
+ validateFormatter(formatter);
+ formats.put(value, formatter);
}
}
super.init(args);
@@ -172,4 +182,52 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
return (null == type) || type instanceof DateValueFieldType;
};
}
+
+ public static void validateFormatter(DateTimeFormatter formatter) {
+ // check it's valid via round-trip
+ try {
+ parseInstant(formatter, formatter.format(Instant.now()));
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Bad or unsupported pattern: " + formatter.toFormat().toString(), e);
+ }
+ }
+
+ // see https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8177021 which is fixed in Java 9.
+ // The upshot is that trying to use parse(Instant::from) is unreliable in the event that
+ // the input string contains a timezone/offset that differs from the "override zone"
+ // (which we configure in DEFAULT_TIME_ZONE). Besides, we need the code below which handles
+ // the optionality of time. Were it not for that, we truly could do formatter.parse(Instant::from).
+ private static Instant parseInstant(DateTimeFormatter formatter, String dateStr) {
+ final TemporalAccessor temporal = formatter.parse(dateStr);
+ // Get Date; mandatory
+ LocalDate date = temporal.query(TemporalQueries.localDate());//mandatory
+ if (date == null) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Date (year, month, day) is mandatory: " + formatter.toFormat().toString());
+ }
+ // Get Time; optional
+ LocalTime time = temporal.query(TemporalQueries.localTime());
+ if (time == null) {
+ time = LocalTime.MIN;
+ }
+
+ final LocalDateTime localDateTime = LocalDateTime.of(date, time);
+
+ // Get Zone Offset; optional
+ ZoneOffset offset = temporal.query(TemporalQueries.offset());
+ if (offset == null) {
+ // no Zone offset; get Zone ID
+ ZoneId zoneId = temporal.query(TemporalQueries.zone());
+ if (zoneId == null) {
+ zoneId = formatter.getZone();
+ if (zoneId == null) {
+ zoneId = ZoneOffset.UTC;
+ }
+ }
+ return localDateTime.atZone(zoneId).toInstant();
+ } else {
+ return localDateTime.toInstant(offset);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
index b0b63df..4394d5c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml
@@ -179,22 +179,10 @@
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory"/>
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
index 5930d37..83be4ee 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml
@@ -29,35 +29,35 @@
<updateRequestProcessorChain name="parse-date">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-explicit-not-in-schema-selector-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<bool name="fieldNameMatchesSchemaField">false</bool>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-explicit-typeclass-point-selector-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="typeClass">solr.DatePointField</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
<updateRequestProcessorChain name="parse-date-explicit-typeclass-selector-no-run-processor">
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="typeClass">solr.TrieDateField</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
</processor>
</updateRequestProcessorChain>
@@ -65,8 +65,7 @@
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="defaultTimeZone">America/New_York</str>
<str name="locale">en_US</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSS</str>
+ <str name="format">yyyy-MM-dd'T'HH:mm:ss.SSS[z]</str>
</processor>
<processor class="solr.RunUpdateProcessorFactory" />
</updateRequestProcessorChain>
@@ -75,7 +74,7 @@
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<str name="defaultTimeZone">America/Los_Angeles</str>
<arr name="format">
- <str>MM/dd/yyyy</str>
+ <str>M/d/yyyy</str>
</arr>
</processor>
</updateRequestProcessorChain>
@@ -85,30 +84,16 @@
<str name="defaultTimeZone">UTC</str>
<str name="locale">en_US</str>
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
- <str>yyyy-MM-dd hh:mm a</str>
- <str>yyyy-MM-dd hh:mma</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
- <str>EEE MMM dd HH:mm:ss Z yyyy</str>
- <str>EEE MMM dd HH:mm:ss yyyy Z</str>
- <str>EEE MMM dd HH:mm:ss yyyy</str>
- <str>EEE, dd MMM yyyy HH:mm:ss Z</str>
- <str>EEEE, dd-MMM-yy HH:mm:ss Z</str>
+ <str>yyyy-MM-dd hh:mm[ ]a</str>
+ <str>EEE MMM dd HH:mm:ss ZZZ yyyy</str>
+ <str>EEE MMM dd HH:mm:ss yyyy[ z]</str>
+ <str>EEE, dd MMM yyyy HH:mm:ss z</str>
+ <str>EEEE, dd-MMM-yy HH:mm:ss z</str>
<str>EEEE, MMMM dd, yyyy</str>
<str>MMMM dd, yyyy</str>
<str>MMM. dd, yyyy</str>
@@ -229,7 +214,7 @@
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
<str>yyyy-MM-dd</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
+ <str>yyyy-MM-dd'T'HH:mm:ss.SSSz</str>
<str>yyyy-MM-dd'T'HH:mm</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
index 4f10a8e..446f3f0 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-schemaless.xml
@@ -61,22 +61,10 @@
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory"/>
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
index 51f2c63..d393316 100644
--- a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml
@@ -1141,22 +1141,10 @@
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
index d1a3a53..96cc676 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/AddSchemaFieldsUpdateProcessorFactoryTest.java
@@ -17,17 +17,18 @@
package org.apache.solr.update.processor;
import java.io.File;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Date;
+import java.util.Locale;
import org.apache.commons.io.FileUtils;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.schema.IndexSchema;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
import org.junit.After;
import org.junit.Before;
@@ -62,9 +63,7 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe
IndexSchema schema = h.getCore().getLatestSchema();
final String fieldName = "newfield1";
assertNull(schema.getFieldOrNull(fieldName));
- String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- Date date = dateTimeFormatter.parseDateTime(dateString).toDate();
+ Date date = Date.from(Instant.now());
SolrInputDocument d = processAdd("add-fields-no-run-processor", doc(f("id", "1"), f(fieldName, date)));
assertNotNull(d);
schema = h.getCore().getLatestSchema();
@@ -202,11 +201,11 @@ public class AddSchemaFieldsUpdateProcessorFactoryTest extends UpdateProcessorTe
String field3String2 = "-5.28E-3";
Double field3Value2 = -5.28E-3;
String field4String1 = "1999-04-17 17:42";
- DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm").withZoneUTC();
- DateTime dateTime = dateTimeFormatter.parseDateTime(field4String1);
- Date field4Value1 = dateTime.toDate();
- DateTimeFormatter dateTimeFormatter2 = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss").withZoneUTC();
- String field4Value1String = dateTimeFormatter2.print(dateTime) + "Z";
+ DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm", Locale.ROOT).withZone(ZoneOffset.UTC);
+ LocalDateTime dateTime = LocalDateTime.parse(field4String1, dateTimeFormatter);
+ Date field4Value1 = Date.from(dateTime.atZone(ZoneOffset.UTC).toInstant());
+ DateTimeFormatter dateTimeFormatter2 = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT).withZone(ZoneOffset.UTC);
+ String field4Value1String = dateTime.format(dateTimeFormatter2) + "Z";
SolrInputDocument d = processAdd
("parse-and-add-fields", doc(f("id", "6"), f(fieldName1, field1String1, field1String2, field1String3),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java b/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
index 3aeb1fb..e26ca41 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/ParsingFieldUpdateProcessorsTest.java
@@ -16,28 +16,35 @@
*/
package org.apache.solr.update.processor;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.schema.IndexSchema;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
-import org.junit.BeforeClass;
-
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.TemporalAccessor;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.schema.IndexSchema;
+import org.junit.BeforeClass;
+
/**
* Tests for the field mutating update processors
* that parse Dates, Longs, Doubles, and Booleans.
*/
public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
private static final double EPSILON = 1E-15;
+ private static final DateTimeFormatter isoDateOptionalTimeFormatter =
+ DateTimeFormatter.ofPattern("yyyy-MM-dd['T'HH:mm[:ss[.SSS]]][z", Locale.ROOT).withZone(ZoneOffset.UTC);
@BeforeClass
public static void beforeClass() throws Exception {
@@ -50,10 +57,8 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
String dateString = "2010-11-12T13:14:15.168Z";
SolrInputDocument d = processAdd("parse-date", doc(f("id", "9"), f("date_dt", dateString)));
assertNotNull(d);
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
assertTrue(d.getFieldValue("date_dt") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_dt")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date) d.getFieldValue("date_dt")).toInstant());
assertU(commit());
assertQ(req("id:9"), "//date[@name='date_dt'][.='" + dateString + "']");
}
@@ -64,10 +69,8 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
String dateString = "2010-11-12T13:14:15.168Z";
SolrInputDocument d = processAdd("parse-date", doc(f("id", "39"), f("date_tdt", dateString)));
assertNotNull(d);
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
assertTrue(d.getFieldValue("date_tdt") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date) d.getFieldValue("date_tdt")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date) d.getFieldValue("date_tdt")).toInstant());
assertU(commit());
assertQ(req("id:39"), "//date[@name='date_tdt'][.='" + dateString + "']");
}
@@ -77,14 +80,12 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-date-no-run-processor",
doc(f("id", "18"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("not_in_schema")).toInstant());
d = processAdd("parse-date-no-run-processor",
doc(f("id", "36"), f("not_in_schema", "not a date", dateString)));
@@ -116,12 +117,8 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
("parse-date-non-UTC-defaultTimeZone", doc(f("id", "99"), f("dateUTC_dt", dateStringUTC),
f("dateNoTimeZone_dt", dateStringNoTimeZone)));
assertNotNull(d);
- String pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
- DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(pattern);
- DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(dateStringUTC);
assertTrue(d.getFieldValue("dateUTC_dt") instanceof Date);
assertTrue(d.getFieldValue("dateNoTimeZone_dt") instanceof Date);
- assertEquals(dateTimeUTC.getMillis(), ((Date) d.getFieldValue("dateUTC_dt")).getTime());
assertU(commit());
assertQ(req("id:99")
,"//date[@name='dateUTC_dt'][.='" + dateStringUTC + "']"
@@ -132,22 +129,18 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-date-explicit-not-in-schema-selector-no-run-processor",
doc(f("id", "88"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("not_in_schema")).toInstant());
}
public void testParseDateExplicitTypeClassSelector() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("date_dt"));
String dateString = "2010-11-12T13:14:15.168Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d;
if (schema.getField("date_dt").getType().isPointField()) {
d = processAdd("parse-date-explicit-typeclass-point-selector-no-run-processor",
@@ -159,71 +152,66 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
assertNotNull(d);
assertTrue(d.getFieldValue("date_dt") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("date_dt")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("date_dt")).toInstant());
}
public void testParseUSPacificDate() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
String dateString = "8/9/2010"; // Interpreted as 00:00 US Pacific Daylight Time = UTC+07:00
- String dateStringUTC = "2010-08-09T07:00:00.000Z";
SolrInputDocument d = processAdd("US-Pacific-parse-date-no-run-processor",
doc(f("id", "288"), f("not_in_schema", dateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateStringUTC,
- (new DateTime(((Date)d.getFieldValue("not_in_schema")).getTime(),DateTimeZone.UTC)).toString());
+ assertEquals(Instant.parse("2010-08-09T07:00:00.000Z"), ((Date)d.getFieldValue("not_in_schema")).toInstant());
}
public void testParseDateFormats() throws Exception {
String[] formatExamples = {
- "yyyy-MM-dd'T'HH:mm:ss.SSSZ", "2010-01-15T00:00:00.000Z",
- "yyyy-MM-dd'T'HH:mm:ss,SSSZ", "2010-01-15T00:00:00,000Z",
- "yyyy-MM-dd'T'HH:mm:ss.SSS", "2010-01-15T00:00:00.000",
- "yyyy-MM-dd'T'HH:mm:ss,SSS", "2010-01-15T00:00:00,000",
- "yyyy-MM-dd'T'HH:mm:ssZ", "2010-01-15T00:00:00Z",
- "yyyy-MM-dd'T'HH:mm:ss", "2010-01-15T00:00:00",
- "yyyy-MM-dd'T'HH:mmZ", "2010-01-15T00:00Z",
- "yyyy-MM-dd'T'HH:mm", "2010-01-15T00:00",
- "yyyy-MM-dd HH:mm:ss.SSSZ", "2010-01-15 00:00:00.000Z",
- "yyyy-MM-dd HH:mm:ss,SSSZ", "2010-01-15 00:00:00,000Z",
- "yyyy-MM-dd HH:mm:ss.SSS", "2010-01-15 00:00:00.000",
- "yyyy-MM-dd HH:mm:ss,SSS", "2010-01-15 00:00:00,000",
- "yyyy-MM-dd HH:mm:ssZ", "2010-01-15 00:00:00Z",
- "yyyy-MM-dd HH:mm:ss", "2010-01-15 00:00:00",
- "yyyy-MM-dd HH:mmZ", "2010-01-15 00:00Z",
- "yyyy-MM-dd HH:mm", "2010-01-15 00:00",
- "yyyy-MM-dd hh:mm a", "2010-01-15 12:00 AM",
- "yyyy-MM-dd hh:mma", "2010-01-15 12:00AM",
- "yyyy-MM-dd", "2010-01-15",
- "EEE MMM dd HH:mm:ss Z yyyy", "Fri Jan 15 00:00:00 +0000 2010",
- "EEE MMM dd HH:mm:ss yyyy Z", "Fri Jan 15 00:00:00 2010 +00:00",
- "EEE MMM dd HH:mm:ss yyyy", "Fri Jan 15 00:00:00 2010",
- "EEE, dd MMM yyyy HH:mm:ss Z", "Fri, 15 Jan 2010 00:00:00 +00:00",
- "EEEE, dd-MMM-yy HH:mm:ss Z", "Friday, 15-Jan-10 00:00:00 +00:00",
- "EEEE, MMMM dd, yyyy", "Friday, January 15, 2010",
- "MMMM dd, yyyy", "January 15, 2010",
- "MMM. dd, yyyy", "Jan. 15, 2010"
+ "2010-01-15T00:00:00.000Z",
+ "2010-01-15T00:00:00,000Z",
+ "2010-01-15T00:00:00.000",
+ "2010-01-15T00:00:00,000",
+ "2010-01-15T00:00:00Z",
+ "2010-01-15T00:00:00",
+ "2010-01-15T00:00Z",
+ "2010-01-15T00:00",
+ "2010-01-15 00:00:00.000Z",
+ "2010-01-15 00:00:00,000Z",
+ "2010-01-15 00:00:00.000",
+ "2010-01-15 00:00:00,000",
+ "2010-01-15 00:00:00Z",
+ "2010-01-15 00:00:00",
+ "2010-01-15 00:00Z",
+ "2010-01-15 00:00",
+ "2010-01-15 12:00 AM",
+ "2010-01-15 12:00AM",
+ "2010-01-15",
+ "Fri Jan 15 00:00:00 +0000 2010",
+ "Fri Jan 15 00:00:00 2010 +00:00",
+ "Fri Jan 15 00:00:00 2010",
+ "Fri, 15 Jan 2010 00:00:00 +00:00",
+ "Friday, 15-Jan-10 00:00:00 +00:00",
+ "Friday, January 15, 2010",
+ "January 15, 2010",
+ "Jan. 15, 2010"
};
IndexSchema schema = h.getCore().getLatestSchema();
assertNotNull(schema.getFieldOrNull("dateUTC_dt")); // should match "*_dt" dynamic field
- String dateTimePattern = "yyyy-MM-dd'T'HH:mm:ss.SSSZ";
- DateTimeFormatter dateTimeFormatterUTC = DateTimeFormat.forPattern(dateTimePattern);
- DateTime dateTimeUTC = dateTimeFormatterUTC.parseDateTime(formatExamples[1]);
+ Instant expectedInstant = Instant.parse(formatExamples[0]);
- for (int i = 0 ; i < formatExamples.length ; i += 2) {
- String format = formatExamples[i];
- String dateString = formatExamples[i + 1];
+ for (int i = 0 ; i < formatExamples.length ; ++i) {
+ String dateString = formatExamples[i];
String id = "95" + i;
SolrInputDocument d = processAdd("parse-date-UTC-defaultTimeZone-no-run-processor",
doc(f("id", id), f("dateUTC_dt", dateString)));
assertNotNull(d);
- assertTrue("date '" + dateString + "' with format '" + format + "' is not mutated to a Date",
+ assertTrue("index: " + i + " date '" + dateString + "' is not mutated to a Date",
d.getFieldValue("dateUTC_dt") instanceof Date);
- assertEquals("date '" + dateString + "' with format '" + format + "' mismatched milliseconds",
- dateTimeUTC.getMillis(), ((Date)d.getFieldValue("dateUTC_dt")).getTime());
+ assertEquals("date '" + dateString + "' mismatched milliseconds",
+ expectedInstant, ((Date)d.getFieldValue("dateUTC_dt")).toInstant());
}
}
@@ -232,23 +220,20 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
assertNull(schema.getFieldOrNull("not_in_schema"));
String frenchDateString = "le vendredi 15 janvier 2010";
String dateString = "2010-01-15T00:00:00.000Z";
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime();
- DateTime dateTime = dateTimeFormatter.parseDateTime(dateString);
SolrInputDocument d = processAdd("parse-french-date-UTC-defaultTimeZone-no-run-processor",
doc(f("id", "88"), f("not_in_schema", frenchDateString)));
assertNotNull(d);
assertTrue(d.getFieldValue("not_in_schema") instanceof Date);
- assertEquals(dateTime.getMillis(), ((Date)d.getFieldValue("not_in_schema")).getTime());
+ assertEquals(Instant.parse(dateString), ((Date)d.getFieldValue("not_in_schema")).toInstant());
}
public void testFailedParseMixedDate() throws Exception {
IndexSchema schema = h.getCore().getLatestSchema();
assertNull(schema.getFieldOrNull("not_in_schema"));
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Object,Object> mixed = new HashMap<>();
String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
- mixed.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
+ mixed.put(parse(isoDateOptionalTimeFormatter, dateString), dateString);
}
Double extraDouble = 29.554d;
mixed.put(extraDouble, extraDouble); // Double-typed field value
@@ -805,11 +790,10 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
longs.remove(o);
}
- DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Date,String> dates = new HashMap<>();
String[] dateStrings = { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
- dates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
+ dates.put(parse(isoDateOptionalTimeFormatter, dateString), dateString);
}
d = processAdd(chain, doc(f("id", "343"), f(fieldName, dates.values())));
assertNotNull(d);
@@ -896,13 +880,12 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
}
assertTrue(mixedBooleans.isEmpty());
- dateTimeFormatter = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC();
Map<Date,Object> mixedDates = new HashMap<>();
dateStrings = new String[] { "2020-05-13T18:47", "1989-12-14", "1682-07-22T18:33:00.000Z" };
for (String dateString : dateStrings) {
- mixedDates.put(dateTimeFormatter.parseDateTime(dateString).toDate(), dateString);
+ mixedDates.put(parse(isoDateOptionalTimeFormatter, dateString), dateString);
}
- Date extraDate = dateTimeFormatter.parseDateTime("2003-04-24").toDate();
+ Date extraDate = parse(isoDateOptionalTimeFormatter, "2003-04-24");
mixedDates.put(extraDate, extraDate); // Date-typed field value
d = processAdd(chain, doc(f("id", "3395"), f(fieldName, mixedDates.values())));
assertNotNull(d);
@@ -912,4 +895,24 @@ public class ParsingFieldUpdateProcessorsTest extends UpdateProcessorTestBase {
}
assertTrue(mixedDates.isEmpty());
}
+
+ private Date parse(DateTimeFormatter dateTimeFormatter, String dateString) {
+ final TemporalAccessor temporalAccessor = dateTimeFormatter.parseBest(dateString, OffsetDateTime::from,
+ ZonedDateTime::from, LocalDateTime::from, LocalDate::from, Instant::from);
+ return temporalToDate(temporalAccessor, dateTimeFormatter.getZone());
+ }
+
+ private Date temporalToDate(TemporalAccessor in, ZoneId timeZoneId) {
+ if (in instanceof OffsetDateTime) {
+ return Date.from(((OffsetDateTime) in).toInstant());
+ } else if (in instanceof ZonedDateTime) {
+ return Date.from(((ZonedDateTime) in).toInstant());
+ } else if (in instanceof LocalDateTime) {
+ return Date.from(((LocalDateTime) in).atZone(timeZoneId).toInstant());
+ } else if (in instanceof Instant) {
+ return Date.from((Instant) in);
+ } else {
+ return Date.from(((LocalDate) in).atStartOfDay(timeZoneId).toInstant());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/example/files/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/example/files/conf/solrconfig.xml b/solr/example/files/conf/solrconfig.xml
index bca0df8..7bc8d34 100644
--- a/solr/example/files/conf/solrconfig.xml
+++ b/solr/example/files/conf/solrconfig.xml
@@ -1161,22 +1161,10 @@
<processor class="solr.ParseDoubleFieldUpdateProcessorFactory"/>
<processor class="solr.ParseDateFieldUpdateProcessorFactory">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</processor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/licenses/joda-time-2.2.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/joda-time-2.2.jar.sha1 b/solr/licenses/joda-time-2.2.jar.sha1
deleted file mode 100644
index 5e68639..0000000
--- a/solr/licenses/joda-time-2.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a5f29a7acaddea3f4af307e8cf2d0cc82645fd7d
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/licenses/joda-time-LICENSE-ASL.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/joda-time-LICENSE-ASL.txt b/solr/licenses/joda-time-LICENSE-ASL.txt
deleted file mode 100644
index d645695..0000000
--- a/solr/licenses/joda-time-LICENSE-ASL.txt
+++ /dev/null
@@ -1,202 +0,0 @@
-
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/licenses/joda-time-NOTICE.txt
----------------------------------------------------------------------
diff --git a/solr/licenses/joda-time-NOTICE.txt b/solr/licenses/joda-time-NOTICE.txt
deleted file mode 100644
index dffbcf3..0000000
--- a/solr/licenses/joda-time-NOTICE.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-=============================================================================
-= NOTICE file corresponding to section 4d of the Apache License Version 2.0 =
-=============================================================================
-This product includes software developed by
-Joda.org (http://www.joda.org/).
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/server/solr/configsets/_default/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml
index 51f2c63..d393316 100644
--- a/solr/server/solr/configsets/_default/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml
@@ -1141,22 +1141,10 @@
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c3887b35/solr/solr-ref-guide/src/schemaless-mode.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/schemaless-mode.adoc b/solr/solr-ref-guide/src/schemaless-mode.adoc
index b62a92e..0a2f555 100644
--- a/solr/solr-ref-guide/src/schemaless-mode.adoc
+++ b/solr/solr-ref-guide/src/schemaless-mode.adoc
@@ -103,22 +103,10 @@ To start, you should define it as follows (see the javadoc links below for updat
<updateProcessor class="solr.ParseDoubleFieldUpdateProcessorFactory" name="parse-double"/>
<updateProcessor class="solr.ParseDateFieldUpdateProcessorFactory" name="parse-date">
<arr name="format">
- <str>yyyy-MM-dd'T'HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd'T'HH:mm:ssZ</str>
- <str>yyyy-MM-dd'T'HH:mm:ss</str>
- <str>yyyy-MM-dd'T'HH:mmZ</str>
- <str>yyyy-MM-dd'T'HH:mm</str>
- <str>yyyy-MM-dd HH:mm:ss.SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss,SSSZ</str>
- <str>yyyy-MM-dd HH:mm:ss.SSS</str>
- <str>yyyy-MM-dd HH:mm:ss,SSS</str>
- <str>yyyy-MM-dd HH:mm:ssZ</str>
- <str>yyyy-MM-dd HH:mm:ss</str>
- <str>yyyy-MM-dd HH:mmZ</str>
- <str>yyyy-MM-dd HH:mm</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd'T'HH:mm[:ss[,SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[.SSS]][z</str>
+ <str>yyyy-MM-dd HH:mm[:ss[,SSS]][z</str>
<str>yyyy-MM-dd</str>
</arr>
</updateProcessor>
[16/17] lucene-solr:jira/solr-12470: SOLR-11834: rollback [subquery]
fields documentation
Posted by ab...@apache.org.
SOLR-11834: rollback [subquery] fields documentation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/63fc1246
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/63fc1246
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/63fc1246
Branch: refs/heads/jira/solr-12470
Commit: 63fc1246f7d30c73bff0fdbf344216a34efdc554
Parents: 6c11284
Author: Mikhail Khludnev <mk...@apache.org>
Authored: Thu Aug 9 20:27:35 2018 +0300
Committer: Mikhail Khludnev <mk...@apache.org>
Committed: Thu Aug 9 20:28:45 2018 +0300
----------------------------------------------------------------------
solr/solr-ref-guide/src/transforming-result-documents.adoc | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/63fc1246/solr/solr-ref-guide/src/transforming-result-documents.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/transforming-result-documents.adoc b/solr/solr-ref-guide/src/transforming-result-documents.adoc
index f837baf..f9fe805 100644
--- a/solr/solr-ref-guide/src/transforming-result-documents.adoc
+++ b/solr/solr-ref-guide/src/transforming-result-documents.adoc
@@ -251,12 +251,15 @@ Here is how it looks like in various formats:
==== Subquery Result Fields
-To appear in subquery document list, a field should be specified in subquery's `fl` parameter e.g., `foo.fl` (it is not necessary to specify in main one's `fl`). Of course, you can use wildcard in this parameter. For example, if field title should appear in categories subquery, it can be done via one of these ways.
+To appear in subquery document list, a field should be specified both fl parameters, in main one fl (despite the main result documents have no this field) and in subquery's one e.g., `foo.fl`.
+Of course, you can use wildcard in any or both of these parameters. For example, if field `title` should appear in categories subquery, it can be done via one of these ways.
[source,plain]
----
-fl=...id,categories:[subquery]&categories.fl=title&categories.q=...
-fl=...id,categories:[subquery]&categories.fl=*&categories.q=...
+fl=...title,categories:[subquery]&categories.fl=title&categories.q=...
+fl=...title,categories:[subquery]&categories.fl=*&categories.q=...
+fl=...*,categories:[subquery]&categories.fl=title&categories.q=...
+fl=...*,categories:[subquery]&categories.fl=*&categories.q=...
----
==== Subquery Parameters Shift
[12/17] lucene-solr:jira/solr-12470: SOLR-11008: Use a lighter config
for MetricsHandlerTest and ensure the core is up before the test starts
Posted by ab...@apache.org.
SOLR-11008: Use a lighter config for MetricsHandlerTest and ensure the core is up before the test starts
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7c4584bd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7c4584bd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7c4584bd
Branch: refs/heads/jira/solr-12470
Commit: 7c4584bd4f14917116b96a0931bab242d433e47a
Parents: 37e8fea
Author: Varun Thacker <va...@apache.org>
Authored: Tue Aug 7 13:19:44 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Aug 8 13:28:00 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../test/org/apache/solr/handler/admin/MetricsHandlerTest.java | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c4584bd/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7468451..9c80773 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -282,6 +282,8 @@ Other Changes
* SOLR-12617: Remove Commons BeanUtils as a dependency (Varun Thacker)
+* SOLR-11008: Use a lighter config for MetricsHandlerTest and ensure the core is up before the test starts (Varun Thacker)
+
================== 7.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7c4584bd/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 0fe5ad7..1730355 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -35,7 +35,9 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml", "schema.xml");
+ initCore("solrconfig-minimal.xml", "schema.xml");
+ h.getCoreContainer().waitForLoadingCoresToFinish(30000);
+
// manually register some metrics in solr.jvm and solr.jetty - TestHarness doesn't init them
Counter c = h.getCoreContainer().getMetricManager().counter(null, "solr.jvm", "foo");
c.inc();
[03/17] lucene-solr:jira/solr-12470: SOLR-12485: Solr-Update XML
format now accepts child documents under a field thus providing a semantic
relationship. (like allowed for JSON). Improved XML.java a bit Closes #430
Posted by ab...@apache.org.
SOLR-12485: Solr-Update XML format now accepts child documents under a field
thus providing a semantic relationship. (like allowed for JSON).
Improved XML.java a bit
Closes #430
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/38bf976c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/38bf976c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/38bf976c
Branch: refs/heads/jira/solr-12470
Commit: 38bf976cd4b9e324c21664bd7ae3d554df803705
Parents: c3887b3
Author: Moshe <mo...@mail.com>
Authored: Tue Aug 7 13:52:11 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Aug 7 13:52:11 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../apache/solr/handler/loader/XMLLoader.java | 18 ++-
.../apache/solr/update/AddBlockUpdateTest.java | 148 +++++++++++++++++++
.../solr/client/solrj/util/ClientUtils.java | 20 ++-
.../java/org/apache/solr/common/util/XML.java | 98 +++---------
.../org/apache/solr/util/BaseTestHarness.java | 23 +--
6 files changed, 215 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 94960bb..3ede98b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -150,6 +150,9 @@ New Features
* SOLR-12592: support #EQUAL function, range operator, decimal and percentage in cores in autoscaling policies (noble)
+* SOLR-12485: Uploading docs in XML now supports child documents as field values, thus providing a label to the
+ relationship instead of the current "anonymous" relationship. (Moshe Bla, David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
index a07aff2..724a40c 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/XMLLoader.java
@@ -404,6 +404,7 @@ public class XMLLoader extends ContentStreamLoader {
StringBuilder text = new StringBuilder();
String name = null;
boolean isNull = false;
+ boolean isLabeledChildDoc = false;
String update = null;
Collection<SolrInputDocument> subDocs = null;
Map<String, Map<String, Object>> updateMap = null;
@@ -453,7 +454,13 @@ public class XMLLoader extends ContentStreamLoader {
}
break;
}
- doc.addField(name, v);
+ if(!isLabeledChildDoc){
+ // only add data if this is not a childDoc, since it was added already
+ doc.addField(name, v);
+ } else {
+ // reset so next field is not treated as child doc
+ isLabeledChildDoc = false;
+ }
// field is over
name = null;
}
@@ -463,6 +470,15 @@ public class XMLLoader extends ContentStreamLoader {
text.setLength(0);
String localName = parser.getLocalName();
if ("doc".equals(localName)) {
+ if(name != null) {
+ // flag to prevent spaces after doc from being added
+ isLabeledChildDoc = true;
+ if(!doc.containsKey(name)) {
+ doc.setField(name, Lists.newArrayList());
+ }
+ doc.addField(name, readDoc(parser));
+ break;
+ }
if (subDocs == null)
subDocs = Lists.newArrayList();
subDocs.add(readDoc(parser));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
index a302d58..50c0c39 100644
--- a/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
+++ b/solr/core/src/test/org/apache/solr/update/AddBlockUpdateTest.java
@@ -502,6 +502,154 @@ public class AddBlockUpdateTest extends SolrTestCaseJ4 {
}
@Test
+ public void testXMLMultiLevelLabeledChildren() throws IOException, XMLStreamException {
+ String xml_doc1 =
+ "<doc >" +
+ " <field name=\"id\">1</field>" +
+ " <field name=\"empty_s\"></field>" +
+ " <field name=\"parent_s\">X</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >2</field>" +
+ " <field name=\"child_s\">y</field>" +
+ " </doc>" +
+ " <doc> " +
+ " <field name=\"id\" >3</field>" +
+ " <field name=\"child_s\">z</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ String xml_doc2 =
+ "<doc >" +
+ " <field name=\"id\">4</field>" +
+ " <field name=\"parent_s\">A</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >5</field>" +
+ " <field name=\"child_s\">b</field>" +
+ " <field name=\"grandChild\">" +
+ " <doc> " +
+ " <field name=\"id\" >7</field>" +
+ " <field name=\"child_s\">d</field>" +
+ " </doc>" +
+ " </field>" +
+ " </doc>" +
+ " </field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >6</field>" +
+ " <field name=\"child_s\">c</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ XMLStreamReader parser =
+ inputFactory.createXMLStreamReader(new StringReader(xml_doc1));
+ parser.next(); // read the START document...
+ //null for the processor is all right here
+ XMLLoader loader = new XMLLoader();
+ SolrInputDocument document1 = loader.readDoc(parser);
+
+ XMLStreamReader parser2 =
+ inputFactory.createXMLStreamReader(new StringReader(xml_doc2));
+ parser2.next(); // read the START document...
+ //null for the processor is all right here
+ //XMLLoader loader = new XMLLoader();
+ SolrInputDocument document2 = loader.readDoc(parser2);
+
+ assertFalse(document1.hasChildDocuments());
+ assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test",
+ sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString());
+
+ assertFalse(document2.hasChildDocuments());
+ assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test",
+ sdocs(sdoc("id", "5", "child_s", "b", "grandChild", Collections.singleton(sdoc("id", "7", "child_s", "d"))),
+ sdoc("id", "6", "child_s", "c"))).toString());
+ }
+
+ @Test
+ public void testXMLLabeledChildren() throws IOException, XMLStreamException {
+ UpdateRequest req = new UpdateRequest();
+
+ List<SolrInputDocument> docs = new ArrayList<>();
+
+ String xml_doc1 =
+ "<doc >" +
+ " <field name=\"id\">1</field>" +
+ " <field name=\"empty_s\"></field>" +
+ " <field name=\"parent_s\">X</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >2</field>" +
+ " <field name=\"child_s\">y</field>" +
+ " </doc>"+
+ " <doc> " +
+ " <field name=\"id\" >3</field>" +
+ " <field name=\"child_s\">z</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ String xml_doc2 =
+ "<doc >" +
+ " <field name=\"id\">4</field>" +
+ " <field name=\"parent_s\">A</field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >5</field>" +
+ " <field name=\"child_s\">b</field>" +
+ " </doc>"+
+ " </field>" +
+ " <field name=\"test\">" +
+ " <doc> " +
+ " <field name=\"id\" >6</field>" +
+ " <field name=\"child_s\">c</field>" +
+ " </doc>" +
+ " </field> " +
+ "</doc>";
+
+ XMLStreamReader parser =
+ inputFactory.createXMLStreamReader( new StringReader( xml_doc1 ) );
+ parser.next(); // read the START document...
+ //null for the processor is all right here
+ XMLLoader loader = new XMLLoader();
+ SolrInputDocument document1 = loader.readDoc( parser );
+
+ XMLStreamReader parser2 =
+ inputFactory.createXMLStreamReader( new StringReader( xml_doc2 ) );
+ parser2.next(); // read the START document...
+ //null for the processor is all right here
+ //XMLLoader loader = new XMLLoader();
+ SolrInputDocument document2 = loader.readDoc( parser2 );
+
+ assertFalse(document1.hasChildDocuments());
+ assertEquals(document1.toString(), sdoc("id", "1", "empty_s", "", "parent_s", "X", "test",
+ sdocs(sdoc("id", "2", "child_s", "y"), sdoc("id", "3", "child_s", "z"))).toString());
+
+ assertFalse(document2.hasChildDocuments());
+ assertEquals(document2.toString(), sdoc("id", "4", "parent_s", "A", "test",
+ sdocs(sdoc("id", "5", "child_s", "b"), sdoc("id", "6", "child_s", "c"))).toString());
+
+ docs.add(document1);
+ docs.add(document2);
+
+ Collections.shuffle(docs, random());
+ req.add(docs);
+
+ RequestWriter requestWriter = new RequestWriter();
+ OutputStream os = new ByteArrayOutputStream();
+ requestWriter.write(req, os);
+ assertBlockU(os.toString());
+ assertU(commit());
+
+ final SolrIndexSearcher searcher = getSearcher();
+ assertSingleParentOf(searcher, one("yz"), "X");
+ assertSingleParentOf(searcher, one("bc"), "A");
+
+ }
+
+ @Test
public void testJavaBinCodecNestedRelation() throws IOException {
SolrInputDocument topDocument = new SolrInputDocument();
topDocument.addField("parent_f1", "v1");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
index 54986db..26a188d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/util/ClientUtils.java
@@ -72,7 +72,9 @@ public class ClientUtils
for( Object v : field ) {
String update = null;
- if (v instanceof Map) {
+ if(v instanceof SolrInputDocument) {
+ writeVal(writer, name, v , null);
+ } else if (v instanceof Map) {
// currently only supports a single value
for (Entry<Object,Object> entry : ((Map<Object,Object>)v).entrySet()) {
update = entry.getKey().toString();
@@ -112,20 +114,28 @@ public class ClientUtils
v = Base64.byteArrayToBase64(bytes.array(), bytes.position(),bytes.limit() - bytes.position());
}
+ XML.Writable valWriter = null;
+ if(v instanceof SolrInputDocument) {
+ final SolrInputDocument solrDoc = (SolrInputDocument) v;
+ valWriter = (writer1) -> writeXML(solrDoc, writer1);
+ } else if(v != null) {
+ final Object val = v;
+ valWriter = (writer1) -> XML.escapeCharData(val.toString(), writer1);
+ }
+
if (update == null) {
if (v != null) {
- XML.writeXML(writer, "field", v.toString(), "name", name );
+ XML.writeXML(writer, "field", valWriter, "name", name);
}
} else {
if (v == null) {
- XML.writeXML(writer, "field", null, "name", name, "update", update, "null", true);
+ XML.writeXML(writer, "field", (XML.Writable) null, "name", name, "update", update, "null", true);
} else {
- XML.writeXML(writer, "field", v.toString(), "name", name, "update", update);
+ XML.writeXML(writer, "field", valWriter, "name", name, "update", update);
}
}
}
-
public static String toXML( SolrInputDocument doc )
{
StringWriter str = new StringWriter();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/solrj/src/java/org/apache/solr/common/util/XML.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/XML.java b/solr/solrj/src/java/org/apache/solr/common/util/XML.java
index c6e5205..9d1b8a8 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/XML.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/XML.java
@@ -16,9 +16,10 @@
*/
package org.apache.solr.common.util;
-import java.io.Writer;
import java.io.IOException;
+import java.io.Writer;
import java.util.Map;
+import java.util.stream.Stream;
/**
*
@@ -37,9 +38,7 @@ public class XML {
private static final String[] attribute_escapes=
{"#0;","#1;","#2;","#3;","#4;","#5;","#6;","#7;","#8;",null,null,"#11;","#12;",null,"#14;","#15;","#16;","#17;","#18;","#19;","#20;","#21;","#22;","#23;","#24;","#25;","#26;","#27;","#28;","#29;","#30;","#31;",null,null,""",null,null,null,"&",null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,"<"};
-
-
- /*****************************************
+ /*
#Simple python script used to generate the escape table above. -YCS
#
#use individual char arrays or one big char array for better efficiency
@@ -65,13 +64,9 @@ public class XML {
result += val + ','
print result
- ****************************************/
+ */
-/*********
- *
- * @throws IOException If there is a low-level I/O error.
- */
public static void escapeCharData(String str, Writer out) throws IOException {
escape(str, out, chardata_escapes);
}
@@ -84,53 +79,28 @@ public class XML {
escape(chars, start, length, out, attribute_escapes);
}
+ /** does NOT escape character data in val; it must already be valid XML. Attributes are always escaped. */
+ public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ writeXML(out, tag, (writer1) -> writer1.write(val), attrs);
+ }
- public final static void writeXML(Writer out, String tag, String val) throws IOException {
- out.write('<');
- out.write(tag);
- if (val == null) {
- out.write('/');
- out.write('>');
- } else {
- out.write('>');
- escapeCharData(val,out);
- out.write('<');
- out.write('/');
- out.write(tag);
- out.write('>');
- }
+ /** escapes character data in val and attributes */
+ public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ final Writable writable = val != null ? (writer1) -> XML.escapeCharData(val, writer1) : null;
+ writeXML(out, tag, writable, attrs);
}
- /** does NOT escape character data in val, must already be valid XML */
- public final static void writeUnescapedXML(Writer out, String tag, String val, Object... attrs) throws IOException {
- out.write('<');
- out.write(tag);
- for (int i=0; i<attrs.length; i++) {
- out.write(' ');
- out.write(attrs[i++].toString());
- out.write('=');
- out.write('"');
- out.write(attrs[i].toString());
- out.write('"');
- }
- if (val == null) {
- out.write('/');
- out.write('>');
- } else {
- out.write('>');
- out.write(val);
- out.write('<');
- out.write('/');
- out.write(tag);
- out.write('>');
- }
+ /** escapes character data in val and attributes */
+ public static void writeXML(Writer out, String tag, String val, Map<String, String> attrs) throws IOException {
+ writeXML(out, tag, val, attrs.entrySet().stream().flatMap((entry) -> Stream.of(entry.getKey(), entry.getValue())).toArray());
}
- /** escapes character data in val */
- public final static void writeXML(Writer out, String tag, String val, Object... attrs) throws IOException {
+ /** @lucene.internal */
+ public final static void writeXML(Writer out, String tag, Writable valWritable, Object... attrs) throws IOException {
out.write('<');
out.write(tag);
- for (int i=0; i<attrs.length; i++) {
+ final int attrsLen = attrs == null ? 0 : attrs.length;
+ for (int i = 0; i< attrsLen; i++) {
out.write(' ');
out.write(attrs[i++].toString());
out.write('=');
@@ -138,12 +108,12 @@ public class XML {
escapeAttributeValue(attrs[i].toString(), out);
out.write('"');
}
- if (val == null) {
+ if (valWritable == null) {
out.write('/');
out.write('>');
} else {
out.write('>');
- escapeCharData(val,out);
+ valWritable.write(out);
out.write('<');
out.write('/');
out.write(tag);
@@ -151,29 +121,9 @@ public class XML {
}
}
- /** escapes character data in val */
- public static void writeXML(Writer out, String tag, String val, Map<String, String> attrs) throws IOException {
- out.write('<');
- out.write(tag);
- for (Map.Entry<String, String> entry : attrs.entrySet()) {
- out.write(' ');
- out.write(entry.getKey());
- out.write('=');
- out.write('"');
- escapeAttributeValue(entry.getValue(), out);
- out.write('"');
- }
- if (val == null) {
- out.write('/');
- out.write('>');
- } else {
- out.write('>');
- escapeCharData(val,out);
- out.write('<');
- out.write('/');
- out.write(tag);
- out.write('>');
- }
+ @FunctionalInterface
+ public interface Writable {
+ void write(Writer w) throws IOException;
}
private static void escape(char [] chars, int offset, int length, Writer out, String [] escapes) throws IOException{
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/38bf976c/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
index a84d6d1..ad1d38e 100644
--- a/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/BaseTestHarness.java
@@ -15,11 +15,6 @@
* limitations under the License.
*/
package org.apache.solr.util;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.util.XML;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
import javax.xml.namespace.QName;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -27,13 +22,17 @@ import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
-
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.io.UnsupportedEncodingException;
import java.nio.charset.StandardCharsets;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.XML;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
+
abstract public class BaseTestHarness {
private static final ThreadLocal<DocumentBuilder> builderTL = new ThreadLocal<>();
private static final ThreadLocal<XPath> xpathTL = new ThreadLocal<>();
@@ -200,15 +199,9 @@ abstract public class BaseTestHarness {
public static String simpleTag(String tag, String... args) {
try {
- StringWriter r = new StringWriter();
-
- // this is annoying
- if (null == args || 0 == args.length) {
- XML.writeXML(r, tag, null);
- } else {
- XML.writeXML(r, tag, null, (Object[])args);
- }
- return r.getBuffer().toString();
+ StringWriter writer = new StringWriter();
+ XML.writeXML(writer, tag, (String) null, (Object[])args);
+ return writer.getBuffer().toString();
} catch (IOException e) {
throw new RuntimeException
("this should never happen with a StringWriter", e);
[06/17] lucene-solr:jira/solr-12470: SOLR-8207: Collections with
underscores in name no longer cause a crash the Cloud->Nodes UI
Posted by ab...@apache.org.
SOLR-8207: Collections with underscores in name no longer cause a crash the Cloud->Nodes UI
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6845bbff
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6845bbff
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6845bbff
Branch: refs/heads/jira/solr-12470
Commit: 6845bbff55707ca219fd29f9c4d3d9e3c20175ec
Parents: 9306922
Author: Jan Høydahl <ja...@apache.org>
Authored: Wed Aug 8 15:25:43 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Wed Aug 8 15:25:43 2018 +0200
----------------------------------------------------------------------
solr/webapp/web/js/angular/controllers/cloud.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6845bbff/solr/webapp/web/js/angular/controllers/cloud.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js
index 8cbd4db..8c65037 100644
--- a/solr/webapp/web/js/angular/controllers/cloud.js
+++ b/solr/webapp/web/js/angular/controllers/cloud.js
@@ -407,13 +407,13 @@ var nodesSubController = function($scope, Collections, System, Metrics) {
if (cores) {
for (coreId in cores) {
var core = cores[coreId];
- var keyName = "solr.core." + core['core'].replace('_', '.').replace('_', '.');
+ var keyName = "solr.core." + core['core'].replace(/(.*?)_(shard\d+)_(replica_.\d+)/, '\$1.\$2.\$3');
var nodeMetric = m.metrics[keyName];
var size = nodeMetric['INDEX.sizeInBytes'];
size = (typeof size !== 'undefined') ? size : 0;
core['sizeInBytes'] = size;
core['size'] = bytesToSize(size);
- core['label'] = core['core'].replace('_shard', '_s').replace(/_replica_./, 'r');
+ core['label'] = core['core'].replace(/(.*?)_shard(\d+)_replica_.(\d+)/, '\$1_s\$2r\$3');
indexSizeTotal += size;
var numDocs = nodeMetric['SEARCHER.searcher.numDocs'];
numDocs = (typeof numDocs !== 'undefined') ? numDocs : 0;
[08/17] lucene-solr:jira/solr-12470: Ref Guide: fix typo in
MODIFYCOLLECTION example
Posted by ab...@apache.org.
Ref Guide: fix typo in MODIFYCOLLECTION example
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2674c538
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2674c538
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2674c538
Branch: refs/heads/jira/solr-12470
Commit: 2674c53809a24527be007000e1753ec1121a9a20
Parents: 49e3cca
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Aug 8 08:52:18 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Aug 8 08:53:15 2018 -0500
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-auto-add-replicas.adoc | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2674c538/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
index 87e50f5..622ce41 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-auto-add-replicas.adoc
@@ -27,11 +27,11 @@ The boolean `autoAddReplicas` parameter can be passed to the CREATE command of t
[source,text]
http://localhost:8983/solr/admin/collections?action=CREATE&name=my_collection&numShards=1&replicationFactor=5&autoAddReplicas=true
-The MODIFYCOLLECTION command can be used to enable/disable this feature for any collection.
+The MODIFYCOLLECTION command can be used to enable or disable this feature for any collection.
.Modify collection to disable autoAddReplicas
[source,text]
-http://localhost:8983/solr/admin/collections?action=MODIFYCOLLECTION&name=my_collection&autoAddReplicas=false
+http://localhost:8983/solr/admin/collections?action=MODIFYCOLLECTION&collection=my_collection&autoAddReplicas=false
== Implementation Using .autoAddReplicas Trigger
@@ -39,7 +39,7 @@ A Trigger named `.autoAddReplicas` is automatically created whenever any collect
Only one trigger is sufficient to serve all collections having this feature enabled. The `.autoAddReplicas` trigger watches for nodes that are lost from the cluster and uses the default `TriggerActions` to create new replicas to replace the ones which were hosted by the lost node. If the old node comes back online, it unloads the moved replicas and the node is free to host other replicas as and when required.
-Since the trigger provides the autoAddReplicas feature for all collections, the `suspend-trigger` and `resume-trigger` Autoscaling API commands can be used to disable and enable this feature for all collections in one API call.
+Since the trigger provides the autoAddReplicas feature for all collections, the `suspend-trigger` and `resume-trigger` Autoscaling API commands can be used to disable and enable this feature for all collections in one API call.
.Suspending autoAddReplicas for all collections
[source,json]
[07/17] lucene-solr:jira/solr-12470: LUCENE-8204: Boolean queries
with a mix of required and optional clauses are now faster if the total hit
count is not required
Posted by ab...@apache.org.
LUCENE-8204: Boolean queries with a mix of required and optional clauses are now faster if the total hit count is not required
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/49e3cca7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/49e3cca7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/49e3cca7
Branch: refs/heads/jira/solr-12470
Commit: 49e3cca77f868dd8dcf766a528ab23c686a6edfc
Parents: 6845bbf
Author: Jim Ferenczi <ji...@apache.org>
Authored: Wed Aug 8 15:49:58 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed Aug 8 15:49:58 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +-
.../lucene/search/Boolean2ScorerSupplier.java | 2 +-
.../apache/lucene/search/ReqOptSumScorer.java | 175 +++++++++-----
.../lucene/search/TestReqOptSumScorer.java | 241 +++++++++++++++++--
.../lucene/search/RandomApproximationQuery.java | 10 +-
5 files changed, 352 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f0360a9..31c8988 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -135,7 +135,10 @@ Optimizations
to run faster. (Adrien Grand)
* LUCENE-8439: Disjunction max queries can skip blocks to select the top documents
- when the total hit count is not required. (Jim Ferenczi, Adrien Grand)
+ if the total hit count is not required. (Jim Ferenczi, Adrien Grand)
+
+* LUCENE-8204: Boolean queries with a mix of required and optional clauses are
+ now faster if the total hit count is not required. (Jim Ferenczi, Adrien Grand)
======================= Lucene 7.5.0 =======================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
index bae7941..a13547f 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
@@ -111,7 +111,7 @@ final class Boolean2ScorerSupplier extends ScorerSupplier {
assert scoreMode.needsScores();
return new ReqOptSumScorer(
excl(req(subs.get(Occur.FILTER), subs.get(Occur.MUST), leadCost), subs.get(Occur.MUST_NOT), leadCost),
- opt(subs.get(Occur.SHOULD), minShouldMatch, scoreMode, leadCost));
+ opt(subs.get(Occur.SHOULD), minShouldMatch, scoreMode, leadCost), scoreMode);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
index 2463e6a..dc3ee02 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
@@ -18,108 +18,160 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
+import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
+
/** A Scorer for queries with a required part and an optional part.
* Delays skipTo() on the optional part until a score() is needed.
*/
class ReqOptSumScorer extends Scorer {
- /** The scorers passed from the constructor.
- * These are set to null as soon as their next() or skipTo() returns false.
- */
private final Scorer reqScorer;
private final Scorer optScorer;
- private final float reqMaxScore;
+ private final DocIdSetIterator reqApproximation;
private final DocIdSetIterator optApproximation;
private final TwoPhaseIterator optTwoPhase;
- private boolean optIsRequired;
private final DocIdSetIterator approximation;
private final TwoPhaseIterator twoPhase;
- final MaxScoreSumPropagator maxScorePropagator;
- /** Construct a <code>ReqOptScorer</code>.
+ private float minScore = 0;
+ private float reqMaxScore;
+ private boolean optIsRequired;
+
+ /**
+ * Construct a <code>ReqOptScorer</code>.
+ *
* @param reqScorer The required scorer. This must match.
* @param optScorer The optional scorer. This is used for scoring only.
+ * @param scoreMode How the produced scorers will be consumed.
*/
- public ReqOptSumScorer(
- Scorer reqScorer,
- Scorer optScorer) throws IOException
- {
+ public ReqOptSumScorer(Scorer reqScorer, Scorer optScorer, ScoreMode scoreMode) throws IOException {
super(reqScorer.weight);
assert reqScorer != null;
assert optScorer != null;
this.reqScorer = reqScorer;
this.optScorer = optScorer;
- reqScorer.advanceShallow(0);
- this.reqMaxScore = reqScorer.getMaxScore(DocIdSetIterator.NO_MORE_DOCS);
- this.maxScorePropagator = new MaxScoreSumPropagator(Arrays.asList(reqScorer, optScorer));
-
final TwoPhaseIterator reqTwoPhase = reqScorer.twoPhaseIterator();
this.optTwoPhase = optScorer.twoPhaseIterator();
- final DocIdSetIterator reqApproximation;
if (reqTwoPhase == null) {
reqApproximation = reqScorer.iterator();
} else {
- reqApproximation= reqTwoPhase.approximation();
+ reqApproximation = reqTwoPhase.approximation();
}
if (optTwoPhase == null) {
optApproximation = optScorer.iterator();
} else {
- optApproximation= optTwoPhase.approximation();
+ optApproximation = optTwoPhase.approximation();
}
+ if (scoreMode != ScoreMode.TOP_SCORES) {
+ approximation = reqApproximation;
+ this.reqMaxScore = Float.POSITIVE_INFINITY;
+ } else {
+ reqScorer.advanceShallow(0);
+ optScorer.advanceShallow(0);
+ this.reqMaxScore = reqScorer.getMaxScore(NO_MORE_DOCS);
+ this.approximation = new DocIdSetIterator() {
+ int upTo = -1;
+ float maxScore;
- approximation = new DocIdSetIterator() {
+ private void moveToNextBlock(int target) throws IOException {
+ upTo = advanceShallow(target);
+ float reqMaxScoreBlock = reqScorer.getMaxScore(upTo);
+ maxScore = getMaxScore(upTo);
- private int nextCommonDoc(int reqDoc) throws IOException {
- int optDoc = optApproximation.docID();
- if (optDoc > reqDoc) {
- reqDoc = reqApproximation.advance(optDoc);
+ // Potentially move to a conjunction
+ optIsRequired = reqMaxScoreBlock < minScore;
}
- while (true) { // invariant: reqDoc >= optDoc
- if (reqDoc == optDoc) {
- return reqDoc;
+ private int advanceImpacts(int target) throws IOException {
+ if (target > upTo) {
+ moveToNextBlock(target);
}
- optDoc = optApproximation.advance(reqDoc);
- if (optDoc == reqDoc) {
- return reqDoc;
+ while (true) {
+ if (maxScore >= minScore) {
+ return target;
+ }
+
+ if (upTo == NO_MORE_DOCS) {
+ return NO_MORE_DOCS;
+ }
+
+ target = upTo + 1;
+
+ moveToNextBlock(target);
}
- reqDoc = reqApproximation.advance(optDoc);
}
- }
- @Override
- public int nextDoc() throws IOException {
- int doc = reqApproximation.nextDoc();
- if (optIsRequired) {
- doc = nextCommonDoc(doc);
+ @Override
+ public int nextDoc() throws IOException {
+ return advanceInternal(reqApproximation.docID()+1);
}
- return doc;
- }
- @Override
- public int advance(int target) throws IOException {
- int doc = reqApproximation.advance(target);
- if (optIsRequired) {
- doc = nextCommonDoc(doc);
+ @Override
+ public int advance(int target) throws IOException {
+ return advanceInternal(target);
}
- return doc;
- }
- @Override
- public int docID() {
- return reqApproximation.docID();
- }
+ private int advanceInternal(int target) throws IOException {
+ if (target == NO_MORE_DOCS) {
+ reqApproximation.advance(target);
+ return NO_MORE_DOCS;
+ }
+ int reqDoc = target;
+ advanceHead: for (;;) {
+ if (minScore != 0) {
+ reqDoc = advanceImpacts(reqDoc);
+ }
+ if (reqApproximation.docID() < reqDoc) {
+ reqDoc = reqApproximation.advance(reqDoc);
+ }
+ if (reqDoc == NO_MORE_DOCS || optIsRequired == false) {
+ return reqDoc;
+ }
- @Override
- public long cost() {
- return reqApproximation.cost();
- }
+ int upperBound = reqMaxScore < minScore ? NO_MORE_DOCS : upTo;
+ if (reqDoc > upperBound) {
+ continue;
+ }
- };
+ // Find the next common doc within the current block
+ for (;;) { // invariant: reqDoc >= optDoc
+ int optDoc = optApproximation.docID();
+ if (optDoc < reqDoc) {
+ optDoc = optApproximation.advance(reqDoc);
+ }
+ if (optDoc > upperBound) {
+ reqDoc = upperBound + 1;
+ continue advanceHead;
+ }
+
+ if (optDoc != reqDoc) {
+ reqDoc = reqApproximation.advance(optDoc);
+ if (reqDoc > upperBound) {
+ continue advanceHead;
+ }
+ }
+
+ if (reqDoc == NO_MORE_DOCS || optDoc == reqDoc) {
+ return reqDoc;
+ }
+ }
+ }
+ }
+
+ @Override
+ public int docID() {
+ return reqApproximation.docID();
+ }
+
+ @Override
+ public long cost() {
+ return reqApproximation.cost();
+ }
+ };
+ }
if (reqTwoPhase == null && optTwoPhase == null) {
this.twoPhase = null;
@@ -212,10 +264,13 @@ class ReqOptSumScorer extends Scorer {
@Override
public int advanceShallow(int target) throws IOException {
- if (optScorer.docID() < target) {
- optScorer.advanceShallow(target);
+ int upTo = reqScorer.advanceShallow(target);
+ if (optScorer.docID() <= target) {
+ upTo = Math.min(upTo, optScorer.advanceShallow(target));
+ } else if (optScorer.docID() != NO_MORE_DOCS) {
+ upTo = Math.min(upTo, optScorer.docID() - 1);
}
- return reqScorer.advanceShallow(target);
+ return upTo;
}
@Override
@@ -229,8 +284,9 @@ class ReqOptSumScorer extends Scorer {
@Override
public void setMinCompetitiveScore(float minScore) {
+ this.minScore = minScore;
// Potentially move to a conjunction
- if (optIsRequired == false && minScore > reqMaxScore) {
+ if (reqMaxScore < minScore) {
optIsRequired = true;
}
}
@@ -242,5 +298,4 @@ class ReqOptSumScorer extends Scorer {
children.add(new ChildScorer(optScorer, "SHOULD"));
return children;
}
-
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
index f8fc6c2..0d4454d 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestReqOptSumScorer.java
@@ -17,11 +17,20 @@
package org.apache.lucene.search;
import java.io.IOException;
+import java.util.Arrays;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermFrequencyAttribute;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -97,6 +106,120 @@ public class TestReqOptSumScorer extends LuceneTestCase {
dir.close();
}
+ public void testMaxBlock() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
+ FieldType ft = new FieldType();
+ ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+ ft.setTokenized(true);
+ ft.freeze();
+
+ for (int i = 0; i < 1024; i++) {
+ // create documents with an increasing number of As and one B
+ Document doc = new Document();
+ doc.add(new Field("foo", new TermFreqTokenStream("a", i+1), ft));
+ if (random().nextFloat() < 0.5f) {
+ doc.add(new Field("foo", new TermFreqTokenStream("b", 1), ft));
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ w.close();
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = newSearcher(reader);
+ searcher.setSimilarity(new TestSimilarity.SimpleSimilarity());
+ // freq == score
+ // searcher.setSimilarity(new TestSimilarity.SimpleSimilarity());
+ final Query reqQ = new TermQuery(new Term("foo", "a"));
+ final Query optQ = new TermQuery(new Term("foo", "b"));
+ final Query boolQ = new BooleanQuery.Builder()
+ .add(reqQ, Occur.MUST)
+ .add(optQ, Occur.SHOULD)
+ .build();
+ Scorer actual = reqOptScorer(searcher, reqQ, optQ, true);
+ Scorer expected = searcher
+ .createWeight(boolQ, ScoreMode.COMPLETE, 1)
+ .scorer(searcher.getIndexReader().leaves().get(0));
+ actual.setMinCompetitiveScore(Math.nextUp(1));
+ // Checks that all blocks are fully visited
+ for (int i = 0; i < 1024; i++) {
+ assertEquals(i, actual.iterator().nextDoc());
+ assertEquals(i, expected.iterator().nextDoc());
+ assertEquals(actual.score(),expected.score(), 0);
+ }
+ reader.close();
+ dir.close();
+ }
+
+ public void testMaxScoreSegment() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(newLogMergePolicy()));
+ for (String[] values : Arrays.asList(
+ new String[]{ "A" }, // 0
+ new String[]{ "A" }, // 1
+ new String[]{ }, // 2
+ new String[]{ "A", "B" }, // 3
+ new String[]{ "A" }, // 4
+ new String[]{ "B" }, // 5
+ new String[]{ "A", "B" }, // 6
+ new String[]{ "B" } // 7
+ )) {
+ Document doc = new Document();
+ for (String value : values) {
+ doc.add(new StringField("foo", value, Store.NO));
+ }
+ w.addDocument(doc);
+ }
+ w.forceMerge(1);
+ w.close();
+
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = newSearcher(reader);
+ final Query reqQ = new ConstantScoreQuery(new TermQuery(new Term("foo", "A")));
+ final Query optQ = new ConstantScoreQuery(new TermQuery(new Term("foo", "B")));
+ Scorer scorer = reqOptScorer(searcher, reqQ, optQ, false);
+ assertEquals(0, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(1, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(3, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(4, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(6, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ scorer = reqOptScorer(searcher, reqQ, optQ, false);
+ scorer.setMinCompetitiveScore(Math.nextDown(1f));
+ assertEquals(0, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(1, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(3, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(4, scorer.iterator().nextDoc());
+ assertEquals(1, scorer.score(), 0);
+ assertEquals(6, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ scorer = reqOptScorer(searcher, reqQ, optQ, false);
+ scorer.setMinCompetitiveScore(Math.nextUp(1f));
+ assertEquals(3, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(6, scorer.iterator().nextDoc());
+ assertEquals(2, scorer.score(), 0);
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ scorer = reqOptScorer(searcher, reqQ, optQ, true);
+ scorer.setMinCompetitiveScore(Math.nextUp(2f));
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, scorer.iterator().nextDoc());
+
+ reader.close();
+ dir.close();
+ }
+
public void testRandomFrequentOpt() throws IOException {
doTestRandom(0.5);
}
@@ -150,41 +273,125 @@ public class TestReqOptSumScorer extends LuceneTestCase {
searcher.search(query, coll);
ScoreDoc[] expectedFiltered = coll.topDocs().scoreDocs;
- for (int i = 0; i < 4; ++i) {
- Query must = mustTerm;
- if (i % 2 == 1) {
- must = new RandomApproximationQuery(must, random());
- }
- Query should = shouldTerm;
- if (i >= 2) {
- should = new RandomApproximationQuery(should, random());
- }
-
- query = new BooleanQuery.Builder()
- .add(must, Occur.MUST)
- .add(should, Occur.SHOULD)
+ CheckHits.checkTopScores(random(), query, searcher);
+
+ {
+ Query q = new BooleanQuery.Builder()
+ .add(new RandomApproximationQuery(mustTerm, random()), Occur.MUST)
+ .add(shouldTerm, Occur.SHOULD)
.build();
coll = TopScoreDocCollector.create(10, null, 1);
- searcher.search(query, coll);
+ searcher.search(q, coll);
ScoreDoc[] actual = coll.topDocs().scoreDocs;
-
CheckHits.checkEqual(query, expected, actual);
+ q = new BooleanQuery.Builder()
+ .add(mustTerm, Occur.MUST)
+ .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD)
+ .build();
+ coll = TopScoreDocCollector.create(10, null, 1);
+ searcher.search(q, coll);
+ actual = coll.topDocs().scoreDocs;
+ CheckHits.checkEqual(q, expected, actual);
+
+ q = new BooleanQuery.Builder()
+ .add(new RandomApproximationQuery(mustTerm, random()), Occur.MUST)
+ .add(new RandomApproximationQuery(shouldTerm, random()), Occur.SHOULD)
+ .build();
+ coll = TopScoreDocCollector.create(10, null, 1);
+ searcher.search(q, coll);
+ actual = coll.topDocs().scoreDocs;
+ CheckHits.checkEqual(q, expected, actual);
+ }
+
+ {
+ Query nestedQ = new BooleanQuery.Builder()
+ .add(query, Occur.MUST)
+ .add(new TermQuery(new Term("f", "C")), Occur.FILTER)
+ .build();
+ CheckHits.checkTopScores(random(), nestedQ, searcher);
+
query = new BooleanQuery.Builder()
.add(query, Occur.MUST)
.add(new RandomApproximationQuery(new TermQuery(new Term("f", "C")), random()), Occur.FILTER)
.build();
coll = TopScoreDocCollector.create(10, null, 1);
- searcher.search(query, coll);
+ searcher.search(nestedQ, coll);
ScoreDoc[] actualFiltered = coll.topDocs().scoreDocs;
+ CheckHits.checkEqual(nestedQ, expectedFiltered, actualFiltered);
+ }
- CheckHits.checkEqual(query, expectedFiltered, actualFiltered);
+ {
+ query = new BooleanQuery.Builder()
+ .add(query, Occur.MUST)
+ .add(new TermQuery(new Term("f", "C")), Occur.SHOULD)
+ .build();
+
+ CheckHits.checkTopScores(random(), query, searcher);
+
+ query = new BooleanQuery.Builder()
+ .add(new TermQuery(new Term("f", "C")), Occur.MUST)
+ .add(query, Occur.SHOULD)
+ .build();
+
+ CheckHits.checkTopScores(random(), query, searcher);
}
r.close();
dir.close();
}
+ private static Scorer reqOptScorer(IndexSearcher searcher, Query reqQ, Query optQ, boolean withBlockScore) throws IOException {
+ Scorer reqScorer = searcher
+ .createWeight(reqQ, ScoreMode.TOP_SCORES, 1)
+ .scorer(searcher.getIndexReader().leaves().get(0));
+ Scorer optScorer = searcher
+ .createWeight(optQ, ScoreMode.TOP_SCORES, 1)
+ .scorer(searcher.getIndexReader().leaves().get(0));
+ if (withBlockScore) {
+ return new ReqOptSumScorer(reqScorer, optScorer, ScoreMode.TOP_SCORES);
+ } else {
+ return new ReqOptSumScorer(reqScorer, optScorer, ScoreMode.TOP_SCORES) {
+ @Override
+ public float getMaxScore(int upTo) {
+ return Float.POSITIVE_INFINITY;
+ }
+ };
+ }
+ }
+
+ private static class TermFreqTokenStream extends TokenStream {
+ private final String term;
+ private final int termFreq;
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final TermFrequencyAttribute termFreqAtt = addAttribute(TermFrequencyAttribute.class);
+ private boolean finish;
+
+ public TermFreqTokenStream(String term, int termFreq) {
+ this.term = term;
+ this.termFreq = termFreq;
+ }
+
+ @Override
+ public boolean incrementToken() {
+ if (finish) {
+ return false;
+ }
+
+ clearAttributes();
+
+ termAtt.append(term);
+ termFreqAtt.setTermFrequency(termFreq);
+
+ finish = true;
+ return true;
+ }
+
+ @Override
+ public void reset() {
+ finish = false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/49e3cca7/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java b/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
index a050b50..b43ccc4 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/RandomApproximationQuery.java
@@ -110,6 +110,12 @@ public class RandomApproximationQuery extends Query {
@Override
public int advanceShallow(int target) throws IOException {
+ if (scorer.docID() > target && twoPhaseView.approximation.docID() != scorer.docID()) {
+ // The random approximation can return doc ids that are not present in the underlying
+ // scorer. These additional doc ids are always *before* the next matching doc so we
+ // cannot use them to shallow advance the main scorer which is already ahead.
+ target = scorer.docID();
+ }
return scorer.advanceShallow(target);
}
@@ -120,12 +126,12 @@ public class RandomApproximationQuery extends Query {
@Override
public int docID() {
- return scorer.docID();
+ return twoPhaseView.approximation().docID();
}
@Override
public DocIdSetIterator iterator() {
- return scorer.iterator();
+ return TwoPhaseIterator.asDocIdSetIterator(twoPhaseView);
}
}
[13/17] lucene-solr:jira/solr-12470: SOLR-12616: Optimize Export
writer upto 4 sort fields to get better performance. This was removed in
SOLR-11598 but brought back in the same version
Posted by ab...@apache.org.
SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance. This was removed in SOLR-11598 but brought back in the same version
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e9f3a3ce
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e9f3a3ce
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e9f3a3ce
Branch: refs/heads/jira/solr-12470
Commit: e9f3a3ce1d482bd90ba8aca6e8cb7fe6c86756eb
Parents: 7c4584b
Author: Varun Thacker <va...@apache.org>
Authored: Wed Aug 8 11:10:21 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Aug 8 13:28:00 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../solr/handler/export/DoubleValueSortDoc.java | 93 ++++++++++++++
.../solr/handler/export/ExportWriter.java | 13 +-
.../solr/handler/export/QuadValueSortDoc.java | 126 +++++++++++++++++++
.../solr/handler/export/SingleValueSortDoc.java | 82 ++++++++++++
.../org/apache/solr/handler/export/SortDoc.java | 7 +-
.../solr/handler/export/TripleValueSortDoc.java | 110 ++++++++++++++++
7 files changed, 431 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9c80773..9e6ece5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -253,6 +253,9 @@ Optimizations
* SOLR-11881: Retry update requests sent by leaders to it's followers (Varun Thacker, Mark Miller, Tomás Fernández Löbbe)
+* SOLR-12616: Optimize Export writer upto 4 sort fields to get better performance.
+ This was removed in SOLR-11598 but brought back in the same version (Amrit Sarkar, Varun Thacker)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
new file mode 100644
index 0000000..30fff33
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/DoubleValueSortDoc.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class DoubleValueSortDoc extends SingleValueSortDoc {
+
+ protected SortValue value2;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((DoubleValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((DoubleValueSortDoc)sortDoc).value2);
+ }
+
+ public DoubleValueSortDoc(SortValue value1, SortValue value2) {
+ super(value1);
+ this.value2 = value2;
+ }
+
+ public SortDoc copy() {
+ return new DoubleValueSortDoc(value1.copy(), value2.copy());
+ }
+
+ public boolean lessThan(Object o) {
+ DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if (comp == 0) {
+ return value2.compareTo(sd.value2);
+ } else {
+ return comp;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
index 57ce600..9cacfcc 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/ExportWriter.java
@@ -431,7 +431,18 @@ public class ExportWriter implements SolrCore.RawWriter, Closeable {
throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
}
}
-
+ //SingleValueSortDoc etc are specialized classes which don't have array lookups. On benchmarking large datasets
+ //This is faster than the using an array in SortDoc . So upto 4 sort fields we still want to keep specialized classes.
+ //SOLR-12616 has more details
+ if (sortValues.length == 1) {
+ return new SingleValueSortDoc(sortValues[0]);
+ } else if (sortValues.length == 2) {
+ return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
+ } else if (sortValues.length == 3) {
+ return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
+ } else if (sortValues.length == 4) {
+ return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
+ }
return new SortDoc(sortValues);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
new file mode 100644
index 0000000..c2dfcba
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/QuadValueSortDoc.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class QuadValueSortDoc extends TripleValueSortDoc {
+
+ protected SortValue value4;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ value3.setNextReader(context);
+ value4.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ value3.reset();
+ value4.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ value3.setCurrentValue(docId);
+ value4.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((QuadValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((QuadValueSortDoc)sortDoc).value2);
+ value3.setCurrentValue(((QuadValueSortDoc)sortDoc).value3);
+ value4.setCurrentValue(((QuadValueSortDoc)sortDoc).value4);
+ }
+
+ public QuadValueSortDoc(SortValue value1, SortValue value2, SortValue value3, SortValue value4) {
+ super(value1, value2, value3);
+ this.value4 = value4;
+ }
+
+ public SortDoc copy() {
+ return new QuadValueSortDoc(value1.copy(), value2.copy(), value3.copy(), value4.copy());
+ }
+
+ public boolean lessThan(Object o) {
+
+ QuadValueSortDoc sd = (QuadValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value3.compareTo(sd.value3);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value4.compareTo(sd.value4);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ QuadValueSortDoc sd = (QuadValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == 0) {
+ comp = value2.compareTo(sd.value2);
+ if(comp == 0) {
+ comp = value3.compareTo(sd.value3);
+ if(comp == 0) {
+ return value4.compareTo(sd.value4);
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
new file mode 100644
index 0000000..88fc799
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/SingleValueSortDoc.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class SingleValueSortDoc extends SortDoc {
+
+ protected SortValue value1;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ this.value1.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((SingleValueSortDoc)sortDoc).value1);
+ }
+
+ public SingleValueSortDoc(SortValue value1) {
+ super();
+ this.value1 = value1;
+ }
+
+ public SortDoc copy() {
+ return new SingleValueSortDoc(value1.copy());
+ }
+
+ public boolean lessThan(Object o) {
+ SingleValueSortDoc sd = (SingleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+
+ public int compareTo(Object o) {
+ SingleValueSortDoc sd = (SingleValueSortDoc)o;
+ return value1.compareTo(sd.value1);
+ }
+
+ public String toString() {
+ return docId+":"+value1.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
index 3cf478b..1302332 100644
--- a/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
+++ b/solr/core/src/java/org/apache/solr/handler/export/SortDoc.java
@@ -32,6 +32,8 @@ class SortDoc {
public SortDoc(SortValue[] sortValues) {
this.sortValues = sortValues;
}
+ public SortDoc() {
+ }
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
@@ -43,6 +45,7 @@ class SortDoc {
public void reset() {
this.docId = -1;
+ this.docBase = -1;
for (SortValue value : sortValues) {
value.reset();
}
@@ -82,9 +85,9 @@ class SortDoc {
SortValue[] sortValues1 = sd.sortValues;
for(int i=0; i<sortValues.length; i++) {
int comp = sortValues[i].compareTo(sortValues1[i]);
- if(comp < 0) {
+ if (comp < 0) {
return true;
- } if(comp > 0) {
+ } else if (comp > 0) {
return false;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e9f3a3ce/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
new file mode 100644
index 0000000..0241ebc
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/export/TripleValueSortDoc.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.export;
+
+import java.io.IOException;
+
+import org.apache.lucene.index.LeafReaderContext;
+
+class TripleValueSortDoc extends DoubleValueSortDoc {
+
+ protected SortValue value3;
+
+ public void setNextReader(LeafReaderContext context) throws IOException {
+ this.ord = context.ord;
+ this.docBase = context.docBase;
+ value1.setNextReader(context);
+ value2.setNextReader(context);
+ value3.setNextReader(context);
+ }
+
+ public void reset() {
+ this.docId = -1;
+ this.docBase = -1;
+ value1.reset();
+ value2.reset();
+ value3.reset();
+ }
+
+ public void setValues(int docId) throws IOException {
+ this.docId = docId;
+ value1.setCurrentValue(docId);
+ value2.setCurrentValue(docId);
+ value3.setCurrentValue(docId);
+ }
+
+ public void setValues(SortDoc sortDoc) {
+ this.docId = sortDoc.docId;
+ this.ord = sortDoc.ord;
+ this.docBase = sortDoc.docBase;
+ value1.setCurrentValue(((TripleValueSortDoc)sortDoc).value1);
+ value2.setCurrentValue(((TripleValueSortDoc)sortDoc).value2);
+ value3.setCurrentValue(((TripleValueSortDoc)sortDoc).value3);
+ }
+
+ public TripleValueSortDoc(SortValue value1, SortValue value2, SortValue value3) {
+ super(value1, value2);
+ this.value3 = value3;
+ }
+
+ public SortDoc copy() {
+ return new TripleValueSortDoc(value1.copy(), value2.copy(), value3.copy());
+ }
+
+ public boolean lessThan(Object o) {
+
+ TripleValueSortDoc sd = (TripleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value2.compareTo(sd.value2);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ comp = value3.compareTo(sd.value3);
+ if(comp == -1) {
+ return true;
+ } else if (comp == 1) {
+ return false;
+ } else {
+ return docId+docBase > sd.docId+sd.docBase;
+ }
+ }
+ }
+ }
+
+ public int compareTo(Object o) {
+ TripleValueSortDoc sd = (TripleValueSortDoc)o;
+ int comp = value1.compareTo(sd.value1);
+ if (comp == 0) {
+ comp = value2.compareTo(sd.value2);
+ if (comp == 0) {
+ return value3.compareTo(sd.value3);
+ } else {
+ return comp;
+ }
+ } else {
+ return comp;
+ }
+ }
+}