You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/10/29 19:07:25 UTC
[01/32] lucene-solr:jira/solr-12730: SOLR-12754: New hl.weightMatches
for UnifiedHighlighter WEIGHT_MATCHES (defaults to true in master/8)
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-12730 3d91b8e27 -> efb73a39b
SOLR-12754: New hl.weightMatches for UnifiedHighlighter WEIGHT_MATCHES
(defaults to true in master/8)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3e89b7a7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3e89b7a7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3e89b7a7
Branch: refs/heads/jira/solr-12730
Commit: 3e89b7a771639aacaed6c21406624a2b27231dd7
Parents: 2e757f6
Author: David Smiley <ds...@apache.org>
Authored: Tue Oct 23 13:28:10 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Oct 23 13:28:10 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 5 +++
.../solr/highlight/UnifiedSolrHighlighter.java | 33 ++++++++++++--------
.../highlight/TestUnifiedSolrHighlighter.java | 14 +++++++--
solr/solr-ref-guide/src/highlighting.adoc | 31 +++++++++++++++---
.../solr/common/params/HighlightParams.java | 1 +
5 files changed, 64 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e89b7a7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c14ac0a..32c8ae2 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -56,6 +56,9 @@ Upgrade Notes
"date.formats" configuration. To ensure date strings are properly parsed, use ParseDateFieldUpdateProcessorFactory
(an URP) commonly registered with the name "parse-date" in "schemaless mode". (David Smiley, Bar Rotstein)
+* SOLR-12754: The UnifiedHighlighter hl.weightMatches now defaults to true. If there are unforseen highlight problems,
+ this may be the culprit.
+
New Features
----------------------
@@ -154,6 +157,8 @@ New Features
* SOLR-5004: Splitshard collections API now supports splitting into more than 2 sub-shards directly i.e. by providing a
numSubShards parameter (Christine Poerschke, Anshum Gupta)
+* SOLR-12754: The UnifiedHighlighter has a new hl.weightMatches param defaulting to false (will be true in 8.0). It's
+ the highest query accuracy mode, and furthermore phrase queries are highlighted as one. (David Smiley)
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e89b7a7/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
index a4beaba..fc43687 100644
--- a/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
+++ b/solr/core/src/java/org/apache/solr/highlight/UnifiedSolrHighlighter.java
@@ -19,6 +19,7 @@ package org.apache.solr.highlight;
import java.io.IOException;
import java.text.BreakIterator;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -43,7 +44,6 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
@@ -80,6 +80,7 @@ import org.apache.solr.util.plugin.PluginInfoInitialized;
* <bool name="hl.usePhraseHighlighter">true</bool>
* <int name="hl.cacheFieldValCharsThreshold">524288</int>
* <str name="hl.offsetSource"></str>
+ * <bool name="hl.weightMatches">true</bool>
* </lst>
* </requestHandler>
* </pre>
@@ -109,6 +110,7 @@ import org.apache.solr.util.plugin.PluginInfoInitialized;
* <li>hl.usePhraseHighlighter (bool) enables phrase highlighting. default is true
* <li>hl.cacheFieldValCharsThreshold (int) controls how many characters from a field are cached. default is 524288 (1MB in 2 byte chars)
* <li>hl.offsetSource (string) specifies which offset source to use, prefers postings, but will use what's available if not specified
+ * <li>hl.weightMatches (bool) enables Lucene Weight Matches mode</li>
* </ul>
*
* @lucene.experimental
@@ -241,12 +243,9 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf
this.setCacheFieldValCharsThreshold(
params.getInt(HighlightParams.CACHE_FIELD_VAL_CHARS_THRESHOLD, DEFAULT_CACHE_CHARS_THRESHOLD));
- // SolrRequestInfo is a thread-local singleton providing access to the ResponseBuilder to code that
- // otherwise can't get it in a nicer way.
- SolrQueryRequest request = SolrRequestInfo.getRequestInfo().getReq();
final RTimerTree timerTree;
- if (request.getRequestTimer() != null) { //It may be null if not used in a search context.
- timerTree = request.getRequestTimer();
+ if (req.getRequestTimer() != null) { //It may be null if not used in a search context.
+ timerTree = req.getRequestTimer();
} else {
timerTree = new RTimerTree(); // since null checks are annoying
}
@@ -394,20 +393,28 @@ public class UnifiedSolrHighlighter extends SolrHighlighter implements PluginInf
}
@Override
- protected boolean shouldHandleMultiTermQuery(String field) {
- return params.getFieldBool(field, HighlightParams.HIGHLIGHT_MULTI_TERM, true);
- }
+ protected Set<HighlightFlag> getFlags(String field) {
+ Set<HighlightFlag> flags = EnumSet.noneOf(HighlightFlag.class);
+ if (params.getFieldBool(field, HighlightParams.HIGHLIGHT_MULTI_TERM, true)) {
+ flags.add(HighlightFlag.MULTI_TERM_QUERY);
+ }
+ if (params.getFieldBool(field, HighlightParams.USE_PHRASE_HIGHLIGHTER, true)) {
+ flags.add(HighlightFlag.PHRASES);
+ }
+ flags.add(HighlightFlag.PASSAGE_RELEVANCY_OVER_SPEED);
- @Override
- protected boolean shouldHighlightPhrasesStrictly(String field) {
- return params.getFieldBool(field, HighlightParams.USE_PHRASE_HIGHLIGHTER, true);
+ if (params.getFieldBool(field, HighlightParams.WEIGHT_MATCHES, true)
+ && flags.contains(HighlightFlag.PHRASES) && flags.contains(HighlightFlag.MULTI_TERM_QUERY)) {
+ flags.add(HighlightFlag.WEIGHT_MATCHES);
+ }
+ return flags;
}
@Override
protected Predicate<String> getFieldMatcher(String field) {
// TODO define hl.queryFieldPattern as a more advanced alternative to hl.requireFieldMatch.
- // note that the UH & PH at Lucene level default to effectively "true"
+ // note that the UH at Lucene level default to effectively "true"
if (params.getFieldBool(field, HighlightParams.FIELD_MATCH, false)) {
return field::equals; // requireFieldMatch
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e89b7a7/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java b/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
index aeffed1..2a3e3a7 100644
--- a/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
+++ b/solr/core/src/test/org/apache/solr/highlight/TestUnifiedSolrHighlighter.java
@@ -103,7 +103,7 @@ public class TestUnifiedSolrHighlighter extends SolrTestCaseJ4 {
assertQ("strict phrase handling",
req("q", "text:\"strict phrases\"", "sort", "id asc", "hl", "true"),
"count(//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/*)=1",
- "//lst[@name='highlighting']/lst[@name='101']/arr/str[1]='<em>Strict</em> <em>phrases</em> should be enabled for phrases'");
+ "//lst[@name='highlighting']/lst[@name='101']/arr/str[1]='<em>Strict phrases</em> should be enabled for phrases'");
}
public void testStrictPhrasesCanBeDisabled() {
@@ -291,5 +291,15 @@ public class TestUnifiedSolrHighlighter extends SolrTestCaseJ4 {
assertQ(req("q", "id:101", "hl", "true", "hl.q", "text:document", "hl.fl", "text3", "hl.requireFieldMatch", "true"),
"count(//lst[@name='highlighting']/lst[@name='101']/arr[@name='text3']/*)=0");
}
-
+
+ public void testWeightMatchesDisabled() {
+ clearIndex();
+ assertU(adoc("text", "alpha bravo charlie", "id", "101"));
+ assertU(commit());
+ assertQ("weight matches disabled, phrase highlights separately",
+ req("q", "text:\"alpha bravo\"", "hl", "true", "hl.weightMatches", "false"),
+ "count(//lst[@name='highlighting']/lst[@name='101']/arr[@name='text']/*)=1",
+ "//lst[@name='highlighting']/lst[@name='101']/arr/str[1]='<em>alpha</em> <em>bravo</em> charlie'");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e89b7a7/solr/solr-ref-guide/src/highlighting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/highlighting.adoc b/solr/solr-ref-guide/src/highlighting.adoc
index 2a832ee..6764343 100644
--- a/solr/solr-ref-guide/src/highlighting.adoc
+++ b/solr/solr-ref-guide/src/highlighting.adoc
@@ -147,19 +147,32 @@ There are many parameters supported by more than one highlighter, and sometimes
There are four highlighters available that can be chosen at runtime with the `hl.method` parameter, in order of general recommendation:
+
<<The Unified Highlighter,Unified Highlighter>>:: (`hl.method=unified`)
+
-The Unified Highlighter is the newest highlighter (as of Solr 6.4), which stands out as the most flexible and performant of the options. We recommend that you try this highlighter even though it isn't the default (yet).
+The Unified Highlighter is the newest highlighter (as of Solr 6.4), which stands out as the most performant and accurate of the options.
+It can handle typical requirements and others possibly via plugins/extension.
+We recommend that you try this highlighter even though it isn't the default (yet).
++
+The UH highlights a query very _accurately_ and thus is true to what the underlying Lucene query actually matches.
+Other highlighters highlight terms more liberally (over-highlight).
+A strong benefit to this highlighter is that you can opt to configure Solr to put more information in the underlying index to speed up highlighting of large documents; multiple configurations are supported, even on a per-field basis.
+There is little or no such flexibility of offset sources for the other highlighters.
+More on this below.
+
-This highlighter supports the most common highlighting parameters and can handle just about any query accurately, even SpanQueries (e.g., as seen from the `surround` parser). A strong benefit to this highlighter is that you can opt to configure Solr to put more information in the underlying index to speed up highlighting of large documents; multiple configurations are supported, even on a per-field basis. There is little or no such flexibility for the other highlighters. More on this below.
+There are some reasons not to choose this highlighter: The `surround` query parser doesn't yet work here -- SOLR-12895.
+Passage scoring does not consider boosts in the query.
+Some people want more/better passage breaking flexibility.
<<The Original Highlighter,Original Highlighter>>:: (`hl.method=original`, the default)
+
-The Original Highlighter, sometimes called the "Standard Highlighter" or "Default Highlighter", is Lucene's original highlighter – a venerable option with a high degree of customization options. Its ability to highlight just about any query accurately is a strength shared with the Unified Highlighter (they share some code for this in fact).
+The Original Highlighter, sometimes called the "Standard Highlighter" or "Default Highlighter", is Lucene's original highlighter – a venerable option with a high degree of customization options.
+It's query accuracy is good enough for most needs, although it's not quite as good/perfect as the Unified Highlighter.
+
-The Original Highlighter will normally analyze stored text on the fly in order to highlight. It will use full term vectors if available, however in this mode it isn't as fast as the Unified Highlighter or FastVector Highlighter.
+The Original Highlighter will normally analyze stored text on the fly in order to highlight. It will use full term vectors if available.
+
-This highlighter is a good choice for a wide variety of search use-cases. Where it falls short is performance; it's often twice as slow as the Unified Highlighter. And despite being the most customizable, it doesn't have a BreakIterator based fragmenter (all the others do), which could pose a challenge for some languages.
+Where this highlighter falls short is performance; it's often twice as slow as the Unified Highlighter. And despite being the most customizable, it doesn't have a BreakIterator based fragmenter (all the others do), which could pose a challenge for some languages.
+
<<The FastVector Highlighter,FastVector Highlighter>>:: (`hl.method=fastVector`)
+
@@ -171,6 +184,7 @@ This highlighter's query-representation is less advanced than the Original or Un
+
Note that both the FastVector and Original Highlighters can be used in conjunction in a search request to highlight some fields with one and some the other. In contrast, the other highlighters can only be chosen exclusively.
+
The Unified Highlighter is exclusively configured via search parameters. In contrast, some settings for the Original and FastVector Highlighters are set in `solrconfig.xml`. There's a robust example of the latter in the "```techproducts```" configset.
In addition to further information below, more information can be found in the {solr-javadocs}/solr-core/org/apache/solr/highlight/package-summary.html[Solr javadocs].
@@ -242,6 +256,13 @@ Indicates which character to break the text on. Use only if you have defined `hl
+
This is useful when the text has already been manipulated in advance to have a special delineation character at desired highlight passage boundaries. This character will still appear in the text as the last character of a passage.
+`hl.weightMatches`::
+Tells the UH to use Lucene's new "Weight Matches" API instead of doing SpanQuery conversion.
+This is the most accurate highlighting mode reflecting the query.
+Furthermore, phrases will be highlighted as a whole instead of word by word.
++
+The default is `true`.
+However if either `hl.usePhraseHighlighter` or `hl.multiTermQuery` are set to false, then this setting is effectively false no matter what you set it to.
== The Original Highlighter
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e89b7a7/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java b/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
index e09a2dc..c4825b7 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/HighlightParams.java
@@ -91,4 +91,5 @@ public interface HighlightParams {
public static final String PHRASE_LIMIT = HIGHLIGHT + ".phraseLimit"; // FVH
public static final String OFFSET_SOURCE = HIGHLIGHT + ".offsetSource"; // UH
public static final String CACHE_FIELD_VAL_CHARS_THRESHOLD = HIGHLIGHT + ".cacheFieldValCharsThreshold"; // UH
+ public static final String WEIGHT_MATCHES = HIGHLIGHT + ".weightMatches"; // UH
}
[24/32] lucene-solr:jira/solr-12730: SOLR-12913: RefGuide formatting
Posted by ab...@apache.org.
SOLR-12913: RefGuide formatting
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0c8675dc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0c8675dc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0c8675dc
Branch: refs/heads/jira/solr-12730
Commit: 0c8675dcccbcd2486db1089aed0d5e183a855b38
Parents: 4821b30
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Oct 26 13:41:09 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Oct 26 13:41:09 2018 -0400
----------------------------------------------------------------------
solr/solr-ref-guide/src/dsp.adoc | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c8675dc/solr/solr-ref-guide/src/dsp.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/dsp.adoc b/solr/solr-ref-guide/src/dsp.adoc
index 8f3b24b..e9d4fd6 100644
--- a/solr/solr-ref-guide/src/dsp.adoc
+++ b/solr/solr-ref-guide/src/dsp.adoc
@@ -28,6 +28,7 @@ the more advanced DSP functions its useful to develop a deeper intuition of the
The dot product operation is performed in two steps:
1) Element-by-element multiplication of two vectors which produces a vector of products.
+
2) Sum the vector of products to produce a scalar result.
This simple bit of math has a number of important applications.
@@ -146,7 +147,8 @@ When this expression is sent to the `/stream` handler it responds with:
----
In the example above two arrays were combined in a way that produced the mean of the first. In the second array
-each value was set to ".2". Another way of looking at this is that each value in the second array has the same weight.
+each value was set to .2. Another way of looking at this is that each value in the second array is
+applying the same weight to the values in the first array.
By varying the weights in the second array we can produce a different result.
For example if the first array represents a time series,
the weights in the second array can be set to add more weight to a particular element in the first array.
[11/32] lucene-solr:jira/solr-12730: SOLR-12793: Move
TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test
package
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
new file mode 100644
index 0000000..91b912f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java
@@ -0,0 +1,855 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.facet;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.cloud.TestCloudPivotFacet;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * Tests randomized JSON Facets, sometimes using query 'join' domain transfers and/or domain 'filter' options
+ * </p>
+ * <p>
+ * The results of each facet constraint count will be compared with a verification query using an equivalent filter
+ * </p>
+ *
+ * @see TestCloudPivotFacet
+ */
+public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
+ private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
+
+ private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
+ private static final int MAX_FIELD_NUM = 15;
+ private static final int UNIQUE_FIELD_VALS = 20;
+
+ // NOTE: set to 'true' to see if refinement testing is adequate (should get fails occasionally)
+ private static final boolean FORCE_DISABLE_REFINEMENT = false;
+
+ /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
+ /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
+
+ /** A basic client for operations at the cloud level, default collection will be set */
+ private static CloudSolrClient CLOUD_CLIENT;
+ /** One client per node */
+ private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
+
+ @BeforeClass
+ private static void createMiniSolrCloudCluster() throws Exception {
+ // sanity check constants
+ assertTrue("bad test constants: some suffixes will never be tested",
+ (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
+
+ // we need DVs on point fields to compute stats & facets
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
+
+ // multi replicas should not matter...
+ final int repFactor = usually() ? 1 : 2;
+ // ... but we definitely want to test multiple shards
+ final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
+ final int numNodes = (numShards * repFactor);
+
+ final String configName = DEBUG_LABEL + "_config-set";
+ final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
+
+ configureCluster(numNodes).addConfig(configName, configDir).configure();
+
+ Map<String, String> collectionProperties = new LinkedHashMap<>();
+ collectionProperties.put("config", "solrconfig-tlog.xml");
+ collectionProperties.put("schema", "schema_latest.xml");
+ CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
+ .setProperties(collectionProperties)
+ .process(cluster.getSolrClient());
+
+ CLOUD_CLIENT = cluster.getSolrClient();
+ CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
+
+ waitForRecoveriesToFinish(CLOUD_CLIENT);
+
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
+ }
+
+ final int numDocs = atLeast(100);
+ for (int id = 0; id < numDocs; id++) {
+ SolrInputDocument doc = sdoc("id", ""+id);
+ for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
+ // NOTE: some docs may have no value in a field
+ final int numValsThisDoc = TestUtil.nextInt(random(), 0, (usually() ? 3 : 6));
+ for (int v = 0; v < numValsThisDoc; v++) {
+ final String fieldValue = randFieldValue(fieldNum);
+
+ // for each fieldNum, there are actaully two fields: one string, and one integer
+ doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
+ doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
+ }
+ }
+ CLOUD_CLIENT.add(doc);
+ if (random().nextInt(100) < 1) {
+ CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
+ }
+ if (random().nextInt(100) < 5) {
+ CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
+ }
+ }
+ CLOUD_CLIENT.commit();
+ }
+
+ /**
+ * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
+ * uses that number and one of hte specified suffixes in it's name.
+ *
+ * @see #STR_FIELD_SUFFIXES
+ * @see #INT_FIELD_SUFFIXES
+ * @see #MAX_FIELD_NUM
+ * @see #randFieldValue
+ */
+ private static String field(final String[] suffixes, final int fieldNum) {
+ assert fieldNum < MAX_FIELD_NUM;
+
+ final String suffix = suffixes[fieldNum % suffixes.length];
+ return "field_" + fieldNum + suffix;
+ }
+ private static String strfield(final int fieldNum) {
+ return field(STR_FIELD_SUFFIXES, fieldNum);
+ }
+ private static String intfield(final int fieldNum) {
+ return field(INT_FIELD_SUFFIXES, fieldNum);
+ }
+
+ /**
+ * Given a (random) field number, returns a random (integer based) value for that field.
+ * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
+ * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
+ * will match fewer documents based on how far apart the field numbers are.
+ *
+ * @see #UNIQUE_FIELD_VALS
+ * @see #field
+ */
+ private static String randFieldValue(final int fieldNum) {
+ return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
+ }
+
+
+ @AfterClass
+ private static void afterClass() throws Exception {
+ CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+ for (HttpSolrClient client : CLIENTS) {
+ client.close();
+ }
+ CLIENTS = null;
+ }
+
+ /** Sanity check that malformed requests produce errors */
+ public void testMalformedGivesError() throws Exception {
+
+ ignoreException(".*'join' domain change.*");
+
+ for (String join : Arrays.asList("bogus",
+ "{ }",
+ "{ from:null, to:foo_s }",
+ "{ from:foo_s }",
+ "{ from:foo_s, to:foo_s, bogus:'what what?' }",
+ "{ to:foo_s, bogus:'what what?' }")) {
+ SolrException e = expectThrows(SolrException.class, () -> {
+ final SolrParams req = params("q", "*:*", "json.facet",
+ "{ x : { type:terms, field:x_s, domain: { join:"+join+" } } }");
+ final NamedList trash = getRandClient(random()).request(new QueryRequest(req));
+ });
+ assertEquals(join + " -> " + e, SolrException.ErrorCode.BAD_REQUEST.code, e.code());
+ assertTrue(join + " -> " + e, e.getMessage().contains("'join' domain change"));
+ }
+ }
+
+ public void testSanityCheckDomainMethods() throws Exception {
+ {
+ final JoinDomain empty = new JoinDomain(null, null, null);
+ assertEquals(null, empty.toJSONFacetParamValue());
+ final SolrParams out = empty.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals(null, out.get("safe_key"));
+ assertEquals("qqq", out.get("q"));
+ }
+ {
+ final JoinDomain join = new JoinDomain("xxx", "yyy", null);
+ assertEquals("domain:{join:{from:xxx,to:yyy}}", join.toJSONFacetParamValue().toString());
+ final SolrParams out = join.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals("qqq", out.get("safe_key"));
+ assertEquals("{!join from=xxx to=yyy v=$safe_key}", out.get("q"));
+
+ }
+ {
+ final JoinDomain filter = new JoinDomain(null, null, "zzz");
+ assertEquals("domain:{filter:'zzz'}", filter.toJSONFacetParamValue().toString());
+ final SolrParams out = filter.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals(null, out.get("safe_key"));
+ assertEquals("zzz AND qqq", out.get("q"));
+ }
+ {
+ final JoinDomain both = new JoinDomain("xxx", "yyy", "zzz");
+ assertEquals("domain:{join:{from:xxx,to:yyy},filter:'zzz'}", both.toJSONFacetParamValue().toString());
+ final SolrParams out = both.applyDomainToQuery("safe_key", params("q","qqq"));
+ assertNotNull(out);
+ assertEquals("qqq", out.get("safe_key"));
+ assertEquals("zzz AND {!join from=xxx to=yyy v=$safe_key}", out.get("q"));
+ }
+ }
+
+ /**
+ * Test some small, hand crafted, but non-trivial queries that are
+ * easier to trace/debug then a pure random monstrosity.
+ * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
+ */
+ public void testBespoke() throws Exception {
+
+ { // sanity check our test methods can handle a query matching no docs
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
+ top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
+ facets.put("empty_top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, strfield(7) + ":bogus");
+ assertEquals("Empty search result shouldn't have found a single bucket",
+ UNIQUE_FIELD_VALS, maxBuckets.get());
+ }
+
+ { // sanity check our test methods can handle a query where a facet filter prevents any doc from having terms
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, "-*:*"));
+ top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
+ facets.put("filtered_top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
+ assertEquals("Empty join filter shouldn't have found a single bucket",
+ UNIQUE_FIELD_VALS, maxBuckets.get());
+ }
+
+ { // sanity check our test methods can handle a query where a facet filter prevents any doc from having sub-terms
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(8), strfield(8), null));
+ top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(null, null, "-*:*")));
+ facets.put("filtered_top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+
+ { // strings
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
+ top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ }
+
+ { // ints
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(intfield(9), new JoinDomain(intfield(5), intfield(9), null));
+ facets.put("top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+intfield(7)+":16 OR "+intfield(3)+":13)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ }
+
+ { // some domains with filter only, no actual join
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, strfield(9)+":[* TO *]"));
+ top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(null, null, strfield(3)+":[* TO 5]")));
+ facets.put("top", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
+
+ }
+
+ { // low limits, explicit refinement
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9),
+ new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
+ 5, 0, true);
+ top.subFacets.put("facet_5", new TermFacet(strfield(11),
+ new JoinDomain(strfield(8), strfield(8), null),
+ 10, 0, true));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
+ }
+
+ { // low limit, high overrequest
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9),
+ new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
+ 5, UNIQUE_FIELD_VALS + 10, false);
+ top.subFacets.put("facet_5", new TermFacet(strfield(11),
+ new JoinDomain(strfield(8), strfield(8), null),
+ 10, UNIQUE_FIELD_VALS + 10, false));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
+ }
+
+ { // low limit, low overrequest, explicit refinement
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9),
+ new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
+ 5, 7, true);
+ top.subFacets.put("facet_5", new TermFacet(strfield(11),
+ new JoinDomain(strfield(8), strfield(8), null),
+ 10, 7, true));
+ facets.put("facet_4", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
+ assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
+ }
+
+ }
+
+ public void testTheTestRandomRefineParam() {
+ // sanity check that randomRefineParam never violates isRefinementNeeded
+ // (should be imposisble ... unless someone changes/breaks the randomization logic in the future)
+ final int numIters = atLeast(100);
+ for (int iter = 0; iter < numIters; iter++) {
+ final Integer limit = TermFacet.randomLimitParam(random());
+ final Integer overrequest = TermFacet.randomOverrequestParam(random());
+ final Boolean refine = TermFacet.randomRefineParam(random(), limit, overrequest);
+ if (TermFacet.isRefinementNeeded(limit, overrequest)) {
+ assertEquals("limit: " + limit + ", overrequest: " + overrequest + ", refine: " + refine,
+ Boolean.TRUE, refine);
+ }
+ }
+ }
+
+ public void testTheTestTermFacetShouldFreakOutOnBadRefineOptions() {
+ expectThrows(AssertionError.class, () -> {
+ final TermFacet bogus = new TermFacet("foo", null, 5, 0, false);
+ });
+ }
+
+ public void testRandom() throws Exception {
+
+ // we put a safety valve in place on the maximum number of buckets that we are willing to verify
+ // across *all* the queries that we do.
+ // that way if the randomized queries we build all have relatively small facets, so be it, but if
+ // we get a really big one early on, we can test as much as possible, skip other iterations.
+ //
+ // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
+ final int maxBucketsAllowed = atLeast(2000);
+ final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
+
+ final int numIters = atLeast(20);
+ for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
+ assertFacetCountsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(), buildRandomQuery());
+ }
+ assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
+ }
+
+ /**
+ * Generates a random query string across the randomized fields/values in the index
+ *
+ * @see #randFieldValue
+ * @see #field
+ */
+ private static String buildRandomQuery() {
+ if (0 == TestUtil.nextInt(random(), 0,10)) {
+ return "*:*";
+ }
+ final int numClauses = TestUtil.nextInt(random(), 3, 10);
+ List<String> clauses = new ArrayList<String>(numClauses);
+ for (int c = 0; c < numClauses; c++) {
+ final int fieldNum = random().nextInt(MAX_FIELD_NUM);
+ // keep queries simple, just use str fields - not point of test
+ clauses.add(strfield(fieldNum) + ":" + randFieldValue(fieldNum));
+ }
+ return "(" + StringUtils.join(clauses, " OR ") + ")";
+ }
+
+ /**
+ * Given a set of (potentially nested) term facets, and a base query string, asserts that
+ * the actual counts returned when executing that query with those facets match the expected results
+ * of filtering on the equivalent facet terms+domain
+ */
+ private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ Map<String,TermFacet> expected,
+ final String query) throws SolrServerException, IOException {
+
+ final SolrParams baseParams = params("q", query, "rows","0");
+ final SolrParams facetParams = params("json.facet", ""+TermFacet.toJSONFacetParamValue(expected));
+ final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
+
+ log.info("Doing full run: {}", initParams);
+
+ QueryResponse rsp = null;
+ // JSON Facets not (currently) available from QueryResponse...
+ NamedList topNamedList = null;
+ try {
+ rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
+ assertNotNull(initParams + " is null rsp?", rsp);
+ topNamedList = rsp.getResponse();
+ assertNotNull(initParams + " is null topNamedList?", topNamedList);
+ } catch (Exception e) {
+ throw new RuntimeException("init query failed: " + initParams + ": " +
+ e.getMessage(), e);
+ }
+ try {
+ final NamedList facetResponse = (NamedList) topNamedList.get("facets");
+ assertNotNull("null facet results?", facetResponse);
+ assertEquals("numFound mismatch with top count?",
+ rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
+ if (0 == rsp.getResults().getNumFound()) {
+ // when the query matches nothing, we should expect no top level facets
+ expected = Collections.emptyMap();
+ }
+ assertFacetCountsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
+ } catch (AssertionError e) {
+ throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
+ } finally {
+ log.info("Ending full run");
+ }
+ }
+
+ /**
+ * Recursive Helper method that walks the actual facet response, comparing the counts to the expected output
+ * based on the equivalent filters generated from the original TermFacet.
+ */
+ private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ final Map<String,TermFacet> expected,
+ final SolrParams baseParams,
+ final NamedList actualFacetResponse) throws SolrServerException, IOException {
+
+ for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
+ final String facetKey = entry.getKey();
+ final TermFacet facet = entry.getValue();
+ final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
+ assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
+ final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
+ assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
+
+ if (buckets.isEmpty()) {
+ // should only happen if the baseParams query does not match any docs with our field X
+ final long docsWithField = getRandClient(random()).query
+ (facet.applyValueConstraintAndDomain(baseParams, facetKey, "[* TO *]")).getResults().getNumFound();
+ assertEquals(facetKey + " has no buckets, but docs in query exist with field: " + facet.field,
+ 0, docsWithField);
+ }
+
+ for (NamedList bucket : buckets) {
+ final long count = ((Number) bucket.get("count")).longValue();
+ final String fieldVal = bucket.get("val").toString(); // int or stringified int
+
+ // change our query to filter on the fieldVal, and wrap in the facet domain (if any)
+ final SolrParams verifyParams = facet.applyValueConstraintAndDomain(baseParams, facetKey, fieldVal);
+
+ // check the count for this bucket
+ assertEquals(facetKey + ": " + verifyParams,
+ count, getRandClient(random()).query(verifyParams).getResults().getNumFound());
+
+ if (maxBucketsToCheck.decrementAndGet() <= 0) {
+ return;
+ }
+
+ // recursively check subFacets
+ if (! facet.subFacets.isEmpty()) {
+ assertFacetCountsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
+ }
+ }
+ }
+ assertTrue("facets have unexpected keys left over: " + actualFacetResponse,
+ // should alwasy be a count, maybe a 'val' if we're a subfacet
+ (actualFacetResponse.size() == expected.size() + 1) ||
+ (actualFacetResponse.size() == expected.size() + 2));
+ }
+
+
+ /**
+ * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
+ *
+ * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
+ */
+ private static final class TermFacet {
+ public final String field;
+ public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
+ public final JoinDomain domain; // may be null
+ public final Integer limit; // may be null
+ public final Integer overrequest; // may be null
+ public final Boolean refine; // may be null
+
+ /** Simplified constructor asks for limit = # unique vals */
+ public TermFacet(String field, JoinDomain domain) {
+ this(field, domain, UNIQUE_FIELD_VALS, 0, false);
+ }
+ public TermFacet(String field, JoinDomain domain, Integer limit, Integer overrequest, Boolean refine) {
+ assert null != field;
+ this.field = field;
+ this.domain = domain;
+ this.limit = limit;
+ this.overrequest = overrequest;
+ this.refine = refine;
+ if (isRefinementNeeded(limit, overrequest)) {
+ assertEquals("Invalid refine param based on limit & overrequest: " + this.toString(),
+ Boolean.TRUE, refine);
+ }
+ }
+
+ /**
+ * Returns new SolrParams that:
+ * <ul>
+ * <li>copy the original SolrParams</li>
+ * <li>modify/wrap the original "q" param to capture the domain change for this facet (if any)</li>
+ * <li>add a filter query against this field with the specified value</li>
+ * </ul>
+ *
+ * @see JoinDomain#applyDomainToQuery
+ */
+ public SolrParams applyValueConstraintAndDomain(SolrParams orig, String facetKey, String facetVal) {
+ // first wrap our original query in the domain if there is one...
+ if (null != domain) {
+ orig = domain.applyDomainToQuery(facetKey + "_q", orig);
+ }
+ // then filter by the facet value we need to test...
+ final ModifiableSolrParams out = new ModifiableSolrParams(orig);
+ out.set("q", field + ":" + facetVal + " AND " + orig.get("q"));
+
+ return out;
+ }
+
+ /**
+ * recursively generates the <code>json.facet</code> param value to use for testing this facet
+ */
+ private CharSequence toJSONFacetParamValue() {
+ final String limitStr = (null == limit) ? "" : (", limit:" + limit);
+ final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
+ final String refineStr = (null == refine) ? "" : ", refine:" + refine;
+ final StringBuilder sb = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + refineStr);
+ if (! subFacets.isEmpty()) {
+ sb.append(", facet:");
+ sb.append(toJSONFacetParamValue(subFacets));
+ }
+ if (null != domain) {
+ CharSequence ds = domain.toJSONFacetParamValue();
+ if (null != ds) {
+ sb.append(", ").append(ds);
+ }
+ }
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
+ * use for testing them against in a solr request.
+ */
+ public static CharSequence toJSONFacetParamValue(Map<String,TermFacet> facets) {
+ assert null != facets;
+ assert 0 < facets.size();
+ StringBuilder sb = new StringBuilder("{");
+ for (String key : facets.keySet()) {
+ sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
+ sb.append(" ,");
+ }
+ sb.setLength(sb.length() - 1);
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Factory method for generating some random (nested) facets.
+ *
+ * For simplicity, each facet will have a unique key name, regardless of it's depth under other facets
+ *
+ * @see JoinDomain
+ */
+ public static Map<String,TermFacet> buildRandomFacets() {
+ // for simplicity, use a unique facet key regardless of depth - simplifies verification
+ AtomicInteger keyCounter = new AtomicInteger(0);
+ final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
+ return buildRandomFacets(keyCounter, maxDepth);
+ }
+
+ /**
+ * picks a random value for the "limit" param, biased in favor of interesting test cases
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Integer randomLimitParam(Random r) {
+ final int limit = 1 + r.nextInt(UNIQUE_FIELD_VALS * 2);
+ if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
+ return -1; // unlimited
+ } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
+ return null; // sometimes, don't specify limit if it's the default
+ }
+ return limit;
+ }
+
+ /**
+ * picks a random value for the "overrequest" param, biased in favor of interesting test cases
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Integer randomOverrequestParam(Random r) {
+ switch(r.nextInt(10)) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ return 0; // 40% of the time, no overrequest to better stress refinement
+ case 4:
+ case 5:
+ return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
+ case 6:
+ return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
+ default: break;
+ }
+ // else.... either leave param unspecified (or redundently specify the -1 default)
+ return r.nextBoolean() ? null : -1;
+ }
+
+ /**
+ * picks a random value for the "refine" param, that is garunteed to be suitable for
+ * the specified limit & overrequest params.
+ *
+ * @return a value to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #randomLimitParam
+ * @see #randomOverrequestParam
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Boolean randomRefineParam(Random r, Integer limitParam, Integer overrequestParam) {
+ if (isRefinementNeeded(limitParam, overrequestParam)) {
+ return true;
+ }
+
+ // refinement is not required
+ if (0 == r.nextInt(10)) { // once in a while, turn on refinement even if it isn't needed.
+ return true;
+ }
+ // explicitly or implicitly indicate refinement is not needed
+ return r.nextBoolean() ? false : null;
+ }
+
+ /**
+ * Deterministicly identifies if the specified limit & overrequest params <b>require</b>
+ * a "refine:true" param be used in the the request, in order for the counts to be 100% accurate.
+ *
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static boolean isRefinementNeeded(Integer limitParam, Integer overrequestParam) {
+
+ if (FORCE_DISABLE_REFINEMENT) {
+ return false;
+ }
+
+ // use the "effective" values if the params are null
+ final int limit = null == limitParam ? DEFAULT_LIMIT : limitParam;
+ final int overrequest = null == overrequestParam ? 0 : overrequestParam;
+
+ return
+ // don't presume how much overrequest will be done by default, just check the limit
+ (overrequest < 0 && limit < UNIQUE_FIELD_VALS)
+ // if the user specified overrequest is not "enough" to get all unique values
+ || (overrequest >= 0 && (long)limit + overrequest < UNIQUE_FIELD_VALS);
+ }
+
+ /**
+ * recursive helper method for building random facets
+ *
+ * @param keyCounter used to ensure every generated facet has a unique key name
+ * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
+ */
+ private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
+ final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
+ Map<String,TermFacet> results = new LinkedHashMap<>();
+ for (int i = 0; i < numFacets; i++) {
+ final JoinDomain domain = JoinDomain.buildRandomDomain();
+ assert null != domain;
+ final Integer limit = randomLimitParam(random());
+ final Integer overrequest = randomOverrequestParam(random());
+ final TermFacet facet = new TermFacet(field(random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES,
+ random().nextInt(MAX_FIELD_NUM)),
+ domain, limit, overrequest,
+ randomRefineParam(random(), limit, overrequest));
+ results.put("facet_" + keyCounter.incrementAndGet(), facet);
+ if (0 < maxDepth) {
+ // if we're going wide, don't go deep
+ final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
+ facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
+ }
+ }
+ return results;
+ }
+ }
+
+
+ /**
+ * Models a Domain Change which includes either a 'join' or a 'filter' or both
+ */
+ private static final class JoinDomain {
+ public final String from;
+ public final String to;
+ public final String filter; // not bothering with more then 1 filter, not the point of the test
+
+ /**
+ * @param from left side of join field name, null if domain involves no joining
+ * @param to right side of join field name, null if domain involves no joining
+ * @param filter filter to apply to domain, null if domain involves no filtering
+ */
+ public JoinDomain(String from, String to, String filter) {
+ assert ! ((null == from) ^ (null == to)) : "if from is null, to must be null";
+ this.from = from;
+ this.to = to;
+ this.filter = filter;
+ }
+
+ /**
+ * @return the JSON string representing this domain for use in a facet param, or null if no domain should be used
+ * */
+ public CharSequence toJSONFacetParamValue() {
+ if (null == from && null == filter) {
+ return null;
+ }
+ StringBuilder sb = new StringBuilder("domain:{");
+ if (null != from) {
+ assert null != to;
+ sb. append("join:{from:").append(from).append(",to:").append(to).append("}");
+ if (null != filter){
+ sb.append(",");
+ }
+
+ }
+ if (null != filter) {
+ sb.append("filter:'").append(filter).append("'");
+ }
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Given some original SolrParams, returns new SolrParams where the original "q" param is wrapped
+ * as needed to apply the equivalent transformation to a query as this domain would to a facet
+ */
+ public SolrParams applyDomainToQuery(String safeKey, SolrParams in) {
+ assert null == in.get(safeKey); // shouldn't be possible if every facet uses a unique key string
+
+ String q = in.get("q");
+ final ModifiableSolrParams out = new ModifiableSolrParams(in);
+ if (null != from) {
+ out.set(safeKey, in.get("q"));
+ q = "{!join from="+from+" to="+to+" v=$"+safeKey+"}";
+ }
+ if (null != filter) {
+ q = filter + " AND " + q;
+ }
+ out.set("q", q);
+ return out;
+ }
+
+ /**
+ * Factory method for creating a random domain change to use with a facet - may return an 'noop' JoinDomain,
+ * but will never return null.
+ */
+ public static JoinDomain buildRandomDomain() {
+
+ // use consistent type on both sides of join
+ final String[] suffixes = random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES;
+
+ final boolean noJoin = random().nextBoolean();
+
+ String from = null;
+ String to = null;
+ for (;;) {
+ if (noJoin) break;
+ from = field(suffixes, random().nextInt(MAX_FIELD_NUM));
+ to = field(suffixes, random().nextInt(MAX_FIELD_NUM));
+ // HACK: joined numeric point fields need docValues.. for now just skip _is fields if we are dealing with points.
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP) && (from.endsWith("_is") || to.endsWith("_is")))
+ {
+ continue;
+ }
+ break;
+ }
+
+ // keep it simple, only filter on string fields - not point of test
+ final String filterField = strfield(random().nextInt(MAX_FIELD_NUM));
+
+ final String filter = random().nextBoolean() ? null : filterField+":[* TO *]";
+ return new JoinDomain(from, to, filter);
+ }
+ }
+
+ /**
+ * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
+ * at a node in our cluster
+ */
+ public static SolrClient getRandClient(Random rand) {
+ int numClients = CLIENTS.size();
+ int idx = TestUtil.nextInt(rand, 0, numClients);
+
+ return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
+ }
+
+ public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
+ assert null != client.getDefaultCollection();
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
+ client.getZkStateReader(),
+ true, true, 330);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java
new file mode 100644
index 0000000..e212993
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java
@@ -0,0 +1,678 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search.facet;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.lang.StringUtils;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.AbstractDistribZkTestBase;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import static org.apache.solr.search.facet.RelatednessAgg.computeRelatedness;
+import static org.apache.solr.search.facet.RelatednessAgg.roundTo5Digits;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>
+ * A randomized test of nested facets using the <code>relatedness()</code> function, that asserts the
+ * accuracy the results for all the buckets returned using verification queries of the (expected)
+ * foreground & background queries based on the nested facet terms.
+ * <p>
+ * Note that unlike normal facet "count" verification, using a high limit + overrequest isn't a substitute
+ * for refinement in order to ensure accurate "skg" computation across shards. For that reason, this
+ * tests forces <code>refine: true</code> (unlike {@link TestCloudJSONFacetJoinDomain}) and specifices a
+ * <code>domain: { 'query':'*:*' }</code> for every facet, in order to garuntee that all shards
+ * participate in all facets, so that the popularity & relatedness values returned can be proven
+ * with validation requests.
+ * </p>
+ * <p>
+ * (Refinement alone is not enough. Using the '*:*' query as the facet domain is neccessary to
+ * prevent situations where a single shardX may return candidate bucket with no child-buckets due to
+ * the normal facet intersections, but when refined on other shardY(s), can produce "high scoring"
+ * SKG child-buckets, which would then be missing the foreground/background "size" contributions from
+ * shardX.
+ * </p>
+ *
+ * @see TestCloudJSONFacetJoinDomain
+ */
+@Slow
+public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
+ private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
+
+ private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
+ private static final int MAX_FIELD_NUM = 15;
+ private static final int UNIQUE_FIELD_VALS = 50;
+
+ /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
+ /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
+ private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
+
+ /** A basic client for operations at the cloud level, default collection will be set */
+ private static CloudSolrClient CLOUD_CLIENT;
+ /** One client per node */
+ private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
+
+ @BeforeClass
+ private static void createMiniSolrCloudCluster() throws Exception {
+ // sanity check constants
+ assertTrue("bad test constants: some suffixes will never be tested",
+ (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
+
+ // we need DVs on point fields to compute stats & facets
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
+
+ // multi replicas should not matter...
+ final int repFactor = usually() ? 1 : 2;
+ // ... but we definitely want to test multiple shards
+ final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
+ final int numNodes = (numShards * repFactor);
+
+ final String configName = DEBUG_LABEL + "_config-set";
+ final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
+
+ configureCluster(numNodes).addConfig(configName, configDir).configure();
+
+ Map<String, String> collectionProperties = new LinkedHashMap<>();
+ collectionProperties.put("config", "solrconfig-tlog.xml");
+ collectionProperties.put("schema", "schema_latest.xml");
+ CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
+ .setProperties(collectionProperties)
+ .process(cluster.getSolrClient());
+
+ CLOUD_CLIENT = cluster.getSolrClient();
+ CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
+
+ waitForRecoveriesToFinish(CLOUD_CLIENT);
+
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
+ }
+
+ final int numDocs = atLeast(100);
+ for (int id = 0; id < numDocs; id++) {
+ SolrInputDocument doc = sdoc("id", ""+id);
+ for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
+ // NOTE: we ensure every doc has at least one value in each field
+ // that way, if a term is returned for a parent there there is garunteed to be at least one
+ // one term in the child facet as well.
+ //
+ // otherwise, we'd face the risk of a single shardX returning parentTermX as a top term for
+ // the parent facet, but having no child terms -- meanwhile on refinement another shardY that
+ // did *not* returned parentTermX in phase#1, could return some *new* child terms under
+ // parentTermX, but their stats would not include the bgCount from shardX.
+ //
+ // in normal operation, this is an edge case that isn't a big deal because the ratios &
+ // relatedness scores are statistically approximate, but for the purpose of this test where
+ // we verify correctness via exactness we need all shards to contribute to the SKG statistics
+ final int numValsThisDoc = TestUtil.nextInt(random(), 1, (usually() ? 5 : 10));
+ for (int v = 0; v < numValsThisDoc; v++) {
+ final String fieldValue = randFieldValue(fieldNum);
+
+ // for each fieldNum, there are actaully two fields: one string, and one integer
+ doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
+ doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
+ }
+ }
+ CLOUD_CLIENT.add(doc);
+ if (random().nextInt(100) < 1) {
+ CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
+ }
+ if (random().nextInt(100) < 5) {
+ CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
+ }
+ }
+ CLOUD_CLIENT.commit();
+ }
+
+ /**
+ * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
+ * uses that number and one of hte specified suffixes in it's name.
+ *
+ * @see #STR_FIELD_SUFFIXES
+ * @see #INT_FIELD_SUFFIXES
+ * @see #MAX_FIELD_NUM
+ * @see #randFieldValue
+ */
+ private static String field(final String[] suffixes, final int fieldNum) {
+ assert fieldNum < MAX_FIELD_NUM;
+
+ final String suffix = suffixes[fieldNum % suffixes.length];
+ return "field_" + fieldNum + suffix;
+ }
+ private static String strfield(final int fieldNum) {
+ return field(STR_FIELD_SUFFIXES, fieldNum);
+ }
+ private static String intfield(final int fieldNum) {
+ return field(INT_FIELD_SUFFIXES, fieldNum);
+ }
+
+ /**
+ * Given a (random) field number, returns a random (integer based) value for that field.
+ * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
+ * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
+ * will match fewer documents based on how far apart the field numbers are.
+ *
+ * @see #UNIQUE_FIELD_VALS
+ * @see #field
+ */
+ private static String randFieldValue(final int fieldNum) {
+ return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
+ }
+
+
+ @AfterClass
+ private static void afterClass() throws Exception {
+ CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+ for (HttpSolrClient client : CLIENTS) {
+ client.close();
+ }
+ CLIENTS = null;
+ }
+
+ /**
+ * Test some small, hand crafted, but non-trivial queries that are
+ * easier to trace/debug then a pure random monstrosity.
+ * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
+ */
+ public void testBespoke() throws Exception {
+ { // trivial single level facet
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, null);
+ facets.put("top1", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+
+ { // trivial single level facet w/sorting on skg
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, "skg desc");
+ facets.put("top2", top);
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+
+ { // trivial single level facet w/ 2 diff ways to request "limit = (effectively) Infinite"
+ // to sanity check refinement of buckets missing from other shard in both cases
+
+ // NOTE that these two queries & facets *should* effectively identical given that the
+ // very large limit value is big enough no shard will ever return that may terms,
+ // but the "limit=-1" case it actaully triggers slightly different code paths
+ // because it causes FacetField.returnsPartial() to be "true"
+ for (int limit : new int[] { 999999999, -1 }) {
+ Map<String,TermFacet> facets = new LinkedHashMap<>();
+ facets.put("top_facet_limit__" + limit, new TermFacet(strfield(9), limit, 0, "skg desc"));
+ final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
+ assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
+ assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
+ }
+ }
+ }
+
+ public void testRandom() throws Exception {
+
+ // since the "cost" of verifying the stats for each bucket is so high (see TODO in verifySKGResults())
+ // we put a safety valve in place on the maximum number of buckets that we are willing to verify
+ // across *all* the queries that we do.
+ // that way if the randomized queries we build all have relatively small facets, so be it, but if
+ // we get a really big one early on, we can test as much as possible, skip other iterations.
+ //
+ // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
+ final int maxBucketsAllowed = atLeast(2000);
+ final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
+
+ final int numIters = atLeast(10);
+ for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
+ assertFacetSKGsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(),
+ buildRandomQuery(), buildRandomQuery(), buildRandomQuery());
+ }
+ assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
+
+
+ }
+
+ /**
+ * Generates a random query string across the randomized fields/values in the index
+ *
+ * @see #randFieldValue
+ * @see #field
+ */
+ private static String buildRandomQuery() {
+ if (0 == TestUtil.nextInt(random(), 0,10)) {
+ return "*:*";
+ }
+ final int numClauses = TestUtil.nextInt(random(), 3, 10);
+ final String[] clauses = new String[numClauses];
+ for (int c = 0; c < numClauses; c++) {
+ final int fieldNum = random().nextInt(MAX_FIELD_NUM);
+ // keep queries simple, just use str fields - not point of test
+ clauses[c] = strfield(fieldNum) + ":" + randFieldValue(fieldNum);
+ }
+ return buildORQuery(clauses);
+ }
+
+ private static String buildORQuery(String... clauses) {
+ assert 0 < clauses.length;
+ return "(" + StringUtils.join(clauses, " OR ") + ")";
+ }
+
+ /**
+ * Given a set of term facets, and top level query strings, asserts that
+ * the SKG stats for each facet term returned when executing that query with those foreground/background
+ * queries match the expected results of executing the equivalent queries in isolation.
+ *
+ * @see #verifySKGResults
+ */
+ private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ Map<String,TermFacet> expected,
+ final String query,
+ final String foreQ,
+ final String backQ) throws SolrServerException, IOException {
+ final SolrParams baseParams = params("rows","0", "fore", foreQ, "back", backQ);
+
+ final SolrParams facetParams = params("q", query,
+ "json.facet", ""+TermFacet.toJSONFacetParamValue(expected,null));
+ final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
+
+ log.info("Doing full run: {}", initParams);
+
+ QueryResponse rsp = null;
+ // JSON Facets not (currently) available from QueryResponse...
+ NamedList topNamedList = null;
+ try {
+ rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
+ assertNotNull(initParams + " is null rsp?", rsp);
+ topNamedList = rsp.getResponse();
+ assertNotNull(initParams + " is null topNamedList?", topNamedList);
+ } catch (Exception e) {
+ throw new RuntimeException("init query failed: " + initParams + ": " +
+ e.getMessage(), e);
+ }
+ try {
+ final NamedList facetResponse = (NamedList) topNamedList.get("facets");
+ assertNotNull("null facet results?", facetResponse);
+ assertEquals("numFound mismatch with top count?",
+ rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
+
+ // Note: even if the query has numFound=0, our explicit background query domain should
+ // still force facet results
+ // (even if the background query matches nothing, that just means there will be no
+ // buckets in those facets)
+ assertFacetSKGsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
+
+ } catch (AssertionError e) {
+ throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
+ } finally {
+ log.info("Ending full run");
+ }
+ }
+
+ /**
+ * Recursive helper method that walks the actual facet response, comparing the SKG results to
+ * the expected output based on the equivalent filters generated from the original TermFacet.
+ */
+ private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
+ final Map<String,TermFacet> expected,
+ final SolrParams baseParams,
+ final NamedList actualFacetResponse) throws SolrServerException, IOException {
+
+ for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
+ final String facetKey = entry.getKey();
+ final TermFacet facet = entry.getValue();
+ final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
+ assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
+ final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
+ assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
+
+ if (buckets.isEmpty()) {
+ // should only happen if the background query does not match any docs with field X
+ final long docsWithField = getNumFound(params("_trace", "noBuckets",
+ "rows", "0",
+ "q", facet.field+":[* TO *]",
+ "fq", baseParams.get("back")));
+
+ assertEquals(facetKey + " has no buckets, but docs in background exist with field: " + facet.field,
+ 0, docsWithField);
+ }
+
+ // NOTE: it's important that we do this depth first -- not just because it's the easiest way to do it,
+ // but because it means that our maxBucketsToCheck will ensure we do a lot of deep sub-bucket checking,
+ // not just all the buckets of the top level(s) facet(s)
+ for (NamedList bucket : buckets) {
+ final String fieldVal = bucket.get("val").toString(); // int or stringified int
+
+ verifySKGResults(facetKey, facet, baseParams, fieldVal, bucket);
+ if (maxBucketsToCheck.decrementAndGet() <= 0) {
+ return;
+ }
+
+ final SolrParams verifyParams = SolrParams.wrapAppended(baseParams,
+ params("fq", facet.field + ":" + fieldVal));
+
+ // recursively check subFacets
+ if (! facet.subFacets.isEmpty()) {
+ assertFacetSKGsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
+ }
+ }
+ }
+
+ { // make sure we don't have any facet keys we don't expect
+ // a little hackish because subfacets have extra keys...
+ final LinkedHashSet expectedKeys = new LinkedHashSet(expected.keySet());
+ expectedKeys.add("count");
+ if (0 <= actualFacetResponse.indexOf("val",0)) {
+ expectedKeys.add("val");
+ expectedKeys.add("skg");
+ }
+ assertEquals("Unexpected keys in facet response",
+ expectedKeys, actualFacetResponse.asShallowMap().keySet());
+ }
+ }
+
+ /**
+ * Verifies that the popularity & relatedness values containined in a single SKG bucket
+ * match the expected values based on the facet field & bucket value, as well the existing
+ * filterParams.
+ *
+ * @see #assertFacetSKGsAreCorrect
+ */
+ private void verifySKGResults(String facetKey, TermFacet facet, SolrParams filterParams,
+ String fieldVal, NamedList<Object> bucket)
+ throws SolrServerException, IOException {
+
+ final String bucketQ = facet.field+":"+fieldVal;
+ final NamedList<Object> skgBucket = (NamedList<Object>) bucket.get("skg");
+ assertNotNull(facetKey + "/bucket:" + bucket.toString(), skgBucket);
+
+ // TODO: make this more efficient?
+ // ideally we'd do a single query w/4 facet.queries, one for each count
+ // but formatting the queries is a pain, currently we leverage the accumulated fq's
+ final long fgSize = getNumFound(SolrParams.wrapAppended(params("_trace", "fgSize",
+ "rows","0",
+ "q","{!query v=$fore}"),
+ filterParams));
+ final long bgSize = getNumFound(params("_trace", "bgSize",
+ "rows","0",
+ "q", filterParams.get("back")));
+
+ final long fgCount = getNumFound(SolrParams.wrapAppended(params("_trace", "fgCount",
+ "rows","0",
+ "q","{!query v=$fore}",
+ "fq", bucketQ),
+ filterParams));
+ final long bgCount = getNumFound(params("_trace", "bgCount",
+ "rows","0",
+ "q", bucketQ,
+ "fq", filterParams.get("back")));
+
+ assertEquals(facetKey + "/bucket:" + bucket + " => fgPop should be: " + fgCount + " / " + bgSize,
+ roundTo5Digits((double) fgCount / bgSize),
+ skgBucket.get("foreground_popularity"));
+ assertEquals(facetKey + "/bucket:" + bucket + " => bgPop should be: " + bgCount + " / " + bgSize,
+ roundTo5Digits((double) bgCount / bgSize),
+ skgBucket.get("background_popularity"));
+ assertEquals(facetKey + "/bucket:" + bucket + " => relatedness is wrong",
+ roundTo5Digits(computeRelatedness(fgCount, fgSize, bgCount, bgSize)),
+ skgBucket.get("relatedness"));
+
+ }
+
+
+ /**
+ * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
+ *
+ * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
+ */
+ private static final class TermFacet {
+ public final String field;
+ public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
+ public final Integer limit; // may be null
+ public final Integer overrequest; // may be null
+ public final String sort; // may be null
+ /** Simplified constructor asks for limit = # unique vals */
+ public TermFacet(String field) {
+ this(field, UNIQUE_FIELD_VALS, 0, "skg desc");
+
+ }
+ public TermFacet(String field, Integer limit, Integer overrequest, String sort) {
+ assert null != field;
+ this.field = field;
+ this.limit = limit;
+ this.overrequest = overrequest;
+ this.sort = sort;
+ }
+
+ /**
+ * recursively generates the <code>json.facet</code> param value to use for testing this facet
+ */
+ private CharSequence toJSONFacetParamValue() {
+ final String limitStr = (null == limit) ? "" : (", limit:" + limit);
+ final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
+ final String sortStr = (null == sort) ? "" : (", sort: '" + sort + "'");
+ final StringBuilder sb
+ = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + sortStr);
+
+ // see class javadocs for why we always use refine:true & the query:'*:*' domain for this test.
+ sb.append(", refine: true, domain: { query: '*:*' }, facet:");
+ sb.append(toJSONFacetParamValue(subFacets, "skg : 'relatedness($fore,$back)'"));
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
+ * use for testing them against in a solr request.
+ */
+ public static CharSequence toJSONFacetParamValue(final Map<String,TermFacet> facets,
+ final String extraJson) {
+ assert null != facets;
+ if (0 == facets.size() && null == extraJson) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder("{ processEmpty: true, ");
+ for (String key : facets.keySet()) {
+ sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
+ sb.append(" ,");
+ }
+ if (null == extraJson) {
+ sb.setLength(sb.length() - 1);
+ } else {
+ sb.append(extraJson);
+ }
+ sb.append("}");
+ return sb;
+ }
+
+ /**
+ * Factory method for generating some random facets.
+ *
+ * For simplicity, each facet will have a unique key name.
+ */
+ public static Map<String,TermFacet> buildRandomFacets() {
+ // for simplicity, use a unique facet key regardless of depth - simplifies verification
+ // and le's us enforce a hard limit on the total number of facets in a request
+ AtomicInteger keyCounter = new AtomicInteger(0);
+
+ final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
+ return buildRandomFacets(keyCounter, maxDepth);
+ }
+
+ /**
+ * picks a random value for the "sort" param, biased in favor of interesting test cases
+ *
+ * @return a sort string (w/direction), or null to specify nothing (trigger default behavior)
+ * @see #randomLimitParam
+ */
+ public static String randomSortParam(Random r) {
+
+ // IMPORTANT!!!
+ // if this method is modified to produce new sorts, make sure to update
+ // randomLimitParam to account for them if they are impacted by SOLR-12556
+ final String dir = random().nextBoolean() ? "asc" : "desc";
+ switch(r.nextInt(4)) {
+ case 0: return null;
+ case 1: return "count " + dir;
+ case 2: return "skg " + dir;
+ case 3: return "index " + dir;
+ default: throw new RuntimeException("Broken case statement");
+ }
+ }
+ /**
+ * picks a random value for the "limit" param, biased in favor of interesting test cases
+ *
+ * <p>
+ * <b>NOTE:</b> Due to SOLR-12556, we have to force an overrequest of "all" possible terms for
+ * some sort values.
+ * </p>
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ * @see #randomSortParam
+ */
+ public static Integer randomLimitParam(Random r, final String sort) {
+ if (null != sort) {
+ if (sort.equals("count asc") || sort.startsWith("skg")) {
+ // of the known types of sorts produced, these are at risk of SOLR-12556
+ // so request (effectively) unlimited num buckets
+ return r.nextBoolean() ? UNIQUE_FIELD_VALS : -1;
+ }
+ }
+ final int limit = 1 + r.nextInt((int) (UNIQUE_FIELD_VALS * 1.5F));
+ if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
+ return -1; // unlimited
+ } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
+ return null; // sometimes, don't specify limit if it's the default
+ }
+ return limit;
+ }
+
+ /**
+ * picks a random value for the "overrequest" param, biased in favor of interesting test cases.
+ *
+ * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
+ * @see #UNIQUE_FIELD_VALS
+ */
+ public static Integer randomOverrequestParam(Random r) {
+ switch(r.nextInt(10)) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ return 0; // 40% of the time, disable overrequest to better stress refinement
+ case 4:
+ case 5:
+ return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
+ case 6:
+ return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
+ default: break;
+ }
+ // else.... either leave param unspecified (or redundently specify the -1 default)
+ return r.nextBoolean() ? null : -1;
+ }
+
+ /**
+ * recursive helper method for building random facets
+ *
+ * @param keyCounter used to ensure every generated facet has a unique key name
+ * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
+ */
+ private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
+ final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
+ Map<String,TermFacet> results = new LinkedHashMap<>();
+ for (int i = 0; i < numFacets; i++) {
+ if (keyCounter.get() < 3) { // a hard limit on the total number of facets (regardless of depth) to reduce OOM risk
+
+ final String sort = randomSortParam(random());
+ final Integer limit = randomLimitParam(random(), sort);
+ final Integer overrequest = randomOverrequestParam(random());
+ final TermFacet facet = new TermFacet(field((random().nextBoolean()
+ ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES),
+ random().nextInt(MAX_FIELD_NUM)),
+ limit, overrequest, sort);
+ results.put("facet_" + keyCounter.incrementAndGet(), facet);
+ if (0 < maxDepth) {
+ // if we're going wide, don't go deep
+ final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
+ facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
+ }
+ }
+ }
+ return results;
+ }
+ }
+
+ /**
+ * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
+ * at a node in our cluster
+ */
+ public static SolrClient getRandClient(Random rand) {
+ int numClients = CLIENTS.size();
+ int idx = TestUtil.nextInt(rand, 0, numClients);
+
+ return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
+ }
+
+ /**
+ * Uses a random SolrClient to execture a request and returns only the numFound
+ * @see #getRandClient
+ */
+ public static long getNumFound(final SolrParams req) throws SolrServerException, IOException {
+ return getRandClient(random()).query(req).getResults().getNumFound();
+ }
+
+ public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
+ assert null != client.getDefaultCollection();
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
+ client.getZkStateReader(),
+ true, true, 330);
+ }
+
+}
[09/32] lucene-solr:jira/solr-12730: SOLR-9317: cleanup a couple of
typos; add some description to addreplica examples
Posted by ab...@apache.org.
SOLR-9317: cleanup a couple of typos; add some description to addreplica examples
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b72f05da
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b72f05da
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b72f05da
Branch: refs/heads/jira/solr-12730
Commit: b72f05dac63f88929056627525857f45b303154b
Parents: e62fe45
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Oct 24 14:21:35 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Oct 24 14:41:36 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/collections-api.adoc | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b72f05da/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index a5021da..47a09d0 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -1056,14 +1056,14 @@ If the exact shard name is not known, users may pass the `\_route_` value and th
Ignored if the `shard` parameter is also specified.
`node`::
-The name of the node where the replica should be created (optional)
+The name of the node where the replica should be created (optional).
`createNodeSet`::
A comma-separated list of nodes among which the best ones will be chosen to place the replicas (optional)
+
The format is a comma-separated list of node_names, such as `localhost:8983_solr,localhost:8984_solr,localhost:8985_solr`.
-If neither `node`, nor `createNodeSet` is specified then the best node(s) from among all the live nodes in the cluster are chosen.
+NOTE: If neither `node` nor `createNodeSet` are specified then the best node(s) from among all the live nodes in the cluster are chosen.
`instanceDir`::
The instanceDir for the core that will be created.
@@ -1074,11 +1074,9 @@ The directory in which the core should be created.
`type`::
The type of replica to create. These possible values are allowed:
+
---
* `nrt`: The NRT type maintains a transaction log and updates its index locally. This is the default and the most commonly used.
* `tlog`: The TLOG type maintains a transaction log but only updates its index via replication.
* `pull`: The PULL type does not maintain a transaction log and only updates its index via replication. This type is not eligible to become a leader.
---
+
See the section <<shards-and-indexing-data-in-solrcloud.adoc#types-of-replicas,Types of Replicas>> for more information about replica type options.
@@ -1104,6 +1102,8 @@ Request ID to track this action which will be <<Asynchronous Calls,processed asy
*Input*
+Create a replica for the "test" collection on the node "192.167.1.2:8983_solr".
+
[source,text]
----
http://localhost:8983/solr/admin/collections?action=ADDREPLICA&collection=test2&shard=shard2&node=192.167.1.2:8983_solr&wt=xml
@@ -1130,6 +1130,10 @@ http://localhost:8983/solr/admin/collections?action=ADDREPLICA&collection=test2&
</response>
----
+*Input*
+
+Create a replica for the "gettingstarted" collection with one PULL replica and one TLOG replica.
+
[source,text]
----
http://localhost:8983/solr/admin/collections?action=addreplica&collection=gettingstarted&shard=shard1&tlogReplicas=1&pullReplicas=1
[06/32] lucene-solr:jira/solr-12730: SOLR-12873: Replace few
remaining occurrences of LUCENE_CURRENT with LATEST for luceneMatchVersion.
Posted by ab...@apache.org.
SOLR-12873: Replace few remaining occurrences of LUCENE_CURRENT with LATEST for luceneMatchVersion.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c277674f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c277674f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c277674f
Branch: refs/heads/jira/solr-12730
Commit: c277674f0ed5ab810432a1ad18c174f75dd4a9be
Parents: 7fc91de
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:28:12 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:28:12 2018 +0100
----------------------------------------------------------------------
.../conf/solrconfig-distrib-update-processor-chains.xml | 2 +-
.../solr/collection1/conf/solrconfig-doctransformers.xml | 2 +-
solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml | 2 +-
.../solr/collection1/conf/solrconfig-master-throttled.xml | 2 +-
.../src/test-files/solr/collection1/conf/solrconfig-paramset.xml | 2 +-
solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../src/test-files/solr/collection1/conf/solrconfig-tagger.xml | 2 +-
.../src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml | 2 +-
.../test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml | 2 +-
10 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
index f91be75..c9c351f 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml
@@ -20,7 +20,7 @@
<config>
<jmx />
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}">
<!-- used to keep RAM reqs down for HdfsDirectoryFactory -->
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
index 5a7d65e..f3a0bd5 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-doctransformers.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
index cad70d8..6600f7c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-hash.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
index 4a53776..1a37f3f 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-master-throttled.xml
@@ -18,7 +18,7 @@
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.NRTCachingDirectoryFactory}"/>
<schemaFactory class="ClassicIndexSchemaFactory"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
index 7eefc9d..a9e71f6 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml
@@ -20,7 +20,7 @@
<!-- a basic solrconfig that tests can use when they want simple minimal solrconfig/schema
DO NOT ADD THINGS TO THIS CONFIG! -->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
index 8ec6489..ac8ea62 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sql.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
index e0d3677..c97ce08 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml
@@ -24,7 +24,7 @@
<!-- a basic solrconfig that tests can use when they want simple minimal solrconfig/schema
DO NOT ADD THINGS TO THIS CONFIG! -->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<dataDir>${solr.data.dir:}</dataDir>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.RAMDirectoryFactory}"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
index af659ad..129cc48 100644
--- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
+++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-sql.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
index b002a30..0120f35 100644
--- a/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
+++ b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/solrconfig.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c277674f/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
index 70e9403..0e13a5a 100644
--- a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
+++ b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/solrconfig.xml
@@ -21,7 +21,7 @@
It is *not* a good example to work from.
-->
<config>
- <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
<indexConfig>
<useCompoundFile>${useCompoundFile:false}</useCompoundFile>
</indexConfig>
[30/32] lucene-solr:jira/solr-12730: SOLR-12862: Update CHANGES.txt
Posted by ab...@apache.org.
SOLR-12862: Update CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5fc4d516
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5fc4d516
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5fc4d516
Branch: refs/heads/jira/solr-12730
Commit: 5fc4d516b11c1c203d5affbdaeaf0d63e070f617
Parents: 416cc16
Author: Joel Bernstein <jb...@apache.org>
Authored: Sun Oct 28 17:08:26 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sun Oct 28 17:08:26 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5fc4d516/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a21a949..2c79aaa 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -168,6 +168,8 @@ New Features
* SOLR-12840: Add pairSort Stream Evaluator (Joel Bernstein)
+* SOLR-12862: Add log10 Stream Evaluator and allow the pow Stream Evaluator to accept a vector of exponents (Joel Bernstein)
+
Other Changes
----------------------
[02/32] lucene-solr:jira/solr-12730: Add log.warn when a replica
become leader after timeout
Posted by ab...@apache.org.
Add log.warn when a replica become leader after timeout
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e083b150
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e083b150
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e083b150
Branch: refs/heads/jira/solr-12730
Commit: e083b1501ebe21abaf95bcd93f89af12142fd1ee
Parents: 3e89b7a
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed Oct 24 08:56:01 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed Oct 24 08:56:01 2018 +0700
----------------------------------------------------------------------
solr/core/src/java/org/apache/solr/cloud/ElectionContext.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e083b150/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
index d4f84f9..6d17de4 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ElectionContext.java
@@ -522,6 +522,9 @@ final class ShardLeaderElectionContext extends ShardLeaderElectionContextBase {
long timeoutAt = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS);
while (!isClosed && !cc.isShutDown()) {
if (System.nanoTime() > timeoutAt) {
+ log.warn("After waiting for {}ms, no other potential leader was found, {} try to become leader anyway (" +
+ "core_term:{}, highest_term:{})",
+ timeout, coreNodeName, zkShardTerms.getTerm(coreNodeName), zkShardTerms.getHighestTerm());
return true;
}
if (replicasWithHigherTermParticipated(zkShardTerms, coreNodeName)) {
[28/32] lucene-solr:jira/solr-12730: SOLR-12811: Update CHANGES.txt
Posted by ab...@apache.org.
SOLR-12811: Update CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f8ffc1af
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f8ffc1af
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f8ffc1af
Branch: refs/heads/jira/solr-12730
Commit: f8ffc1afd66f22906a4011045ba105da8ee9ed97
Parents: 7a63e13
Author: Joel Bernstein <jb...@apache.org>
Authored: Sun Oct 28 13:47:12 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sun Oct 28 13:47:12 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f8ffc1af/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0d2693d..3b9092f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -164,6 +164,8 @@ New Features
* SOLR-11907: Add convexHull and associated geometric Stream Evaluators. (Joel Bernstein)
+* SOLR-12811: Add enclosingDisk and associated geometric Stream Evaluators. (Joel Bernstein)
+
Other Changes
----------------------
[27/32] lucene-solr:jira/solr-12730: SOLR-11907: Update CHANGES.txt
Posted by ab...@apache.org.
SOLR-11907: Update CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7a63e13a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7a63e13a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7a63e13a
Branch: refs/heads/jira/solr-12730
Commit: 7a63e13a3bf1be8419a289e480b49aed71b63e47
Parents: 1abc38b
Author: Joel Bernstein <jb...@apache.org>
Authored: Sun Oct 28 13:42:29 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sun Oct 28 13:42:29 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a63e13a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index febff90..0d2693d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -162,6 +162,8 @@ New Features
* SOLR-12828: Add oscillate Stream Evaluator to support sine wave analysis. (Joel Bernstein)
+* SOLR-11907: Add convexHull and associated geometric Stream Evaluators. (Joel Bernstein)
+
Other Changes
----------------------
[17/32] lucene-solr:jira/solr-12730: SOLR-12423: fix Tika version in
CHANGES
Posted by ab...@apache.org.
SOLR-12423: fix Tika version in CHANGES
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8d109393
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8d109393
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8d109393
Branch: refs/heads/jira/solr-12730
Commit: 8d109393492924cdde9663b9b9c4da00daaae433
Parents: 93ccdce
Author: Cassandra Targett <ct...@apache.org>
Authored: Thu Oct 25 11:56:19 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu Oct 25 11:56:19 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8d109393/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index bb8f86d..7715efa 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -85,7 +85,7 @@ Other Changes
* SOLR-12586: Upgrade ParseDateFieldUpdateProcessorFactory (present in "schemaless mode") to use Java 8's
java.time.DateTimeFormatter instead of Joda time (see upgrade notes). "Lenient" is enabled. Removed Joda Time dependency.
(David Smiley, Bar Rotstein)
-
+
* SOLR-5163: edismax now throws an exception when qf refers to a nonexistent field (Charles Sanders, David Smiley)
* SOLR-12805: Store previous term (generation) of replica when start recovery process (Cao Manh Dat)
@@ -96,7 +96,7 @@ Other Changes
* SOLR-11812: Remove backward compatibility of old LIR implementation in 8.0 (Cao Manh Dat)
-* SOLR-12620: Remove the Admin UI Cloud -> Graph (Radial) view (janhoy)
+* SOLR-12620: Remove the Admin UI Cloud -> Graph (Radial) view (janhoy)
================== 7.6.0 ==================
@@ -104,7 +104,7 @@ Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this r
Versions of Major Components
---------------------
-Apache Tika 1.18
+Apache Tika 1.19.1
Carrot2 3.16.0
Velocity 1.7 and Velocity Tools 2.0
Apache ZooKeeper 3.4.11
@@ -582,7 +582,7 @@ Other Changes
* SOLR-8742: In HdfsDirectoryTest replace RAMDirectory usages with ByteBuffersDirectory.
(hossman, Mark Miller, Andrzej Bialecki, Steve Rowe)
-
+
* SOLR-12771: Improve Autoscaling Policy and Preferences documentation. (hossman, Steve Rowe)
================== 7.4.0 ==================
[20/32] lucene-solr:jira/solr-12730: Added an arabic snowball stemmer
and test dataset
Posted by ab...@apache.org.
Added an arabic snowball stemmer and test dataset
This change adds an Arabic snowball stemmer based on snowballstem.org
as well as an arabic test dataset in `TestSnowballVocabData.zip`
It also updates the `ant patch-snowball` target to be compatible with
the java classes generated by the last snowball version (tree:
1964ce688cbeca505263c8f77e16ed923296ce7a). The `ant patch-snowball` target
is retro-compatible with the version of snowball stemmers used in
lucene 7.x and ignores already patched classes.
Signed-off-by: Jim Ferenczi <ji...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5c567d4f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5c567d4f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5c567d4f
Branch: refs/heads/jira/solr-12730
Commit: 5c567d4fbcab276a7b6ae4132c3c03af95a75df9
Parents: 6f291d4
Author: Ryadh Dahimene <da...@gmail.com>
Authored: Thu Sep 13 13:26:34 2018 +0100
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Oct 26 10:53:45 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
lucene/analysis/common/build.xml | 108 +-
.../tartarus/snowball/ext/ArabicStemmer.java | 1912 ++++++++++++++++++
.../analysis/snowball/TestSnowballVocab.java | 1 +
.../analysis/snowball/TestSnowballVocabData.zip | Bin 3128133 -> 3568843 bytes
5 files changed, 2000 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c567d4f/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 35dc4ec..9b0382b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -226,6 +226,10 @@ New Features
* LUCENE-8538: Add a Simple WKT Shape Parser for creating Lucene Geometries (Polygon, Line,
Rectangle) from WKT format. (Nick Knize)
+* LUCENE-8462: Adds an Arabic snowball stemmer based on
+ https://github.com/snowballstem/snowball/blob/master/algorithms/arabic.sbl
+ (Ryadh Dahimene via Jim Ferenczi)
+
Improvements:
* LUCENE-8521: Change LatLonShape encoding to 7 dimensions instead of 6; where the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c567d4f/lucene/analysis/common/build.xml
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/build.xml b/lucene/analysis/common/build.xml
index 6e4f118..2064e19 100644
--- a/lucene/analysis/common/build.xml
+++ b/lucene/analysis/common/build.xml
@@ -7,9 +7,9 @@
The ASF licenses this file to You under the Apache License, Version 2.0
the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
-
+
http://www.apache.org/licenses/LICENSE-2.0
-
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -17,10 +17,10 @@
limitations under the License.
-->
-<project name="analyzers-common" default="default">
+<project name="analyzers-common" default="default" xmlns:rsel="antlib:org.apache.tools.ant.types.resources.selectors">
<description>
- Analyzers for indexing content in different languages and domains.
+ Analyzers for indexing content in different languages and domains.
</description>
<!-- some files for testing that do not have license headers -->
@@ -28,9 +28,9 @@
<property name="rat.additional-includes" value="src/tools/**"/>
<import file="../analysis-module-build.xml"/>
-
- <property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>
-
+
+ <property name="snowball.programs.dir" location="src/java/org/tartarus/snowball/ext"/>
+
<property name="unicode-props-file" location="src/java/org/apache/lucene/analysis/util/UnicodeProps.java"/>
<target name="jflex" depends="-install-jflex,clean-jflex,-jflex-ClassicAnalyzer,-jflex-UAX29URLEmailTokenizer,
@@ -66,7 +66,7 @@
<run-jflex-and-disable-buffer-expansion
dir="src/java/org/apache/lucene/analysis/standard" name="UAX29URLEmailTokenizerImpl"/>
</target>
-
+
<target name="-jflex-ClassicAnalyzer" depends="init,-install-jflex">
<run-jflex dir="src/java/org/apache/lucene/analysis/standard" name="ClassicTokenizerImpl"/>
</target>
@@ -84,28 +84,28 @@
</fileset>
</delete>
</target>
-
+
<target xmlns:ivy="antlib:org.apache.ivy.ant" name="-resolve-icu4j" unless="icu4j.resolved" depends="ivy-availability-check,ivy-configure">
<loadproperties prefix="ivyversions" srcFile="${common.dir}/ivy-versions.properties"/>
<ivy:cachepath organisation="com.ibm.icu" module="icu4j" revision="${ivyversions./com.ibm.icu/icu4j}"
- inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
+ inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
<property name="icu4j.resolved" value="true"/>
</target>
-
+
<target name="unicode-data" depends="-resolve-icu4j,resolve-groovy">
<groovy classpathref="icu4j.classpath" src="src/tools/groovy/generate-unicode-data.groovy"/>
<fixcrlf file="${unicode-props-file}" encoding="UTF-8"/>
</target>
-
+
<property name="tld.zones" value="http://www.internic.net/zones/root.zone"/>
<property name="tld.output" location="src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro"/>
<target name="gen-tlds" depends="compile-tools">
<java
- classname="org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"
- dir="."
- fork="true"
- failonerror="true">
+ classname="org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"
+ dir="."
+ fork="true"
+ failonerror="true">
<classpath>
<pathelement location="${build.dir}/classes/tools"/>
</classpath>
@@ -117,8 +117,8 @@
<target name="compile-tools" depends="common.compile-tools">
<compile
- srcdir="src/tools/java"
- destdir="${build.dir}/classes/tools">
+ srcdir="src/tools/java"
+ destdir="${build.dir}/classes/tools">
<classpath refid="classpath"/>
</compile>
</target>
@@ -126,15 +126,73 @@
<target name="javadocs" depends="module-build.javadocs"/>
<target name="regenerate" depends="jflex,unicode-data"/>
-
+
<target name="patch-snowball" description="Patches all snowball programs in '${snowball.programs.dir}' to make them work with MethodHandles">
- <fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
- <replaceregexp match="^public class \w+Stemmer\b" replace="@SuppressWarnings("unused") \0" flags="m" encoding="UTF-8">
+ <fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
+
+ <replaceregexp match="^public class \w+Stemmer\b" replace="@SuppressWarnings("unused") \0" flags="m" encoding="UTF-8">
+ <restrict>
+ <fileset refid="snowball.programs"/>
+ <rsel:not>
+ <rsel:contains text="patched"/>
+ </rsel:not>
+ </restrict>
+ </replaceregexp>
+
+ <replaceregexp match="new Among\(([^,]*,[^,]*,[^,]*?)(?=\))" replace="\0, "", methodObject" flags="g" encoding="UTF-8">
+ <restrict>
+ <fileset refid="snowball.programs"/>
+ <rsel:not>
+ <rsel:contains text="patched"/>
+ </rsel:not>
+ </restrict>
+ </replaceregexp>
+
+ <replaceregexp match="(new Among\([^,]*,[^,]*,[^,]*,[^,]*,)[^,]*?(?=\))" replace="\1 methodObject" flags="g" encoding="UTF-8">
+ <restrict>
<fileset refid="snowball.programs"/>
- </replaceregexp>
- <replaceregexp match="private final static \w+Stemmer methodObject\b.*$" replace="/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();" flags="m" encoding="UTF-8">
+ <rsel:not>
+ <rsel:contains text="patched"/>
+ </rsel:not>
+ </restrict>
+ </replaceregexp>
+
+ <replaceregexp match="(?:find_among(?:|_b)\()(.*?)(?=\))" replace="\0, \1.length" flags="g" encoding="UTF-8">
+ <restrict>
+ <fileset refid="snowball.programs"/>
+ <rsel:not>
+ <rsel:contains text="patched"/>
+ </rsel:not>
+ </restrict>
+ </replaceregexp>
+
+ <replaceregexp match="current" replace="getCurrent()" flags="g" encoding="UTF-8">
+ <restrict>
<fileset refid="snowball.programs"/>
- </replaceregexp>
- <fixcrlf srcdir="${snowball.programs.dir}" includes="*Stemmer.java" tab="remove" tablength="2" encoding="UTF-8" javafiles="yes" fixlast="yes"/>
+ <rsel:not>
+ <rsel:contains text="patched"/>
+ </rsel:not>
+ </restrict>
+ </replaceregexp>
+
+ <replaceregexp match="(?:eq_s(?:|_b)\()(.*?)(?=\))" replace="\0.length(),\1" flags="g" encoding="UTF-8">
+ <restrict>
+ <fileset refid="snowball.programs"/>
+ <rsel:not>
+ <rsel:contains text="patched"/>
+ </rsel:not>
+ </restrict>
+ </replaceregexp>
+
+ <replaceregexp match="private static final long serialVersionUID(.*)" replace="private static final long serialVersionUID = 1L; ${line.separator}${line.separator} /* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();" flags="m" encoding="UTF-8">
+ <restrict>
+ <fileset refid="snowball.programs"/>
+ <rsel:not>
+ <rsel:contains text="patched"/>
+ </rsel:not>
+ </restrict>
+ </replaceregexp>
+
+ <fixcrlf srcdir="${snowball.programs.dir}" includes="*Stemmer.java" tab="remove" tablength="2" encoding="UTF-8" javafiles="yes" fixlast="yes"/>
</target>
</project>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c567d4f/lucene/analysis/common/src/java/org/tartarus/snowball/ext/ArabicStemmer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/tartarus/snowball/ext/ArabicStemmer.java b/lucene/analysis/common/src/java/org/tartarus/snowball/ext/ArabicStemmer.java
new file mode 100644
index 0000000..2b907b2
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/tartarus/snowball/ext/ArabicStemmer.java
@@ -0,0 +1,1912 @@
+// This file was generated automatically by the Snowball to Java compiler
+// http://snowballstem.org/
+
+package org.tartarus.snowball.ext;
+
+import org.tartarus.snowball.Among;
+
+/**
+ * This class was automatically generated by a Snowball to Java compiler
+ * It implements the stemming algorithm defined by a snowball script.
+ */
+
+@SuppressWarnings("unused") public class ArabicStemmer extends org.tartarus.snowball.SnowballProgram {
+
+ private static final long serialVersionUID = 1L;
+
+ /* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();
+
+ private final static Among a_0[] = {
+ new Among("\u0640", -1, 2, "", methodObject),
+ new Among("\u064B", -1, 1, "", methodObject),
+ new Among("\u064C", -1, 1, "", methodObject),
+ new Among("\u064D", -1, 1, "", methodObject),
+ new Among("\u064E", -1, 1, "", methodObject),
+ new Among("\u064F", -1, 1, "", methodObject),
+ new Among("\u0650", -1, 1, "", methodObject),
+ new Among("\u0651", -1, 1, "", methodObject),
+ new Among("\u0652", -1, 1, "", methodObject),
+ new Among("\u0660", -1, 3, "", methodObject),
+ new Among("\u0661", -1, 4, "", methodObject),
+ new Among("\u0662", -1, 5, "", methodObject),
+ new Among("\u0663", -1, 6, "", methodObject),
+ new Among("\u0664", -1, 7, "", methodObject),
+ new Among("\u0665", -1, 8, "", methodObject),
+ new Among("\u0666", -1, 9, "", methodObject),
+ new Among("\u0667", -1, 10, "", methodObject),
+ new Among("\u0668", -1, 11, "", methodObject),
+ new Among("\u0669", -1, 12, "", methodObject),
+ new Among("\uFE80", -1, 13, "", methodObject),
+ new Among("\uFE81", -1, 17, "", methodObject),
+ new Among("\uFE82", -1, 17, "", methodObject),
+ new Among("\uFE83", -1, 14, "", methodObject),
+ new Among("\uFE84", -1, 14, "", methodObject),
+ new Among("\uFE85", -1, 18, "", methodObject),
+ new Among("\uFE86", -1, 18, "", methodObject),
+ new Among("\uFE87", -1, 15, "", methodObject),
+ new Among("\uFE88", -1, 15, "", methodObject),
+ new Among("\uFE89", -1, 16, "", methodObject),
+ new Among("\uFE8A", -1, 16, "", methodObject),
+ new Among("\uFE8B", -1, 16, "", methodObject),
+ new Among("\uFE8C", -1, 16, "", methodObject),
+ new Among("\uFE8D", -1, 19, "", methodObject),
+ new Among("\uFE8E", -1, 19, "", methodObject),
+ new Among("\uFE8F", -1, 20, "", methodObject),
+ new Among("\uFE90", -1, 20, "", methodObject),
+ new Among("\uFE91", -1, 20, "", methodObject),
+ new Among("\uFE92", -1, 20, "", methodObject),
+ new Among("\uFE93", -1, 21, "", methodObject),
+ new Among("\uFE94", -1, 21, "", methodObject),
+ new Among("\uFE95", -1, 22, "", methodObject),
+ new Among("\uFE96", -1, 22, "", methodObject),
+ new Among("\uFE97", -1, 22, "", methodObject),
+ new Among("\uFE98", -1, 22, "", methodObject),
+ new Among("\uFE99", -1, 23, "", methodObject),
+ new Among("\uFE9A", -1, 23, "", methodObject),
+ new Among("\uFE9B", -1, 23, "", methodObject),
+ new Among("\uFE9C", -1, 23, "", methodObject),
+ new Among("\uFE9D", -1, 24, "", methodObject),
+ new Among("\uFE9E", -1, 24, "", methodObject),
+ new Among("\uFE9F", -1, 24, "", methodObject),
+ new Among("\uFEA0", -1, 24, "", methodObject),
+ new Among("\uFEA1", -1, 25, "", methodObject),
+ new Among("\uFEA2", -1, 25, "", methodObject),
+ new Among("\uFEA3", -1, 25, "", methodObject),
+ new Among("\uFEA4", -1, 25, "", methodObject),
+ new Among("\uFEA5", -1, 26, "", methodObject),
+ new Among("\uFEA6", -1, 26, "", methodObject),
+ new Among("\uFEA7", -1, 26, "", methodObject),
+ new Among("\uFEA8", -1, 26, "", methodObject),
+ new Among("\uFEA9", -1, 27, "", methodObject),
+ new Among("\uFEAA", -1, 27, "", methodObject),
+ new Among("\uFEAB", -1, 28, "", methodObject),
+ new Among("\uFEAC", -1, 28, "", methodObject),
+ new Among("\uFEAD", -1, 29, "", methodObject),
+ new Among("\uFEAE", -1, 29, "", methodObject),
+ new Among("\uFEAF", -1, 30, "", methodObject),
+ new Among("\uFEB0", -1, 30, "", methodObject),
+ new Among("\uFEB1", -1, 31, "", methodObject),
+ new Among("\uFEB2", -1, 31, "", methodObject),
+ new Among("\uFEB3", -1, 31, "", methodObject),
+ new Among("\uFEB4", -1, 31, "", methodObject),
+ new Among("\uFEB5", -1, 32, "", methodObject),
+ new Among("\uFEB6", -1, 32, "", methodObject),
+ new Among("\uFEB7", -1, 32, "", methodObject),
+ new Among("\uFEB8", -1, 32, "", methodObject),
+ new Among("\uFEB9", -1, 33, "", methodObject),
+ new Among("\uFEBA", -1, 33, "", methodObject),
+ new Among("\uFEBB", -1, 33, "", methodObject),
+ new Among("\uFEBC", -1, 33, "", methodObject),
+ new Among("\uFEBD", -1, 34, "", methodObject),
+ new Among("\uFEBE", -1, 34, "", methodObject),
+ new Among("\uFEBF", -1, 34, "", methodObject),
+ new Among("\uFEC0", -1, 34, "", methodObject),
+ new Among("\uFEC1", -1, 35, "", methodObject),
+ new Among("\uFEC2", -1, 35, "", methodObject),
+ new Among("\uFEC3", -1, 35, "", methodObject),
+ new Among("\uFEC4", -1, 35, "", methodObject),
+ new Among("\uFEC5", -1, 36, "", methodObject),
+ new Among("\uFEC6", -1, 36, "", methodObject),
+ new Among("\uFEC7", -1, 36, "", methodObject),
+ new Among("\uFEC8", -1, 36, "", methodObject),
+ new Among("\uFEC9", -1, 37, "", methodObject),
+ new Among("\uFECA", -1, 37, "", methodObject),
+ new Among("\uFECB", -1, 37, "", methodObject),
+ new Among("\uFECC", -1, 37, "", methodObject),
+ new Among("\uFECD", -1, 38, "", methodObject),
+ new Among("\uFECE", -1, 38, "", methodObject),
+ new Among("\uFECF", -1, 38, "", methodObject),
+ new Among("\uFED0", -1, 38, "", methodObject),
+ new Among("\uFED1", -1, 39, "", methodObject),
+ new Among("\uFED2", -1, 39, "", methodObject),
+ new Among("\uFED3", -1, 39, "", methodObject),
+ new Among("\uFED4", -1, 39, "", methodObject),
+ new Among("\uFED5", -1, 40, "", methodObject),
+ new Among("\uFED6", -1, 40, "", methodObject),
+ new Among("\uFED7", -1, 40, "", methodObject),
+ new Among("\uFED8", -1, 40, "", methodObject),
+ new Among("\uFED9", -1, 41, "", methodObject),
+ new Among("\uFEDA", -1, 41, "", methodObject),
+ new Among("\uFEDB", -1, 41, "", methodObject),
+ new Among("\uFEDC", -1, 41, "", methodObject),
+ new Among("\uFEDD", -1, 42, "", methodObject),
+ new Among("\uFEDE", -1, 42, "", methodObject),
+ new Among("\uFEDF", -1, 42, "", methodObject),
+ new Among("\uFEE0", -1, 42, "", methodObject),
+ new Among("\uFEE1", -1, 43, "", methodObject),
+ new Among("\uFEE2", -1, 43, "", methodObject),
+ new Among("\uFEE3", -1, 43, "", methodObject),
+ new Among("\uFEE4", -1, 43, "", methodObject),
+ new Among("\uFEE5", -1, 44, "", methodObject),
+ new Among("\uFEE6", -1, 44, "", methodObject),
+ new Among("\uFEE7", -1, 44, "", methodObject),
+ new Among("\uFEE8", -1, 44, "", methodObject),
+ new Among("\uFEE9", -1, 45, "", methodObject),
+ new Among("\uFEEA", -1, 45, "", methodObject),
+ new Among("\uFEEB", -1, 45, "", methodObject),
+ new Among("\uFEEC", -1, 45, "", methodObject),
+ new Among("\uFEED", -1, 46, "", methodObject),
+ new Among("\uFEEE", -1, 46, "", methodObject),
+ new Among("\uFEEF", -1, 47, "", methodObject),
+ new Among("\uFEF0", -1, 47, "", methodObject),
+ new Among("\uFEF1", -1, 48, "", methodObject),
+ new Among("\uFEF2", -1, 48, "", methodObject),
+ new Among("\uFEF3", -1, 48, "", methodObject),
+ new Among("\uFEF4", -1, 48, "", methodObject),
+ new Among("\uFEF5", -1, 52, "", methodObject),
+ new Among("\uFEF6", -1, 52, "", methodObject),
+ new Among("\uFEF7", -1, 50, "", methodObject),
+ new Among("\uFEF8", -1, 50, "", methodObject),
+ new Among("\uFEF9", -1, 51, "", methodObject),
+ new Among("\uFEFA", -1, 51, "", methodObject),
+ new Among("\uFEFB", -1, 49, "", methodObject),
+ new Among("\uFEFC", -1, 49, "", methodObject)
+ };
+
+ private final static Among a_1[] = {
+ new Among("\u0622", -1, 1, "", methodObject),
+ new Among("\u0623", -1, 1, "", methodObject),
+ new Among("\u0624", -1, 2, "", methodObject),
+ new Among("\u0625", -1, 1, "", methodObject),
+ new Among("\u0626", -1, 3, "", methodObject)
+ };
+
+ private final static Among a_2[] = {
+ new Among("\u0622", -1, 1, "", methodObject),
+ new Among("\u0623", -1, 1, "", methodObject),
+ new Among("\u0624", -1, 2, "", methodObject),
+ new Among("\u0625", -1, 1, "", methodObject),
+ new Among("\u0626", -1, 3, "", methodObject)
+ };
+
+ private final static Among a_3[] = {
+ new Among("\u0627\u0644", -1, 2, "", methodObject),
+ new Among("\u0628\u0627\u0644", -1, 1, "", methodObject),
+ new Among("\u0643\u0627\u0644", -1, 1, "", methodObject),
+ new Among("\u0644\u0644", -1, 2, "", methodObject)
+ };
+
+ private final static Among a_4[] = {
+ new Among("\u0623\u0622", -1, 2, "", methodObject),
+ new Among("\u0623\u0623", -1, 1, "", methodObject),
+ new Among("\u0623\u0624", -1, 3, "", methodObject),
+ new Among("\u0623\u0625", -1, 5, "", methodObject),
+ new Among("\u0623\u0627", -1, 4, "", methodObject)
+ };
+
+ private final static Among a_5[] = {
+ new Among("\u0641", -1, 1, "", methodObject),
+ new Among("\u0648", -1, 2, "", methodObject)
+ };
+
+ private final static Among a_6[] = {
+ new Among("\u0627\u0644", -1, 2, "", methodObject),
+ new Among("\u0628\u0627\u0644", -1, 1, "", methodObject),
+ new Among("\u0643\u0627\u0644", -1, 1, "", methodObject),
+ new Among("\u0644\u0644", -1, 2, "", methodObject)
+ };
+
+ private final static Among a_7[] = {
+ new Among("\u0628", -1, 1, "", methodObject),
+ new Among("\u0628\u0628", 0, 2, "", methodObject),
+ new Among("\u0643\u0643", -1, 3, "", methodObject)
+ };
+
+ private final static Among a_8[] = {
+ new Among("\u0633\u0623", -1, 4, "", methodObject),
+ new Among("\u0633\u062A", -1, 2, "", methodObject),
+ new Among("\u0633\u0646", -1, 3, "", methodObject),
+ new Among("\u0633\u064A", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_9[] = {
+ new Among("\u062A\u0633\u062A", -1, 1, "", methodObject),
+ new Among("\u0646\u0633\u062A", -1, 1, "", methodObject),
+ new Among("\u064A\u0633\u062A", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_10[] = {
+ new Among("\u0643\u0645\u0627", -1, 3, "", methodObject),
+ new Among("\u0647\u0645\u0627", -1, 3, "", methodObject),
+ new Among("\u0646\u0627", -1, 2, "", methodObject),
+ new Among("\u0647\u0627", -1, 2, "", methodObject),
+ new Among("\u0643", -1, 1, "", methodObject),
+ new Among("\u0643\u0645", -1, 2, "", methodObject),
+ new Among("\u0647\u0645", -1, 2, "", methodObject),
+ new Among("\u0647\u0646", -1, 2, "", methodObject),
+ new Among("\u0647", -1, 1, "", methodObject),
+ new Among("\u064A", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_11[] = {
+ new Among("\u0646", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_12[] = {
+ new Among("\u0627", -1, 1, "", methodObject),
+ new Among("\u0648", -1, 1, "", methodObject),
+ new Among("\u064A", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_13[] = {
+ new Among("\u0627\u062A", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_14[] = {
+ new Among("\u062A", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_15[] = {
+ new Among("\u0629", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_16[] = {
+ new Among("\u064A", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_17[] = {
+ new Among("\u0643\u0645\u0627", -1, 3, "", methodObject),
+ new Among("\u0647\u0645\u0627", -1, 3, "", methodObject),
+ new Among("\u0646\u0627", -1, 2, "", methodObject),
+ new Among("\u0647\u0627", -1, 2, "", methodObject),
+ new Among("\u0643", -1, 1, "", methodObject),
+ new Among("\u0643\u0645", -1, 2, "", methodObject),
+ new Among("\u0647\u0645", -1, 2, "", methodObject),
+ new Among("\u0643\u0646", -1, 2, "", methodObject),
+ new Among("\u0647\u0646", -1, 2, "", methodObject),
+ new Among("\u0647", -1, 1, "", methodObject),
+ new Among("\u0643\u0645\u0648", -1, 3, "", methodObject),
+ new Among("\u0646\u064A", -1, 2, "", methodObject)
+ };
+
+ private final static Among a_18[] = {
+ new Among("\u0627", -1, 2, "", methodObject),
+ new Among("\u062A\u0627", 0, 3, "", methodObject),
+ new Among("\u062A\u0645\u0627", 0, 5, "", methodObject),
+ new Among("\u0646\u0627", 0, 3, "", methodObject),
+ new Among("\u062A", -1, 1, "", methodObject),
+ new Among("\u0646", -1, 2, "", methodObject),
+ new Among("\u0627\u0646", 5, 4, "", methodObject),
+ new Among("\u062A\u0646", 5, 3, "", methodObject),
+ new Among("\u0648\u0646", 5, 4, "", methodObject),
+ new Among("\u064A\u0646", 5, 4, "", methodObject),
+ new Among("\u064A", -1, 2, "", methodObject)
+ };
+
+ private final static Among a_19[] = {
+ new Among("\u0648\u0627", -1, 1, "", methodObject),
+ new Among("\u062A\u0645", -1, 1, "", methodObject)
+ };
+
+ private final static Among a_20[] = {
+ new Among("\u0648", -1, 1, "", methodObject),
+ new Among("\u062A\u0645\u0648", 0, 2, "", methodObject)
+ };
+
+ private final static Among a_21[] = {
+ new Among("\u0649", -1, 1, "", methodObject)
+ };
+
+ private boolean B_is_defined;
+ private boolean B_is_verb;
+ private boolean B_is_noun;
+ private int I_word_len;
+
+
+ private boolean r_Normalize_pre() {
+ int among_var;
+ // (, line 251
+ // loop, line 252
+ for (int v_1 = getCurrent().length(); v_1 > 0; v_1--)
+ {
+ // (, line 252
+ // or, line 316
+ lab0: do {
+ int v_2 = cursor;
+ lab1: do {
+ // (, line 253
+ // [, line 254
+ bra = cursor;
+ // substring, line 254
+ among_var = find_among(a_0, a_0.length);
+ if (among_var == 0)
+ {
+ break lab1;
+ }
+ // ], line 254
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ break lab1;
+ case 1:
+ // (, line 255
+ // delete, line 255
+ slice_del();
+ break;
+ case 2:
+ // (, line 256
+ // delete, line 256
+ slice_del();
+ break;
+ case 3:
+ // (, line 259
+ // <-, line 259
+ slice_from("0");
+ break;
+ case 4:
+ // (, line 260
+ // <-, line 260
+ slice_from("1");
+ break;
+ case 5:
+ // (, line 261
+ // <-, line 261
+ slice_from("2");
+ break;
+ case 6:
+ // (, line 262
+ // <-, line 262
+ slice_from("3");
+ break;
+ case 7:
+ // (, line 263
+ // <-, line 263
+ slice_from("4");
+ break;
+ case 8:
+ // (, line 264
+ // <-, line 264
+ slice_from("5");
+ break;
+ case 9:
+ // (, line 265
+ // <-, line 265
+ slice_from("6");
+ break;
+ case 10:
+ // (, line 266
+ // <-, line 266
+ slice_from("7");
+ break;
+ case 11:
+ // (, line 267
+ // <-, line 267
+ slice_from("8");
+ break;
+ case 12:
+ // (, line 268
+ // <-, line 268
+ slice_from("9");
+ break;
+ case 13:
+ // (, line 271
+ // <-, line 271
+ slice_from("\u0621");
+ break;
+ case 14:
+ // (, line 272
+ // <-, line 272
+ slice_from("\u0623");
+ break;
+ case 15:
+ // (, line 273
+ // <-, line 273
+ slice_from("\u0625");
+ break;
+ case 16:
+ // (, line 274
+ // <-, line 274
+ slice_from("\u0626");
+ break;
+ case 17:
+ // (, line 275
+ // <-, line 275
+ slice_from("\u0622");
+ break;
+ case 18:
+ // (, line 276
+ // <-, line 276
+ slice_from("\u0624");
+ break;
+ case 19:
+ // (, line 277
+ // <-, line 277
+ slice_from("\u0627");
+ break;
+ case 20:
+ // (, line 278
+ // <-, line 278
+ slice_from("\u0628");
+ break;
+ case 21:
+ // (, line 279
+ // <-, line 279
+ slice_from("\u0629");
+ break;
+ case 22:
+ // (, line 280
+ // <-, line 280
+ slice_from("\u062A");
+ break;
+ case 23:
+ // (, line 281
+ // <-, line 281
+ slice_from("\u062B");
+ break;
+ case 24:
+ // (, line 282
+ // <-, line 282
+ slice_from("\u062C");
+ break;
+ case 25:
+ // (, line 283
+ // <-, line 283
+ slice_from("\u062D");
+ break;
+ case 26:
+ // (, line 284
+ // <-, line 284
+ slice_from("\u062E");
+ break;
+ case 27:
+ // (, line 285
+ // <-, line 285
+ slice_from("\u062F");
+ break;
+ case 28:
+ // (, line 286
+ // <-, line 286
+ slice_from("\u0630");
+ break;
+ case 29:
+ // (, line 287
+ // <-, line 287
+ slice_from("\u0631");
+ break;
+ case 30:
+ // (, line 288
+ // <-, line 288
+ slice_from("\u0632");
+ break;
+ case 31:
+ // (, line 289
+ // <-, line 289
+ slice_from("\u0633");
+ break;
+ case 32:
+ // (, line 290
+ // <-, line 290
+ slice_from("\u0634");
+ break;
+ case 33:
+ // (, line 291
+ // <-, line 291
+ slice_from("\u0635");
+ break;
+ case 34:
+ // (, line 292
+ // <-, line 292
+ slice_from("\u0636");
+ break;
+ case 35:
+ // (, line 293
+ // <-, line 293
+ slice_from("\u0637");
+ break;
+ case 36:
+ // (, line 294
+ // <-, line 294
+ slice_from("\u0638");
+ break;
+ case 37:
+ // (, line 295
+ // <-, line 295
+ slice_from("\u0639");
+ break;
+ case 38:
+ // (, line 296
+ // <-, line 296
+ slice_from("\u063A");
+ break;
+ case 39:
+ // (, line 297
+ // <-, line 297
+ slice_from("\u0641");
+ break;
+ case 40:
+ // (, line 298
+ // <-, line 298
+ slice_from("\u0642");
+ break;
+ case 41:
+ // (, line 299
+ // <-, line 299
+ slice_from("\u0643");
+ break;
+ case 42:
+ // (, line 300
+ // <-, line 300
+ slice_from("\u0644");
+ break;
+ case 43:
+ // (, line 301
+ // <-, line 301
+ slice_from("\u0645");
+ break;
+ case 44:
+ // (, line 302
+ // <-, line 302
+ slice_from("\u0646");
+ break;
+ case 45:
+ // (, line 303
+ // <-, line 303
+ slice_from("\u0647");
+ break;
+ case 46:
+ // (, line 304
+ // <-, line 304
+ slice_from("\u0648");
+ break;
+ case 47:
+ // (, line 305
+ // <-, line 305
+ slice_from("\u0649");
+ break;
+ case 48:
+ // (, line 306
+ // <-, line 306
+ slice_from("\u064A");
+ break;
+ case 49:
+ // (, line 309
+ // <-, line 309
+ slice_from("\u0644\u0627");
+ break;
+ case 50:
+ // (, line 310
+ // <-, line 310
+ slice_from("\u0644\u0623");
+ break;
+ case 51:
+ // (, line 311
+ // <-, line 311
+ slice_from("\u0644\u0625");
+ break;
+ case 52:
+ // (, line 312
+ // <-, line 312
+ slice_from("\u0644\u0622");
+ break;
+ }
+ break lab0;
+ } while (false);
+ cursor = v_2;
+ // next, line 317
+ if (cursor >= limit)
+ {
+ return false;
+ }
+ cursor++;
+ } while (false);
+ }
+ return true;
+ }
+
+ private boolean r_Normalize_post() {
+ int among_var;
+ // (, line 321
+ // do, line 323
+ int v_1 = cursor;
+ lab0: do {
+ // (, line 323
+ // backwards, line 325
+ limit_backward = cursor;
+ cursor = limit;
+ // (, line 325
+ // [, line 326
+ ket = cursor;
+ // substring, line 326
+ among_var = find_among_b(a_1, a_1.length);
+ if (among_var == 0)
+ {
+ break lab0;
+ }
+ // ], line 326
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ break lab0;
+ case 1:
+ // (, line 327
+ // <-, line 327
+ slice_from("\u0621");
+ break;
+ case 2:
+ // (, line 328
+ // <-, line 328
+ slice_from("\u0621");
+ break;
+ case 3:
+ // (, line 329
+ // <-, line 329
+ slice_from("\u0621");
+ break;
+ }
+ cursor = limit_backward;
+ } while (false);
+ cursor = v_1;
+ // do, line 334
+ int v_2 = cursor;
+ lab1: do {
+ // loop, line 334
+ for (int v_3 = I_word_len; v_3 > 0; v_3--)
+ {
+ // (, line 334
+ // or, line 343
+ lab2: do {
+ int v_4 = cursor;
+ lab3: do {
+ // (, line 335
+ // [, line 337
+ bra = cursor;
+ // substring, line 337
+ among_var = find_among(a_2, a_2.length);
+ if (among_var == 0)
+ {
+ break lab3;
+ }
+ // ], line 337
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ break lab3;
+ case 1:
+ // (, line 338
+ // <-, line 338
+ slice_from("\u0627");
+ break;
+ case 2:
+ // (, line 339
+ // <-, line 339
+ slice_from("\u0648");
+ break;
+ case 3:
+ // (, line 340
+ // <-, line 340
+ slice_from("\u064A");
+ break;
+ }
+ break lab2;
+ } while (false);
+ cursor = v_4;
+ // next, line 344
+ if (cursor >= limit)
+ {
+ break lab1;
+ }
+ cursor++;
+ } while (false);
+ }
+ } while (false);
+ cursor = v_2;
+ return true;
+ }
+
+ private boolean r_Checks1() {
+ int among_var;
+ // (, line 349
+ I_word_len = getCurrent().length();
+ // [, line 351
+ bra = cursor;
+ // substring, line 351
+ among_var = find_among(a_3, a_3.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 351
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 352
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // set is_noun, line 352
+ B_is_noun = true;
+ // unset is_verb, line 352
+ B_is_verb = false;
+ // set is_defined, line 352
+ B_is_defined = true;
+ break;
+ case 2:
+ // (, line 353
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // set is_noun, line 353
+ B_is_noun = true;
+ // unset is_verb, line 353
+ B_is_verb = false;
+ // set is_defined, line 353
+ B_is_defined = true;
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Prefix_Step1() {
+ int among_var;
+ // (, line 359
+ I_word_len = getCurrent().length();
+ // [, line 361
+ bra = cursor;
+ // substring, line 361
+ among_var = find_among(a_4, a_4.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 361
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 362
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // <-, line 362
+ slice_from("\u0623");
+ break;
+ case 2:
+ // (, line 363
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // <-, line 363
+ slice_from("\u0622");
+ break;
+ case 3:
+ // (, line 364
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // <-, line 364
+ slice_from("\u0623");
+ break;
+ case 4:
+ // (, line 365
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // <-, line 365
+ slice_from("\u0627");
+ break;
+ case 5:
+ // (, line 366
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // <-, line 366
+ slice_from("\u0625");
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Prefix_Step2() {
+ int among_var;
+ // (, line 371
+ I_word_len = getCurrent().length();
+ // not, line 373
+ {
+ int v_1 = cursor;
+ lab0: do {
+ // literal, line 373
+ if (!(eq_s("\u0641\u0627".length(),"\u0641\u0627")))
+ {
+ break lab0;
+ }
+ return false;
+ } while (false);
+ cursor = v_1;
+ }
+ // not, line 374
+ {
+ int v_2 = cursor;
+ lab1: do {
+ // literal, line 374
+ if (!(eq_s("\u0648\u0627".length(),"\u0648\u0627")))
+ {
+ break lab1;
+ }
+ return false;
+ } while (false);
+ cursor = v_2;
+ }
+ // [, line 375
+ bra = cursor;
+ // substring, line 375
+ among_var = find_among(a_5, a_5.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 375
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 376
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // delete, line 376
+ slice_del();
+ break;
+ case 2:
+ // (, line 377
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // delete, line 377
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Prefix_Step3a_Noun() {
+ int among_var;
+ // (, line 381
+ I_word_len = getCurrent().length();
+ // [, line 383
+ bra = cursor;
+ // substring, line 383
+ among_var = find_among(a_6, a_6.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 383
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 384
+ if (!(I_word_len > 5))
+ {
+ return false;
+ }
+ // delete, line 384
+ slice_del();
+ break;
+ case 2:
+ // (, line 385
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // delete, line 385
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Prefix_Step3b_Noun() {
+ int among_var;
+ // (, line 389
+ I_word_len = getCurrent().length();
+ // not, line 391
+ {
+ int v_1 = cursor;
+ lab0: do {
+ // literal, line 391
+ if (!(eq_s("\u0628\u0627".length(),"\u0628\u0627")))
+ {
+ break lab0;
+ }
+ return false;
+ } while (false);
+ cursor = v_1;
+ }
+ // [, line 392
+ bra = cursor;
+ // substring, line 392
+ among_var = find_among(a_7, a_7.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 392
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 393
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // delete, line 393
+ slice_del();
+ break;
+ case 2:
+ // (, line 395
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // <-, line 395
+ slice_from("\u0628");
+ break;
+ case 3:
+ // (, line 396
+ if (!(I_word_len > 3))
+ {
+ return false;
+ }
+ // <-, line 396
+ slice_from("\u0643");
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Prefix_Step3_Verb() {
+ int among_var;
+ // (, line 401
+ I_word_len = getCurrent().length();
+ // [, line 403
+ bra = cursor;
+ // substring, line 403
+ among_var = find_among(a_8, a_8.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 403
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 405
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // <-, line 405
+ slice_from("\u064A");
+ break;
+ case 2:
+ // (, line 406
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // <-, line 406
+ slice_from("\u062A");
+ break;
+ case 3:
+ // (, line 407
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // <-, line 407
+ slice_from("\u0646");
+ break;
+ case 4:
+ // (, line 408
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // <-, line 408
+ slice_from("\u0623");
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Prefix_Step4_Verb() {
+ int among_var;
+ // (, line 412
+ I_word_len = getCurrent().length();
+ // [, line 414
+ bra = cursor;
+ // substring, line 414
+ among_var = find_among(a_9, a_9.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 414
+ ket = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 415
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // set is_verb, line 415
+ B_is_verb = true;
+ // unset is_noun, line 415
+ B_is_noun = false;
+ // <-, line 415
+ slice_from("\u0627\u0633\u062A");
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Noun_Step1a() {
+ int among_var;
+ // (, line 422
+ I_word_len = getCurrent().length();
+ // [, line 424
+ ket = cursor;
+ // substring, line 424
+ among_var = find_among_b(a_10, a_10.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 424
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 425
+ if (!(I_word_len >= 4))
+ {
+ return false;
+ }
+ // delete, line 425
+ slice_del();
+ break;
+ case 2:
+ // (, line 426
+ if (!(I_word_len >= 5))
+ {
+ return false;
+ }
+ // delete, line 426
+ slice_del();
+ break;
+ case 3:
+ // (, line 427
+ if (!(I_word_len >= 6))
+ {
+ return false;
+ }
+ // delete, line 427
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Noun_Step1b() {
+ int among_var;
+ // (, line 430
+ I_word_len = getCurrent().length();
+ // [, line 432
+ ket = cursor;
+ // substring, line 432
+ among_var = find_among_b(a_11, a_11.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 432
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 433
+ if (!(I_word_len > 5))
+ {
+ return false;
+ }
+ // delete, line 433
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Noun_Step2a() {
+ int among_var;
+ // (, line 437
+ I_word_len = getCurrent().length();
+ // [, line 439
+ ket = cursor;
+ // substring, line 439
+ among_var = find_among_b(a_12, a_12.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 439
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 440
+ if (!(I_word_len > 4))
+ {
+ return false;
+ }
+ // delete, line 440
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Noun_Step2b() {
+ int among_var;
+ // (, line 444
+ I_word_len = getCurrent().length();
+ // [, line 446
+ ket = cursor;
+ // substring, line 446
+ among_var = find_among_b(a_13, a_13.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 446
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 447
+ if (!(I_word_len >= 5))
+ {
+ return false;
+ }
+ // delete, line 447
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Noun_Step2c1() {
+ int among_var;
+ // (, line 451
+ I_word_len = getCurrent().length();
+ // [, line 453
+ ket = cursor;
+ // substring, line 453
+ among_var = find_among_b(a_14, a_14.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 453
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 454
+ if (!(I_word_len >= 4))
+ {
+ return false;
+ }
+ // delete, line 454
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Noun_Step2c2() {
+ int among_var;
+ // (, line 457
+ I_word_len = getCurrent().length();
+ // [, line 459
+ ket = cursor;
+ // substring, line 459
+ among_var = find_among_b(a_15, a_15.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 459
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 460
+ if (!(I_word_len >= 4))
+ {
+ return false;
+ }
+ // delete, line 460
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Noun_Step3() {
+ int among_var;
+ // (, line 463
+ I_word_len = getCurrent().length();
+ // [, line 465
+ ket = cursor;
+ // substring, line 465
+ among_var = find_among_b(a_16, a_16.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 465
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 466
+ if (!(I_word_len >= 3))
+ {
+ return false;
+ }
+ // delete, line 466
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Verb_Step1() {
+ int among_var;
+ // (, line 470
+ I_word_len = getCurrent().length();
+ // [, line 472
+ ket = cursor;
+ // substring, line 472
+ among_var = find_among_b(a_17, a_17.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 472
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 473
+ if (!(I_word_len >= 4))
+ {
+ return false;
+ }
+ // delete, line 473
+ slice_del();
+ break;
+ case 2:
+ // (, line 474
+ if (!(I_word_len >= 5))
+ {
+ return false;
+ }
+ // delete, line 474
+ slice_del();
+ break;
+ case 3:
+ // (, line 475
+ if (!(I_word_len >= 6))
+ {
+ return false;
+ }
+ // delete, line 475
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Verb_Step2a() {
+ int among_var;
+ // (, line 478
+ I_word_len = getCurrent().length();
+ // [, line 480
+ ket = cursor;
+ // substring, line 480
+ among_var = find_among_b(a_18, a_18.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 480
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 481
+ if (!(I_word_len >= 4))
+ {
+ return false;
+ }
+ // delete, line 481
+ slice_del();
+ break;
+ case 2:
+ // (, line 482
+ if (!(I_word_len >= 4))
+ {
+ return false;
+ }
+ // delete, line 482
+ slice_del();
+ break;
+ case 3:
+ // (, line 483
+ if (!(I_word_len >= 5))
+ {
+ return false;
+ }
+ // delete, line 483
+ slice_del();
+ break;
+ case 4:
+ // (, line 484
+ if (!(I_word_len > 5))
+ {
+ return false;
+ }
+ // delete, line 484
+ slice_del();
+ break;
+ case 5:
+ // (, line 485
+ if (!(I_word_len >= 6))
+ {
+ return false;
+ }
+ // delete, line 485
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Verb_Step2b() {
+ int among_var;
+ // (, line 489
+ I_word_len = getCurrent().length();
+ // [, line 491
+ ket = cursor;
+ // substring, line 491
+ among_var = find_among_b(a_19, a_19.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 491
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 492
+ if (!(I_word_len >= 5))
+ {
+ return false;
+ }
+ // delete, line 492
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_Verb_Step2c() {
+ int among_var;
+ // (, line 497
+ I_word_len = getCurrent().length();
+ // [, line 499
+ ket = cursor;
+ // substring, line 499
+ among_var = find_among_b(a_20, a_20.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 499
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 500
+ if (!(I_word_len >= 4))
+ {
+ return false;
+ }
+ // delete, line 500
+ slice_del();
+ break;
+ case 2:
+ // (, line 501
+ if (!(I_word_len >= 6))
+ {
+ return false;
+ }
+ // delete, line 501
+ slice_del();
+ break;
+ }
+ return true;
+ }
+
+ private boolean r_Suffix_All_alef_maqsura() {
+ int among_var;
+ // (, line 505
+ I_word_len = getCurrent().length();
+ // [, line 507
+ ket = cursor;
+ // substring, line 507
+ among_var = find_among_b(a_21, a_21.length);
+ if (among_var == 0)
+ {
+ return false;
+ }
+ // ], line 507
+ bra = cursor;
+ switch (among_var) {
+ case 0:
+ return false;
+ case 1:
+ // (, line 508
+ // <-, line 508
+ slice_from("\u064A");
+ break;
+ }
+ return true;
+ }
+
+ public boolean stem() {
+ // (, line 515
+ // set is_noun, line 517
+ B_is_noun = true;
+ // set is_verb, line 518
+ B_is_verb = true;
+ // unset is_defined, line 519
+ B_is_defined = false;
+ // do, line 522
+ int v_1 = cursor;
+ lab0: do {
+ // call Checks1, line 522
+ if (!r_Checks1())
+ {
+ break lab0;
+ }
+ } while (false);
+ cursor = v_1;
+ // do, line 525
+ int v_2 = cursor;
+ lab1: do {
+ // call Normalize_pre, line 525
+ if (!r_Normalize_pre())
+ {
+ break lab1;
+ }
+ } while (false);
+ cursor = v_2;
+ // backwards, line 528
+ limit_backward = cursor;
+ cursor = limit;
+ // (, line 528
+ // do, line 530
+ int v_3 = limit - cursor;
+ lab2: do {
+ // (, line 530
+ // or, line 544
+ lab3: do {
+ int v_4 = limit - cursor;
+ lab4: do {
+ // (, line 532
+ // Boolean test is_verb, line 533
+ if (!(B_is_verb))
+ {
+ break lab4;
+ }
+ // (, line 534
+ // or, line 539
+ lab5: do {
+ int v_5 = limit - cursor;
+ lab6: do {
+ // (, line 535
+ // (, line 536
+ // atleast, line 536
+ {
+ int v_6 = 1;
+ // atleast, line 536
+ replab7: while(true)
+ {
+ int v_7 = limit - cursor;
+ lab8: do {
+ // call Suffix_Verb_Step1, line 536
+ if (!r_Suffix_Verb_Step1())
+ {
+ break lab8;
+ }
+ v_6--;
+ continue replab7;
+ } while (false);
+ cursor = limit - v_7;
+ break replab7;
+ }
+ if (v_6 > 0)
+ {
+ break lab6;
+ }
+ }
+ // (, line 537
+ // or, line 537
+ lab9: do {
+ int v_8 = limit - cursor;
+ lab10: do {
+ // call Suffix_Verb_Step2a, line 537
+ if (!r_Suffix_Verb_Step2a())
+ {
+ break lab10;
+ }
+ break lab9;
+ } while (false);
+ cursor = limit - v_8;
+ lab11: do {
+ // call Suffix_Verb_Step2c, line 537
+ if (!r_Suffix_Verb_Step2c())
+ {
+ break lab11;
+ }
+ break lab9;
+ } while (false);
+ cursor = limit - v_8;
+ // next, line 537
+ if (cursor <= limit_backward)
+ {
+ break lab6;
+ }
+ cursor--;
+ } while (false);
+ break lab5;
+ } while (false);
+ cursor = limit - v_5;
+ lab12: do {
+ // call Suffix_Verb_Step2b, line 539
+ if (!r_Suffix_Verb_Step2b())
+ {
+ break lab12;
+ }
+ break lab5;
+ } while (false);
+ cursor = limit - v_5;
+ // call Suffix_Verb_Step2a, line 540
+ if (!r_Suffix_Verb_Step2a())
+ {
+ break lab4;
+ }
+ } while (false);
+ break lab3;
+ } while (false);
+ cursor = limit - v_4;
+ lab13: do {
+ // (, line 544
+ // Boolean test is_noun, line 545
+ if (!(B_is_noun))
+ {
+ break lab13;
+ }
+ // (, line 546
+ // try, line 548
+ int v_9 = limit - cursor;
+ lab14: do {
+ // (, line 548
+ // or, line 550
+ lab15: do {
+ int v_10 = limit - cursor;
+ lab16: do {
+ // call Suffix_Noun_Step2c2, line 549
+ if (!r_Suffix_Noun_Step2c2())
+ {
+ break lab16;
+ }
+ break lab15;
+ } while (false);
+ cursor = limit - v_10;
+ lab17: do {
+ // (, line 550
+ // not, line 550
+ lab18: do {
+ // Boolean test is_defined, line 550
+ if (!(B_is_defined))
+ {
+ break lab18;
+ }
+ break lab17;
+ } while (false);
+ // call Suffix_Noun_Step1a, line 550
+ if (!r_Suffix_Noun_Step1a())
+ {
+ break lab17;
+ }
+ // (, line 550
+ // or, line 552
+ lab19: do {
+ int v_12 = limit - cursor;
+ lab20: do {
+ // call Suffix_Noun_Step2a, line 551
+ if (!r_Suffix_Noun_Step2a())
+ {
+ break lab20;
+ }
+ break lab19;
+ } while (false);
+ cursor = limit - v_12;
+ lab21: do {
+ // call Suffix_Noun_Step2b, line 552
+ if (!r_Suffix_Noun_Step2b())
+ {
+ break lab21;
+ }
+ break lab19;
+ } while (false);
+ cursor = limit - v_12;
+ lab22: do {
+ // call Suffix_Noun_Step2c1, line 553
+ if (!r_Suffix_Noun_Step2c1())
+ {
+ break lab22;
+ }
+ break lab19;
+ } while (false);
+ cursor = limit - v_12;
+ // next, line 554
+ if (cursor <= limit_backward)
+ {
+ break lab17;
+ }
+ cursor--;
+ } while (false);
+ break lab15;
+ } while (false);
+ cursor = limit - v_10;
+ lab23: do {
+ // (, line 555
+ // call Suffix_Noun_Step1b, line 555
+ if (!r_Suffix_Noun_Step1b())
+ {
+ break lab23;
+ }
+ // (, line 555
+ // or, line 557
+ lab24: do {
+ int v_13 = limit - cursor;
+ lab25: do {
+ // call Suffix_Noun_Step2a, line 556
+ if (!r_Suffix_Noun_Step2a())
+ {
+ break lab25;
+ }
+ break lab24;
+ } while (false);
+ cursor = limit - v_13;
+ lab26: do {
+ // call Suffix_Noun_Step2b, line 557
+ if (!r_Suffix_Noun_Step2b())
+ {
+ break lab26;
+ }
+ break lab24;
+ } while (false);
+ cursor = limit - v_13;
+ // call Suffix_Noun_Step2c1, line 558
+ if (!r_Suffix_Noun_Step2c1())
+ {
+ break lab23;
+ }
+ } while (false);
+ break lab15;
+ } while (false);
+ cursor = limit - v_10;
+ lab27: do {
+ // (, line 559
+ // not, line 559
+ lab28: do {
+ // Boolean test is_defined, line 559
+ if (!(B_is_defined))
+ {
+ break lab28;
+ }
+ break lab27;
+ } while (false);
+ // call Suffix_Noun_Step2a, line 559
+ if (!r_Suffix_Noun_Step2a())
+ {
+ break lab27;
+ }
+ break lab15;
+ } while (false);
+ cursor = limit - v_10;
+ // (, line 560
+ // call Suffix_Noun_Step2b, line 560
+ if (!r_Suffix_Noun_Step2b())
+ {
+ cursor = limit - v_9;
+ break lab14;
+ }
+ } while (false);
+ } while (false);
+ // call Suffix_Noun_Step3, line 562
+ if (!r_Suffix_Noun_Step3())
+ {
+ break lab13;
+ }
+ break lab3;
+ } while (false);
+ cursor = limit - v_4;
+ // call Suffix_All_alef_maqsura, line 568
+ if (!r_Suffix_All_alef_maqsura())
+ {
+ break lab2;
+ }
+ } while (false);
+ } while (false);
+ cursor = limit - v_3;
+ cursor = limit_backward;
+ // do, line 573
+ int v_15 = cursor;
+ lab29: do {
+ // (, line 573
+ // try, line 574
+ int v_16 = cursor;
+ lab30: do {
+ // call Prefix_Step1, line 574
+ if (!r_Prefix_Step1())
+ {
+ cursor = v_16;
+ break lab30;
+ }
+ } while (false);
+ // try, line 575
+ int v_17 = cursor;
+ lab31: do {
+ // call Prefix_Step2, line 575
+ if (!r_Prefix_Step2())
+ {
+ cursor = v_17;
+ break lab31;
+ }
+ } while (false);
+ // (, line 576
+ // or, line 577
+ lab32: do {
+ int v_18 = cursor;
+ lab33: do {
+ // call Prefix_Step3a_Noun, line 576
+ if (!r_Prefix_Step3a_Noun())
+ {
+ break lab33;
+ }
+ break lab32;
+ } while (false);
+ cursor = v_18;
+ lab34: do {
+ // (, line 577
+ // Boolean test is_noun, line 577
+ if (!(B_is_noun))
+ {
+ break lab34;
+ }
+ // call Prefix_Step3b_Noun, line 577
+ if (!r_Prefix_Step3b_Noun())
+ {
+ break lab34;
+ }
+ break lab32;
+ } while (false);
+ cursor = v_18;
+ // (, line 578
+ // Boolean test is_verb, line 578
+ if (!(B_is_verb))
+ {
+ break lab29;
+ }
+ // try, line 578
+ int v_19 = cursor;
+ lab35: do {
+ // call Prefix_Step3_Verb, line 578
+ if (!r_Prefix_Step3_Verb())
+ {
+ cursor = v_19;
+ break lab35;
+ }
+ } while (false);
+ // call Prefix_Step4_Verb, line 578
+ if (!r_Prefix_Step4_Verb())
+ {
+ break lab29;
+ }
+ } while (false);
+ } while (false);
+ cursor = v_15;
+ // do, line 583
+ int v_20 = cursor;
+ lab36: do {
+ // call Normalize_post, line 583
+ if (!r_Normalize_post())
+ {
+ break lab36;
+ }
+ } while (false);
+ cursor = v_20;
+ return true;
+ }
+
+ public boolean equals( Object o ) {
+ return o instanceof ArabicStemmer;
+ }
+
+ public int hashCode() {
+ return ArabicStemmer.class.getName().hashCode();
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c567d4f/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
index d215e02..bba8d33 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
@@ -36,6 +36,7 @@ public class TestSnowballVocab extends LuceneTestCase {
* Run all languages against their snowball vocabulary tests.
*/
public void testStemmers() throws IOException {
+ assertCorrectOutput("Arabic", "arabic");
assertCorrectOutput("Danish", "danish");
assertCorrectOutput("Dutch", "dutch");
assertCorrectOutput("English", "english");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5c567d4f/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocabData.zip
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocabData.zip b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocabData.zip
index 8831d8a..e3cae65 100644
Binary files a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocabData.zip and b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocabData.zip differ
[04/32] lucene-solr:jira/solr-12730: Fix typo in
stream-decorator-reference.adoc file.
Posted by ab...@apache.org.
Fix typo in stream-decorator-reference.adoc file.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cebf7039
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cebf7039
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cebf7039
Branch: refs/heads/jira/solr-12730
Commit: cebf7039ee047302d4c52fdb282f43c58ed65595
Parents: bd714ca
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:22:51 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:22:51 2018 +0100
----------------------------------------------------------------------
solr/solr-ref-guide/src/stream-decorator-reference.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cebf7039/solr/solr-ref-guide/src/stream-decorator-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-decorator-reference.adoc b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
index c2d6160..5549fc2 100644
--- a/solr/solr-ref-guide/src/stream-decorator-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
@@ -612,7 +612,7 @@ eval(expr)
----
In the example above the `eval` expression reads the first tuple from the underlying expression. It then compiles and
-executes the string Streaming Expression in the epxr_s field.
+executes the string Streaming Expression in the expr_s field.
== executor
[32/32] lucene-solr:jira/solr-12730: SOLR-12730: add more debug
logging.
Posted by ab...@apache.org.
SOLR-12730: add more debug logging.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/efb73a39
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/efb73a39
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/efb73a39
Branch: refs/heads/jira/solr-12730
Commit: efb73a39b03b3f4ac2fa139d4e0816a35c5d8b8d
Parents: b47df35
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Oct 29 20:06:31 2018 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Oct 29 20:06:31 2018 +0100
----------------------------------------------------------------------
.../cloud/autoscaling/IndexSizeTrigger.java | 19 +++++++++++---
.../src/java/org/apache/solr/core/SolrCore.java | 26 ++++++++++++++++++++
.../solr/handler/admin/MetricsHandler.java | 8 ++++++
.../apache/solr/update/SolrIndexSplitter.java | 4 +--
.../cloud/autoscaling/IndexSizeTriggerTest.java | 19 ++++++++++++--
5 files changed, 69 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/efb73a39/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 533ec53..e41297e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -252,7 +252,7 @@ public class IndexSizeTrigger extends TriggerBase {
DocCollection docCollection = clusterState.getCollection(coll);
shards.forEach((sh, replicas) -> {
- // check only the leader of a replica in active shard
+ // check only the leader replica in an active shard
Slice s = docCollection.getSlice(sh);
if (s.getState() != Slice.State.ACTIVE) {
return;
@@ -262,6 +262,10 @@ public class IndexSizeTrigger extends TriggerBase {
if (r == null) {
return;
}
+ // not on this node
+ if (!r.getNodeName().equals(node)) {
+ return;
+ }
// find ReplicaInfo
ReplicaInfo info = null;
for (ReplicaInfo ri : replicas) {
@@ -282,6 +286,7 @@ public class IndexSizeTrigger extends TriggerBase {
String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null);
String tag = "metrics:" + registry + ":INDEX.sizeInBytes";
metricTags.put(tag, info);
+ metricTags.put("metrics:" + registry + ":INDEX.sizeDetails", info);
tag = "metrics:" + registry + ":SEARCHER.searcher.numDocs";
metricTags.put(tag, info);
});
@@ -297,14 +302,14 @@ public class IndexSizeTrigger extends TriggerBase {
} else {
// verify that it's a Number
if (!(size instanceof Number)) {
- log.warn("invalid size value - not a number: '" + size + "' is " + size.getClass().getName());
+ log.warn("invalid size value for tag " + tag + " - not a number: '" + size + "' is " + size.getClass().getName());
return;
}
ReplicaInfo currentInfo = currentSizes.computeIfAbsent(info.getCore(), k -> (ReplicaInfo)info.clone());
if (tag.contains("INDEX")) {
currentInfo.getVariables().put(BYTES_SIZE_PROP, ((Number) size).longValue());
- } else {
+ } else if (tag.contains("SEARCHER")) {
currentInfo.getVariables().put(DOCS_SIZE_PROP, ((Number) size).longValue());
}
}
@@ -458,6 +463,14 @@ public class IndexSizeTrigger extends TriggerBase {
if (ops.isEmpty()) {
return;
}
+ try {
+ ClusterState cs = cloudManager.getClusterStateProvider().getClusterState();
+ cs.forEachCollection(coll -> {
+ log.debug("##== Collection: {}", coll);
+ });
+ } catch (IOException e) {
+ throw new RuntimeException("oops: ", e);
+ }
if (processor.process(new IndexSizeEvent(getName(), eventTime.get(), ops, aboveSize, belowSize))) {
// update last event times
aboveSize.forEach((coll, replicas) -> {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/efb73a39/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 6e13039..abc4af6 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -443,6 +443,31 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
return size;
}
+ String getIndexSizeDetails() {
+ Directory dir;
+ StringBuilder sb = new StringBuilder();
+ try {
+ if (directoryFactory.exists(getIndexDir())) {
+ dir = directoryFactory.get(getIndexDir(), DirContext.DEFAULT, solrConfig.indexConfig.lockType);
+ try {
+ String[] files = dir.listAll();
+ Arrays.sort(files);
+ for (String file : files) {
+ sb.append('\n');
+ sb.append(file);
+ sb.append('\t');
+ sb.append(String.valueOf(dir.fileLength(file)));
+ }
+ } finally {
+ directoryFactory.release(dir);
+ }
+ }
+ } catch (IOException e) {
+ SolrException.log(log, "IO error while trying to get the details of the Directory", e);
+ }
+ return sb.toString();
+ }
+
@Override
public String getName() {
return name;
@@ -1161,6 +1186,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
manager.registerGauge(this, registry, () -> resourceLoader.getInstancePath().toString(), getMetricTag(), true, "instanceDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : getIndexDir(), getMetricTag(), true, "indexDir", Category.CORE.toString());
manager.registerGauge(this, registry, () -> isClosed() ? 0 : getIndexSize(), getMetricTag(), true, "sizeInBytes", Category.INDEX.toString());
+ manager.registerGauge(this, registry, () -> isClosed() ? "" : getIndexSizeDetails(), getMetricTag(), true, "sizeDetails", Category.INDEX.toString());
manager.registerGauge(this, registry, () -> isClosed() ? "(closed)" : NumberUtils.readableSize(getIndexSize()), getMetricTag(), true, "size", Category.INDEX.toString());
if (coreContainer != null) {
manager.registerGauge(this, registry, () -> coreContainer.getNamesForCore(this), getMetricTag(), true, "aliases", Category.CORE.toString());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/efb73a39/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index 752e021..9b9f948 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -17,6 +17,7 @@
package org.apache.solr.handler.admin;
+import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@ -41,6 +42,7 @@ import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.metrics.SolrMetricManager;
@@ -49,11 +51,15 @@ import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.security.AuthorizationContext;
import org.apache.solr.security.PermissionNameProvider;
import org.apache.solr.util.stats.MetricUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Request handler to return metrics
*/
public class MetricsHandler extends RequestHandlerBase implements PermissionNameProvider {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
final SolrMetricManager metricManager;
public static final String COMPACT_PARAM = "compact";
@@ -99,6 +105,8 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
}
handleRequest(req.getParams(), (k, v) -> rsp.add(k, v));
+ log.debug("##== Req: {}", req);
+ log.debug("##== Rsp: {}", Utils.toJSONString(rsp.getValues()));
}
public void handleRequest(SolrParams params, BiConsumer<String, Object> consumer) throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/efb73a39/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
index 334a29d..6ed0b1f 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrIndexSplitter.java
@@ -174,7 +174,7 @@ public class SolrIndexSplitter {
log.error("Original error closing IndexWriter:", e);
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reopening IndexWriter after failed close", e1);
}
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error closing current IndexWriter, aborting offline split...", e);
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error closing current IndexWriter, aborting 'link' split...", e);
}
}
boolean success = false;
@@ -195,7 +195,7 @@ public class SolrIndexSplitter {
t = timings.sub("parentApplyBufferedUpdates");
ulog.applyBufferedUpdates();
t.stop();
- log.info("Splitting in 'offline' mode " + (success? "finished" : "FAILED") +
+ log.info("Splitting in 'link' mode " + (success? "finished" : "FAILED") +
": re-opened parent IndexWriter.");
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/efb73a39/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index c933c0a..934ec20 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -67,7 +67,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_P
/**
*
*/
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.handler.admin.MetricsHandler=DEBUG")
public class IndexSizeTriggerTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -90,6 +90,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
@BeforeClass
public static void setupCluster() throws Exception {
+ System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
@@ -503,7 +504,8 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
int aboveBytes = maxSize * 9 / 10;
- long waitForSeconds = 3 + random().nextInt(5);
+ // need to wait for recovery after splitting
+ long waitForSeconds = 10 + random().nextInt(5);
// the trigger is initially disabled so that we have time to add listeners
// and have them capture all events once the trigger is enabled
@@ -562,6 +564,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
"'name' : 'index_size_trigger4'" +
"}" +
"}";
+ log.info("-- resuming trigger");
req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -570,6 +573,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
assertTrue("did not finish processing in time", await);
+ log.info("-- suspending trigger");
// suspend the trigger to avoid generating more events
String suspendTriggerCommand = "{" +
"'suspend-trigger' : {" +
@@ -624,6 +628,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
finished = new CountDownLatch(1);
+ log.info("-- deleting documents");
for (int j = 0; j < 10; j++) {
UpdateRequest ureq = new UpdateRequest();
ureq.setParam("collection", collectionName);
@@ -632,6 +637,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
}
solrClient.request(ureq);
}
+ cloudManager.getTimeSource().sleep(5000);
// make sure the actual index size is reduced by deletions, otherwise we may still violate aboveBytes
UpdateRequest ur = new UpdateRequest();
ur.setParam(UpdateParams.COMMIT, "true");
@@ -640,14 +646,17 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
ur.setParam(UpdateParams.MAX_OPTIMIZE_SEGMENTS, "1");
ur.setParam(UpdateParams.WAIT_SEARCHER, "true");
ur.setParam(UpdateParams.OPEN_SEARCHER, "true");
+ log.info("-- requesting optimize / expungeDeletes / commit");
solrClient.request(ur, collectionName);
// wait for the segments to merge to reduce the index size
cloudManager.getTimeSource().sleep(50000);
+ log.info("-- requesting commit");
solrClient.commit(collectionName, true, true);
// resume the trigger
+ log.info("-- resuming trigger");
req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -656,6 +665,12 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
assertTrue("did not finish processing in time", await);
+ log.info("-- suspending trigger");
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ System.exit(-1);
+
assertEquals(1, listenerEvents.size());
events = listenerEvents.get("capturing4");
assertNotNull("'capturing4' events not found", events);
[19/32] lucene-solr:jira/solr-12730: LUCENE-8524: Add the Hangul
Letter Araea (interpunct) as a separator in Nori's tokenizer. This change
also removes empty terms and trim surface form in Nori's Korean dictionary.
Posted by ab...@apache.org.
LUCENE-8524: Add the Hangul Letter Araea (interpunct) as a separator in Nori's tokenizer.
This change also removes empty terms and trim surface form in Nori's Korean dictionary.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6f291d40
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6f291d40
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6f291d40
Branch: refs/heads/jira/solr-12730
Commit: 6f291d402b93ca534eccfef620fa392d0cd2b892
Parents: f33be7a
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Oct 26 10:28:37 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Oct 26 10:28:37 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../lucene/analysis/ko/KoreanTokenizer.java | 4 +++
.../ko/dict/TokenInfoDictionary$buffer.dat | Bin 7245625 -> 7245613 bytes
.../ko/dict/TokenInfoDictionary$fst.dat | Bin 5640925 -> 5640903 bytes
.../ko/dict/TokenInfoDictionary$targetMap.dat | Bin 811783 -> 811783 bytes
.../lucene/analysis/ko/TestKoreanTokenizer.java | 8 +++++
.../ko/dict/TestTokenInfoDictionary.java | 4 +++
.../ko/util/BinaryDictionaryWriter.java | 29 ++++++++++---------
.../ko/util/TokenInfoDictionaryBuilder.java | 17 ++++++-----
9 files changed, 44 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4097ce0..35dc4ec 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -212,6 +212,9 @@ Bug fixes:
in the graph if the slop is greater than 0. Span queries cannot be used in this case because
they don't handle slop the same way than phrase queries. (Steve Rowe, Uwe Schindler, Jim Ferenczi)
+* LUCENE-8524: Add the Hangul Letter Araea (interpunct) as a separator in Nori's tokenizer.
+ This change also removes empty terms and trim surface form in Nori's Korean dictionary. (Trey Jones, Jim Ferenczi)
+
New Features
* LUCENE-8496: Selective indexing - modify BKDReader/BKDWriter to allow users
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
index 822853b..ab3205f 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/KoreanTokenizer.java
@@ -932,6 +932,10 @@ public final class KoreanTokenizer extends Tokenizer {
}
private static boolean isPunctuation(char ch) {
+ // special case for Hangul Letter Araea (interpunct)
+ if (ch == 0x318D) {
+ return true;
+ }
switch(Character.getType(ch)) {
case Character.SPACE_SEPARATOR:
case Character.LINE_SEPARATOR:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat
index 6958664..d7cc866 100644
Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$buffer.dat differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat
index 17b531f..fa0cb32 100644
Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$fst.dat differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat
index 7c0823c..4661bf8 100644
Binary files a/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat and b/lucene/analysis/nori/src/resources/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary$targetMap.dat differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
index 0471e5f..7c204fa 100644
--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
@@ -289,6 +289,14 @@ public class TestKoreanTokenizer extends BaseTokenStreamTestCase {
);
}
+ public void testInterpunct() throws IOException {
+ assertAnalyzesTo(analyzer, "도로ㆍ지반ㆍ수자원ㆍ건설환경ㆍ건축ㆍ화재설비연구",
+ new String[]{"도로", "지반", "수자원", "건설", "환경", "건축", "화재", "설비", "연구"},
+ new int[]{0, 3, 6, 10, 12, 15, 18, 20, 22},
+ new int[]{2, 5, 9, 12, 14, 17, 20, 22, 24},
+ new int[]{1, 1, 1, 1, 1, 1, 1, 1, 1}
+ );
+ }
/** blast some random strings through the tokenizer */
public void testRandomStrings() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
index d278841..3457de1 100644
--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
@@ -48,6 +48,8 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
chars[i] = (char)input.ints[input.offset+i];
}
String surfaceForm = new String(chars);
+ assertFalse(surfaceForm.isEmpty());
+ assertEquals(surfaceForm.trim(), surfaceForm);
assertTrue(UnicodeUtil.validUTF16String(surfaceForm));
Long output = mapping.output;
@@ -96,6 +98,8 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
int offset = 0;
for (Dictionary.Morpheme morph : decompound) {
assertTrue(UnicodeUtil.validUTF16String(morph.surfaceForm));
+ assertFalse(morph.surfaceForm.isEmpty());
+ assertEquals(morph.surfaceForm.trim(), morph.surfaceForm);
if (type != POS.Type.INFLECT) {
assertEquals(morph.surfaceForm, surfaceForm.substring(offset, offset + morph.surfaceForm.length()));
offset += morph.surfaceForm.length();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java b/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
index 35c16ae..b77d1ba 100644
--- a/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
+++ b/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/BinaryDictionaryWriter.java
@@ -26,6 +26,7 @@ import java.nio.channels.Channels;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import org.apache.lucene.analysis.ko.POS;
import org.apache.lucene.analysis.ko.dict.Dictionary;
@@ -109,23 +110,23 @@ public abstract class BinaryDictionaryWriter {
assert existing == null || existing.equals(fullPOSData);
posDict.set(leftId, fullPOSData);
- final Dictionary.Morpheme[] morphemes;
+ final List<Dictionary.Morpheme> morphemes = new ArrayList<>();
// true if the POS and decompounds of the token are all the same.
boolean hasSinglePOS = (leftPOS == rightPOS);
if (posType != POS.Type.MORPHEME && expression.length() > 0) {
String[] exprTokens = expression.split("\\+");
- morphemes = new Dictionary.Morpheme[exprTokens.length];
for (int i = 0; i < exprTokens.length; i++) {
String[] tokenSplit = exprTokens[i].split("\\/");
assert tokenSplit.length == 3;
- POS.Tag exprTag = POS.resolveTag(tokenSplit[1]);
- morphemes[i] = new Dictionary.Morpheme(exprTag, tokenSplit[0]);
- if (leftPOS != exprTag) {
- hasSinglePOS = false;
+ String surfaceForm = tokenSplit[0].trim();
+ if (surfaceForm.isEmpty() == false) {
+ POS.Tag exprTag = POS.resolveTag(tokenSplit[1]);
+ morphemes.add(new Dictionary.Morpheme(exprTag, tokenSplit[0]));
+ if (leftPOS != exprTag) {
+ hasSinglePOS = false;
+ }
}
}
- } else {
- morphemes = new Dictionary.Morpheme[0];
}
int flags = 0;
@@ -151,17 +152,17 @@ public abstract class BinaryDictionaryWriter {
if (hasSinglePOS == false) {
buffer.put((byte) rightPOS.ordinal());
}
- buffer.put((byte) morphemes.length);
+ buffer.put((byte) morphemes.size());
int compoundOffset = 0;
- for (int i = 0; i < morphemes.length; i++) {
+ for (Dictionary.Morpheme morpheme : morphemes) {
if (hasSinglePOS == false) {
- buffer.put((byte) morphemes[i].posTag.ordinal());
+ buffer.put((byte) morpheme.posTag.ordinal());
}
if (posType != POS.Type.INFLECT) {
- buffer.put((byte) morphemes[i].surfaceForm.length());
- compoundOffset += morphemes[i].surfaceForm.length();
+ buffer.put((byte) morpheme.surfaceForm.length());
+ compoundOffset += morpheme.surfaceForm.length();
} else {
- writeString(morphemes[i].surfaceForm);
+ writeString(morpheme.surfaceForm);
}
assert compoundOffset <= entry[0].length() : Arrays.toString(entry);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6f291d40/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java b/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java
index de60daa..d5fb73f 100644
--- a/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java
+++ b/lucene/analysis/nori/src/tools/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java
@@ -116,6 +116,10 @@ public class TokenInfoDictionaryBuilder {
// build tokeninfo dictionary
for (String[] entry : lines) {
+ String surfaceForm = entry[0].trim();
+ if (surfaceForm.isEmpty()) {
+ continue;
+ }
int next = dictionary.put(entry);
if(next == offset){
@@ -123,15 +127,14 @@ public class TokenInfoDictionaryBuilder {
continue;
}
- String token = entry[0];
- if (!token.equals(lastValue)) {
+ if (!surfaceForm.equals(lastValue)) {
// new word to add to fst
ord++;
- lastValue = token;
- scratch.grow(token.length());
- scratch.setLength(token.length());
- for (int i = 0; i < token.length(); i++) {
- scratch.setIntAt(i, (int) token.charAt(i));
+ lastValue = surfaceForm;
+ scratch.grow(surfaceForm.length());
+ scratch.setLength(surfaceForm.length());
+ for (int i = 0; i < surfaceForm.length(); i++) {
+ scratch.setIntAt(i, (int) surfaceForm.charAt(i));
}
fstBuilder.add(scratch.get(), ord);
}
[31/32] lucene-solr:jira/solr-12730: Merge branch 'master' into
jira/solr-12730
Posted by ab...@apache.org.
Merge branch 'master' into jira/solr-12730
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b47df35d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b47df35d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b47df35d
Branch: refs/heads/jira/solr-12730
Commit: b47df35d526073142a32cf9f7fc57c0ea8e4b1b0
Parents: 3d91b8e 5fc4d51
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Oct 29 12:41:15 2018 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Oct 29 12:41:15 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 10 +
lucene/analysis/common/README.txt | 2 +
lucene/analysis/common/build.xml | 20 +-
.../tartarus/snowball/ext/ArabicStemmer.java | 1912 ++++++++++++++++++
.../analysis/snowball/TestSnowballVocab.java | 1 +
.../analysis/snowball/TestSnowballVocabData.zip | Bin 3128133 -> 3568843 bytes
.../lucene/analysis/ko/KoreanTokenizer.java | 4 +
.../ko/dict/TokenInfoDictionary$buffer.dat | Bin 7245625 -> 7245613 bytes
.../ko/dict/TokenInfoDictionary$fst.dat | Bin 5640925 -> 5640903 bytes
.../ko/dict/TokenInfoDictionary$targetMap.dat | Bin 811783 -> 811783 bytes
.../lucene/analysis/ko/TestKoreanTokenizer.java | 8 +
.../ko/dict/TestTokenInfoDictionary.java | 4 +
.../ko/util/BinaryDictionaryWriter.java | 29 +-
.../ko/util/TokenInfoDictionaryBuilder.java | 17 +-
lucene/common-build.xml | 2 +-
.../apache/lucene/geo/SimpleWKTShapeParser.java | 406 ++++
.../lucene/geo/TestSimpleWKTShapeParsing.java | 206 ++
solr/CHANGES.txt | 29 +-
.../java/org/apache/solr/api/V2HttpCall.java | 5 +-
.../org/apache/solr/cloud/ElectionContext.java | 3 +
.../solr/highlight/UnifiedSolrHighlighter.java | 33 +-
.../org/apache/solr/servlet/HttpSolrCall.java | 4 +-
.../solr/collection1/conf/schema15.xml | 10 +
...lrconfig-distrib-update-processor-chains.xml | 2 +-
.../conf/solrconfig-doctransformers.xml | 2 +-
.../solr/collection1/conf/solrconfig-hash.xml | 2 +-
.../conf/solrconfig-master-throttled.xml | 2 +-
.../collection1/conf/solrconfig-paramset.xml | 2 +-
.../solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../solr/collection1/conf/solrconfig-tagger.xml | 2 +-
.../cloud/TestCloudJSONFacetJoinDomain.java | 853 --------
.../solr/cloud/TestCloudJSONFacetSKG.java | 677 -------
.../TestSolrConfigHandlerConcurrent.java | 4 +-
.../highlight/TestUnifiedSolrHighlighter.java | 14 +-
.../apache/solr/search/QueryEqualityTest.java | 12 +
.../org/apache/solr/search/facet/DebugAgg.java | 2 +-
.../facet/TestCloudJSONFacetJoinDomain.java | 855 ++++++++
.../search/facet/TestCloudJSONFacetSKG.java | 678 +++++++
solr/solr-ref-guide/src/collections-api.adoc | 36 +-
.../src/computational-geometry.adoc | 188 ++
solr/solr-ref-guide/src/curve-fitting.adoc | 101 +
solr/solr-ref-guide/src/dsp.adoc | 124 +-
solr/solr-ref-guide/src/highlighting.adoc | 31 +-
.../src/images/math-expressions/sinewave.png | Bin 0 -> 273253 bytes
.../src/images/math-expressions/sinewave256.png | Bin 0 -> 369866 bytes
solr/solr-ref-guide/src/math-expressions.adoc | 15 +-
solr/solr-ref-guide/src/matrix-math.adoc | 53 +
solr/solr-ref-guide/src/statistics.adoc | 87 +
.../src/stream-decorator-reference.adoc | 50 +-
.../src/stream-evaluator-reference.adoc | 5 +-
solr/solr-ref-guide/src/vector-math.adoc | 38 +
.../impl/StreamingBinaryResponseParser.java | 14 +-
.../solr/common/params/HighlightParams.java | 1 +
.../solr/collection1/conf/solrconfig-sql.xml | 2 +-
.../solr/configsets/ml/conf/solrconfig.xml | 2 +-
.../configsets/streaming/conf/solrconfig.xml | 2 +-
.../SolrExampleStreamingBinaryTest.java | 56 +
.../client/solrj/request/TestV2Request.java | 46 +-
.../solr/cloud/MultiSolrCloudTestCase.java | 1 +
59 files changed, 5007 insertions(+), 1659 deletions(-)
----------------------------------------------------------------------
[21/32] lucene-solr:jira/solr-12730: SOLR-7557: Fix parsing of child
documents using queryAndStreamResponse
Posted by ab...@apache.org.
SOLR-7557: Fix parsing of child documents using queryAndStreamResponse
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6c419454
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6c419454
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6c419454
Branch: refs/heads/jira/solr-12730
Commit: 6c419454a23dd8149b4dc90f8e5ad9427a2750bd
Parents: 5c567d4
Author: Marvin Bredal Lillehaug <ma...@gmail.com>
Authored: Fri Oct 26 14:59:31 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Fri Oct 26 14:59:31 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../impl/StreamingBinaryResponseParser.java | 14 ++++-
.../SolrExampleStreamingBinaryTest.java | 56 ++++++++++++++++++++
3 files changed, 70 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c419454/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 03c97fd..1e0e530 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -212,6 +212,8 @@ Bug Fixes
* SOLR-12868: Request forwarding for v2 API is broken (noble)
+* SOLR-7557: Fix parsing of child documents using queryAndStreamResponse (Marvin Bredal Lillehaug/Stian Østerhaug via janhoy)
+
Improvements
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c419454/solr/solrj/src/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
index 5f88672..b70daee 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/StreamingBinaryResponseParser.java
@@ -47,11 +47,21 @@ public class StreamingBinaryResponseParser extends BinaryResponseParser {
public NamedList<Object> processResponse(InputStream body, String encoding) {
try (JavaBinCodec codec = new JavaBinCodec() {
+ private int nestedLevel;
+
@Override
public SolrDocument readSolrDocument(DataInputInputStream dis) throws IOException {
+ nestedLevel++;
SolrDocument doc = super.readSolrDocument(dis);
- callback.streamSolrDocument( doc );
- return null;
+ nestedLevel--;
+ if (nestedLevel == 0) {
+ // parent document
+ callback.streamSolrDocument(doc);
+ return null;
+ } else {
+ // child document
+ return doc;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6c419454/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java
index 1428054..bcab567 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/SolrExampleStreamingBinaryTest.java
@@ -16,12 +16,21 @@
*/
package org.apache.solr.client.solrj.embedded;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.StreamingResponseCallback;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.Test;
@Slow
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
@@ -34,4 +43,51 @@ public class SolrExampleStreamingBinaryTest extends SolrExampleStreamingTest {
client.setRequestWriter(new BinaryRequestWriter());
return client;
}
+
+ @Test
+ public void testQueryAndStreamResponse() throws Exception {
+ // index a simple document with one child
+ SolrClient client = getSolrClient();
+ client.deleteByQuery("*:*");
+
+ SolrInputDocument child = new SolrInputDocument();
+ child.addField("id", "child");
+ child.addField("type_s", "child");
+ child.addField("text_s", "text");
+
+ SolrInputDocument parent = new SolrInputDocument();
+ parent.addField("id", "parent");
+ parent.addField("type_s", "parent");
+ parent.addChildDocument(child);
+
+ client.add(parent);
+ client.commit();
+
+ // create a query with child doc transformer
+ SolrQuery query = new SolrQuery("{!parent which='type_s:parent'}text_s:text");
+ query.addField("*,[child parentFilter='type_s:parent']");
+
+ // test regular query
+ QueryResponse response = client.query(query);
+ assertEquals(1, response.getResults().size());
+ SolrDocument parentDoc = response.getResults().get(0);
+ assertEquals(1, parentDoc.getChildDocumentCount());
+
+ // test streaming
+ final List<SolrDocument> docs = new ArrayList<>();
+ client.queryAndStreamResponse(query, new StreamingResponseCallback() {
+ @Override
+ public void streamSolrDocument(SolrDocument doc) {
+ docs.add(doc);
+ }
+
+ @Override
+ public void streamDocListInfo(long numFound, long start, Float maxScore) {
+ }
+ });
+
+ assertEquals(1, docs.size());
+ parentDoc = docs.get(0);
+ assertEquals(1, parentDoc.getChildDocumentCount());
+ }
}
[23/32] lucene-solr:jira/solr-12730: SOLR-12913: RefGuide revisions
Posted by ab...@apache.org.
SOLR-12913: RefGuide revisions
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4821b305
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4821b305
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4821b305
Branch: refs/heads/jira/solr-12730
Commit: 4821b30598a92d9a6a5c6b3525a392c9ed9dedf0
Parents: 4fa99c2
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Oct 26 13:31:22 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Oct 26 13:31:53 2018 -0400
----------------------------------------------------------------------
.../src/computational-geometry.adoc | 12 ++---
solr/solr-ref-guide/src/curve-fitting.adoc | 9 ++--
solr/solr-ref-guide/src/dsp.adoc | 48 ++++++++++++++------
solr/solr-ref-guide/src/matrix-math.adoc | 4 +-
solr/solr-ref-guide/src/statistics.adoc | 2 +-
.../src/stream-decorator-reference.adoc | 46 +++++++++----------
solr/solr-ref-guide/src/vector-math.adoc | 4 +-
7 files changed, 75 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4821b305/solr/solr-ref-guide/src/computational-geometry.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/computational-geometry.adoc b/solr/solr-ref-guide/src/computational-geometry.adoc
index 263f6d0..abcdb08 100644
--- a/solr/solr-ref-guide/src/computational-geometry.adoc
+++ b/solr/solr-ref-guide/src/computational-geometry.adoc
@@ -24,14 +24,14 @@ functions.
A convex hull is the smallest convex set of points that encloses a data set. Math expressions has support for computing
the convex hull of a 2D data set. Once a convex hull has been calculated, a set of math expression functions
-can be used to geometrically describe the convex hull.
+can be applied to geometrically describe the convex hull.
The `convexHull` function finds the convex hull of an observation matrix of 2D vectors.
-Each row of the matrix contains a 2D observation.
+Each row of the matrix is a 2D observation.
In the example below a convex hull is calculated for a randomly generated set of 100 2D observations.
-Then the following functions are called on the convex result:
+Then the following functions are called on the convex hull:
-`getBaryCenter`: Returns the 2D point that is the bary center of the convex hull.
@@ -39,7 +39,7 @@ Then the following functions are called on the convex result:
-`getBoundarySize`: Returns the boundary size of the convex hull.
--`getVertices`: Returns 2D points that are the vertices of the convex hull.
+-`getVertices`: Returns a set of 2D points that are the vertices of the convex hull.
[source,text]
@@ -126,11 +126,11 @@ When this expression is sent to the `/stream` handler it responds with:
The `enclosingDisk` function finds the smallest enclosing circle the encloses a 2D data set.
Once an enclosing disk has been calculated, a set of math expression functions
-can be used to geometrically describe the enclosing disk.
+can be applied to geometrically describe the enclosing disk.
In the example below an enclosing disk is calculated for a randomly generated set of 1000 2D observations.
-Then the following functions are called on the enclosing disk result:
+Then the following functions are called on the enclosing disk:
-`getCenter`: Returns the 2D point that is the center of the disk.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4821b305/solr/solr-ref-guide/src/curve-fitting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/curve-fitting.adoc b/solr/solr-ref-guide/src/curve-fitting.adoc
index dc66eeb..c2086b1 100644
--- a/solr/solr-ref-guide/src/curve-fitting.adoc
+++ b/solr/solr-ref-guide/src/curve-fitting.adoc
@@ -230,10 +230,11 @@ for the x-axis.
The example below shows `harmfit` fitting a single oscillation of a sine wave. `harmfit`
returns the smoothed values at each control point. The return value is also a model which can be used by
-the `predict`, `derivative` and `integrate` function. There are also three helper functions that can be used to
-retrieve the estimated parameters of the fitted model:
+the `predict`, `derivative` and `integrate` functions.
-* `getAmplitude`: Returns the amplitude of sine wave.
+There are also three helper functions that can be used to retrieve the estimated parameters of the fitted model:
+
+* `getAmplitude`: Returns the amplitude of the sine wave.
* `getAngularFrequency`: Returns the angular frequency of the sine wave.
* `getPhase`: Returns the phase of the sine wave.
@@ -241,7 +242,7 @@ retrieve the estimated parameters of the fitted model:
oscillations. This is particularly true if the sine wave has noise. After the curve has been fit it can be
extrapolated to any point in time in the past or future.
-In example below the `harmfit` function fits control points, provided as x and y axes and the
+In the example below the `harmfit` function fits control points, provided as x and y axes, and then the
angular frequency, phase and amplitude are retrieved from the fitted model.
[source,text]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4821b305/solr/solr-ref-guide/src/dsp.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/dsp.adoc b/solr/solr-ref-guide/src/dsp.adoc
index 04524dc..8f3b24b 100644
--- a/solr/solr-ref-guide/src/dsp.adoc
+++ b/solr/solr-ref-guide/src/dsp.adoc
@@ -21,19 +21,43 @@ Digital Signal Processing (DSP).
== Dot Product
-The `dotProduct` function is used to calculate the dot product of two arrays.
+The `dotProduct` function is used to calculate the dot product of two numeric arrays.
The dot product is a fundamental calculation for the DSP functions discussed in this section. Before diving into
-the more advanced DSP functions, its useful to get a better understanding of how the dot product calculation works.
+the more advanced DSP functions its useful to develop a deeper intuition of the dot product.
-=== Combining Two Arrays
+The dot product operation is performed in two steps:
-The `dotProduct` function can be used to combine two arrays into a single product. A simple example can help
-illustrate this concept.
+1) Element-by-element multiplication of two vectors which produces a vector of products.
+2) Sum the vector of products to produce a scalar result.
+
+This simple bit of math has a number of important applications.
+
+=== Representing Linear Combinations
+
+The `dotProduct` performs the math of a Linear Combination. A Linear Combination has the following form:
+
+[source,text]
+----
+(a1*v1)+(a2*v2)...
+----
+
+In the above example a1 and a2 are random variables that change. v1 and v2 are *constant values*.
+
+When computing the dot product the elements of two vectors are multiplied together and the results are added.
+If the first vector contains random variables and the second vector contains *constant values*
+then the dot product is performing a linear combination.
+
+This scenario comes up again and again in machine learning. For example both linear and logistic regression
+solve for a vector of constant weights. In order to perform a prediction, a dot product is calculated
+between a random observation vector and the constant weight vector. That dot product is a linear combination because
+one of the vectors holds constant weights.
+
+Lets look at simple example of how a linear combination can be used to find the *mean* of a vector of numbers.
In the example below two arrays are set to variables *`a`* and *`b`* and then operated on by the `dotProduct` function.
The output of the `dotProduct` function is set to variable *`c`*.
-Then the `mean` function is then used to compute the mean of the first array which is set to the variable *`d`*.
+The `mean` function is then used to compute the mean of the first array which is set to the variable *`d`*.
Both the dot product and the mean are included in the output.
@@ -527,14 +551,12 @@ When this expression is sent to the `/stream` handler it responds with:
== Oscillate (Sine Wave)
-The `oscillate` function generates a periodic oscillating signal based
-on a parameters. The `oscillate` function can be used to study,
-combine and model sine waves.
+The `oscillate` function generates a periodic oscillating signal which can be used to model and study sine waves.
-The `oscillate` function takes three parameters: amplitude, angular frequency
-and phase and returns a vector contain the y axis points of sine wave.
+The `oscillate` function takes three parameters: *amplitude*, *angular frequency*
+and *phase* and returns a vector containing the y-axis points of a sine wave.
-The y axis points were generated from a sequence 0-127.
+The y-axis points were generated from an x-axis sequence of 0-127.
Below is an example of the `oscillate` function called with an amplitude of
1, and angular frequency of .28 and phase of 1.57.
@@ -551,7 +573,7 @@ image::images/math-expressions/sinewave.png[]
=== Sine Wave Interpolation, Extrapolation
The `oscillate` function returns a function which can be used by the `predict` function to interpolate or extrapolate a sine wave.
-The example below extrapolates the sine wave to a sequence from 0-256.
+The example below extrapolates the sine wave to an x-axis sequence of 0-256.
[source,text]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4821b305/solr/solr-ref-guide/src/matrix-math.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/matrix-math.adoc b/solr/solr-ref-guide/src/matrix-math.adoc
index 36191c4..b5cce75 100644
--- a/solr/solr-ref-guide/src/matrix-math.adoc
+++ b/solr/solr-ref-guide/src/matrix-math.adoc
@@ -103,7 +103,7 @@ responds with:
}
----
-== Pair sorting vectors
+== Pair Sorting Vectors
The `pairSort` function can be used to sort two vectors based on the values in
the first vector. The sorting operation maintains the pairing between
@@ -115,7 +115,7 @@ the second row in the matrix is the second vector.
The individual vectors can then be accessed using the `rowAt` function.
-The example below performs a pair sort on two vectors and returns the
+The example below performs a pair sort of two vectors and returns the
matrix containing the sorted vectors.
----
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4821b305/solr/solr-ref-guide/src/statistics.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/statistics.adoc b/solr/solr-ref-guide/src/statistics.adoc
index 2d7a548..324a8a6 100644
--- a/solr/solr-ref-guide/src/statistics.adoc
+++ b/solr/solr-ref-guide/src/statistics.adoc
@@ -578,7 +578,7 @@ When this expression is sent to the `/stream` handler it responds with:
== Back Transformations
-Vectors that have been transformed with `log`, `log10`, `sqrt` and `cbrt` functions
+Vectors that have been transformed with the `log`, `log10`, `sqrt` and `cbrt` functions
can be back transformed using the `pow` function.
The example below shows how to back transform data that has been transformed by the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4821b305/solr/solr-ref-guide/src/stream-decorator-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-decorator-reference.adoc b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
index aa8c55d..d9186c9 100644
--- a/solr/solr-ref-guide/src/stream-decorator-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
@@ -735,29 +735,6 @@ leftOuterJoin(
)
----
-[#list_expression]
-== list
-
-The `list` function wraps N Stream Expressions and opens and iterates each stream sequentially.
-This has the effect of concatenating the results of multiple Streaming Expressions.
-
-=== list Parameters
-
-* StreamExpressions ...: N Streaming Expressions
-
-=== list Syntax
-
-[source,text]
-----
-list(tuple(a="hello world"), tuple(a="HELLO WORLD"))
-
-list(search(collection1, q="*:*", fl="id, prod_ss", sort="id asc"),
- search(collection2, q="*:*", fl="id, prod_ss", sort="id asc"))
-
-list(tuple(a=search(collection1, q="*:*", fl="id, prod_ss", sort="id asc")),
- tuple(a=search(collection2, q="*:*", fl="id, prod_ss", sort="id asc")))
-----
-
== hashJoin
The `hashJoin` function wraps two streams, Left and Right, and for every tuple in Left which exists in Right will emit a tuple containing the fields of both tuples. This supports one-to-one, one-to-many, many-to-one, and many-to-many inner join scenarios. The tuples are emitted in the order in which they appear in the Left stream. The order of the streams does not matter. If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.
@@ -863,6 +840,29 @@ intersect(
)
----
+[#list_expression]
+== list
+
+The `list` function wraps N Stream Expressions and opens and iterates each stream sequentially.
+This has the effect of concatenating the results of multiple Streaming Expressions.
+
+=== list Parameters
+
+* StreamExpressions ...: N Streaming Expressions
+
+=== list Syntax
+
+[source,text]
+----
+list(tuple(a="hello world"), tuple(a="HELLO WORLD"))
+
+list(search(collection1, q="*:*", fl="id, prod_ss", sort="id asc"),
+ search(collection2, q="*:*", fl="id, prod_ss", sort="id asc"))
+
+list(tuple(a=search(collection1, q="*:*", fl="id, prod_ss", sort="id asc")),
+ tuple(a=search(collection2, q="*:*", fl="id, prod_ss", sort="id asc")))
+----
+
== merge
The `merge` function merges two or more streaming expressions and maintains the ordering of the underlying streams. Because the order is maintained, the sorts of the underlying streams must line up with the on parameter provided to the merge function.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4821b305/solr/solr-ref-guide/src/vector-math.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/vector-math.adoc b/solr/solr-ref-guide/src/vector-math.adoc
index d610d4e..6171d77 100644
--- a/solr/solr-ref-guide/src/vector-math.adoc
+++ b/solr/solr-ref-guide/src/vector-math.adoc
@@ -145,7 +145,9 @@ When this expression is sent to the `/stream` handler it responds with:
== Vector Sorting
-An array can be sorted in natural ascending order with `asc` function.
+An array can be sorted in natural ascending order with the `asc` function.
+
+The example below shows the `asc` function sorting an array:
[source,text]
----
[18/32] lucene-solr:jira/solr-12730: SOLR-12868: Request forwarding
for v2 API is broken
Posted by ab...@apache.org.
SOLR-12868: Request forwarding for v2 API is broken
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f33be7a1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f33be7a1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f33be7a1
Branch: refs/heads/jira/solr-12730
Commit: f33be7a172d7b4596530d8cb925ba6dd1f1f53f0
Parents: 8d10939
Author: Noble Paul <no...@apache.org>
Authored: Fri Oct 26 12:50:45 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Fri Oct 26 12:50:45 2018 +1100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../java/org/apache/solr/api/V2HttpCall.java | 5 ++-
.../org/apache/solr/servlet/HttpSolrCall.java | 4 +-
.../client/solrj/request/TestV2Request.java | 44 ++++++++++++++++++++
4 files changed, 51 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f33be7a1/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7715efa..03c97fd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -210,6 +210,8 @@ Bug Fixes
* SOLR-12874: Java 9+ GC Logging filesize parameter should use a unit. (Tim Underwood via Uwe Schindler)
+* SOLR-12868: Request forwarding for v2 API is broken (noble)
+
Improvements
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f33be7a1/solr/core/src/java/org/apache/solr/api/V2HttpCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java
index d2b891e..c318cb0 100644
--- a/solr/core/src/java/org/apache/solr/api/V2HttpCall.java
+++ b/solr/core/src/java/org/apache/solr/api/V2HttpCall.java
@@ -121,9 +121,10 @@ public class V2HttpCall extends HttpSolrCall {
core = getCoreByCollection(collection.getName(), isPreferLeader);
if (core == null) {
//this collection exists , but this node does not have a replica for that collection
- extractRemotePath(collection.getName(), origCorename);
+ extractRemotePath(collection.getName(), collection.getName());
if (action == REMOTEQUERY) {
- this.path = path = path.substring(prefix.length() + origCorename.length() + 2);
+ coreUrl = coreUrl.replace("/solr/", "/solr/____v2/c/");
+ this.path = path = path.substring(prefix.length() + collection.getName().length() + 2);
return;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f33be7a1/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index 4a3c34f..64dc3dd 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -133,7 +133,7 @@ import static org.apache.solr.servlet.SolrDispatchFilter.Action.RETURN;
public class HttpSolrCall {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- static final Random random;
+ public static final Random random;
static {
// We try to make things reproducible in the context of our tests by initializing the random instance
// based on the current seed
@@ -877,7 +877,7 @@ public class HttpSolrCall {
}
}
- private String getRemotCoreUrl(String collectionName, String origCorename) {
+ protected String getRemotCoreUrl(String collectionName, String origCorename) {
ClusterState clusterState = cores.getZkController().getClusterState();
final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName);
Slice[] slices = (docCollection != null) ? docCollection.getActiveSlicesArr() : null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f33be7a1/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
index 12b3271..4aa1b64 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
@@ -18,18 +18,25 @@
package org.apache.solr.client.solrj.request;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.List;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.V2Response;
import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.util.NamedList;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TestV2Request extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
@BeforeClass
public static void setupCluster() throws Exception {
@@ -99,4 +106,41 @@ public class TestV2Request extends SolrCloudTestCase {
}
+ public void testV2Forwarding() throws Exception {
+ SolrClient client = cluster.getSolrClient();
+ assertSuccess(client, new V2Request.Builder("/collections")
+ .withMethod(SolrRequest.METHOD.POST)
+ .withPayload("{" +
+ " 'create' : {" +
+ " 'name' : 'v2forward'," +
+ " 'numShards' : 1," +
+ " 'replicationFactor' : 1," +
+ " 'config' : 'config'" +
+ " }" +
+ "}").build());
+
+ ClusterState cs = cluster.getSolrClient().getClusterStateProvider().getClusterState();
+ System.out.println("livenodes: " + cs.getLiveNodes());
+
+ String[] node = new String[1];
+ cs.getCollection("v2forward").forEachReplica((s, replica) -> node[0] = replica.getNodeName());
+
+ //find a node that does not have a replica for this collection
+ final String[] testNode = new String[1];
+ cs.getLiveNodes().forEach(s -> {
+ if (!s.equals(node[0])) testNode[0] = s;
+ });
+
+ String testServer = cluster.getSolrClient().getZkStateReader().getBaseUrlForNodeName(testNode[0]);
+ V2Request v2r = new V2Request.Builder("/c/v2forward/_introspect")
+ .withMethod(SolrRequest.METHOD.GET).build();
+
+ try(HttpSolrClient client1 = new HttpSolrClient.Builder()
+ .withBaseSolrUrl(testServer)
+ .build()) {
+ V2Response rsp = v2r.process(client1);
+ assertEquals("0",rsp._getStr("responseHeader/status", null));
+ }
+ }
+
}
[15/32] lucene-solr:jira/solr-12730: SOLR-12913: Streaming
Expressions / Math Expressions docs for 7.6 release
Posted by ab...@apache.org.
SOLR-12913: Streaming Expressions / Math Expressions docs for 7.6 release
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7952cec9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7952cec9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7952cec9
Branch: refs/heads/jira/solr-12730
Commit: 7952cec99aae943e89c7fd9f7329962a7285c3dd
Parents: 26e1498
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Oct 25 09:16:54 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Oct 25 09:17:25 2018 -0400
----------------------------------------------------------------------
.../src/computational-geometry.adoc | 188 +++++++++++++++++++
solr/solr-ref-guide/src/curve-fitting.adoc | 100 ++++++++++
solr/solr-ref-guide/src/dsp.adoc | 86 +++++++--
.../src/images/math-expressions/sinewave.png | Bin 0 -> 273253 bytes
.../src/images/math-expressions/sinewave256.png | Bin 0 -> 369866 bytes
solr/solr-ref-guide/src/math-expressions.adoc | 15 +-
solr/solr-ref-guide/src/matrix-math.adoc | 53 ++++++
solr/solr-ref-guide/src/statistics.adoc | 87 +++++++++
.../src/stream-decorator-reference.adoc | 48 +++++
solr/solr-ref-guide/src/vector-math.adoc | 36 ++++
10 files changed, 587 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/computational-geometry.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/computational-geometry.adoc b/solr/solr-ref-guide/src/computational-geometry.adoc
new file mode 100644
index 0000000..263f6d0
--- /dev/null
+++ b/solr/solr-ref-guide/src/computational-geometry.adoc
@@ -0,0 +1,188 @@
+= Computational Geometry
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+This section of the math expressions user guide covers computational geometry
+functions.
+
+== Convex Hull
+
+A convex hull is the smallest convex set of points that encloses a data set. Math expressions has support for computing
+the convex hull of a 2D data set. Once a convex hull has been calculated, a set of math expression functions
+can be used to geometrically describe the convex hull.
+
+The `convexHull` function finds the convex hull of an observation matrix of 2D vectors.
+Each row of the matrix contains a 2D observation.
+
+In the example below a convex hull is calculated for a randomly generated set of 100 2D observations.
+
+Then the following functions are called on the convex result:
+
+-`getBaryCenter`: Returns the 2D point that is the bary center of the convex hull.
+
+-`getArea`: Returns the area of the convex hull.
+
+-`getBoundarySize`: Returns the boundary size of the convex hull.
+
+-`getVertices`: Returns 2D points that are the vertices of the convex hull.
+
+
+[source,text]
+----
+let(echo="baryCenter, area, boundarySize, vertices",
+ x=sample(normalDistribution(0, 20), 100),
+ y=sample(normalDistribution(0, 10), 100),
+ observations=transpose(matrix(x,y)),
+ chull=convexHull(observations),
+ baryCenter=getBaryCenter(chull),
+ area=getArea(chull),
+ boundarySize=getBoundarySize(chull),
+ vertices=getVertices(chull))
+----
+
+When this expression is sent to the `/stream` handler it responds with:
+
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "baryCenter": [
+ -3.0969292101230343,
+ 1.2160948182691975
+ ],
+ "area": 3477.480599967595,
+ "boundarySize": 267.52419019533664,
+ "vertices": [
+ [
+ -66.17632818958485,
+ -8.394931552315256
+ ],
+ [
+ -47.556667594765216,
+ -16.940434013651263
+ ],
+ [
+ -33.13582183446102,
+ -17.30914425443977
+ ],
+ [
+ -9.97459859015698,
+ -17.795012801599654
+ ],
+ [
+ 27.7705917246824,
+ -14.487224686587767
+ ],
+ [
+ 54.689432954170236,
+ -1.3333371984299605
+ ],
+ [
+ 35.97568654458672,
+ 23.054169251772556
+ ],
+ [
+ -15.539456215337585,
+ 19.811330468093704
+ ],
+ [
+ -17.05125031092752,
+ 19.53581741341663
+ ],
+ [
+ -35.92010024412891,
+ 15.126430698395572
+ ]
+ ]
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 3
+ }
+ ]
+ }
+}
+----
+
+== Enclosing Disk
+
+The `enclosingDisk` function finds the smallest enclosing circle the encloses a 2D data set.
+Once an enclosing disk has been calculated, a set of math expression functions
+can be used to geometrically describe the enclosing disk.
+
+In the example below an enclosing disk is calculated for a randomly generated set of 1000 2D observations.
+
+Then the following functions are called on the enclosing disk result:
+
+-`getCenter`: Returns the 2D point that is the center of the disk.
+
+-`getRadius`: Returns the radius of the disk.
+
+-`getSupportPoints`: Returns the support points of the disk.
+
+[source,text]
+----
+let(echo="center, radius, support",
+ x=sample(normalDistribution(0, 20), 1000),
+ y=sample(normalDistribution(0, 20), 1000),
+ observations=transpose(matrix(x,y)),
+ disk=enclosingDisk(observations),
+ center=getCenter(disk),
+ radius=getRadius(disk),
+ support=getSupportPoints(disk))
+----
+
+When this expression is sent to the `/stream` handler it responds with:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "center": [
+ -6.668825009733749,
+ -2.9825450908240025
+ ],
+ "radius": 72.66109546907208,
+ "support": [
+ [
+ 20.350992271739464,
+ 64.46791279377014
+ ],
+ [
+ 33.02079953093981,
+ 57.880978456420365
+ ],
+ [
+ -44.7273247899923,
+ -64.87911518353323
+ ]
+ ]
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 8
+ }
+ ]
+ }
+}
+----
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/curve-fitting.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/curve-fitting.adoc b/solr/solr-ref-guide/src/curve-fitting.adoc
index a86f7ea..dc66eeb 100644
--- a/solr/solr-ref-guide/src/curve-fitting.adoc
+++ b/solr/solr-ref-guide/src/curve-fitting.adoc
@@ -221,6 +221,106 @@ responds with:
}
----
+== Harmonic Curve Fitting
+
+The `harmonicFit` function or `harmfit` (for short) fits a smooth line through control points of a sine wave.
+The `harmfit` function is passed x- and y-axes and fits a smooth curve to the data.
+If only a single array is provided it is treated as the y-axis and a sequence is generated
+for the x-axis.
+
+The example below shows `harmfit` fitting a single oscillation of a sine wave. `harmfit`
+returns the smoothed values at each control point. The return value is also a model which can be used by
+the `predict`, `derivative` and `integrate` function. There are also three helper functions that can be used to
+retrieve the estimated parameters of the fitted model:
+
+* `getAmplitude`: Returns the amplitude of sine wave.
+* `getAngularFrequency`: Returns the angular frequency of the sine wave.
+* `getPhase`: Returns the phase of the sine wave.
+
+*Note*: The `harmfit` function works best when run on a single oscillation rather then a long sequence of
+oscillations. This is particularly true if the sine wave has noise. After the curve has been fit it can be
+extrapolated to any point in time in the past or future.
+
+In example below the `harmfit` function fits control points, provided as x and y axes and the
+angular frequency, phase and amplitude are retrieved from the fitted model.
+
+[source,text]
+----
+let(echo="freq, phase, amp",
+ x=array(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19),
+ y=array(-0.7441113653915925,-0.8997532112139415, -0.9853140681578838, -0.9941296760805463,
+ -0.9255133950087844, -0.7848096869247675, -0.5829778403072583, -0.33573836075915076,
+ -0.06234851460699166, 0.215897602691855, 0.47732764497752245, 0.701579055431586,
+ 0.8711850882773975, 0.9729352782968976, 0.9989043923858761, 0.9470697190130273,
+ 0.8214686154479715, 0.631884041542757, 0.39308257356494, 0.12366424851680227),
+ model=harmfit(x, y),
+ freq=getAngularFrequency(model),
+ phase=getPhase(model),
+ amp=getAmplitude(model))
+----
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "freq": 0.28,
+ "phase": 2.4100000000000006,
+ "amp": 0.9999999999999999
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 0
+ }
+ ]
+ }
+}
+----
+
+=== Interpolation and Extrapolation
+
+The `harmfit` function returns a fitted model of the sine wave that can used by the `predict` function to
+interpolate or extrapolate the sine wave.
+
+The example below uses the fitted model to extrapolate the sine wave beyond the control points
+to the x-axis points 20, 21, 22, 23.
+
+
+[source,text]
+----
+let(x=array(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19),
+ y=array(-0.7441113653915925,-0.8997532112139415, -0.9853140681578838, -0.9941296760805463,
+ -0.9255133950087844, -0.7848096869247675, -0.5829778403072583, -0.33573836075915076,
+ -0.06234851460699166, 0.215897602691855, 0.47732764497752245, 0.701579055431586,
+ 0.8711850882773975, 0.9729352782968976, 0.9989043923858761, 0.9470697190130273,
+ 0.8214686154479715, 0.631884041542757, 0.39308257356494, 0.12366424851680227),
+ model=harmfit(x, y),
+ extrapolation=predict(model, array(20, 21, 22, 23)))
+----
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "extrapolation": [
+ -0.1553861764415666,
+ -0.42233370833176975,
+ -0.656386037906838,
+ -0.8393130343914845
+ ]
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 0
+ }
+ ]
+ }
+}
+----
+
== Gaussian Curve Fitting
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/dsp.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/dsp.adoc b/solr/solr-ref-guide/src/dsp.adoc
index 60a6594..04524dc 100644
--- a/solr/solr-ref-guide/src/dsp.adoc
+++ b/solr/solr-ref-guide/src/dsp.adoc
@@ -525,6 +525,46 @@ When this expression is sent to the `/stream` handler it responds with:
}
----
+== Oscillate (Sine Wave)
+
+The `oscillate` function generates a periodic oscillating signal based
+on a parameters. The `oscillate` function can be used to study,
+combine and model sine waves.
+
+The `oscillate` function takes three parameters: amplitude, angular frequency
+and phase and returns a vector contain the y axis points of sine wave.
+
+The y axis points were generated from a sequence 0-127.
+
+Below is an example of the `oscillate` function called with an amplitude of
+1, and angular frequency of .28 and phase of 1.57.
+
+[source,text]
+----
+oscillate(1, 0.28, 1.57)
+----
+
+The result of the `oscillate` function is plotted below:
+
+image::images/math-expressions/sinewave.png[]
+
+=== Sine Wave Interpolation, Extrapolation
+
+The `oscillate` function returns a function which can be used by the `predict` function to interpolate or extrapolate a sine wave.
+The example below extrapolates the sine wave to a sequence from 0-256.
+
+
+[source,text]
+----
+let(a=oscillate(1, 0.28, 1.57),
+ b=predict(a, sequence(256, 0, 1)))
+----
+
+The extrapolated sine wave is plotted below:
+
+image::images/math-expressions/sinewave256.png[]
+
+
== Autocorrelation
Autocorrelation measures the degree to which a signal is correlated with itself. Autocorrelation is used to determine
@@ -532,15 +572,16 @@ if a vector contains a signal or is purely random.
A few examples, with plots, will help to understand the concepts.
-In the first example the `sin` function is wrapped around a `sequence` function to generate a sine wave. The result of this
+The first example simply revisits the example above of an extrapolated sine wave. The result of this
is plotted in the image below. Notice that there is a structure to the plot that is clearly not random.
[source,text]
----
-sin(sequence(256, 0, 6))
+let(a=oscillate(1, 0.28, 1.57),
+ b=predict(a, sequence(256, 0, 1)))
----
-image::images/math-expressions/signal.png[]
+image::images/math-expressions/sinewave256.png[]
In the next example the `sample` function is used to draw 256 samples from a `uniformDistribution` to create a
@@ -562,9 +603,10 @@ becomes more dense it can become harder to see a pattern hidden within noise.
[source,text]
----
-let(a=sin(sequence(256, 0, 6)),
- b=sample(uniformDistribution(-1.5, 1.5), 256),
- c=ebeAdd(a, b))
+let(a=oscillate(1, 0.28, 1.57),
+ b=predict(a, sequence(256, 0, 1)),
+ c=sample(uniformDistribution(-1.5, 1.5), 256),
+ d=ebeAdd(b,c))
----
image::images/math-expressions/hidden-signal.png[]
@@ -585,8 +627,9 @@ This is the autocorrelation plot of a pure signal.
[source,text]
----
-let(a=sin(sequence(256, 0, 6)),
- b=conv(a, rev(a)),
+let(a=oscillate(1, 0.28, 1.57),
+ b=predict(a, sequence(256, 0, 1)),
+ c=conv(b, rev(b)))
----
image::images/math-expressions/signal-autocorrelation.png[]
@@ -615,10 +658,11 @@ strongly that there is an underlying signal hidden within the noise.
[source,text]
----
-let(a=sin(sequence(256, 0, 6)),
- b=sample(uniformDistribution(-1.5, 1.5), 256),
- c=ebeAdd(a, b),
- d=conv(c, rev(c))
+let(a=oscillate(1, 0.28, 1.57),
+ b=predict(a, sequence(256, 0, 1)),
+ c=sample(uniformDistribution(-1.5, 1.5), 256),
+ d=ebeAdd(b, c),
+ e=conv(d, rev(d)))
----
image::images/math-expressions/hidden-signal-autocorrelation.png[]
@@ -675,9 +719,10 @@ associated with them. This `fft` shows a clear signal with very low levels of no
[source,text]
----
-let(a=sin(sequence(256, 0, 6)),
- b=fft(a),
- c=rowAt(b, 0))
+let(a=oscillate(1, 0.28, 1.57),
+ b=predict(a, sequence(256, 0, 1)),
+ c=fft(b),
+ d=rowAt(c, 0))
----
@@ -709,11 +754,12 @@ shows that there is considerable noise along with the signal.
[source,text]
----
-let(a=sin(sequence(256, 0, 6)),
- b=sample(uniformDistribution(-1.5, 1.5), 256),
- c=ebeAdd(a, b),
- d=fft(c),
- e=rowAt(d, 0))
+let(a=oscillate(1, 0.28, 1.57),
+ b=predict(a, sequence(256, 0, 1)),
+ c=sample(uniformDistribution(-1.5, 1.5), 256),
+ d=ebeAdd(b, c),
+ e=fft(d),
+ f=rowAt(e, 0))
----
image::images/math-expressions/hidden-signal-fft.png[]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/images/math-expressions/sinewave.png
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/images/math-expressions/sinewave.png b/solr/solr-ref-guide/src/images/math-expressions/sinewave.png
new file mode 100644
index 0000000..19d9b93
Binary files /dev/null and b/solr/solr-ref-guide/src/images/math-expressions/sinewave.png differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/images/math-expressions/sinewave256.png
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/images/math-expressions/sinewave256.png b/solr/solr-ref-guide/src/images/math-expressions/sinewave256.png
new file mode 100644
index 0000000..e821057
Binary files /dev/null and b/solr/solr-ref-guide/src/images/math-expressions/sinewave256.png differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/math-expressions.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/math-expressions.adoc b/solr/solr-ref-guide/src/math-expressions.adoc
index 3974989..595a7b1 100644
--- a/solr/solr-ref-guide/src/math-expressions.adoc
+++ b/solr/solr-ref-guide/src/math-expressions.adoc
@@ -1,5 +1,5 @@
= Math Expressions
-:page-children: scalar-math, vector-math, variables, matrix-math, vectorization, term-vectors, statistics, probability-distributions, simulations, time-series, regression, numerical-analysis, curve-fitting, dsp, machine-learning
+:page-children: scalar-math, vector-math, variables, matrix-math, vectorization, term-vectors, statistics, probability-distributions, simulations, time-series, regression, numerical-analysis, curve-fitting, dsp, machine-learning, computational-geometry
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
@@ -44,18 +44,21 @@ record in your Solr Cloud cluster computable.
*<<statistics.adoc#statistics,Statistics>>*: Statistical functions in math expressions.
-*<<probability-distributions.adoc#probability-distributions,Probability>>*: Mathematical models for probability.
+*<<probability-distributions.adoc#probability-distributions,Probability>>*: Mathematical models of probability.
-*<<simulations.adoc#simulations,Monte Carlo Simulations>>*: Performing correlated and uncorrelated Monte Carlo simulations.
+*<<simulations.adoc#simulations,Monte Carlo Simulations>>*: Performing uncorrelated and correlated Monte Carlo simulations.
*<<regression.adoc#regression,Linear Regression>>*: Simple and multivariate linear regression.
*<<numerical-analysis.adoc#numerical-analysis,Interpolation, Derivatives and Integrals>>*: Numerical analysis math expressions.
-*<<curve-fitting.adoc#curve-fitting,Curve Fitting>>*: Polynomial and Gaussian curve fitting.
-
*<<dsp.adoc#dsp,Digital Signal Processing>>*: Functions commonly used with digital signal processing.
-*<<time-series.adoc#time-series,Time Series>>*: Aggregation, smoothing, differencing, modeling and anomaly detection for time series.
+*<<curve-fitting.adoc#curve-fitting,Curve Fitting>>*: Polynomial, Harmonic and Gaussian curve fitting.
+
+*<<time-series.adoc#time-series,Time Series>>*: Aggregation, smoothing and differencing of time series.
*<<machine-learning.adoc#machine-learning,Machine Learning>>*: Functions used in machine learning.
+
+*<<computational-geometry.adoc#computational-geometry,Computational Geometry>>*: Convex Hulls and Enclosing Disks.
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/matrix-math.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/matrix-math.adoc b/solr/solr-ref-guide/src/matrix-math.adoc
index b73f01a..36191c4 100644
--- a/solr/solr-ref-guide/src/matrix-math.adoc
+++ b/solr/solr-ref-guide/src/matrix-math.adoc
@@ -103,6 +103,59 @@ responds with:
}
----
+== Pair sorting vectors
+
+The `pairSort` function can be used to sort two vectors based on the values in
+the first vector. The sorting operation maintains the pairing between
+the two vectors during the sort.
+
+The `pairSort` function returns a matrix containing the
+pair sorted vectors. The first row in the matrix is the first vector,
+the second row in the matrix is the second vector.
+
+The individual vectors can then be accessed using the `rowAt` function.
+
+The example below performs a pair sort on two vectors and returns the
+matrix containing the sorted vectors.
+
+----
+let(a=array(10, 2, 1),
+ b=array(100, 200, 300),
+ c=pairSort(a, b))
+----
+
+When this expression is sent to the `/stream` handler it
+responds with:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "c": [
+ [
+ 1,
+ 2,
+ 10
+ ],
+ [
+ 300,
+ 200,
+ 100
+ ]
+ ]
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 1
+ }
+ ]
+ }
+}
+----
+
+
== Row and Column Labels
A matrix can have column and rows and labels. The functions
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/statistics.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/statistics.adoc b/solr/solr-ref-guide/src/statistics.adoc
index af908d3..2d7a548 100644
--- a/solr/solr-ref-guide/src/statistics.adoc
+++ b/solr/solr-ref-guide/src/statistics.adoc
@@ -537,6 +537,8 @@ array.
* `log`: Returns a numeric array with the natural log of each element of the original array.
+* `log10`: Returns a numeric array with the base 10 log of each element of the original array.
+
* `sqrt`: Returns a numeric array with the square root of each element of the original array.
* `cbrt`: Returns a numeric array with the cube root of each element of the original array.
@@ -574,6 +576,91 @@ When this expression is sent to the `/stream` handler it responds with:
}
----
+== Back Transformations
+
+Vectors that have been transformed with `log`, `log10`, `sqrt` and `cbrt` functions
+can be back transformed using the `pow` function.
+
+The example below shows how to back transform data that has been transformed by the
+`sqrt` function.
+
+
+[source,text]
+----
+let(echo="b,c",
+ a=array(100, 200, 300),
+ b=sqrt(a),
+ c=pow(b, 2))
+----
+
+When this expression is sent to the `/stream` handler it responds with:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "b": [
+ 10,
+ 14.142135623730951,
+ 17.320508075688775
+ ],
+ "c": [
+ 100,
+ 200.00000000000003,
+ 300.00000000000006
+ ]
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 0
+ }
+ ]
+ }
+}
+----
+
+The example below shows how to back transform data that has been transformed by the
+`log10` function.
+
+
+[source,text]
+----
+let(echo="b,c",
+ a=array(100, 200, 300),
+ b=log10(a),
+ c=pow(10, b))
+----
+
+When this expression is sent to the `/stream` handler it responds with:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "b": [
+ 2,
+ 2.3010299956639813,
+ 2.4771212547196626
+ ],
+ "c": [
+ 100,
+ 200.00000000000003,
+ 300.0000000000001
+ ]
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 0
+ }
+ ]
+ }
+}
+----
+
== Z-scores
The `zscores` function converts a numeric array to an array of z-scores. The z-score
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/stream-decorator-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-decorator-reference.adoc b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
index 5549fc2..aa8c55d 100644
--- a/solr/solr-ref-guide/src/stream-decorator-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
@@ -735,6 +735,29 @@ leftOuterJoin(
)
----
+[#list_expression]
+== list
+
+The `list` function wraps N Stream Expressions and opens and iterates each stream sequentially.
+This has the effect of concatenating the results of multiple Streaming Expressions.
+
+=== list Parameters
+
+* StreamExpressions ...: N Streaming Expressions
+
+=== list Syntax
+
+[source,text]
+----
+list(tuple(a="hello world"), tuple(a="HELLO WORLD"))
+
+list(search(collection1, q="*:*", fl="id, prod_ss", sort="id asc"),
+ search(collection2, q="*:*", fl="id, prod_ss", sort="id asc"))
+
+list(tuple(a=search(collection1, q="*:*", fl="id, prod_ss", sort="id asc")),
+ tuple(a=search(collection2, q="*:*", fl="id, prod_ss", sort="id asc")))
+----
+
== hashJoin
The `hashJoin` function wraps two streams, Left and Right, and for every tuple in Left which exists in Right will emit a tuple containing the fields of both tuples. This supports one-to-one, one-to-many, many-to-one, and many-to-many inner join scenarios. The tuples are emitted in the order in which they appear in the Left stream. The order of the streams does not matter. If both tuples contain a field of the same name then the value from the Right stream will be used in the emitted tuple.
@@ -1028,6 +1051,31 @@ The following is a `solrconfig.xml` snippet for 2 workers and "year_i" as the `p
----
====
+== plist
+
+The `plist` function wraps N Stream Expressions and opens the streams in parallel
+and iterates each stream sequentially. The difference between the `list` and `plist` is that
+the streams are opened in parallel. Since many streams such as
+`facet`, `stats` and `significantTerms` push down heavy operations to Solr when they are opened,
+the plist function can dramatically improve performance by doing these operations in parallel.
+
+=== plist Parameters
+
+* StreamExpressions ...: N Streaming Expressions
+
+=== plist Syntax
+
+[source,text]
+----
+plist(tuple(a="hello world"), tuple(a="HELLO WORLD"))
+
+plist(search(collection1, q="*:*", fl="id, prod_ss", sort="id asc"),
+ search(collection2, q="*:*", fl="id, prod_ss", sort="id asc"))
+
+plist(tuple(a=search(collection1, q="*:*", fl="id, prod_ss", sort="id asc")),
+ tuple(a=search(collection2, q="*:*", fl="id, prod_ss", sort="id asc")))
+----
+
== priority
The `priority` function is a simple priority scheduler for the <<executor>> function. The `executor` function doesn't directly have a concept of task prioritization; instead it simply executes tasks in the order that they are read from it's underlying stream. The `priority` function provides the ability to schedule a higher priority task ahead of lower priority tasks that were submitted earlier.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7952cec9/solr/solr-ref-guide/src/vector-math.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/vector-math.adoc b/solr/solr-ref-guide/src/vector-math.adoc
index e06008d..d610d4e 100644
--- a/solr/solr-ref-guide/src/vector-math.adoc
+++ b/solr/solr-ref-guide/src/vector-math.adoc
@@ -143,6 +143,42 @@ When this expression is sent to the `/stream` handler it responds with:
}
----
+== Vector Sorting
+
+An array can be sorted in natural ascending order with `asc` function.
+
+[source,text]
+----
+asc(array(10,1,2,3,4,5,6))
+----
+
+When this expression is sent to the `/stream` handler it responds with:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "return-value": [
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 6,
+ 10
+ ]
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 1
+ }
+ ]
+ }
+}
+----
+
== Vector Summarizations and Norms
There are a set of functions that perform summarizations and return norms of arrays. These functions
[12/32] lucene-solr:jira/solr-12730: SOLR-12793: Move
TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test
package
Posted by ab...@apache.org.
SOLR-12793: Move TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test package
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/71988c75
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/71988c75
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/71988c75
Branch: refs/heads/jira/solr-12730
Commit: 71988c756b76a96cb96fc8f86183219a4a008389
Parents: 16ee847
Author: Varun Thacker <va...@apache.org>
Authored: Wed Oct 24 13:57:28 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Oct 24 13:57:35 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../cloud/TestCloudJSONFacetJoinDomain.java | 853 ------------------
.../solr/cloud/TestCloudJSONFacetSKG.java | 677 ---------------
.../org/apache/solr/search/facet/DebugAgg.java | 2 +-
.../facet/TestCloudJSONFacetJoinDomain.java | 855 +++++++++++++++++++
.../search/facet/TestCloudJSONFacetSKG.java | 678 +++++++++++++++
6 files changed, 1536 insertions(+), 1531 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 32c8ae2..bb8f86d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -178,6 +178,8 @@ Other Changes
* SOLR-12423: Upgrade to Tika 1.19.1 when available (Tim Allison via Erick Erickson)
+* SOLR-12793: Move TestCloudJSONFacetJoinDomain amd TestCloudJSONFacetSKG to the facet test package (Varun Thacker)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
deleted file mode 100644
index 150a08d..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
+++ /dev/null
@@ -1,853 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.lucene.util.TestUtil;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.search.facet.FacetField;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * Tests randomized JSON Facets, sometimes using query 'join' domain transfers and/or domain 'filter' options
- * </p>
- * <p>
- * The results of each facet constraint count will be compared with a verification query using an equivalent filter
- * </p>
- *
- * @see TestCloudPivotFacet
- */
-public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
- private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
-
- private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
- private static final int MAX_FIELD_NUM = 15;
- private static final int UNIQUE_FIELD_VALS = 20;
-
- // NOTE: set to 'true' to see if refinement testing is adequate (should get fails occasionally)
- private static final boolean FORCE_DISABLE_REFINEMENT = false;
-
- /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
- /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
-
- /** A basic client for operations at the cloud level, default collection will be set */
- private static CloudSolrClient CLOUD_CLIENT;
- /** One client per node */
- private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
-
- @BeforeClass
- private static void createMiniSolrCloudCluster() throws Exception {
- // sanity check constants
- assertTrue("bad test constants: some suffixes will never be tested",
- (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
-
- // we need DVs on point fields to compute stats & facets
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
-
- // multi replicas should not matter...
- final int repFactor = usually() ? 1 : 2;
- // ... but we definitely want to test multiple shards
- final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
- final int numNodes = (numShards * repFactor);
-
- final String configName = DEBUG_LABEL + "_config-set";
- final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
-
- configureCluster(numNodes).addConfig(configName, configDir).configure();
-
- Map<String, String> collectionProperties = new LinkedHashMap<>();
- collectionProperties.put("config", "solrconfig-tlog.xml");
- collectionProperties.put("schema", "schema_latest.xml");
- CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
- .setProperties(collectionProperties)
- .process(cluster.getSolrClient());
-
- CLOUD_CLIENT = cluster.getSolrClient();
- CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
-
- waitForRecoveriesToFinish(CLOUD_CLIENT);
-
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
- }
-
- final int numDocs = atLeast(100);
- for (int id = 0; id < numDocs; id++) {
- SolrInputDocument doc = sdoc("id", ""+id);
- for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
- // NOTE: some docs may have no value in a field
- final int numValsThisDoc = TestUtil.nextInt(random(), 0, (usually() ? 3 : 6));
- for (int v = 0; v < numValsThisDoc; v++) {
- final String fieldValue = randFieldValue(fieldNum);
-
- // for each fieldNum, there are actaully two fields: one string, and one integer
- doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
- doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
- }
- }
- CLOUD_CLIENT.add(doc);
- if (random().nextInt(100) < 1) {
- CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
- }
- if (random().nextInt(100) < 5) {
- CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
- }
- }
- CLOUD_CLIENT.commit();
- }
-
- /**
- * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
- * uses that number and one of hte specified suffixes in it's name.
- *
- * @see #STR_FIELD_SUFFIXES
- * @see #INT_FIELD_SUFFIXES
- * @see #MAX_FIELD_NUM
- * @see #randFieldValue
- */
- private static String field(final String[] suffixes, final int fieldNum) {
- assert fieldNum < MAX_FIELD_NUM;
-
- final String suffix = suffixes[fieldNum % suffixes.length];
- return "field_" + fieldNum + suffix;
- }
- private static String strfield(final int fieldNum) {
- return field(STR_FIELD_SUFFIXES, fieldNum);
- }
- private static String intfield(final int fieldNum) {
- return field(INT_FIELD_SUFFIXES, fieldNum);
- }
-
- /**
- * Given a (random) field number, returns a random (integer based) value for that field.
- * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
- * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
- * will match fewer documents based on how far apart the field numbers are.
- *
- * @see #UNIQUE_FIELD_VALS
- * @see #field
- */
- private static String randFieldValue(final int fieldNum) {
- return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
- }
-
-
- @AfterClass
- private static void afterClass() throws Exception {
- CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
- for (HttpSolrClient client : CLIENTS) {
- client.close();
- }
- CLIENTS = null;
- }
-
- /** Sanity check that malformed requests produce errors */
- public void testMalformedGivesError() throws Exception {
-
- ignoreException(".*'join' domain change.*");
-
- for (String join : Arrays.asList("bogus",
- "{ }",
- "{ from:null, to:foo_s }",
- "{ from:foo_s }",
- "{ from:foo_s, to:foo_s, bogus:'what what?' }",
- "{ to:foo_s, bogus:'what what?' }")) {
- SolrException e = expectThrows(SolrException.class, () -> {
- final SolrParams req = params("q", "*:*", "json.facet",
- "{ x : { type:terms, field:x_s, domain: { join:"+join+" } } }");
- final NamedList trash = getRandClient(random()).request(new QueryRequest(req));
- });
- assertEquals(join + " -> " + e, SolrException.ErrorCode.BAD_REQUEST.code, e.code());
- assertTrue(join + " -> " + e, e.getMessage().contains("'join' domain change"));
- }
- }
-
- public void testSanityCheckDomainMethods() throws Exception {
- {
- final JoinDomain empty = new JoinDomain(null, null, null);
- assertEquals(null, empty.toJSONFacetParamValue());
- final SolrParams out = empty.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals(null, out.get("safe_key"));
- assertEquals("qqq", out.get("q"));
- }
- {
- final JoinDomain join = new JoinDomain("xxx", "yyy", null);
- assertEquals("domain:{join:{from:xxx,to:yyy}}", join.toJSONFacetParamValue().toString());
- final SolrParams out = join.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals("qqq", out.get("safe_key"));
- assertEquals("{!join from=xxx to=yyy v=$safe_key}", out.get("q"));
-
- }
- {
- final JoinDomain filter = new JoinDomain(null, null, "zzz");
- assertEquals("domain:{filter:'zzz'}", filter.toJSONFacetParamValue().toString());
- final SolrParams out = filter.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals(null, out.get("safe_key"));
- assertEquals("zzz AND qqq", out.get("q"));
- }
- {
- final JoinDomain both = new JoinDomain("xxx", "yyy", "zzz");
- assertEquals("domain:{join:{from:xxx,to:yyy},filter:'zzz'}", both.toJSONFacetParamValue().toString());
- final SolrParams out = both.applyDomainToQuery("safe_key", params("q","qqq"));
- assertNotNull(out);
- assertEquals("qqq", out.get("safe_key"));
- assertEquals("zzz AND {!join from=xxx to=yyy v=$safe_key}", out.get("q"));
- }
- }
-
- /**
- * Test some small, hand crafted, but non-trivial queries that are
- * easier to trace/debug then a pure random monstrosity.
- * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
- */
- public void testBespoke() throws Exception {
-
- { // sanity check our test methods can handle a query matching no docs
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
- top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
- facets.put("empty_top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, strfield(7) + ":bogus");
- assertEquals("Empty search result shouldn't have found a single bucket",
- UNIQUE_FIELD_VALS, maxBuckets.get());
- }
-
- { // sanity check our test methods can handle a query where a facet filter prevents any doc from having terms
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, "-*:*"));
- top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
- facets.put("filtered_top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
- assertEquals("Empty join filter shouldn't have found a single bucket",
- UNIQUE_FIELD_VALS, maxBuckets.get());
- }
-
- { // sanity check our test methods can handle a query where a facet filter prevents any doc from having sub-terms
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(8), strfield(8), null));
- top.subFacets.put("sub", new TermFacet(strfield(11), new JoinDomain(null, null, "-*:*")));
- facets.put("filtered_top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
-
- { // strings
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"));
- top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(strfield(8), strfield(8), null)));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- }
-
- { // ints
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(intfield(9), new JoinDomain(intfield(5), intfield(9), null));
- facets.put("top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+intfield(7)+":16 OR "+intfield(3)+":13)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- }
-
- { // some domains with filter only, no actual join
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), new JoinDomain(null, null, strfield(9)+":[* TO *]"));
- top.subFacets.put("facet_5", new TermFacet(strfield(11), new JoinDomain(null, null, strfield(3)+":[* TO 5]")));
- facets.put("top", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":16 OR "+strfield(9)+":16 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS * UNIQUE_FIELD_VALS);
-
- }
-
- { // low limits, explicit refinement
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9),
- new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
- 5, 0, true);
- top.subFacets.put("facet_5", new TermFacet(strfield(11),
- new JoinDomain(strfield(8), strfield(8), null),
- 10, 0, true));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
- }
-
- { // low limit, high overrequest
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9),
- new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
- 5, UNIQUE_FIELD_VALS + 10, false);
- top.subFacets.put("facet_5", new TermFacet(strfield(11),
- new JoinDomain(strfield(8), strfield(8), null),
- 10, UNIQUE_FIELD_VALS + 10, false));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
- }
-
- { // low limit, low overrequest, explicit refinement
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9),
- new JoinDomain(strfield(5), strfield(9), strfield(9)+":[* TO *]"),
- 5, 7, true);
- top.subFacets.put("facet_5", new TermFacet(strfield(11),
- new JoinDomain(strfield(8), strfield(8), null),
- 10, 7, true));
- facets.put("facet_4", top);
- final AtomicInteger maxBuckets = new AtomicInteger(5 * 10);
- assertFacetCountsAreCorrect(maxBuckets, facets, "("+strfield(7)+":6 OR "+strfield(9)+":6 OR "+strfield(6)+":19 OR "+strfield(0)+":11)");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < 5 * 10);
- }
-
- }
-
- public void testTheTestRandomRefineParam() {
- // sanity check that randomRefineParam never violates isRefinementNeeded
- // (should be imposisble ... unless someone changes/breaks the randomization logic in the future)
- final int numIters = atLeast(100);
- for (int iter = 0; iter < numIters; iter++) {
- final Integer limit = TermFacet.randomLimitParam(random());
- final Integer overrequest = TermFacet.randomOverrequestParam(random());
- final Boolean refine = TermFacet.randomRefineParam(random(), limit, overrequest);
- if (TermFacet.isRefinementNeeded(limit, overrequest)) {
- assertEquals("limit: " + limit + ", overrequest: " + overrequest + ", refine: " + refine,
- Boolean.TRUE, refine);
- }
- }
- }
-
- public void testTheTestTermFacetShouldFreakOutOnBadRefineOptions() {
- expectThrows(AssertionError.class, () -> {
- final TermFacet bogus = new TermFacet("foo", null, 5, 0, false);
- });
- }
-
- public void testRandom() throws Exception {
-
- // we put a safety valve in place on the maximum number of buckets that we are willing to verify
- // across *all* the queries that we do.
- // that way if the randomized queries we build all have relatively small facets, so be it, but if
- // we get a really big one early on, we can test as much as possible, skip other iterations.
- //
- // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
- final int maxBucketsAllowed = atLeast(2000);
- final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
-
- final int numIters = atLeast(20);
- for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
- assertFacetCountsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(), buildRandomQuery());
- }
- assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
- }
-
- /**
- * Generates a random query string across the randomized fields/values in the index
- *
- * @see #randFieldValue
- * @see #field
- */
- private static String buildRandomQuery() {
- if (0 == TestUtil.nextInt(random(), 0,10)) {
- return "*:*";
- }
- final int numClauses = TestUtil.nextInt(random(), 3, 10);
- List<String> clauses = new ArrayList<String>(numClauses);
- for (int c = 0; c < numClauses; c++) {
- final int fieldNum = random().nextInt(MAX_FIELD_NUM);
- // keep queries simple, just use str fields - not point of test
- clauses.add(strfield(fieldNum) + ":" + randFieldValue(fieldNum));
- }
- return "(" + StringUtils.join(clauses, " OR ") + ")";
- }
-
- /**
- * Given a set of (potentially nested) term facets, and a base query string, asserts that
- * the actual counts returned when executing that query with those facets match the expected results
- * of filtering on the equivalent facet terms+domain
- */
- private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
- Map<String,TermFacet> expected,
- final String query) throws SolrServerException, IOException {
-
- final SolrParams baseParams = params("q", query, "rows","0");
- final SolrParams facetParams = params("json.facet", ""+TermFacet.toJSONFacetParamValue(expected));
- final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
-
- log.info("Doing full run: {}", initParams);
-
- QueryResponse rsp = null;
- // JSON Facets not (currently) available from QueryResponse...
- NamedList topNamedList = null;
- try {
- rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
- assertNotNull(initParams + " is null rsp?", rsp);
- topNamedList = rsp.getResponse();
- assertNotNull(initParams + " is null topNamedList?", topNamedList);
- } catch (Exception e) {
- throw new RuntimeException("init query failed: " + initParams + ": " +
- e.getMessage(), e);
- }
- try {
- final NamedList facetResponse = (NamedList) topNamedList.get("facets");
- assertNotNull("null facet results?", facetResponse);
- assertEquals("numFound mismatch with top count?",
- rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
- if (0 == rsp.getResults().getNumFound()) {
- // when the query matches nothing, we should expect no top level facets
- expected = Collections.emptyMap();
- }
- assertFacetCountsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
- } catch (AssertionError e) {
- throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
- } finally {
- log.info("Ending full run");
- }
- }
-
- /**
- * Recursive Helper method that walks the actual facet response, comparing the counts to the expected output
- * based on the equivalent filters generated from the original TermFacet.
- */
- private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
- final Map<String,TermFacet> expected,
- final SolrParams baseParams,
- final NamedList actualFacetResponse) throws SolrServerException, IOException {
-
- for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
- final String facetKey = entry.getKey();
- final TermFacet facet = entry.getValue();
- final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
- assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
- final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
- assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
-
- if (buckets.isEmpty()) {
- // should only happen if the baseParams query does not match any docs with our field X
- final long docsWithField = getRandClient(random()).query
- (facet.applyValueConstraintAndDomain(baseParams, facetKey, "[* TO *]")).getResults().getNumFound();
- assertEquals(facetKey + " has no buckets, but docs in query exist with field: " + facet.field,
- 0, docsWithField);
- }
-
- for (NamedList bucket : buckets) {
- final long count = ((Number) bucket.get("count")).longValue();
- final String fieldVal = bucket.get("val").toString(); // int or stringified int
-
- // change our query to filter on the fieldVal, and wrap in the facet domain (if any)
- final SolrParams verifyParams = facet.applyValueConstraintAndDomain(baseParams, facetKey, fieldVal);
-
- // check the count for this bucket
- assertEquals(facetKey + ": " + verifyParams,
- count, getRandClient(random()).query(verifyParams).getResults().getNumFound());
-
- if (maxBucketsToCheck.decrementAndGet() <= 0) {
- return;
- }
-
- // recursively check subFacets
- if (! facet.subFacets.isEmpty()) {
- assertFacetCountsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
- }
- }
- }
- assertTrue("facets have unexpected keys left over: " + actualFacetResponse,
- // should alwasy be a count, maybe a 'val' if we're a subfacet
- (actualFacetResponse.size() == expected.size() + 1) ||
- (actualFacetResponse.size() == expected.size() + 2));
- }
-
-
- /**
- * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
- *
- * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
- */
- private static final class TermFacet {
- public final String field;
- public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
- public final JoinDomain domain; // may be null
- public final Integer limit; // may be null
- public final Integer overrequest; // may be null
- public final Boolean refine; // may be null
-
- /** Simplified constructor asks for limit = # unique vals */
- public TermFacet(String field, JoinDomain domain) {
- this(field, domain, UNIQUE_FIELD_VALS, 0, false);
- }
- public TermFacet(String field, JoinDomain domain, Integer limit, Integer overrequest, Boolean refine) {
- assert null != field;
- this.field = field;
- this.domain = domain;
- this.limit = limit;
- this.overrequest = overrequest;
- this.refine = refine;
- if (isRefinementNeeded(limit, overrequest)) {
- assertEquals("Invalid refine param based on limit & overrequest: " + this.toString(),
- Boolean.TRUE, refine);
- }
- }
-
- /**
- * Returns new SolrParams that:
- * <ul>
- * <li>copy the original SolrParams</li>
- * <li>modify/wrap the original "q" param to capture the domain change for this facet (if any)</li>
- * <li>add a filter query against this field with the specified value</li>
- * </ul>
- *
- * @see JoinDomain#applyDomainToQuery
- */
- public SolrParams applyValueConstraintAndDomain(SolrParams orig, String facetKey, String facetVal) {
- // first wrap our original query in the domain if there is one...
- if (null != domain) {
- orig = domain.applyDomainToQuery(facetKey + "_q", orig);
- }
- // then filter by the facet value we need to test...
- final ModifiableSolrParams out = new ModifiableSolrParams(orig);
- out.set("q", field + ":" + facetVal + " AND " + orig.get("q"));
-
- return out;
- }
-
- /**
- * recursively generates the <code>json.facet</code> param value to use for testing this facet
- */
- private CharSequence toJSONFacetParamValue() {
- final String limitStr = (null == limit) ? "" : (", limit:" + limit);
- final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
- final String refineStr = (null == refine) ? "" : ", refine:" + refine;
- final StringBuilder sb = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + refineStr);
- if (! subFacets.isEmpty()) {
- sb.append(", facet:");
- sb.append(toJSONFacetParamValue(subFacets));
- }
- if (null != domain) {
- CharSequence ds = domain.toJSONFacetParamValue();
- if (null != ds) {
- sb.append(", ").append(ds);
- }
- }
- sb.append("}");
- return sb;
- }
-
- /**
- * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
- * use for testing them against in a solr request.
- */
- public static CharSequence toJSONFacetParamValue(Map<String,TermFacet> facets) {
- assert null != facets;
- assert 0 < facets.size();
- StringBuilder sb = new StringBuilder("{");
- for (String key : facets.keySet()) {
- sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
- sb.append(" ,");
- }
- sb.setLength(sb.length() - 1);
- sb.append("}");
- return sb;
- }
-
- /**
- * Factory method for generating some random (nested) facets.
- *
- * For simplicity, each facet will have a unique key name, regardless of it's depth under other facets
- *
- * @see JoinDomain
- */
- public static Map<String,TermFacet> buildRandomFacets() {
- // for simplicity, use a unique facet key regardless of depth - simplifies verification
- AtomicInteger keyCounter = new AtomicInteger(0);
- final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
- return buildRandomFacets(keyCounter, maxDepth);
- }
-
- /**
- * picks a random value for the "limit" param, biased in favor of interesting test cases
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- */
- public static Integer randomLimitParam(Random r) {
- final int limit = 1 + r.nextInt(UNIQUE_FIELD_VALS * 2);
- if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
- return -1; // unlimited
- } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
- return null; // sometimes, don't specify limit if it's the default
- }
- return limit;
- }
-
- /**
- * picks a random value for the "overrequest" param, biased in favor of interesting test cases
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- */
- public static Integer randomOverrequestParam(Random r) {
- switch(r.nextInt(10)) {
- case 0:
- case 1:
- case 2:
- case 3:
- return 0; // 40% of the time, no overrequest to better stress refinement
- case 4:
- case 5:
- return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
- case 6:
- return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
- default: break;
- }
- // else.... either leave param unspecified (or redundently specify the -1 default)
- return r.nextBoolean() ? null : -1;
- }
-
- /**
- * picks a random value for the "refine" param, that is garunteed to be suitable for
- * the specified limit & overrequest params.
- *
- * @return a value to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #randomLimitParam
- * @see #randomOverrequestParam
- * @see #UNIQUE_FIELD_VALS
- */
- public static Boolean randomRefineParam(Random r, Integer limitParam, Integer overrequestParam) {
- if (isRefinementNeeded(limitParam, overrequestParam)) {
- return true;
- }
-
- // refinement is not required
- if (0 == r.nextInt(10)) { // once in a while, turn on refinement even if it isn't needed.
- return true;
- }
- // explicitly or implicitly indicate refinement is not needed
- return r.nextBoolean() ? false : null;
- }
-
- /**
- * Deterministicly identifies if the specified limit & overrequest params <b>require</b>
- * a "refine:true" param be used in the the request, in order for the counts to be 100% accurate.
- *
- * @see #UNIQUE_FIELD_VALS
- */
- public static boolean isRefinementNeeded(Integer limitParam, Integer overrequestParam) {
-
- if (FORCE_DISABLE_REFINEMENT) {
- return false;
- }
-
- // use the "effective" values if the params are null
- final int limit = null == limitParam ? DEFAULT_LIMIT : limitParam;
- final int overrequest = null == overrequestParam ? 0 : overrequestParam;
-
- return
- // don't presume how much overrequest will be done by default, just check the limit
- (overrequest < 0 && limit < UNIQUE_FIELD_VALS)
- // if the user specified overrequest is not "enough" to get all unique values
- || (overrequest >= 0 && (long)limit + overrequest < UNIQUE_FIELD_VALS);
- }
-
- /**
- * recursive helper method for building random facets
- *
- * @param keyCounter used to ensure every generated facet has a unique key name
- * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
- */
- private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
- final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
- Map<String,TermFacet> results = new LinkedHashMap<>();
- for (int i = 0; i < numFacets; i++) {
- final JoinDomain domain = JoinDomain.buildRandomDomain();
- assert null != domain;
- final Integer limit = randomLimitParam(random());
- final Integer overrequest = randomOverrequestParam(random());
- final TermFacet facet = new TermFacet(field(random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES,
- random().nextInt(MAX_FIELD_NUM)),
- domain, limit, overrequest,
- randomRefineParam(random(), limit, overrequest));
- results.put("facet_" + keyCounter.incrementAndGet(), facet);
- if (0 < maxDepth) {
- // if we're going wide, don't go deep
- final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
- facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
- }
- }
- return results;
- }
- }
-
-
- /**
- * Models a Domain Change which includes either a 'join' or a 'filter' or both
- */
- private static final class JoinDomain {
- public final String from;
- public final String to;
- public final String filter; // not bothering with more then 1 filter, not the point of the test
-
- /**
- * @param from left side of join field name, null if domain involves no joining
- * @param to right side of join field name, null if domain involves no joining
- * @param filter filter to apply to domain, null if domain involves no filtering
- */
- public JoinDomain(String from, String to, String filter) {
- assert ! ((null == from) ^ (null == to)) : "if from is null, to must be null";
- this.from = from;
- this.to = to;
- this.filter = filter;
- }
-
- /**
- * @return the JSON string representing this domain for use in a facet param, or null if no domain should be used
- * */
- public CharSequence toJSONFacetParamValue() {
- if (null == from && null == filter) {
- return null;
- }
- StringBuilder sb = new StringBuilder("domain:{");
- if (null != from) {
- assert null != to;
- sb. append("join:{from:").append(from).append(",to:").append(to).append("}");
- if (null != filter){
- sb.append(",");
- }
-
- }
- if (null != filter) {
- sb.append("filter:'").append(filter).append("'");
- }
- sb.append("}");
- return sb;
- }
-
- /**
- * Given some original SolrParams, returns new SolrParams where the original "q" param is wrapped
- * as needed to apply the equivalent transformation to a query as this domain would to a facet
- */
- public SolrParams applyDomainToQuery(String safeKey, SolrParams in) {
- assert null == in.get(safeKey); // shouldn't be possible if every facet uses a unique key string
-
- String q = in.get("q");
- final ModifiableSolrParams out = new ModifiableSolrParams(in);
- if (null != from) {
- out.set(safeKey, in.get("q"));
- q = "{!join from="+from+" to="+to+" v=$"+safeKey+"}";
- }
- if (null != filter) {
- q = filter + " AND " + q;
- }
- out.set("q", q);
- return out;
- }
-
- /**
- * Factory method for creating a random domain change to use with a facet - may return an 'noop' JoinDomain,
- * but will never return null.
- */
- public static JoinDomain buildRandomDomain() {
-
- // use consistent type on both sides of join
- final String[] suffixes = random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES;
-
- final boolean noJoin = random().nextBoolean();
-
- String from = null;
- String to = null;
- for (;;) {
- if (noJoin) break;
- from = field(suffixes, random().nextInt(MAX_FIELD_NUM));
- to = field(suffixes, random().nextInt(MAX_FIELD_NUM));
- // HACK: joined numeric point fields need docValues.. for now just skip _is fields if we are dealing with points.
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP) && (from.endsWith("_is") || to.endsWith("_is")))
- {
- continue;
- }
- break;
- }
-
- // keep it simple, only filter on string fields - not point of test
- final String filterField = strfield(random().nextInt(MAX_FIELD_NUM));
-
- final String filter = random().nextBoolean() ? null : filterField+":[* TO *]";
- return new JoinDomain(from, to, filter);
- }
- }
-
- /**
- * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
- * at a node in our cluster
- */
- public static SolrClient getRandClient(Random rand) {
- int numClients = CLIENTS.size();
- int idx = TestUtil.nextInt(rand, 0, numClients);
-
- return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
- }
-
- public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
- assert null != client.getDefaultCollection();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
- client.getZkStateReader(),
- true, true, 330);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
deleted file mode 100644
index 24384cc..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
+++ /dev/null
@@ -1,677 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.commons.lang.StringUtils;
-
-import org.apache.lucene.util.LuceneTestCase.Slow;
-import org.apache.lucene.util.TestUtil;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.search.facet.FacetField;
-import static org.apache.solr.search.facet.RelatednessAgg.computeRelatedness;
-import static org.apache.solr.search.facet.RelatednessAgg.roundTo5Digits;
-
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * <p>
- * A randomized test of nested facets using the <code>relatedness()</code> function, that asserts the
- * accuracy the results for all the buckets returned using verification queries of the (expected)
- * foreground & background queries based on the nested facet terms.
- * <p>
- * Note that unlike normal facet "count" verification, using a high limit + overrequest isn't a substitute
- * for refinement in order to ensure accurate "skg" computation across shards. For that reason, this
- * tests forces <code>refine: true</code> (unlike {@link TestCloudJSONFacetJoinDomain}) and specifices a
- * <code>domain: { 'query':'*:*' }</code> for every facet, in order to garuntee that all shards
- * participate in all facets, so that the popularity & relatedness values returned can be proven
- * with validation requests.
- * </p>
- * <p>
- * (Refinement alone is not enough. Using the '*:*' query as the facet domain is neccessary to
- * prevent situations where a single shardX may return candidate bucket with no child-buckets due to
- * the normal facet intersections, but when refined on other shardY(s), can produce "high scoring"
- * SKG child-buckets, which would then be missing the foreground/background "size" contributions from
- * shardX.
- * </p>
- *
- * @see TestCloudJSONFacetJoinDomain
- */
-@Slow
-public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
- private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
-
- private static final int DEFAULT_LIMIT = FacetField.DEFAULT_FACET_LIMIT;
- private static final int MAX_FIELD_NUM = 15;
- private static final int UNIQUE_FIELD_VALS = 50;
-
- /** Multivalued string field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] STR_FIELD_SUFFIXES = new String[] { "_ss", "_sds", "_sdsS" };
- /** Multivalued int field suffixes that can be randomized for testing diff facet/join code paths */
- private static final String[] INT_FIELD_SUFFIXES = new String[] { "_is", "_ids", "_idsS" };
-
- /** A basic client for operations at the cloud level, default collection will be set */
- private static CloudSolrClient CLOUD_CLIENT;
- /** One client per node */
- private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
-
- @BeforeClass
- private static void createMiniSolrCloudCluster() throws Exception {
- // sanity check constants
- assertTrue("bad test constants: some suffixes will never be tested",
- (STR_FIELD_SUFFIXES.length < MAX_FIELD_NUM) && (INT_FIELD_SUFFIXES.length < MAX_FIELD_NUM));
-
- // we need DVs on point fields to compute stats & facets
- if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) System.setProperty(NUMERIC_DOCVALUES_SYSPROP,"true");
-
- // multi replicas should not matter...
- final int repFactor = usually() ? 1 : 2;
- // ... but we definitely want to test multiple shards
- final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
- final int numNodes = (numShards * repFactor);
-
- final String configName = DEBUG_LABEL + "_config-set";
- final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
-
- configureCluster(numNodes).addConfig(configName, configDir).configure();
-
- Map<String, String> collectionProperties = new LinkedHashMap<>();
- collectionProperties.put("config", "solrconfig-tlog.xml");
- collectionProperties.put("schema", "schema_latest.xml");
- CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
- .setProperties(collectionProperties)
- .process(cluster.getSolrClient());
-
- CLOUD_CLIENT = cluster.getSolrClient();
- CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
-
- waitForRecoveriesToFinish(CLOUD_CLIENT);
-
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
- }
-
- final int numDocs = atLeast(100);
- for (int id = 0; id < numDocs; id++) {
- SolrInputDocument doc = sdoc("id", ""+id);
- for (int fieldNum = 0; fieldNum < MAX_FIELD_NUM; fieldNum++) {
- // NOTE: we ensure every doc has at least one value in each field
- // that way, if a term is returned for a parent there there is garunteed to be at least one
- // one term in the child facet as well.
- //
- // otherwise, we'd face the risk of a single shardX returning parentTermX as a top term for
- // the parent facet, but having no child terms -- meanwhile on refinement another shardY that
- // did *not* returned parentTermX in phase#1, could return some *new* child terms under
- // parentTermX, but their stats would not include the bgCount from shardX.
- //
- // in normal operation, this is an edge case that isn't a big deal because the ratios &
- // relatedness scores are statistically approximate, but for the purpose of this test where
- // we verify correctness via exactness we need all shards to contribute to the SKG statistics
- final int numValsThisDoc = TestUtil.nextInt(random(), 1, (usually() ? 5 : 10));
- for (int v = 0; v < numValsThisDoc; v++) {
- final String fieldValue = randFieldValue(fieldNum);
-
- // for each fieldNum, there are actaully two fields: one string, and one integer
- doc.addField(field(STR_FIELD_SUFFIXES, fieldNum), fieldValue);
- doc.addField(field(INT_FIELD_SUFFIXES, fieldNum), fieldValue);
- }
- }
- CLOUD_CLIENT.add(doc);
- if (random().nextInt(100) < 1) {
- CLOUD_CLIENT.commit(); // commit 1% of the time to create new segments
- }
- if (random().nextInt(100) < 5) {
- CLOUD_CLIENT.add(doc); // duplicate the doc 5% of the time to create deleted docs
- }
- }
- CLOUD_CLIENT.commit();
- }
-
- /**
- * Given a (random) number, and a (static) array of possible suffixes returns a consistent field name that
- * uses that number and one of hte specified suffixes in it's name.
- *
- * @see #STR_FIELD_SUFFIXES
- * @see #INT_FIELD_SUFFIXES
- * @see #MAX_FIELD_NUM
- * @see #randFieldValue
- */
- private static String field(final String[] suffixes, final int fieldNum) {
- assert fieldNum < MAX_FIELD_NUM;
-
- final String suffix = suffixes[fieldNum % suffixes.length];
- return "field_" + fieldNum + suffix;
- }
- private static String strfield(final int fieldNum) {
- return field(STR_FIELD_SUFFIXES, fieldNum);
- }
- private static String intfield(final int fieldNum) {
- return field(INT_FIELD_SUFFIXES, fieldNum);
- }
-
- /**
- * Given a (random) field number, returns a random (integer based) value for that field.
- * NOTE: The number of unique values in each field is constant acording to {@link #UNIQUE_FIELD_VALS}
- * but the precise <em>range</em> of values will vary for each unique field number, such that cross field joins
- * will match fewer documents based on how far apart the field numbers are.
- *
- * @see #UNIQUE_FIELD_VALS
- * @see #field
- */
- private static String randFieldValue(final int fieldNum) {
- return "" + (fieldNum + TestUtil.nextInt(random(), 1, UNIQUE_FIELD_VALS));
- }
-
-
- @AfterClass
- private static void afterClass() throws Exception {
- CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
- for (HttpSolrClient client : CLIENTS) {
- client.close();
- }
- CLIENTS = null;
- }
-
- /**
- * Test some small, hand crafted, but non-trivial queries that are
- * easier to trace/debug then a pure random monstrosity.
- * (ie: if something obvious gets broken, this test may fail faster and in a more obvious way then testRandom)
- */
- public void testBespoke() throws Exception {
- { // trivial single level facet
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, null);
- facets.put("top1", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
-
- { // trivial single level facet w/sorting on skg
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- TermFacet top = new TermFacet(strfield(9), UNIQUE_FIELD_VALS, 0, "skg desc");
- facets.put("top2", top);
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
-
- { // trivial single level facet w/ 2 diff ways to request "limit = (effectively) Infinite"
- // to sanity check refinement of buckets missing from other shard in both cases
-
- // NOTE that these two queries & facets *should* effectively identical given that the
- // very large limit value is big enough no shard will ever return that may terms,
- // but the "limit=-1" case it actaully triggers slightly different code paths
- // because it causes FacetField.returnsPartial() to be "true"
- for (int limit : new int[] { 999999999, -1 }) {
- Map<String,TermFacet> facets = new LinkedHashMap<>();
- facets.put("top_facet_limit__" + limit, new TermFacet(strfield(9), limit, 0, "skg desc"));
- final AtomicInteger maxBuckets = new AtomicInteger(UNIQUE_FIELD_VALS);
- assertFacetSKGsAreCorrect(maxBuckets, facets, strfield(7)+":11", strfield(5)+":9", "*:*");
- assertTrue("Didn't check a single bucket???", maxBuckets.get() < UNIQUE_FIELD_VALS);
- }
- }
- }
-
- public void testRandom() throws Exception {
-
- // since the "cost" of verifying the stats for each bucket is so high (see TODO in verifySKGResults())
- // we put a safety valve in place on the maximum number of buckets that we are willing to verify
- // across *all* the queries that we do.
- // that way if the randomized queries we build all have relatively small facets, so be it, but if
- // we get a really big one early on, we can test as much as possible, skip other iterations.
- //
- // (deeply nested facets may contain more buckets then the max, but we won't *check* all of them)
- final int maxBucketsAllowed = atLeast(2000);
- final AtomicInteger maxBucketsToCheck = new AtomicInteger(maxBucketsAllowed);
-
- final int numIters = atLeast(10);
- for (int iter = 0; iter < numIters && 0 < maxBucketsToCheck.get(); iter++) {
- assertFacetSKGsAreCorrect(maxBucketsToCheck, TermFacet.buildRandomFacets(),
- buildRandomQuery(), buildRandomQuery(), buildRandomQuery());
- }
- assertTrue("Didn't check a single bucket???", maxBucketsToCheck.get() < maxBucketsAllowed);
-
-
- }
-
- /**
- * Generates a random query string across the randomized fields/values in the index
- *
- * @see #randFieldValue
- * @see #field
- */
- private static String buildRandomQuery() {
- if (0 == TestUtil.nextInt(random(), 0,10)) {
- return "*:*";
- }
- final int numClauses = TestUtil.nextInt(random(), 3, 10);
- final String[] clauses = new String[numClauses];
- for (int c = 0; c < numClauses; c++) {
- final int fieldNum = random().nextInt(MAX_FIELD_NUM);
- // keep queries simple, just use str fields - not point of test
- clauses[c] = strfield(fieldNum) + ":" + randFieldValue(fieldNum);
- }
- return buildORQuery(clauses);
- }
-
- private static String buildORQuery(String... clauses) {
- assert 0 < clauses.length;
- return "(" + StringUtils.join(clauses, " OR ") + ")";
- }
-
- /**
- * Given a set of term facets, and top level query strings, asserts that
- * the SKG stats for each facet term returned when executing that query with those foreground/background
- * queries match the expected results of executing the equivalent queries in isolation.
- *
- * @see #verifySKGResults
- */
- private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
- Map<String,TermFacet> expected,
- final String query,
- final String foreQ,
- final String backQ) throws SolrServerException, IOException {
- final SolrParams baseParams = params("rows","0", "fore", foreQ, "back", backQ);
-
- final SolrParams facetParams = params("q", query,
- "json.facet", ""+TermFacet.toJSONFacetParamValue(expected,null));
- final SolrParams initParams = SolrParams.wrapAppended(facetParams, baseParams);
-
- log.info("Doing full run: {}", initParams);
-
- QueryResponse rsp = null;
- // JSON Facets not (currently) available from QueryResponse...
- NamedList topNamedList = null;
- try {
- rsp = (new QueryRequest(initParams)).process(getRandClient(random()));
- assertNotNull(initParams + " is null rsp?", rsp);
- topNamedList = rsp.getResponse();
- assertNotNull(initParams + " is null topNamedList?", topNamedList);
- } catch (Exception e) {
- throw new RuntimeException("init query failed: " + initParams + ": " +
- e.getMessage(), e);
- }
- try {
- final NamedList facetResponse = (NamedList) topNamedList.get("facets");
- assertNotNull("null facet results?", facetResponse);
- assertEquals("numFound mismatch with top count?",
- rsp.getResults().getNumFound(), ((Number)facetResponse.get("count")).longValue());
-
- // Note: even if the query has numFound=0, our explicit background query domain should
- // still force facet results
- // (even if the background query matches nothing, that just means there will be no
- // buckets in those facets)
- assertFacetSKGsAreCorrect(maxBucketsToCheck, expected, baseParams, facetResponse);
-
- } catch (AssertionError e) {
- throw new AssertionError(initParams + " ===> " + topNamedList + " --> " + e.getMessage(), e);
- } finally {
- log.info("Ending full run");
- }
- }
-
- /**
- * Recursive helper method that walks the actual facet response, comparing the SKG results to
- * the expected output based on the equivalent filters generated from the original TermFacet.
- */
- private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
- final Map<String,TermFacet> expected,
- final SolrParams baseParams,
- final NamedList actualFacetResponse) throws SolrServerException, IOException {
-
- for (Map.Entry<String,TermFacet> entry : expected.entrySet()) {
- final String facetKey = entry.getKey();
- final TermFacet facet = entry.getValue();
- final NamedList results = (NamedList) actualFacetResponse.get(facetKey);
- assertNotNull(facetKey + " key missing from: " + actualFacetResponse, results);
- final List<NamedList> buckets = (List<NamedList>) results.get("buckets");
- assertNotNull(facetKey + " has null buckets: " + actualFacetResponse, buckets);
-
- if (buckets.isEmpty()) {
- // should only happen if the background query does not match any docs with field X
- final long docsWithField = getNumFound(params("_trace", "noBuckets",
- "rows", "0",
- "q", facet.field+":[* TO *]",
- "fq", baseParams.get("back")));
-
- assertEquals(facetKey + " has no buckets, but docs in background exist with field: " + facet.field,
- 0, docsWithField);
- }
-
- // NOTE: it's important that we do this depth first -- not just because it's the easiest way to do it,
- // but because it means that our maxBucketsToCheck will ensure we do a lot of deep sub-bucket checking,
- // not just all the buckets of the top level(s) facet(s)
- for (NamedList bucket : buckets) {
- final String fieldVal = bucket.get("val").toString(); // int or stringified int
-
- verifySKGResults(facetKey, facet, baseParams, fieldVal, bucket);
- if (maxBucketsToCheck.decrementAndGet() <= 0) {
- return;
- }
-
- final SolrParams verifyParams = SolrParams.wrapAppended(baseParams,
- params("fq", facet.field + ":" + fieldVal));
-
- // recursively check subFacets
- if (! facet.subFacets.isEmpty()) {
- assertFacetSKGsAreCorrect(maxBucketsToCheck, facet.subFacets, verifyParams, bucket);
- }
- }
- }
-
- { // make sure we don't have any facet keys we don't expect
- // a little hackish because subfacets have extra keys...
- final LinkedHashSet expectedKeys = new LinkedHashSet(expected.keySet());
- expectedKeys.add("count");
- if (0 <= actualFacetResponse.indexOf("val",0)) {
- expectedKeys.add("val");
- expectedKeys.add("skg");
- }
- assertEquals("Unexpected keys in facet response",
- expectedKeys, actualFacetResponse.asShallowMap().keySet());
- }
- }
-
- /**
- * Verifies that the popularity & relatedness values containined in a single SKG bucket
- * match the expected values based on the facet field & bucket value, as well the existing
- * filterParams.
- *
- * @see #assertFacetSKGsAreCorrect
- */
- private void verifySKGResults(String facetKey, TermFacet facet, SolrParams filterParams,
- String fieldVal, NamedList<Object> bucket)
- throws SolrServerException, IOException {
-
- final String bucketQ = facet.field+":"+fieldVal;
- final NamedList<Object> skgBucket = (NamedList<Object>) bucket.get("skg");
- assertNotNull(facetKey + "/bucket:" + bucket.toString(), skgBucket);
-
- // TODO: make this more efficient?
- // ideally we'd do a single query w/4 facet.queries, one for each count
- // but formatting the queries is a pain, currently we leverage the accumulated fq's
- final long fgSize = getNumFound(SolrParams.wrapAppended(params("_trace", "fgSize",
- "rows","0",
- "q","{!query v=$fore}"),
- filterParams));
- final long bgSize = getNumFound(params("_trace", "bgSize",
- "rows","0",
- "q", filterParams.get("back")));
-
- final long fgCount = getNumFound(SolrParams.wrapAppended(params("_trace", "fgCount",
- "rows","0",
- "q","{!query v=$fore}",
- "fq", bucketQ),
- filterParams));
- final long bgCount = getNumFound(params("_trace", "bgCount",
- "rows","0",
- "q", bucketQ,
- "fq", filterParams.get("back")));
-
- assertEquals(facetKey + "/bucket:" + bucket + " => fgPop should be: " + fgCount + " / " + bgSize,
- roundTo5Digits((double) fgCount / bgSize),
- skgBucket.get("foreground_popularity"));
- assertEquals(facetKey + "/bucket:" + bucket + " => bgPop should be: " + bgCount + " / " + bgSize,
- roundTo5Digits((double) bgCount / bgSize),
- skgBucket.get("background_popularity"));
- assertEquals(facetKey + "/bucket:" + bucket + " => relatedness is wrong",
- roundTo5Digits(computeRelatedness(fgCount, fgSize, bgCount, bgSize)),
- skgBucket.get("relatedness"));
-
- }
-
-
- /**
- * Trivial data structure for modeling a simple terms facet that can be written out as a json.facet param.
- *
- * Doesn't do any string escaping or quoting, so don't use whitespace or reserved json characters
- */
- private static final class TermFacet {
- public final String field;
- public final Map<String,TermFacet> subFacets = new LinkedHashMap<>();
- public final Integer limit; // may be null
- public final Integer overrequest; // may be null
- public final String sort; // may be null
- /** Simplified constructor asks for limit = # unique vals */
- public TermFacet(String field) {
- this(field, UNIQUE_FIELD_VALS, 0, "skg desc");
-
- }
- public TermFacet(String field, Integer limit, Integer overrequest, String sort) {
- assert null != field;
- this.field = field;
- this.limit = limit;
- this.overrequest = overrequest;
- this.sort = sort;
- }
-
- /**
- * recursively generates the <code>json.facet</code> param value to use for testing this facet
- */
- private CharSequence toJSONFacetParamValue() {
- final String limitStr = (null == limit) ? "" : (", limit:" + limit);
- final String overrequestStr = (null == overrequest) ? "" : (", overrequest:" + overrequest);
- final String sortStr = (null == sort) ? "" : (", sort: '" + sort + "'");
- final StringBuilder sb
- = new StringBuilder("{ type:terms, field:" + field + limitStr + overrequestStr + sortStr);
-
- // see class javadocs for why we always use refine:true & the query:'*:*' domain for this test.
- sb.append(", refine: true, domain: { query: '*:*' }, facet:");
- sb.append(toJSONFacetParamValue(subFacets, "skg : 'relatedness($fore,$back)'"));
- sb.append("}");
- return sb;
- }
-
- /**
- * Given a set of (possibly nested) facets, generates a suitable <code>json.facet</code> param value to
- * use for testing them against in a solr request.
- */
- public static CharSequence toJSONFacetParamValue(final Map<String,TermFacet> facets,
- final String extraJson) {
- assert null != facets;
- if (0 == facets.size() && null == extraJson) {
- return "";
- }
-
- StringBuilder sb = new StringBuilder("{ processEmpty: true, ");
- for (String key : facets.keySet()) {
- sb.append(key).append(" : ").append(facets.get(key).toJSONFacetParamValue());
- sb.append(" ,");
- }
- if (null == extraJson) {
- sb.setLength(sb.length() - 1);
- } else {
- sb.append(extraJson);
- }
- sb.append("}");
- return sb;
- }
-
- /**
- * Factory method for generating some random facets.
- *
- * For simplicity, each facet will have a unique key name.
- */
- public static Map<String,TermFacet> buildRandomFacets() {
- // for simplicity, use a unique facet key regardless of depth - simplifies verification
- // and le's us enforce a hard limit on the total number of facets in a request
- AtomicInteger keyCounter = new AtomicInteger(0);
-
- final int maxDepth = TestUtil.nextInt(random(), 0, (usually() ? 2 : 3));
- return buildRandomFacets(keyCounter, maxDepth);
- }
-
- /**
- * picks a random value for the "sort" param, biased in favor of interesting test cases
- *
- * @return a sort string (w/direction), or null to specify nothing (trigger default behavior)
- * @see #randomLimitParam
- */
- public static String randomSortParam(Random r) {
-
- // IMPORTANT!!!
- // if this method is modified to produce new sorts, make sure to update
- // randomLimitParam to account for them if they are impacted by SOLR-12556
- final String dir = random().nextBoolean() ? "asc" : "desc";
- switch(r.nextInt(4)) {
- case 0: return null;
- case 1: return "count " + dir;
- case 2: return "skg " + dir;
- case 3: return "index " + dir;
- default: throw new RuntimeException("Broken case statement");
- }
- }
- /**
- * picks a random value for the "limit" param, biased in favor of interesting test cases
- *
- * <p>
- * <b>NOTE:</b> Due to SOLR-12556, we have to force an overrequest of "all" possible terms for
- * some sort values.
- * </p>
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- * @see #randomSortParam
- */
- public static Integer randomLimitParam(Random r, final String sort) {
- if (null != sort) {
- if (sort.equals("count asc") || sort.startsWith("skg")) {
- // of the known types of sorts produced, these are at risk of SOLR-12556
- // so request (effectively) unlimited num buckets
- return r.nextBoolean() ? UNIQUE_FIELD_VALS : -1;
- }
- }
- final int limit = 1 + r.nextInt((int) (UNIQUE_FIELD_VALS * 1.5F));
- if (limit >= UNIQUE_FIELD_VALS && r.nextBoolean()) {
- return -1; // unlimited
- } else if (limit == DEFAULT_LIMIT && r.nextBoolean()) {
- return null; // sometimes, don't specify limit if it's the default
- }
- return limit;
- }
-
- /**
- * picks a random value for the "overrequest" param, biased in favor of interesting test cases.
- *
- * @return a number to specify in the request, or null to specify nothing (trigger default behavior)
- * @see #UNIQUE_FIELD_VALS
- */
- public static Integer randomOverrequestParam(Random r) {
- switch(r.nextInt(10)) {
- case 0:
- case 1:
- case 2:
- case 3:
- return 0; // 40% of the time, disable overrequest to better stress refinement
- case 4:
- case 5:
- return r.nextInt(UNIQUE_FIELD_VALS); // 20% ask for less them what's needed
- case 6:
- return r.nextInt(Integer.MAX_VALUE); // 10%: completley random value, statisticaly more then enough
- default: break;
- }
- // else.... either leave param unspecified (or redundently specify the -1 default)
- return r.nextBoolean() ? null : -1;
- }
-
- /**
- * recursive helper method for building random facets
- *
- * @param keyCounter used to ensure every generated facet has a unique key name
- * @param maxDepth max possible depth allowed for the recusion, a lower value may be used depending on how many facets are returned at the current level.
- */
- private static Map<String,TermFacet> buildRandomFacets(AtomicInteger keyCounter, int maxDepth) {
- final int numFacets = Math.max(1, TestUtil.nextInt(random(), -1, 3)); // 3/5th chance of being '1'
- Map<String,TermFacet> results = new LinkedHashMap<>();
- for (int i = 0; i < numFacets; i++) {
- if (keyCounter.get() < 3) { // a hard limit on the total number of facets (regardless of depth) to reduce OOM risk
-
- final String sort = randomSortParam(random());
- final Integer limit = randomLimitParam(random(), sort);
- final Integer overrequest = randomOverrequestParam(random());
- final TermFacet facet = new TermFacet(field((random().nextBoolean()
- ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES),
- random().nextInt(MAX_FIELD_NUM)),
- limit, overrequest, sort);
- results.put("facet_" + keyCounter.incrementAndGet(), facet);
- if (0 < maxDepth) {
- // if we're going wide, don't go deep
- final int nextMaxDepth = Math.max(0, maxDepth - numFacets);
- facet.subFacets.putAll(buildRandomFacets(keyCounter, TestUtil.nextInt(random(), 0, nextMaxDepth)));
- }
- }
- }
- return results;
- }
- }
-
- /**
- * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
- * at a node in our cluster
- */
- public static SolrClient getRandClient(Random rand) {
- int numClients = CLIENTS.size();
- int idx = TestUtil.nextInt(rand, 0, numClients);
-
- return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
- }
-
- /**
- * Uses a random SolrClient to execture a request and returns only the numFound
- * @see #getRandClient
- */
- public static long getNumFound(final SolrParams req) throws SolrServerException, IOException {
- return getRandClient(random()).query(req).getResults().getNumFound();
- }
-
- public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
- assert null != client.getDefaultCollection();
- AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
- client.getZkStateReader(),
- true, true, 330);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/71988c75/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
index 9de0d12..a661198 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/DebugAgg.java
@@ -32,7 +32,7 @@ import org.apache.solr.search.SyntaxError;
import org.apache.solr.search.ValueSourceParser;
-public class DebugAgg extends AggValueSource {
+class DebugAgg extends AggValueSource {
public static AtomicLong parses = new AtomicLong(0);
public static class Parser extends ValueSourceParser {
[07/32] lucene-solr:jira/solr-12730: SOLR-9425: fix
NullPointerException in TestSolrConfigHandlerConcurrent
Posted by ab...@apache.org.
SOLR-9425: fix NullPointerException in TestSolrConfigHandlerConcurrent
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab14cc95
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab14cc95
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab14cc95
Branch: refs/heads/jira/solr-12730
Commit: ab14cc9566e15743eb168b36ca63d2b3197ba0a1
Parents: c277674
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:30:44 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:30:44 2018 +0100
----------------------------------------------------------------------
.../org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab14cc95/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
index 6aa4e1f..c2599a0 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
@@ -66,11 +66,11 @@ public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBa
for (Object o : caches.entrySet()) {
final Map.Entry e = (Map.Entry) o;
if (e.getValue() instanceof Map) {
+ List<String> errs = new ArrayList<>();
+ collectErrors.add(errs);
Map value = (Map) e.getValue();
Thread t = new Thread(() -> {
try {
- List<String> errs = new ArrayList<>();
- collectErrors.add(errs);
invokeBulkCall((String)e.getKey() , errs, value);
} catch (Exception e1) {
e1.printStackTrace();
[03/32] lucene-solr:jira/solr-12730: LUCENE-8538: Add a Simple WKT
Shape Parser for creating Lucene Geometries (Polygon, Line, Rectangle, points,
and multi-variants) from WKT format.
Posted by ab...@apache.org.
LUCENE-8538: Add a Simple WKT Shape Parser for creating Lucene Geometries (Polygon, Line, Rectangle, points, and multi-variants) from WKT format.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bd714ca1
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bd714ca1
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bd714ca1
Branch: refs/heads/jira/solr-12730
Commit: bd714ca1edc681cd871e6d385bd98a9cb9d7ffe6
Parents: e083b15
Author: Nicholas Knize <nk...@gmail.com>
Authored: Mon Oct 22 14:25:22 2018 -0500
Committer: Nicholas Knize <nk...@gmail.com>
Committed: Wed Oct 24 09:27:47 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../apache/lucene/geo/SimpleWKTShapeParser.java | 406 +++++++++++++++++++
.../lucene/geo/TestSimpleWKTShapeParsing.java | 206 ++++++++++
3 files changed, 615 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd714ca1/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 23d807b..4097ce0 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -220,6 +220,9 @@ New Features
may be used to determine how to split the inner nodes, and dimensions N+1 to D
are ignored and stored as data dimensions at the leaves. (Nick Knize)
+* LUCENE-8538: Add a Simple WKT Shape Parser for creating Lucene Geometries (Polygon, Line,
+ Rectangle) from WKT format. (Nick Knize)
+
Improvements:
* LUCENE-8521: Change LatLonShape encoding to 7 dimensions instead of 6; where the
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd714ca1/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java b/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java
new file mode 100644
index 0000000..17b595f
--- /dev/null
+++ b/lucene/sandbox/src/java/org/apache/lucene/geo/SimpleWKTShapeParser.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.geo;
+
+import java.io.IOException;
+import java.io.StreamTokenizer;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+/**
+ * Parses shape geometry represented in WKT format
+ *
+ * complies with OGC® document: 12-063r5 and ISO/IEC 13249-3:2016 standard
+ * located at http://docs.opengeospatial.org/is/12-063r5/12-063r5.html
+ */
+public class SimpleWKTShapeParser {
+ public static final String EMPTY = "EMPTY";
+ public static final String SPACE = " ";
+ public static final String LPAREN = "(";
+ public static final String RPAREN = ")";
+ public static final String COMMA = ",";
+ public static final String NAN = "NaN";
+
+ private static final String NUMBER = "<NUMBER>";
+ private static final String EOF = "END-OF-STREAM";
+ private static final String EOL = "END-OF-LINE";
+
+ // no instance
+ private SimpleWKTShapeParser() {}
+
+ public static Object parse(String wkt) throws IOException, ParseException {
+ return parseExpectedType(wkt, null);
+ }
+
+ public static Object parseExpectedType(String wkt, final ShapeType shapeType) throws IOException, ParseException {
+ try (StringReader reader = new StringReader(wkt)) {
+ // setup the tokenizer; configured to read words w/o numbers
+ StreamTokenizer tokenizer = new StreamTokenizer(reader);
+ tokenizer.resetSyntax();
+ tokenizer.wordChars('a', 'z');
+ tokenizer.wordChars('A', 'Z');
+ tokenizer.wordChars(128 + 32, 255);
+ tokenizer.wordChars('0', '9');
+ tokenizer.wordChars('-', '-');
+ tokenizer.wordChars('+', '+');
+ tokenizer.wordChars('.', '.');
+ tokenizer.whitespaceChars(0, ' ');
+ tokenizer.commentChar('#');
+ Object geometry = parseGeometry(tokenizer, shapeType);
+ checkEOF(tokenizer);
+ return geometry;
+ }
+ }
+
+ /** parse geometry from the stream tokenizer */
+ private static Object parseGeometry(StreamTokenizer stream, ShapeType shapeType) throws IOException, ParseException {
+ final ShapeType type = ShapeType.forName(nextWord(stream));
+ if (shapeType != null && shapeType != ShapeType.GEOMETRYCOLLECTION) {
+ if (type.wktName().equals(shapeType.wktName()) == false) {
+ throw new ParseException("Expected geometry type: [" + shapeType + "], but found: [" + type + "]", stream.lineno());
+ }
+ }
+ switch (type) {
+ case POINT:
+ return parsePoint(stream);
+ case MULTIPOINT:
+ return parseMultiPoint(stream);
+ case LINESTRING:
+ return parseLine(stream);
+ case MULTILINESTRING:
+ return parseMultiLine(stream);
+ case POLYGON:
+ return parsePolygon(stream);
+ case MULTIPOLYGON:
+ return parseMultiPolygon(stream);
+ case ENVELOPE:
+ return parseBBox(stream);
+ case GEOMETRYCOLLECTION:
+ return parseGeometryCollection(stream);
+ default:
+ throw new IllegalArgumentException("Unknown geometry type: " + type);
+ }
+ }
+
+ /** Parses a point as a double array */
+ private static double[] parsePoint(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ double[] pt = new double[]{nextNumber(stream), nextNumber(stream)};
+ if (isNumberNext(stream) == true) {
+ nextNumber(stream);
+ }
+ nextCloser(stream);
+ return pt;
+ }
+
+ /** Parses a list of points into latitude and longitude arraylists */
+ private static void parseCoordinates(StreamTokenizer stream, ArrayList lats, ArrayList lons)
+ throws IOException, ParseException {
+ boolean isOpenParen = false;
+ if (isNumberNext(stream) || (isOpenParen = nextWord(stream).equals(LPAREN))) {
+ parseCoordinate(stream, lats, lons);
+ }
+
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ isOpenParen = false;
+ if (isNumberNext(stream) || (isOpenParen = nextWord(stream).equals(LPAREN))) {
+ parseCoordinate(stream, lats, lons);
+ }
+ if (isOpenParen && nextCloser(stream).equals(RPAREN) == false) {
+ throw new ParseException("expected: [" + RPAREN + "] but found: [" + tokenString(stream) + "]", stream.lineno());
+ }
+ }
+
+ if (isOpenParen && nextCloser(stream).equals(RPAREN) == false) {
+ throw new ParseException("expected: [" + RPAREN + "] but found: [" + tokenString(stream) + "]", stream.lineno());
+ }
+ }
+
+ /** parses a single coordinate, w/ optional 3rd dimension */
+ private static void parseCoordinate(StreamTokenizer stream, ArrayList lats, ArrayList lons)
+ throws IOException, ParseException {
+ lons.add(nextNumber(stream));
+ lats.add(nextNumber(stream));
+ if (isNumberNext(stream)) {
+ nextNumber(stream);
+ }
+ }
+
+ /** parses a MULTIPOINT type */
+ private static double[][] parseMultiPoint(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ double[][] result = new double[lats.size()][2];
+ for (int i = 0; i < lats.size(); ++i) {
+ result[i] = new double[] {lons.get(i), lats.get(i)};
+ }
+ return result;
+ }
+
+ /** parses a LINESTRING */
+ private static Line parseLine(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ return new Line(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray());
+ }
+
+ /** parses a MULTILINESTRING */
+ private static Line[] parseMultiLine(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Line> lines = new ArrayList();
+ lines.add(parseLine(stream));
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ lines.add(parseLine(stream));
+ }
+ return lines.toArray(new Line[lines.size()]);
+ }
+
+ /** parses the hole of a polygon */
+ private static Polygon parsePolygonHole(StreamTokenizer stream) throws IOException, ParseException {
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ return new Polygon(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray());
+ }
+
+ /** parses a POLYGON */
+ private static Polygon parsePolygon(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ nextOpener(stream);
+ ArrayList<Double> lats = new ArrayList();
+ ArrayList<Double> lons = new ArrayList();
+ parseCoordinates(stream, lats, lons);
+ ArrayList<Polygon> holes = new ArrayList<>();
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ holes.add(parsePolygonHole(stream));
+ }
+
+ if (holes.isEmpty() == false) {
+ return new Polygon(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray(), holes.toArray(new Polygon[holes.size()]));
+ }
+ return new Polygon(lats.stream().mapToDouble(i->i).toArray(), lons.stream().mapToDouble(i->i).toArray());
+ }
+
+ /** parses a MULTIPOLYGON */
+ private static Polygon[] parseMultiPolygon(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextEmptyOrOpen(stream);
+ if (token.equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Polygon> polygons = new ArrayList();
+ polygons.add(parsePolygon(stream));
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ polygons.add(parsePolygon(stream));
+ }
+ return polygons.toArray(new Polygon[polygons.size()]);
+ }
+
+ /** parses an ENVELOPE */
+ private static Rectangle parseBBox(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ double minLon = nextNumber(stream);
+ nextComma(stream);
+ double maxLon = nextNumber(stream);
+ nextComma(stream);
+ double maxLat = nextNumber(stream);
+ nextComma(stream);
+ double minLat = nextNumber(stream);
+ nextCloser(stream);
+ return new Rectangle(minLat, maxLat, minLon, maxLon);
+ }
+
+ /** parses a GEOMETRYCOLLECTION */
+ private static Object[] parseGeometryCollection(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextEmptyOrOpen(stream).equals(EMPTY)) {
+ return null;
+ }
+ ArrayList<Object> geometries = new ArrayList<>();
+ geometries.add(parseGeometry(stream, ShapeType.GEOMETRYCOLLECTION));
+ while (nextCloserOrComma(stream).equals(COMMA)) {
+ geometries.add(parseGeometry(stream, null));
+ }
+ return geometries.toArray(new Object[geometries.size()]);
+ }
+
+ /** next word in the stream */
+ private static String nextWord(StreamTokenizer stream) throws ParseException, IOException {
+ switch (stream.nextToken()) {
+ case StreamTokenizer.TT_WORD:
+ final String word = stream.sval;
+ return word.equalsIgnoreCase(EMPTY) ? EMPTY : word;
+ case '(': return LPAREN;
+ case ')': return RPAREN;
+ case ',': return COMMA;
+ }
+ throw new ParseException("expected word but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** next number in the stream */
+ private static double nextNumber(StreamTokenizer stream) throws IOException, ParseException {
+ if (stream.nextToken() == StreamTokenizer.TT_WORD) {
+ if (stream.sval.equalsIgnoreCase(NAN)) {
+ return Double.NaN;
+ } else {
+ try {
+ return Double.parseDouble(stream.sval);
+ } catch (NumberFormatException e) {
+ throw new ParseException("invalid number found: " + stream.sval, stream.lineno());
+ }
+ }
+ }
+ throw new ParseException("expected number but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** next token in the stream */
+ private static String tokenString(StreamTokenizer stream) {
+ switch (stream.ttype) {
+ case StreamTokenizer.TT_WORD: return stream.sval;
+ case StreamTokenizer.TT_EOF: return EOF;
+ case StreamTokenizer.TT_EOL: return EOL;
+ case StreamTokenizer.TT_NUMBER: return NUMBER;
+ }
+ return "'" + (char)stream.ttype + "'";
+ }
+
+ /** checks if the next token is a number */
+ private static boolean isNumberNext(StreamTokenizer stream) throws IOException {
+ final int type = stream.nextToken();
+ stream.pushBack();
+ return type == StreamTokenizer.TT_WORD;
+ }
+
+ /** checks if next token is an EMPTY or open paren */
+ private static String nextEmptyOrOpen(StreamTokenizer stream) throws IOException, ParseException {
+ final String next = nextWord(stream);
+ if (next.equals(EMPTY) || next.equals(LPAREN)) {
+ return next;
+ }
+ throw new ParseException("expected " + EMPTY + " or " + LPAREN
+ + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** checks if next token is a closing paren */
+ private static String nextCloser(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextWord(stream).equals(RPAREN)) {
+ return RPAREN;
+ }
+ throw new ParseException("expected " + RPAREN + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** expects a comma as next token */
+ private static String nextComma(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextWord(stream).equals(COMMA) == true) {
+ return COMMA;
+ }
+ throw new ParseException("expected " + COMMA + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** expects an open RPAREN as the next toke */
+ private static String nextOpener(StreamTokenizer stream) throws IOException, ParseException {
+ if (nextWord(stream).equals(LPAREN)) {
+ return LPAREN;
+ }
+ throw new ParseException("expected " + LPAREN + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** expects either a closing LPAREN or comma as the next token */
+ private static String nextCloserOrComma(StreamTokenizer stream) throws IOException, ParseException {
+ String token = nextWord(stream);
+ if (token.equals(COMMA) || token.equals(RPAREN)) {
+ return token;
+ }
+ throw new ParseException("expected " + COMMA + " or " + RPAREN
+ + " but found: " + tokenString(stream), stream.lineno());
+ }
+
+ /** next word in the stream */
+ private static void checkEOF(StreamTokenizer stream) throws ParseException, IOException {
+ if (stream.nextToken() != StreamTokenizer.TT_EOF) {
+ throw new ParseException("expected end of WKT string but found additional text: "
+ + tokenString(stream), stream.lineno());
+ }
+ }
+
+ /** Enumerated type for Shapes */
+ public enum ShapeType {
+ POINT("point"),
+ MULTIPOINT("multipoint"),
+ LINESTRING("linestring"),
+ MULTILINESTRING("multilinestring"),
+ POLYGON("polygon"),
+ MULTIPOLYGON("multipolygon"),
+ GEOMETRYCOLLECTION("geometrycollection"),
+ ENVELOPE("envelope"); // not part of the actual WKB spec
+
+ private final String shapeName;
+ private static Map<String, ShapeType> shapeTypeMap = new HashMap<>();
+ private static final String BBOX = "BBOX";
+
+ static {
+ for (ShapeType type : values()) {
+ shapeTypeMap.put(type.shapeName, type);
+ }
+ shapeTypeMap.put(ENVELOPE.wktName().toLowerCase(Locale.ROOT), ENVELOPE);
+ }
+
+ ShapeType(String shapeName) {
+ this.shapeName = shapeName;
+ }
+
+ protected String typename() {
+ return shapeName;
+ }
+
+ /** wkt shape name */
+ public String wktName() {
+ return this == ENVELOPE ? BBOX : this.shapeName;
+ }
+
+ public static ShapeType forName(String shapename) {
+ String typename = shapename.toLowerCase(Locale.ROOT);
+ for (ShapeType type : values()) {
+ if(type.shapeName.equals(typename)) {
+ return type;
+ }
+ }
+ throw new IllegalArgumentException("unknown geo_shape ["+shapename+"]");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd714ca1/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java b/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java
new file mode 100644
index 0000000..e941ef4
--- /dev/null
+++ b/lucene/sandbox/src/test/org/apache/lucene/geo/TestSimpleWKTShapeParsing.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.geo;
+
+import org.apache.lucene.geo.SimpleWKTShapeParser.ShapeType;
+import org.apache.lucene.util.LuceneTestCase;
+
+/** simple WKT parsing tests */
+public class TestSimpleWKTShapeParsing extends LuceneTestCase {
+
+ /** test simple Point */
+ public void testPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POINT + "(101.0 10.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof double[]);
+ double[] point = (double[])shape;
+ assertEquals(101d, point[0], 0d); // lon
+ assertEquals(10d, point[1], 1d); // lat
+ }
+
+ /** test POINT EMPTY returns null */
+ public void testEmptyPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POINT + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple MULTIPOINT */
+ public void testMultiPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTIPOINT + "(101.0 10.0, 180.0 90.0, -180.0 -90.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof double[][]);
+ double[][] pts = (double[][])shape;
+ assertEquals(3, pts.length,0);
+ assertEquals(101d, pts[0][0], 0);
+ assertEquals(10d, pts[0][1], 0);
+ assertEquals(180d, pts[1][0], 0);
+ assertEquals(90d, pts[1][1], 0);
+ assertEquals(-180d, pts[2][0], 0);
+ assertEquals(-90d, pts[2][1], 0);
+ }
+
+ /** test MULTIPOINT EMPTY returns null */
+ public void testEmptyMultiPoint() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTIPOINT + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple LINESTRING */
+ public void testLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.LINESTRING + "(101.0 10.0, 180.0 90.0, -180.0 -90.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Line);
+ Line line = (Line)shape;
+ assertEquals(3, line.numPoints(),0);
+ assertEquals(101d, line.getLon(0), 0);
+ assertEquals(10d, line.getLat(0), 0);
+ assertEquals(180d, line.getLon(1), 0);
+ assertEquals(90d, line.getLat(1), 0);
+ assertEquals(-180d, line.getLon(2), 0);
+ assertEquals(-90d, line.getLat(2), 0);
+ }
+
+ /** test empty LINESTRING */
+ public void testEmptyLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.LINESTRING + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple MULTILINESTRING */
+ public void testMultiLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTILINESTRING + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0),");
+ b.append("(10.0 2.0, 11.0 2.0, 11.0 3.0, 10.0 3.0, 10.0 2.0))");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Line[]);
+ Line[] lines = (Line[])shape;
+ assertEquals(2, lines.length, 0);
+ }
+
+ /** test empty MULTILINESTRING */
+ public void testEmptyMultiLine() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTILINESTRING + SimpleWKTShapeParser.SPACE + SimpleWKTShapeParser.EMPTY);
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+ assertNull(shape);
+ }
+
+ /** test simple polygon: POLYGON((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)) */
+ public void testPolygon() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POLYGON + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0))\n");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Polygon);
+ Polygon polygon = (Polygon)shape;
+ assertEquals(new Polygon(new double[] {0.0, 0.0, 1.0, 1.0, 0.0},
+ new double[] {100.0, 101.0, 101.0, 100.0, 100.0}), polygon);
+ }
+
+ /** test polygon with hole */
+ public void testPolygonWithHole() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POLYGON + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0), ");
+ b.append("(100.5 0.5, 100.5 0.75, 100.75 0.75, 100.75 0.5, 100.5 0.5))");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Polygon);
+ Polygon hole = new Polygon(new double[] {0.5, 0.75, 0.75, 0.5, 0.5},
+ new double[] {100.5, 100.5, 100.75, 100.75, 100.5});
+ Polygon expected = new Polygon(new double[] {0.0, 0.0, 1.0, 1.0, 0.0},
+ new double[] {100.0, 101.0, 101.0, 100.0, 100.0}, hole);
+ Polygon polygon = (Polygon)shape;
+
+ assertEquals(expected, polygon);
+ }
+
+ /** test MultiPolygon returns Polygon array */
+ public void testMultiPolygon() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.MULTIPOLYGON + "(((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)),");
+ b.append("((10.0 2.0, 11.0 2.0, 11.0 3.0, 10.0 3.0, 10.0 2.0)))");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Polygon[]);
+ Polygon[] polygons = (Polygon[])shape;
+ assertEquals(2, polygons.length);
+ assertEquals(new Polygon(new double[] {0.0, 0.0, 1.0, 1.0, 0.0},
+ new double[] {100.0, 101.0, 101.0, 100.0, 100.0}), polygons[0]);
+ assertEquals(new Polygon(new double[] {2.0, 2.0, 3.0, 3.0, 2.0},
+ new double[] {10.0, 11.0, 11.0, 10.0, 10.0}), polygons[1]);
+ }
+
+ /** polygon must be closed */
+ public void testPolygonNotClosed() {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.POLYGON + "((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0))\n");
+
+ IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
+ SimpleWKTShapeParser.parse(b.toString());
+ });
+ assertTrue(expected.getMessage(),
+ expected.getMessage().contains("first and last points of the polygon must be the same (it must close itself)"));
+ }
+
+ /** test simple ENVELOPE (minLon, maxLon, maxLat, minLat) */
+ public void testEnvelope() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.ENVELOPE + "(-180.0, 180.0, 90.0, -90.0)");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Rectangle);
+ Rectangle bbox = (Rectangle)shape;
+ assertEquals(-180d, bbox.minLon, 0);
+ assertEquals(180d, bbox.maxLon, 0);
+ assertEquals(-90d, bbox.minLat, 0);
+ assertEquals(90d, bbox.maxLat, 0);
+ }
+
+ /** test simple geometry collection */
+ public void testGeometryCollection() throws Exception {
+ StringBuilder b = new StringBuilder();
+ b.append(ShapeType.GEOMETRYCOLLECTION + "(");
+ b.append(ShapeType.MULTIPOLYGON + "(((100.0 0.0, 101.0 0.0, 101.0 1.0, 100.0 1.0, 100.0 0.0)),");
+ b.append("((10.0 2.0, 11.0 2.0, 11.0 3.0, 10.0 3.0, 10.0 2.0))),");
+ b.append(ShapeType.POINT + "(101.0 10.0),");
+ b.append(ShapeType.LINESTRING + "(101.0 10.0, 180.0 90.0, -180.0 -90.0),");
+ b.append(ShapeType.ENVELOPE + "(-180.0, 180.0, 90.0, -90.0)");
+ b.append(")");
+ Object shape = SimpleWKTShapeParser.parse(b.toString());
+
+ assertTrue(shape instanceof Object[]);
+ Object[] shapes = (Object[]) shape;
+ assertEquals(4, shapes.length);
+ assertTrue(shapes[0] instanceof Polygon[]);
+ assertTrue(shapes[1] instanceof double[]);
+ assertTrue(shapes[2] instanceof Line);
+ assertTrue(shapes[3] instanceof Rectangle);
+ }
+}
[22/32] lucene-solr:jira/solr-12730: LUCENE-8462: Revert
ant-patch-snowball modifications. This modifications only worked with the
latest version of snowball and were specifically applied to handle the new
Arabic stemmer. This change reverts these modifi
Posted by ab...@apache.org.
LUCENE-8462: Revert ant-patch-snowball modifications.
This modifications only worked with the latest version of snowball and were specifically applied to handle the new Arabic stemmer.
This change reverts these modifications since they are not compatible with the other stemmer which are created from revision 502.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4fa99c20
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4fa99c20
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4fa99c20
Branch: refs/heads/jira/solr-12730
Commit: 4fa99c201493b4c689b68911eb97f4e51ecc9afe
Parents: 6c41945
Author: Jim Ferenczi <ji...@apache.org>
Authored: Fri Oct 26 16:03:42 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Fri Oct 26 16:03:42 2018 +0200
----------------------------------------------------------------------
lucene/analysis/common/README.txt | 2 +
lucene/analysis/common/build.xml | 88 ++++++----------------------------
2 files changed, 17 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa99c20/lucene/analysis/common/README.txt
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/README.txt b/lucene/analysis/common/README.txt
index a9fc5f1..f3382f8 100644
--- a/lucene/analysis/common/README.txt
+++ b/lucene/analysis/common/README.txt
@@ -15,6 +15,8 @@ A few changes has been made to the static Snowball code and compiled stemmers:
If you want to add new stemmers, use the exact revision / Git commit above to generate the Java class, place it
in src/java/org/tartarus/snowball/ext, and finally execute "ant patch-snowball". The latter will change the APIs
of the generated class to make it compatible. Already patched classes are not modified.
+The Arabic stemmer has been generated from https://github.com/snowballstem/snowball/blob/master/algorithms/arabic.sbl
+using the latest version of snowball and patched manually.
IMPORTANT NOTICE ON BACKWARDS COMPATIBILITY!
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4fa99c20/lucene/analysis/common/build.xml
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/build.xml b/lucene/analysis/common/build.xml
index 2064e19..b8eb37a 100644
--- a/lucene/analysis/common/build.xml
+++ b/lucene/analysis/common/build.xml
@@ -17,10 +17,10 @@
limitations under the License.
-->
-<project name="analyzers-common" default="default" xmlns:rsel="antlib:org.apache.tools.ant.types.resources.selectors">
+<project name="analyzers-common" default="default">
<description>
- Analyzers for indexing content in different languages and domains.
+ Analyzers for indexing content in different languages and domains.
</description>
<!-- some files for testing that do not have license headers -->
@@ -88,7 +88,7 @@
<target xmlns:ivy="antlib:org.apache.ivy.ant" name="-resolve-icu4j" unless="icu4j.resolved" depends="ivy-availability-check,ivy-configure">
<loadproperties prefix="ivyversions" srcFile="${common.dir}/ivy-versions.properties"/>
<ivy:cachepath organisation="com.ibm.icu" module="icu4j" revision="${ivyversions./com.ibm.icu/icu4j}"
- inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
+ inline="true" conf="default" transitive="true" pathid="icu4j.classpath"/>
<property name="icu4j.resolved" value="true"/>
</target>
@@ -102,10 +102,10 @@
<target name="gen-tlds" depends="compile-tools">
<java
- classname="org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"
- dir="."
- fork="true"
- failonerror="true">
+ classname="org.apache.lucene.analysis.standard.GenerateJflexTLDMacros"
+ dir="."
+ fork="true"
+ failonerror="true">
<classpath>
<pathelement location="${build.dir}/classes/tools"/>
</classpath>
@@ -117,8 +117,8 @@
<target name="compile-tools" depends="common.compile-tools">
<compile
- srcdir="src/tools/java"
- destdir="${build.dir}/classes/tools">
+ srcdir="src/tools/java"
+ destdir="${build.dir}/classes/tools">
<classpath refid="classpath"/>
</compile>
</target>
@@ -128,71 +128,13 @@
<target name="regenerate" depends="jflex,unicode-data"/>
<target name="patch-snowball" description="Patches all snowball programs in '${snowball.programs.dir}' to make them work with MethodHandles">
- <fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
-
- <replaceregexp match="^public class \w+Stemmer\b" replace="@SuppressWarnings("unused") \0" flags="m" encoding="UTF-8">
- <restrict>
- <fileset refid="snowball.programs"/>
- <rsel:not>
- <rsel:contains text="patched"/>
- </rsel:not>
- </restrict>
- </replaceregexp>
-
- <replaceregexp match="new Among\(([^,]*,[^,]*,[^,]*?)(?=\))" replace="\0, "", methodObject" flags="g" encoding="UTF-8">
- <restrict>
- <fileset refid="snowball.programs"/>
- <rsel:not>
- <rsel:contains text="patched"/>
- </rsel:not>
- </restrict>
- </replaceregexp>
-
- <replaceregexp match="(new Among\([^,]*,[^,]*,[^,]*,[^,]*,)[^,]*?(?=\))" replace="\1 methodObject" flags="g" encoding="UTF-8">
- <restrict>
- <fileset refid="snowball.programs"/>
- <rsel:not>
- <rsel:contains text="patched"/>
- </rsel:not>
- </restrict>
- </replaceregexp>
-
- <replaceregexp match="(?:find_among(?:|_b)\()(.*?)(?=\))" replace="\0, \1.length" flags="g" encoding="UTF-8">
- <restrict>
+ <fileset id="snowball.programs" dir="${snowball.programs.dir}" includes="*Stemmer.java"/>
+ <replaceregexp match="^public class \w+Stemmer\b" replace="@SuppressWarnings("unused") \0" flags="m" encoding="UTF-8">
<fileset refid="snowball.programs"/>
- <rsel:not>
- <rsel:contains text="patched"/>
- </rsel:not>
- </restrict>
- </replaceregexp>
-
- <replaceregexp match="current" replace="getCurrent()" flags="g" encoding="UTF-8">
- <restrict>
+ </replaceregexp>
+ <replaceregexp match="private final static \w+Stemmer methodObject\b.*$" replace="/* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();" flags="m" encoding="UTF-8">
<fileset refid="snowball.programs"/>
- <rsel:not>
- <rsel:contains text="patched"/>
- </rsel:not>
- </restrict>
- </replaceregexp>
-
- <replaceregexp match="(?:eq_s(?:|_b)\()(.*?)(?=\))" replace="\0.length(),\1" flags="g" encoding="UTF-8">
- <restrict>
- <fileset refid="snowball.programs"/>
- <rsel:not>
- <rsel:contains text="patched"/>
- </rsel:not>
- </restrict>
- </replaceregexp>
-
- <replaceregexp match="private static final long serialVersionUID(.*)" replace="private static final long serialVersionUID = 1L; ${line.separator}${line.separator} /* patched */ private static final java.lang.invoke.MethodHandles.Lookup methodObject = java.lang.invoke.MethodHandles.lookup();" flags="m" encoding="UTF-8">
- <restrict>
- <fileset refid="snowball.programs"/>
- <rsel:not>
- <rsel:contains text="patched"/>
- </rsel:not>
- </restrict>
- </replaceregexp>
-
- <fixcrlf srcdir="${snowball.programs.dir}" includes="*Stemmer.java" tab="remove" tablength="2" encoding="UTF-8" javafiles="yes" fixlast="yes"/>
+ </replaceregexp>
+ <fixcrlf srcdir="${snowball.programs.dir}" includes="*Stemmer.java" tab="remove" tablength="2" encoding="UTF-8" javafiles="yes" fixlast="yes"/>
</target>
</project>
[05/32] lucene-solr:jira/solr-12730: SOLR-12905:
MultiSolrCloudTestCase now clears static clusterId2cluster in @AfterClass
Posted by ab...@apache.org.
SOLR-12905: MultiSolrCloudTestCase now clears static clusterId2cluster in @AfterClass
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7fc91dea
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7fc91dea
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7fc91dea
Branch: refs/heads/jira/solr-12730
Commit: 7fc91deaba25ae91bc9b2c4ae2875fc74c2c19aa
Parents: cebf703
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:25:10 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:25:10 2018 +0100
----------------------------------------------------------------------
.../src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7fc91dea/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
index 4562c3e..eb0a677 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java
@@ -102,6 +102,7 @@ public abstract class MultiSolrCloudTestCase extends SolrTestCaseJ4 {
for (MiniSolrCloudCluster cluster : clusterId2cluster.values()) {
cluster.shutdown();
}
+ clusterId2cluster.clear();
}
}
[26/32] lucene-solr:jira/solr-12730: SOLR-12828: Update CHANGES.txt
Posted by ab...@apache.org.
SOLR-12828: Update CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1abc38b4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1abc38b4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1abc38b4
Branch: refs/heads/jira/solr-12730
Commit: 1abc38b40ecee661f6f7643cc66975b05babd531
Parents: a298802
Author: Joel Bernstein <jb...@apache.org>
Authored: Sun Oct 28 13:30:43 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sun Oct 28 13:30:43 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1abc38b4/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 1e0e530..febff90 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -160,6 +160,8 @@ New Features
* SOLR-12754: The UnifiedHighlighter has a new hl.weightMatches param defaulting to false (will be true in 8.0). It's
the highest query accuracy mode, and furthermore phrase queries are highlighted as one. (David Smiley)
+* SOLR-12828: Add oscillate Stream Evaluator to support sine wave analysis. (Joel Bernstein)
+
Other Changes
----------------------
[14/32] lucene-solr:jira/solr-12730: SOLR-12879 - added missing test
for min_hash qp to QueryEqualityTest
Posted by ab...@apache.org.
SOLR-12879 - added missing test for min_hash qp to QueryEqualityTest
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/26e14986
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/26e14986
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/26e14986
Branch: refs/heads/jira/solr-12730
Commit: 26e14986af7aa60b72940f611f63b2a50fbb9980
Parents: 3e87499
Author: Tommaso Teofili <te...@adobe.com>
Authored: Thu Oct 25 09:57:16 2018 +0200
Committer: Tommaso Teofili <te...@adobe.com>
Committed: Thu Oct 25 09:57:16 2018 +0200
----------------------------------------------------------------------
.../src/test-files/solr/collection1/conf/schema15.xml | 10 ++++++++++
.../test/org/apache/solr/search/QueryEqualityTest.java | 12 ++++++++++++
2 files changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26e14986/solr/core/src/test-files/solr/collection1/conf/schema15.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
index 361344f..c8328ac 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
@@ -445,6 +445,14 @@
<!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->
<fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
+ <fieldType name="text_min_hash" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.ICUTokenizerFactory"/>
+ <filter class="solr.ICUFoldingFilterFactory"/>
+ <filter class="solr.ShingleFilterFactory" minShingleSize="5" outputUnigrams="false" outputUnigramsIfNoShingles="false" maxShingleSize="5" tokenSeparator=" "/>
+ <filter class="org.apache.lucene.analysis.minhash.MinHashFilterFactory" bucketCount="512" hashSetSize="1" hashCount="1"/>
+ </analyzer>
+ </fieldType>
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
<field name="signatureField" type="string" indexed="true" stored="false"/>
@@ -554,6 +562,8 @@
<field name="multi_int_with_docvals" type="tint" multiValued="true" docValues="true" indexed="false"/>
+ <field name="min_hash_analysed" type="text_min_hash" multiValued="false" indexed="true" stored="false"/>
+
<dynamicField name="*_coordinate" type="tdouble" indexed="true" stored="false"/>
<dynamicField name="*_sI" type="string" indexed="true" stored="false"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/26e14986/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index be7fe91..cabe497 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -327,6 +327,18 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
}
}
+ public void testMinHash() throws Exception {
+ SolrQueryRequest req = req("q","apache lucene is a search library",
+ "df", "min_hash_analyzed");
+
+ try {
+ assertQueryEquals("min_hash", req,
+ "{!min_hash field=\"min_hash_analysed\"}apache lucene is a search library");
+ } finally {
+ req.close();
+ }
+ }
+
public void testQueryNested() throws Exception {
SolrQueryRequest req = req("df", "foo_s");
try {
[25/32] lucene-solr:jira/solr-12730: SOLR-12868: test was making
wrong assumption
Posted by ab...@apache.org.
SOLR-12868: test was making wrong assumption
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a2988025
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a2988025
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a2988025
Branch: refs/heads/jira/solr-12730
Commit: a298802516ad3e4ff7445ea0ccbbc06b150042fd
Parents: 0c8675d
Author: Noble Paul <no...@apache.org>
Authored: Sat Oct 27 11:10:59 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Sat Oct 27 11:11:54 2018 +1100
----------------------------------------------------------------------
.../test/org/apache/solr/client/solrj/request/TestV2Request.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a2988025/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
index 4aa1b64..0cafc63 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestV2Request.java
@@ -102,7 +102,7 @@ public class TestV2Request extends SolrCloudTestCase {
assertSuccess(client, new V2Request.Builder("/c/test").withMethod(SolrRequest.METHOD.DELETE).build());
NamedList<Object> res = client.request(new V2Request.Builder("/c").build());
List collections = (List) res.get("collections");
- assertEquals(0, collections.size());
+ assertFalse( collections.contains("test"));
}
[16/32] lucene-solr:jira/solr-12730: SOLR-5004: put param names and
values in monospace
Posted by ab...@apache.org.
SOLR-5004: put param names and values in monospace
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/93ccdce5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/93ccdce5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/93ccdce5
Branch: refs/heads/jira/solr-12730
Commit: 93ccdce57c85fa652efa6b328344a267ba3319fd
Parents: 7952cec
Author: Cassandra Targett <ct...@apache.org>
Authored: Thu Oct 25 11:06:25 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu Oct 25 11:06:25 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/collections-api.adoc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/93ccdce5/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index 673cb1b..ec8517d 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -288,9 +288,9 @@ This parameter can be used to split a shard using a route key such that all docu
For example, suppose `split.key=A!` hashes to the range `12-15` and belongs to shard 'shard1' with range `0-20`. Splitting by this route key would yield three sub-shards with ranges `0-11`, `12-15` and `16-20`. Note that the sub-shard with the hash range of the route key may also contain documents for other route keys whose hash ranges overlap.
`numSubShards`::
-The number of sub-shards to split the parent shard into. Allowed values for this are in the range of 2-8 and defaults to 2.
+The number of sub-shards to split the parent shard into. Allowed values for this are in the range of `2`-`8` and defaults to `2`.
+
-This parameter can only be used when ranges or split.key are not specified.
+This parameter can only be used when `ranges` or `split.key` are not specified.
`splitMethod`::
Currently two methods of shard splitting are supported:
[10/32] lucene-solr:jira/solr-12730: SOLR-12827: remove monospace in
headings; correct formatting for heading level indicator;
remove redundant "Example #" in the example titles
Posted by ab...@apache.org.
SOLR-12827: remove monospace in headings; correct formatting for heading level indicator; remove redundant "Example #" in the example titles
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/16ee8476
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/16ee8476
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/16ee8476
Branch: refs/heads/jira/solr-12730
Commit: 16ee847663d67d1b003228d809dac6d5e30f2682
Parents: b72f05d
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Oct 24 15:04:33 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Oct 24 15:04:33 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/collections-api.adoc | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/16ee8476/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index 47a09d0..673cb1b 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -1208,13 +1208,11 @@ http://localhost:8983/solr/admin/collections?action=CLUSTERPROP&name=urlScheme&v
</response>
----
-=== Deeply Nested Cluster Properties ===
+=== Setting Cluster-Wide Defaults
-==== `defaults` ====
-It is possible to set cluster-wide default values for certain attributes of a collection.
+It is possible to set cluster-wide default values for certain attributes of a collection, using the `defaults` parameter.
-
-*Example 1: Set/update default values*
+*Set/update default values*
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
@@ -1232,7 +1230,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
}' http://localhost:8983/api/cluster
----
-*Example 2: Unset the value of `nrtReplicas` alone*
+*Unset the only value of `nrtReplicas`*
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
@@ -1247,7 +1245,7 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
}' http://localhost:8983/api/cluster
----
-*Example 2: Unset all values in `defaults`*
+*Unset all values in `defaults`*
[source]
----
curl -X POST -H 'Content-type:application/json' --data-binary '
@@ -1256,10 +1254,10 @@ curl -X POST -H 'Content-type:application/json' --data-binary '
}' http://localhost:8983/api/cluster
----
-NOTE: Until Solr 7.5, cluster properties supported a "collectionDefaults" key which is now deprecated. Using the API
-structure for Solr 7.4 or Solr 7.5 with "collectionDefaults" will continue to work but the format of the properties
-will automatically be converted to the new nested structure. Support for the "collectionDefaults" key will be
-removed in Solr 9.
+NOTE: Until Solr 7.5, cluster properties supported a `collectionDefaults` key which is now deprecated and
+replaced with `defaults`. Using the `collectionDefaults` parameter in Solr 7.4 or 7.5 will continue to work
+ but the format of the properties will automatically be converted to the new nested structure.
+Support for the "collectionDefaults" key will be removed in Solr 9.
[[collectionprop]]
== COLLECTIONPROP: Collection Properties
[29/32] lucene-solr:jira/solr-12730: SOLR-12840: Update CHANGES.txt
Posted by ab...@apache.org.
SOLR-12840: Update CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/416cc163
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/416cc163
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/416cc163
Branch: refs/heads/jira/solr-12730
Commit: 416cc163bad78c3abd516c805ce4e3f764421f02
Parents: f8ffc1a
Author: Joel Bernstein <jb...@apache.org>
Authored: Sun Oct 28 14:26:57 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sun Oct 28 14:26:57 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/416cc163/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3b9092f..a21a949 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -166,6 +166,8 @@ New Features
* SOLR-12811: Add enclosingDisk and associated geometric Stream Evaluators. (Joel Bernstein)
+* SOLR-12840: Add pairSort Stream Evaluator (Joel Bernstein)
+
Other Changes
----------------------
[13/32] lucene-solr:jira/solr-12730: Tweak test-help example for
running tests within a package
Posted by ab...@apache.org.
Tweak test-help example for running tests within a package
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3e87499d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3e87499d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3e87499d
Branch: refs/heads/jira/solr-12730
Commit: 3e87499d72131f2ee54b03fa83ed177413fd3cbc
Parents: 71988c7
Author: Varun Thacker <va...@apache.org>
Authored: Wed Oct 24 15:33:23 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Oct 24 15:33:23 2018 -0700
----------------------------------------------------------------------
lucene/common-build.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e87499d/lucene/common-build.xml
----------------------------------------------------------------------
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index 1a6839b..c9463ea 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -1349,7 +1349,7 @@ ant test "-Dtests.class=*.ClassName"
ant test -Dtestcase=ClassName
# Run all tests in a package and sub-packages
-ant test "-Dtests.class=org.apache.lucene.package.*"
+ant test "-Dtests.class=org.apache.lucene.package.Test*|org.apache.lucene.package.*Test"
# Run any test methods that contain 'esi' (like: ...r*esi*ze...).
ant test "-Dtests.method=*esi*"
[08/32] lucene-solr:jira/solr-12730: SOLR-12772: Correct dotProduct
parameters+syntax documentation
Posted by ab...@apache.org.
SOLR-12772: Correct dotProduct parameters+syntax documentation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e62fe459
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e62fe459
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e62fe459
Branch: refs/heads/jira/solr-12730
Commit: e62fe459834a287cccf01f25ec03b8764f7a5af7
Parents: ab14cc9
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Oct 24 19:49:35 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Oct 24 19:49:35 2018 +0100
----------------------------------------------------------------------
solr/solr-ref-guide/src/stream-evaluator-reference.adoc | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e62fe459/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-evaluator-reference.adoc b/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
index a88915f..bb13e77 100644
--- a/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-evaluator-reference.adoc
@@ -626,11 +626,12 @@ div(fieldA,add(fieldA,fieldB)) // fieldA / (fieldA + fieldB)
== dotProduct
-The `dotProduct` function returns the https://en.wikipedia.org/wiki/Dot_product[dotproduct] of a numeric array.
+The `dotProduct` function returns the https://en.wikipedia.org/wiki/Dot_product[dotproduct] of two numeric arrays.
=== dotProduct Parameters
* `numeric array`
+* `numeric array`
=== dotProduct Returns
@@ -639,7 +640,7 @@ A number.
=== dotProduct Syntax
[source,text]
-dotProduct(numericArray)
+dotProduct(numericArray, numericArray)
== ebeAdd