You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/09/17 09:43:47 UTC
[01/47] lucene-solr:jira/solr-12709: SOLR-11861 baseConfigSet default
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-12709 4a9176b9e -> 8837c2d69
SOLR-11861 baseConfigSet default
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b1b09639
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b1b09639
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b1b09639
Branch: refs/heads/jira/solr-12709
Commit: b1b0963947503dac20e84950ca30511e9aace9e1
Parents: e0eb7ba
Author: David Smiley <ds...@apache.org>
Authored: Tue Sep 4 14:02:46 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Sep 4 14:02:46 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 ++
.../cloud/OverseerConfigSetMessageHandler.java | 30 +++++++++++---------
.../solr/handler/admin/ConfigSetsHandler.java | 20 +++++++------
.../apache/solr/cloud/TestConfigSetsAPI.java | 14 ++++-----
solr/solr-ref-guide/src/configsets-api.adoc | 2 +-
.../solrj/request/ConfigSetAdminRequest.java | 5 ++--
.../apispec/cluster.configs.Commands.json | 2 +-
.../request/TestConfigSetAdminRequest.java | 2 --
8 files changed, 41 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c6e89b1..56f5668 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -198,6 +198,9 @@ New Features
* SOLR-12715: NodeAddedTrigger should support adding replicas to new nodes by setting preferredOperation=addreplica.
(shalin)
+* SOLR-11861: When creating a configSet via the API, the "baseConfigSet" parameter now defaults to "_default".
+ (Amrit Sarkar, David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
index 2f2859f..6812971 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
@@ -47,6 +47,7 @@ import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;
+import static org.apache.solr.handler.admin.ConfigSetsHandlerApi.DEFAULT_CONFIGSET_NAME;
/**
* A {@link OverseerMessageHandler} that handles ConfigSets API related
@@ -95,10 +96,10 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
if (!operation.startsWith(CONFIGSETS_ACTION_PREFIX)) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Operation does not contain proper prefix: " + operation
- + " expected: " + CONFIGSETS_ACTION_PREFIX);
+ + " expected: " + CONFIGSETS_ACTION_PREFIX);
}
operation = operation.substring(CONFIGSETS_ACTION_PREFIX.length());
- log.info("OverseerConfigSetMessageHandler.processMessage : "+ operation + " , "+ message.toString());
+ log.info("OverseerConfigSetMessageHandler.processMessage : " + operation + " , " + message.toString());
ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);
if (action == null) {
@@ -120,7 +121,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
if (configSetName == null) {
SolrException.log(log, "Operation " + operation + " failed", e);
- } else {
+ } else {
SolrException.log(log, "ConfigSet: " + configSetName + " operation: " + operation
+ " failed", e);
}
@@ -128,7 +129,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
results.add("Operation " + operation + " caused exception:", e);
SimpleOrderedMap nl = new SimpleOrderedMap();
nl.add("msg", e.getMessage());
- nl.add("rspCode", e instanceof SolrException ? ((SolrException)e).code() : -1);
+ nl.add("rspCode", e instanceof SolrException ? ((SolrException) e).code() : -1);
results.add("exception", nl);
}
return new OverseerSolrResponse(results);
@@ -210,16 +211,20 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
operation = operation.substring(CONFIGSETS_ACTION_PREFIX.length());
ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);
if (action == CREATE) {
- return message.getStr(BASE_CONFIGSET);
+ String baseConfigSetName = message.getStr(BASE_CONFIGSET);
+ if (baseConfigSetName == null || baseConfigSetName.length() == 0) {
+ baseConfigSetName = DEFAULT_CONFIGSET_NAME;
+ }
+ return baseConfigSetName;
}
}
return null;
}
private NamedList getConfigSetProperties(String path) throws IOException {
- byte [] oldPropsData = null;
+ byte[] oldPropsData = null;
try {
- oldPropsData = zkStateReader.getZkClient().getData(path, null, null, true);
+ oldPropsData = zkStateReader.getZkClient().getData(path, null, null, true);
} catch (KeeperException.NoNodeException e) {
log.info("no existing ConfigSet properties found");
} catch (KeeperException | InterruptedException e) {
@@ -283,10 +288,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
}
- String baseConfigSetName = message.getStr(BASE_CONFIGSET);
- if (baseConfigSetName == null || baseConfigSetName.length() == 0) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Base ConfigSet name not specified");
- }
+ String baseConfigSetName = message.getStr(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);
ZkConfigManager configManager = new ZkConfigManager(zkStateReader.getZkClient());
if (configManager.configExists(configSetName)) {
@@ -303,7 +305,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
Map<String, Object> props = getNewProperties(message);
if (props != null) {
// read the old config properties and do a merge, if necessary
- NamedList oldProps = getConfigSetProperties(getPropertyPath(baseConfigSetName,propertyPath));
+ NamedList oldProps = getConfigSetProperties(getPropertyPath(baseConfigSetName, propertyPath));
if (oldProps != null) {
mergeOldProperties(props, oldProps);
}
@@ -317,7 +319,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
try {
zkStateReader.getZkClient().makePath(
getPropertyPath(configSetName, propertyPath),
- propertyData, CreateMode.PERSISTENT, null, false, true);
+ propertyData, CreateMode.PERSISTENT, null, false, true);
} catch (KeeperException | InterruptedException e) {
throw new IOException("Error writing new properties",
SolrZkClient.checkInterrupted(e));
@@ -365,7 +367,7 @@ public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
NamedList properties = getConfigSetProperties(getPropertyPath(configSetName, propertyPath));
if (properties != null) {
Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);
- boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
+ boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
if (!force && isImmutableConfigSet) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
index f8fc533..fbd0fb6 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
@@ -66,6 +66,7 @@ import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;
import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.DELETE;
import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.LIST;
+import static org.apache.solr.handler.admin.ConfigSetsHandlerApi.DEFAULT_CONFIGSET_NAME;
/**
* A {@link org.apache.solr.request.SolrRequestHandler} for ConfigSets API requests.
@@ -73,7 +74,7 @@ import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.LIST
public class ConfigSetsHandler extends RequestHandlerBase implements PermissionNameProvider {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected final CoreContainer coreContainer;
- public static long DEFAULT_ZK_TIMEOUT = 300*1000;
+ public static long DEFAULT_ZK_TIMEOUT = 300 * 1000;
private final ConfigSetsHandlerApi configSetsHandlerApi = new ConfigSetsHandlerApi(this);
/**
@@ -90,11 +91,11 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
if (coreContainer == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Core container instance missing");
+ "Core container instance missing");
}
// Make sure that the core is ZKAware
- if(!coreContainer.isZooKeeperAware()) {
+ if (!coreContainer.isZooKeeperAware()) {
throw new SolrException(ErrorCode.BAD_REQUEST,
"Solr instance is not running in SolrCloud mode.");
}
@@ -195,7 +196,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
}
private void createZkNodeIfNotExistsAndSetData(SolrZkClient zkClient,
- String filePathInZk, byte[] data) throws Exception {
+ String filePathInZk, byte[] data) throws Exception {
if (!zkClient.exists(filePathInZk, true)) {
zkClient.create(filePathInZk, data, CreateMode.PERSISTENT, true);
} else {
@@ -204,7 +205,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
}
private void handleResponse(String operation, ZkNodeProps m,
- SolrQueryResponse rsp, long timeout) throws KeeperException, InterruptedException {
+ SolrQueryResponse rsp, long timeout) throws KeeperException, InterruptedException {
long time = System.nanoTime();
QueueEvent event = coreContainer.getZkController()
@@ -216,7 +217,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
SimpleOrderedMap exp = (SimpleOrderedMap) response.getResponse().get("exception");
if (exp != null) {
Integer code = (Integer) exp.get("rspCode");
- rsp.setException(new SolrException(code != null && code != -1 ? ErrorCode.getErrorCode(code) : ErrorCode.SERVER_ERROR, (String)exp.get("msg")));
+ rsp.setException(new SolrException(code != null && code != -1 ? ErrorCode.getErrorCode(code) : ErrorCode.SERVER_ERROR, (String) exp.get("msg")));
}
} else {
if (System.nanoTime() - time >= TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {
@@ -236,7 +237,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
}
private static Map<String, Object> copyPropertiesWithPrefix(SolrParams params, Map<String, Object> props, String prefix) {
- Iterator<String> iter = params.getParameterNamesIterator();
+ Iterator<String> iter = params.getParameterNamesIterator();
while (iter.hasNext()) {
String param = iter.next();
if (param.startsWith(prefix)) {
@@ -254,6 +255,7 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
public String getDescription() {
return "Manage SolrCloud ConfigSets";
}
+
@Override
public Category getCategory() {
return Category.ADMIN;
@@ -263,7 +265,9 @@ public class ConfigSetsHandler extends RequestHandlerBase implements PermissionN
CREATE_OP(CREATE) {
@Override
Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {
- Map<String, Object> props = CollectionsHandler.copy(req.getParams().required(), null, NAME, BASE_CONFIGSET);
+ String baseConfigSetName = req.getParams().get(BASE_CONFIGSET, DEFAULT_CONFIGSET_NAME);
+ Map<String, Object> props = CollectionsHandler.copy(req.getParams().required(), null, NAME);
+ props.put(BASE_CONFIGSET, baseConfigSetName);
return copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX + ".");
}
},
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index 71a0c2f..f513645 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -16,11 +16,6 @@
*/
package org.apache.solr.cloud;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.BASE_CONFIGSET;
-import static org.apache.solr.common.params.CommonParams.NAME;
-import static org.apache.solr.core.ConfigSetProperties.DEFAULT_FILENAME;
-
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
@@ -99,6 +94,10 @@ import org.noggit.ObjectBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.core.ConfigSetProperties.DEFAULT_FILENAME;
+
/**
* Simple ConfigSets API tests on user errors and simple success cases.
*/
@@ -136,9 +135,8 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 {
CreateNoErrorChecking create = new CreateNoErrorChecking();
verifyException(solrClient, create, NAME);
- // no base ConfigSet name
+ // set ConfigSet
create.setConfigSetName("configSetName");
- verifyException(solrClient, create, BASE_CONFIGSET);
// ConfigSet already exists
Create alreadyExists = new Create();
@@ -156,7 +154,7 @@ public class TestConfigSetsAPI extends SolrTestCaseJ4 {
@Test
public void testCreate() throws Exception {
// no old, no new
- verifyCreate("baseConfigSet1", "configSet1", null, null);
+ verifyCreate(null, "configSet1", null, null);
// no old, new
verifyCreate("baseConfigSet2", "configSet2",
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/solr-ref-guide/src/configsets-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/configsets-api.adoc b/solr/solr-ref-guide/src/configsets-api.adoc
index 8874f21..1fbab30 100644
--- a/solr/solr-ref-guide/src/configsets-api.adoc
+++ b/solr/solr-ref-guide/src/configsets-api.adoc
@@ -130,7 +130,7 @@ name::
The configset to be created. This parameter is required.
baseConfigSet::
-The name of the configset to copy as a base. This parameter is required.
+The name of the configset to copy as a base. This defaults to `_default`
configSetProp._property_=_value_::
A configset property from the base configset to override in the copied configset.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
index 6916877..3db8589 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java
@@ -135,10 +135,9 @@ public abstract class ConfigSetAdminRequest
@Override
public SolrParams getParams() {
ModifiableSolrParams params = new ModifiableSolrParams(super.getParams());
- if (baseConfigSetName == null) {
- throw new RuntimeException( "no Base ConfigSet specified!" );
+ if (baseConfigSetName != null) {
+ params.set("baseConfigSet", baseConfigSetName);
}
- params.set("baseConfigSet", baseConfigSetName);
if (properties != null) {
for (Map.Entry entry : properties.entrySet()) {
params.set(PROPERTY_PREFIX + "." + entry.getKey().toString(),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/solrj/src/resources/apispec/cluster.configs.Commands.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/resources/apispec/cluster.configs.Commands.json b/solr/solrj/src/resources/apispec/cluster.configs.Commands.json
index 3792686..f8b3b3a 100644
--- a/solr/solrj/src/resources/apispec/cluster.configs.Commands.json
+++ b/solr/solrj/src/resources/apispec/cluster.configs.Commands.json
@@ -28,7 +28,7 @@
"additionalProperties" : true
}
},
- "required" : ["name", "baseConfigSet"]
+ "required" : ["name"]
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b1b09639/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
index a3409bd..50e4f5d 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java
@@ -38,8 +38,6 @@ public class TestConfigSetAdminRequest extends SolrTestCaseJ4 {
ConfigSetAdminRequest.Create create = new ConfigSetAdminRequest.Create();
verifyException(create, "ConfigSet");
create.setConfigSetName("name");
- verifyException(create, "Base ConfigSet");
- create.setBaseConfigSetName("baseConfigSet");
create.getParams();
}
[18/47] lucene-solr:jira/solr-12709: SOLR-12701: Fix knnSearch
RefGuide
Posted by ab...@apache.org.
SOLR-12701: Fix knnSearch RefGuide
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4a15482
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4a15482
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4a15482
Branch: refs/heads/jira/solr-12709
Commit: b4a1548248cb233459069bdeddde8640e6425e82
Parents: 4f05588
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Sep 5 14:09:12 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Sep 5 14:14:59 2018 -0400
----------------------------------------------------------------------
.../src/stream-source-reference.adoc | 24 +++++++++-----------
1 file changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4a15482/solr/solr-ref-guide/src/stream-source-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-source-reference.adoc b/solr/solr-ref-guide/src/stream-source-reference.adoc
index 744193e..e9185e0 100644
--- a/solr/solr-ref-guide/src/stream-source-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-source-reference.adoc
@@ -214,36 +214,34 @@ features(collection1,
The `nodes` function provides breadth-first graph traversal. For details, see the section <<graph-traversal.adoc#graph-traversal,Graph Traversal>>.
-== knn
+== knnSearch
-The `knn` function returns the K nearest neighbors for a document based on text similarity. Under the covers the `knn` function
+The `knnSearch` function returns the K nearest neighbors for a document based on text similarity. Under the covers the `knnSearch` function
use the More Like This query parser plugin.
-=== knn Parameters
+=== knnSearch Parameters
* `collection`: (Mandatory) The collection to perform the search in.
* `id`: (Mandatory) The id of the source document to begin the knn search from.
* `qf`: (Mandatory) The query field used to compare documents.
* `k`: (Mandatory) The number of nearest neighbors to return.
* `fl`: (Mandatory) The field list to return.
-* `mintf`: (Optional) The minimum number of occurrences of the term in the source document to be included in the search.
-* `maxtf`: (Optional) The maximum number of occurrences of the term in the source document to be included in the search.
* `mindf`: (Optional) The minimum number of occurrences in the corpus to be included in the search.
* `maxdf`: (Optional) The maximum number of occurrences in the corpus to be included in the search.
* `minwl`: (Optional) The minimum world length of to be included in the search.
* `maxwl`: (Optional) The maximum world length of to be included in the search.
-=== knn Syntax
+=== knnSearch Syntax
[source,text]
----
-knn(collection1,
- id="doc1",
- qf="text_field",
- k="10",
- fl="id, title",
- mintf="3",
- maxdf="10000000")
+knnSearch(collection1,
+ id="doc1",
+ qf="text_field",
+ k="10",
+ fl="id, title",
+ mintf="3",
+ maxdf="10000000")
----
== model
[17/47] lucene-solr:jira/solr-12709: SOLR-12625: fix typos..."an" ->
"and"
Posted by ab...@apache.org.
SOLR-12625: fix typos..."an" -> "and"
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4f055880
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4f055880
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4f055880
Branch: refs/heads/jira/solr-12709
Commit: 4f0558800786c087391e04828d5e38d7ca7693dc
Parents: e893f34
Author: Cassandra Targett <ct...@apache.org>
Authored: Tue Sep 4 21:39:23 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Sep 5 12:47:08 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/docvalues.adoc | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4f055880/solr/solr-ref-guide/src/docvalues.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/docvalues.adoc b/solr/solr-ref-guide/src/docvalues.adoc
index 1502302..a0e61ef 100644
--- a/solr/solr-ref-guide/src/docvalues.adoc
+++ b/solr/solr-ref-guide/src/docvalues.adoc
@@ -44,16 +44,16 @@ DocValues are only available for specific field types. The types chosen determin
* `StrField`, and `UUIDField`:
** If the field is single-valued (i.e., multi-valued is false), Lucene will use the `SORTED` type.
-** If the field is multi-valued, Lucene will use the `SORTED_SET` type. Entries are kept in sorted order an duplicates are removed.
+** If the field is multi-valued, Lucene will use the `SORTED_SET` type. Entries are kept in sorted order and duplicates are removed.
* `BoolField`:
** If the field is single-valued (i.e., multi-valued is false), Lucene will use the `SORTED` type.
-** If the field is multi-valued, Lucene will use the `SORTED_SET` type. Entries are kept in sorted order an duplicates are removed.
+** If the field is multi-valued, Lucene will use the `SORTED_SET` type. Entries are kept in sorted order and duplicates are removed.
* Any `*PointField` Numeric or Date fields, `EnumFieldType`, and `CurrencyFieldType`:
** If the field is single-valued (i.e., multi-valued is false), Lucene will use the `NUMERIC` type.
** If the field is multi-valued, Lucene will use the `SORTED_NUMERIC` type. Entries are kept in sorted order and duplicates are kept.
* Any of the deprecated `Trie*` Numeric or Date fields, `EnumField` and `CurrencyField`:
** If the field is single-valued (i.e., multi-valued is false), Lucene will use the `NUMERIC` type.
-** If the field is multi-valued, Lucene will use the `SORTED_SET` type. Entries are kept in sorted order an duplicates are removed.
+** If the field is multi-valued, Lucene will use the `SORTED_SET` type. Entries are kept in sorted order and duplicates are removed.
These Lucene types are related to how the {lucene-javadocs}/core/org/apache/lucene/index/DocValuesType.html[values are sorted and stored].
@@ -83,7 +83,7 @@ When `useDocValuesAsStored="false"`, non-stored DocValues fields can still be ex
In cases where the query is returning _only_ docValues fields performance may improve since returning stored fields requires disk reads and decompression whereas returning docValues fields in the fl list only requires memory access.
-When retrieving fields from their docValues form (using the <<exporting-result-sets.adoc#exporting-result-sets,/export handler>>, <<streaming-expressions.adoc#streaming-expressions,streaming expressions>> or if the field is requested in the `fl` parameter), two important differences between regular stored fields and docValues fields must be understood:
+When retrieving fields from their docValues form (such as when using the <<exporting-result-sets.adoc#exporting-result-sets,/export handler>>, <<streaming-expressions.adoc#streaming-expressions,streaming expressions>> or if the field is requested in the `fl` parameter), two important differences between regular stored fields and docValues fields must be understood:
1. Order is _not_ preserved. When retrieving stored fields, the insertion order is the return order. For docValues, it is the _sorted_ order.
-2. For field types using `SORTED_SET` (see above), multiple identical entries are collapsed into a single value. Thus if I insert values 4, 5, 2, 4, 1, my return will be 1, 2, 4, 5.
+2. For field types using `SORTED_SET` (see above), multiple identical entries are collapsed into a single value. Thus if values 4, 5, 2, 4, 1 are inserted, the values returned will be 1, 2, 4, 5.
[03/47] lucene-solr:jira/solr-12709: LUCENE-8476: Optimizations in
UserDictionary (KoreanAnalyzer)
Posted by ab...@apache.org.
LUCENE-8476: Optimizations in UserDictionary (KoreanAnalyzer)
Signed-off-by: Namgyu Kim <kn...@gmail.com>
Signed-off-by: Jim Ferenczi <ji...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/97ccbc73
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/97ccbc73
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/97ccbc73
Branch: refs/heads/jira/solr-12709
Commit: 97ccbc734b004f551383f9c19e1840635fedfdf5
Parents: 3b1a335
Author: Namgyu Kim <kn...@gmail.com>
Authored: Wed Sep 5 00:12:10 2018 +0900
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Wed Sep 5 11:35:55 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../org/apache/lucene/analysis/ko/dict/UserDictionary.java | 6 ++----
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/97ccbc73/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c81ae61..40d1c2e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -316,6 +316,9 @@ Other:
* LUCENE-765: Improved org.apache.lucene.index javadocs. (Mike Sokolov)
+* LUCENE-8476: Remove redundant nullity check and switch to optimized List.sort in the
+ Korean's user dictionary. (Namgyu Kim)
+
======================= Lucene 7.4.1 =======================
Bug Fixes:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/97ccbc73/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
index c5378a9..539b9e7 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
@@ -20,7 +20,6 @@ import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -81,8 +80,7 @@ public final class UserDictionary implements Dictionary {
private UserDictionary(List<String> entries) throws IOException {
final CharacterDefinition charDef = CharacterDefinition.getInstance();
- Collections.sort(entries,
- Comparator.comparing(e -> e.split("\\s+")[0]));
+ entries.sort(Comparator.comparing(e -> e.split("\\s+")[0]));
PositiveIntOutputs fstOutput = PositiveIntOutputs.getSingleton();
Builder<Long> fstBuilder = new Builder<>(FST.INPUT_TYPE.BYTE2, fstOutput);
@@ -95,7 +93,7 @@ public final class UserDictionary implements Dictionary {
for (String entry : entries) {
String[] splits = entry.split("\\s+");
String token = splits[0];
- if (lastToken != null && token.equals(lastToken)) {
+ if (token.equals(lastToken)) {
continue;
}
char lastChar = entry.charAt(entry.length()-1);
[21/47] lucene-solr:jira/solr-12709: LUCENE-8481: Javadocs should no
longer reference RAMDirectory.
Posted by ab...@apache.org.
LUCENE-8481: Javadocs should no longer reference RAMDirectory.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/922295a9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/922295a9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/922295a9
Branch: refs/heads/jira/solr-12709
Commit: 922295a94d537c32681c1d9af4d751e18efe6b4d
Parents: 89d598e
Author: Dawid Weiss <dw...@apache.org>
Authored: Thu Sep 6 09:25:57 2018 +0200
Committer: Dawid Weiss <dw...@apache.org>
Committed: Thu Sep 6 09:34:21 2018 +0200
----------------------------------------------------------------------
.../apache/lucene/collation/package-info.java | 21 +++---
lucene/analysis/icu/src/java/overview.html | 21 +++---
lucene/core/src/java/overview.html | 23 +++----
.../src/test/org/apache/lucene/TestDemo.java | 69 +++++++++++---------
4 files changed, 70 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/922295a9/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java b/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java
index e56071a..5b83ea5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/collation/package-info.java
@@ -49,13 +49,14 @@
* // "fa" Locale is not supported by Sun JDK 1.4 or 1.5
* Collator collator = Collator.getInstance(new Locale("ar"));
* CollationKeyAnalyzer analyzer = new CollationKeyAnalyzer(collator);
- * RAMDirectory ramDir = new RAMDirectory();
- * IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(analyzer));
+ * Path dirPath = Files.createTempDirectory("tempIndex");
+ * Directory dir = FSDirectory.open(dirPath);
+ * IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
* Document doc = new Document();
* doc.add(new TextField("content", "\u0633\u0627\u0628", Field.Store.YES));
* writer.addDocument(doc);
* writer.close();
- * IndexReader ir = DirectoryReader.open(ramDir);
+ * IndexReader ir = DirectoryReader.open(dir);
* IndexSearcher is = new IndexSearcher(ir);
*
* QueryParser aqp = new QueryParser("content", analyzer);
@@ -75,8 +76,9 @@
* <pre class="prettyprint">
* Analyzer analyzer
* = new CollationKeyAnalyzer(Collator.getInstance(new Locale("da", "dk")));
- * RAMDirectory indexStore = new RAMDirectory();
- * IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(analyzer));
+ * Path dirPath = Files.createTempDirectory("tempIndex");
+ * Directory dir = FSDirectory.open(dirPath);
+ * IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
* String[] tracer = new String[] { "A", "B", "C", "D", "E" };
* String[] data = new String[] { "HAT", "HUT", "H\u00C5T", "H\u00D8T", "HOT" };
* String[] sortedTracerOrder = new String[] { "A", "E", "B", "D", "C" };
@@ -87,7 +89,7 @@
* writer.addDocument(doc);
* }
* writer.close();
- * IndexReader ir = DirectoryReader.open(indexStore);
+ * IndexReader ir = DirectoryReader.open(dir);
* IndexSearcher searcher = new IndexSearcher(ir);
* Sort sort = new Sort();
* sort.setSort(new SortField("contents", SortField.STRING));
@@ -104,13 +106,14 @@
* Collator collator = Collator.getInstance(new Locale("tr", "TR"));
* collator.setStrength(Collator.PRIMARY);
* Analyzer analyzer = new CollationKeyAnalyzer(collator);
- * RAMDirectory ramDir = new RAMDirectory();
- * IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(analyzer));
+ * Path dirPath = Files.createTempDirectory("tempIndex");
+ * Directory dir = FSDirectory.open(dirPath);
+ * IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
* Document doc = new Document();
* doc.add(new TextField("contents", "DIGY", Field.Store.NO));
* writer.addDocument(doc);
* writer.close();
- * IndexReader ir = DirectoryReader.open(ramDir);
+ * IndexReader ir = DirectoryReader.open(dir);
* IndexSearcher is = new IndexSearcher(ir);
* QueryParser parser = new QueryParser("contents", analyzer);
* Query query = parser.parse("d\u0131gy"); // U+0131: dotless i
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/922295a9/lucene/analysis/icu/src/java/overview.html
----------------------------------------------------------------------
diff --git a/lucene/analysis/icu/src/java/overview.html b/lucene/analysis/icu/src/java/overview.html
index 6e0a5d7..3af0247 100644
--- a/lucene/analysis/icu/src/java/overview.html
+++ b/lucene/analysis/icu/src/java/overview.html
@@ -115,14 +115,15 @@ algorithm.
<pre class="prettyprint">
Collator collator = Collator.getInstance(new ULocale("ar"));
ICUCollationKeyAnalyzer analyzer = new ICUCollationKeyAnalyzer(collator);
- RAMDirectory ramDir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(analyzer));
+ Path indexPath = Files.createTempDirectory("tempIndex");
+ Directory dir = FSDirectory.open(indexPath);
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
Document doc = new Document();
doc.add(new Field("content", "\u0633\u0627\u0628",
Field.Store.YES, Field.Index.ANALYZED));
writer.addDocument(doc);
writer.close();
- IndexSearcher is = new IndexSearcher(ramDir, true);
+ IndexSearcher is = new IndexSearcher(dir, true);
QueryParser aqp = new QueryParser("content", analyzer);
aqp.setAnalyzeRangeTerms(true);
@@ -141,8 +142,9 @@ algorithm.
<pre class="prettyprint">
Analyzer analyzer
= new ICUCollationKeyAnalyzer(Collator.getInstance(new ULocale("da", "dk")));
- RAMDirectory indexStore = new RAMDirectory();
- IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(analyzer));
+ Path indexPath = Files.createTempDirectory("tempIndex");
+ Directory dir = FSDirectory.open(indexPath);
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
String[] tracer = new String[] { "A", "B", "C", "D", "E" };
String[] data = new String[] { "HAT", "HUT", "H\u00C5T", "H\u00D8T", "HOT" };
String[] sortedTracerOrder = new String[] { "A", "E", "B", "D", "C" };
@@ -153,7 +155,7 @@ algorithm.
writer.addDocument(doc);
}
writer.close();
- IndexSearcher searcher = new IndexSearcher(indexStore, true);
+ IndexSearcher searcher = new IndexSearcher(dir, true);
Sort sort = new Sort();
sort.setSort(new SortField("contents", SortField.STRING));
Query query = new MatchAllDocsQuery();
@@ -169,13 +171,14 @@ algorithm.
Collator collator = Collator.getInstance(new ULocale("tr", "TR"));
collator.setStrength(Collator.PRIMARY);
Analyzer analyzer = new ICUCollationKeyAnalyzer(collator);
- RAMDirectory ramDir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(analyzer));
+ Path indexPath = Files.createTempDirectory("tempIndex");
+ Directory dir = FSDirectory.open(indexPath);
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(analyzer));
Document doc = new Document();
doc.add(new Field("contents", "DIGY", Field.Store.NO, Field.Index.ANALYZED));
writer.addDocument(doc);
writer.close();
- IndexSearcher is = new IndexSearcher(ramDir, true);
+ IndexSearcher is = new IndexSearcher(dir, true);
QueryParser parser = new QueryParser("contents", analyzer);
Query query = parser.parse("d\u0131gy"); // U+0131: dotless i
ScoreDoc[] result = is.search(query, null, 1000).scoreDocs;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/922295a9/lucene/core/src/java/overview.html
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/overview.html b/lucene/core/src/java/overview.html
index e941744..b4f5b81 100644
--- a/lucene/core/src/java/overview.html
+++ b/lucene/core/src/java/overview.html
@@ -24,18 +24,14 @@
Here's a simple example how to use Lucene for indexing and searching (using JUnit
to check if the results are what we expect):</p>
-<!-- code comes from org.apache.lucene.TestDemo: -->
-<!-- ======================================================== -->
-<!-- = Java Sourcecode to HTML automatically converted code = -->
-<!-- = Java2Html Converter 5.0 [2006-03-04] by Markus Gebhard markus@jave.de = -->
-<!-- = Further information: http://www.java2html.de = -->
+<!-- code comes from org.apache.lucene.TestDemo.
+ See LUCENE-8481 for reasons why it's out of sync with the code.
+ -->
<pre class="prettyprint">
Analyzer analyzer = new StandardAnalyzer();
- // Store the index in memory:
- Directory directory = new RAMDirectory();
- // To store an index on disk, use this instead:
- //Directory directory = FSDirectory.open(Paths.get("/tmp/testindex"));
+ Path indexPath = Files.createTempDirectory("tempIndex");
+ Directory directory = FSDirectory.open(indexPath)
IndexWriterConfig config = new IndexWriterConfig(analyzer);
IndexWriter iwriter = new IndexWriter(directory, config);
Document doc = new Document();
@@ -58,8 +54,8 @@ to check if the results are what we expect):</p>
assertEquals("This is the text to be indexed.", hitDoc.get("fieldname"));
}
ireader.close();
- directory.close();</pre>
-<!-- = END of automatically generated HTML code = -->
+ directory.close();
+ IOUtils.rm(indexPath);</pre>
<!-- ======================================================== -->
@@ -112,9 +108,8 @@ query structures from strings or xml.
defines an abstract class for storing persistent data, the {@link org.apache.lucene.store.Directory Directory},
which is a collection of named files written by an {@link org.apache.lucene.store.IndexOutput IndexOutput}
and read by an {@link org.apache.lucene.store.IndexInput IndexInput}.
-Multiple implementations are provided, including {@link org.apache.lucene.store.FSDirectory FSDirectory},
-which uses a file system directory to store files, and {@link org.apache.lucene.store.RAMDirectory RAMDirectory}
-which implements files as memory-resident data structures.</li>
+Multiple implementations are provided, but {@link org.apache.lucene.store.FSDirectory FSDirectory} is generally
+recommended as it tries to use operating system disk buffer caches efficiently.</li>
<li>
<b>{@link org.apache.lucene.util}</b>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/922295a9/lucene/core/src/test/org/apache/lucene/TestDemo.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/TestDemo.java b/lucene/core/src/test/org/apache/lucene/TestDemo.java
index f2d3a61..7f62002 100644
--- a/lucene/core/src/test/org/apache/lucene/TestDemo.java
+++ b/lucene/core/src/test/org/apache/lucene/TestDemo.java
@@ -18,17 +18,22 @@ package org.apache.lucene;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -40,40 +45,40 @@ import org.apache.lucene.util.LuceneTestCase;
public class TestDemo extends LuceneTestCase {
public void testDemo() throws IOException {
- Analyzer analyzer = new MockAnalyzer(random());
-
- // Store the index in memory:
- Directory directory = newDirectory();
- // To store an index on disk, use this instead:
- // Directory directory = FSDirectory.open(new File("/tmp/testindex"));
- RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, analyzer);
- Document doc = new Document();
String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm";
String text = "This is the text to be indexed. " + longTerm;
- doc.add(newTextField("fieldname", text, Field.Store.YES));
- iwriter.addDocument(doc);
- iwriter.close();
-
- // Now search the index:
- IndexReader ireader = DirectoryReader.open(directory); // read-only=true
- IndexSearcher isearcher = newSearcher(ireader);
- assertEquals(1, isearcher.count(new TermQuery(new Term("fieldname", longTerm))));
- Query query = new TermQuery(new Term("fieldname", "text"));
- TopDocs hits = isearcher.search(query, 1);
- assertEquals(1, hits.totalHits.value);
- // Iterate through the results:
- for (int i = 0; i < hits.scoreDocs.length; i++) {
- Document hitDoc = isearcher.doc(hits.scoreDocs[i].doc);
- assertEquals(text, hitDoc.get("fieldname"));
- }
+ Path indexPath = Files.createTempDirectory("tempIndex");
+ try (Directory dir = FSDirectory.open(indexPath)) {
+ Analyzer analyzer = new StandardAnalyzer();
+ try (IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(analyzer))) {
+ Document doc = new Document();
+ doc.add(newTextField("fieldname", text, Field.Store.YES));
+ iw.addDocument(doc);
+ }
+
+ // Now search the index.
+ try (IndexReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher searcher = newSearcher(reader);
+
+ assertEquals(1, searcher.count(new TermQuery(new Term("fieldname", longTerm))));
- // Test simple phrase query
- PhraseQuery phraseQuery = new PhraseQuery("fieldname", "to", "be");
- assertEquals(1, isearcher.count(phraseQuery));
+ Query query = new TermQuery(new Term("fieldname", "text"));
+ TopDocs hits = searcher.search(query, 1);
+ assertEquals(1, hits.totalHits.value);
+
+ // Iterate through the results.
+ for (int i = 0; i < hits.scoreDocs.length; i++) {
+ Document hitDoc = searcher.doc(hits.scoreDocs[i].doc);
+ assertEquals(text, hitDoc.get("fieldname"));
+ }
+
+ // Test simple phrase query.
+ PhraseQuery phraseQuery = new PhraseQuery("fieldname", "to", "be");
+ assertEquals(1, searcher.count(phraseQuery));
+ }
+ }
- ireader.close();
- directory.close();
- analyzer.close();
+ IOUtils.rm(indexPath);
}
}
[40/47] lucene-solr:jira/solr-12709: LUCENE-8491: Adjust maxRadius to
prevent constructing circles that are too big
Posted by ab...@apache.org.
LUCENE-8491: Adjust maxRadius to prevent constructing circles that are too big
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6fbcda60
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6fbcda60
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6fbcda60
Branch: refs/heads/jira/solr-12709
Commit: 6fbcda60a21834d9259e78c97ca71e7d80689c68
Parents: 66c671e
Author: iverase <iv...@apache.org>
Authored: Fri Sep 7 14:48:52 2018 +0200
Committer: iverase <iv...@apache.org>
Committed: Fri Sep 7 14:48:52 2018 +0200
----------------------------------------------------------------------
.../spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6fbcda60/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java
index 94fdde9..e1234a4 100644
--- a/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java
+++ b/lucene/spatial-extras/src/test/org/apache/lucene/spatial/spatial4j/Geo3dShapeWGS84ModelRectRelationTest.java
@@ -38,7 +38,7 @@ public class Geo3dShapeWGS84ModelRectRelationTest extends ShapeRectRelationTestC
Geo3dSpatialContextFactory factory = new Geo3dSpatialContextFactory();
factory.planetModel = planetModel;
this.ctx = factory.newSpatialContext();
- this.maxRadius = 178;
+ this.maxRadius = 175;
((Geo3dShapeFactory)ctx.getShapeFactory()).setCircleAccuracy(1e-12);
}
[41/47] lucene-solr:jira/solr-12709: SOLR-8742: In HdfsDirectoryTest
replace RAMDirectory usages with ByteBuffersDirectory.
Posted by ab...@apache.org.
SOLR-8742: In HdfsDirectoryTest replace RAMDirectory usages with ByteBuffersDirectory.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2c889229
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2c889229
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2c889229
Branch: refs/heads/jira/solr-12709
Commit: 2c88922998ab3bc2e97d9c640bc01fd39d95fb34
Parents: 6fbcda6
Author: Steve Rowe <sa...@apache.org>
Authored: Fri Sep 7 13:19:01 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Fri Sep 7 13:19:01 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java | 7 +++----
2 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c889229/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e5596e1..7d30c0c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -424,6 +424,9 @@ Other Changes
* SOLR-12744: Improve logging messages and verbosity around recoveries (Cao Manh Dat, Varun Thacker)
+* SOLR-8742: In HdfsDirectoryTest replace RAMDirectory usages with ByteBuffersDirectory.
+ (hossman, Mark Miller, Andrzej Bialecki, Steve Rowe)
+
================== 7.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2c889229/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java b/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
index 97b26fd..b9bd029 100644
--- a/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
+++ b/solr/core/src/test/org/apache/solr/store/hdfs/HdfsDirectoryTest.java
@@ -24,11 +24,11 @@ import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.RAMDirectory;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
import org.apache.solr.util.BadHdfsThreadsFilter;
@@ -140,9 +140,8 @@ public class HdfsDirectoryTest extends SolrTestCaseJ4 {
}
@Test
- // 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
public void testEOF() throws IOException {
- Directory fsDir = new RAMDirectory();
+ Directory fsDir = new ByteBuffersDirectory();
String name = "test.eof";
createFile(name, fsDir, directory);
long fsLength = fsDir.fileLength(name);
@@ -168,7 +167,7 @@ public class HdfsDirectoryTest extends SolrTestCaseJ4 {
try {
Set<String> names = new HashSet<>();
for (; i< 10; i++) {
- Directory fsDir = new RAMDirectory();
+ Directory fsDir = new ByteBuffersDirectory();
String name = getName();
System.out.println("Working on pass [" + i +"] contains [" + names.contains(name) + "]");
names.add(name);
[13/47] lucene-solr:jira/solr-12709: Fix 'included' and 'number'
typos in stream-source-reference.adoc file.
Posted by ab...@apache.org.
Fix 'included' and 'number' typos in stream-source-reference.adoc file.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3f9937bc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3f9937bc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3f9937bc
Branch: refs/heads/jira/solr-12709
Commit: 3f9937bca2d8d296e3b7f1678f5bc63d2d1130da
Parents: f385f02
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Sep 5 17:47:48 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Sep 5 17:51:00 2018 +0100
----------------------------------------------------------------------
solr/solr-ref-guide/src/stream-source-reference.adoc | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3f9937bc/solr/solr-ref-guide/src/stream-source-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-source-reference.adoc b/solr/solr-ref-guide/src/stream-source-reference.adoc
index fc9f71a..744193e 100644
--- a/solr/solr-ref-guide/src/stream-source-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-source-reference.adoc
@@ -226,12 +226,12 @@ use the More Like This query parser plugin.
* `qf`: (Mandatory) The query field used to compare documents.
* `k`: (Mandatory) The number of nearest neighbors to return.
* `fl`: (Mandatory) The field list to return.
-* `mintf`: (Optional) The minimum numer of occurrences of the term in the source document to be inlcued in the search.
-* `maxtf`: (Optional) The maximum numer of occurrences of the term in the source document to be inlcued in the search.
-* `mindf`: (Optional) The minimum numer of occurrences in the corpus to be inlcued in the search.
-* `maxdf`: (Optional) The maximum numer of occurrences in the corpus to be inlcued in the search.
-* `minwl`: (Optional) The minimum world length of to be inlcued in the search.
-* `maxwl`: (Optional) The maximum world length of to be inlcued in the search.
+* `mintf`: (Optional) The minimum number of occurrences of the term in the source document to be included in the search.
+* `maxtf`: (Optional) The maximum number of occurrences of the term in the source document to be included in the search.
+* `mindf`: (Optional) The minimum number of occurrences in the corpus to be included in the search.
+* `maxdf`: (Optional) The maximum number of occurrences in the corpus to be included in the search.
+* `minwl`: (Optional) The minimum world length of to be included in the search.
+* `maxwl`: (Optional) The maximum world length of to be included in the search.
=== knn Syntax
[08/47] lucene-solr:jira/solr-12709: LUCENE-8484: Only drop fully
deleted segments in SoftDeletesDirectoryReaderWrapper
Posted by ab...@apache.org.
LUCENE-8484: Only drop fully deleted segments in SoftDeletesDirectoryReaderWrapper
This specializes the change in LUCENE-8484 since there are valid usecases
where filtered reader should not modify the number of readers if a higher
level reader wants to expose the deleted or soft-deleted docs.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c8b47e20
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c8b47e20
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c8b47e20
Branch: refs/heads/jira/solr-12709
Commit: c8b47e2024ddef066b297b7bd67983ebf458b060
Parents: 7223a8b
Author: Simon Willnauer <si...@apache.org>
Authored: Wed Sep 5 15:26:37 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Wed Sep 5 16:00:54 2018 +0200
----------------------------------------------------------------------
.../apache/lucene/index/FilterDirectoryReader.java | 17 ++++++++++-------
.../index/SoftDeletesDirectoryReaderWrapper.java | 14 ++++++++++++++
2 files changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8b47e20/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
index 1efd482..4a9cd68 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
/**
@@ -51,16 +50,20 @@ public abstract class FilterDirectoryReader extends DirectoryReader {
*/
public static abstract class SubReaderWrapper {
- private LeafReader[] wrap(List<? extends LeafReader> readers) {
- List<LeafReader> wrapped = new ArrayList<>(readers.size());
+ /**
+ * Wraps a list of LeafReaders
+ * @return an array of wrapped LeafReaders. The returned array might contain less elements compared to the given
+ * reader list if an entire reader is filtered out.
+ */
+ protected LeafReader[] wrap(List<? extends LeafReader> readers) {
+ LeafReader[] wrapped = new LeafReader[readers.size()];
+ int i = 0;
for (LeafReader reader : readers) {
LeafReader wrap = wrap(reader);
assert wrap != null;
- if (wrap.numDocs() > 0) {
- wrapped.add(wrap);
- }
+ wrapped[i++] = wrap;
}
- return wrapped.toArray(new LeafReader[0]);
+ return wrapped;
}
/** Constructor */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c8b47e20/lucene/core/src/java/org/apache/lucene/index/SoftDeletesDirectoryReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/SoftDeletesDirectoryReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SoftDeletesDirectoryReaderWrapper.java
index cf3e437..21aac96 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SoftDeletesDirectoryReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SoftDeletesDirectoryReaderWrapper.java
@@ -19,8 +19,10 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.UncheckedIOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -86,6 +88,18 @@ public final class SoftDeletesDirectoryReaderWrapper extends FilterDirectoryRead
this.field = field;
}
+ protected LeafReader[] wrap(List<? extends LeafReader> readers) {
+ List<LeafReader> wrapped = new ArrayList<>(readers.size());
+ for (LeafReader reader : readers) {
+ LeafReader wrap = wrap(reader);
+ assert wrap != null;
+ if (wrap.numDocs() != 0) {
+ wrapped.add(wrap);
+ }
+ }
+ return wrapped.toArray(new LeafReader[0]);
+ }
+
@Override
public LeafReader wrap(LeafReader reader) {
CacheHelper readerCacheHelper = reader.getReaderCacheHelper();
[33/47] lucene-solr:jira/solr-12709: SOLR-12612: Accept custom keys
in cluster properties (doc changes)
Posted by ab...@apache.org.
SOLR-12612: Accept custom keys in cluster properties (doc changes)
Also added missing known cluster properties
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ccd9f6fc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ccd9f6fc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ccd9f6fc
Branch: refs/heads/jira/solr-12709
Commit: ccd9f6fccb2fe7312150cb2844dbd4fbfaf1e7e6
Parents: 0af269f
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Thu Sep 6 14:38:35 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Thu Sep 6 14:38:35 2018 -0700
----------------------------------------------------------------------
solr/solr-ref-guide/src/collections-api.adoc | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccd9f6fc/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index 80ce941..43825e8 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -1085,7 +1085,8 @@ Add, edit or delete a cluster-wide property.
=== CLUSTERPROP Parameters
`name`::
-The name of the property. Supported properties names are `urlScheme` and `autoAddReplicas and location`. Other names are rejected with an error.
+The name of the property. Supported properties names are `autoAddReplicas`, `legacyCloud` , `location`, `maxCoresPerNode` and `urlScheme`. Other properties can be set
+(for example, if you need them for custom plugins) but they must begin with the prefix `ext.`. Unknown properties that don't begin with `ext.` will be rejected.
`val`::
The value of the property. If the value is empty or null, the property is unset.
[29/47] lucene-solr:jira/solr-12709: SOLR-11943: Add machine learning
functions for location data
Posted by ab...@apache.org.
SOLR-11943: Add machine learning functions for location data
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b8e87a10
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b8e87a10
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b8e87a10
Branch: refs/heads/jira/solr-12709
Commit: b8e87a101017711d634733242d5563eef836365e
Parents: 597bd5d
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Sep 6 14:00:38 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Sep 6 14:01:09 2018 -0400
----------------------------------------------------------------------
.../solr/handler/HaversineMetersEvaluator.java | 59 +++++++++++
.../solr/handler/SolrDefaultStreamFactory.java | 1 +
.../org/apache/solr/client/solrj/io/Lang.java | 1 +
.../solrj/io/eval/LocationVectorsEvaluator.java | 105 +++++++++++++++++++
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 46 ++++++++
6 files changed, 213 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8e87a10/solr/core/src/java/org/apache/solr/handler/HaversineMetersEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/HaversineMetersEvaluator.java b/solr/core/src/java/org/apache/solr/handler/HaversineMetersEvaluator.java
new file mode 100644
index 0000000..2e30555
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/HaversineMetersEvaluator.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler;
+
+import java.io.IOException;
+
+import org.apache.commons.math3.exception.DimensionMismatchException;
+import org.apache.commons.math3.ml.distance.DistanceMeasure;
+import org.apache.lucene.util.SloppyMath;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.eval.RecursiveEvaluator;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class HaversineMetersEvaluator extends RecursiveEvaluator {
+ protected static final long serialVersionUID = 1L;
+
+ public HaversineMetersEvaluator(StreamExpression expression, StreamFactory factory) throws IOException{
+ super(expression, factory);
+ }
+
+
+ @Override
+ public Object evaluate(Tuple tuple) throws IOException {
+ return new HaversineDistance();
+ }
+
+ @Override
+ public Object doWork(Object... values) throws IOException {
+ // Nothing to do here
+ throw new IOException("This call should never occur");
+ }
+
+ public static class HaversineDistance implements DistanceMeasure {
+ private static final long serialVersionUID = -9108154600539125566L;
+
+ public HaversineDistance() {
+ }
+
+ public double compute(double[] a, double[] b) throws DimensionMismatchException {
+ return SloppyMath.haversinMeters(a[0], a[1], b[0], b[1]);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8e87a10/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java b/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java
index 0b375f4..c072f0b 100644
--- a/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java
+++ b/solr/core/src/java/org/apache/solr/handler/SolrDefaultStreamFactory.java
@@ -35,6 +35,7 @@ public class SolrDefaultStreamFactory extends DefaultStreamFactory {
super();
this.withFunctionName("analyze", AnalyzeEvaluator.class);
this.withFunctionName("classify", ClassifyStream.class);
+ this.withFunctionName("haversineMeters", HaversineMetersEvaluator.class);
}
public SolrDefaultStreamFactory withSolrResourceLoader(SolrResourceLoader solrResourceLoader) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8e87a10/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index 32ee6fc..9568ecb 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -255,6 +255,7 @@ public class Lang {
.withFunctionName("removeCache", RemoveCacheEvaluator.class)
.withFunctionName("listCache", ListCacheEvaluator.class)
.withFunctionName("zscores", NormalizeEvaluator.class)
+ .withFunctionName("locationVectors", LocationVectorsEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8e87a10/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java
new file mode 100644
index 0000000..0c1ba99
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class LocationVectorsEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ private String field;
+
+ public LocationVectorsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+
+ List<StreamExpressionNamedParameter> namedParams = factory.getNamedOperands(expression);
+
+ for (StreamExpressionNamedParameter namedParam : namedParams) {
+ if(namedParam.getName().equals("field")) {
+ this.field = namedParam.getParameter().toString();
+ } else {
+ throw new IOException("Unexpected named parameter:" + namedParam.getName());
+ }
+ }
+
+ if(field == null) {
+ throw new IOException("The named parameter \"field\" must be set for the locationVectors function.");
+ }
+ }
+
+ @Override
+ public Object doWork(Object... objects) throws IOException {
+
+ if (objects.length == 1) {
+ //Just docs
+ if(!(objects[0] instanceof List)) {
+ throw new IOException("The locationVectors function expects a list of Tuples as a parameter.");
+ } else {
+ List list = (List)objects[0];
+ if(list.size() > 0) {
+ Object o = list.get(0);
+ if(!(o instanceof Tuple)) {
+ throw new IOException("The locationVectors function expects a list of Tuples as a parameter.");
+ }
+ } else {
+ throw new IOException("Empty list was passed as a parameter to termVectors function.");
+ }
+ }
+
+ List<Tuple> tuples = (List<Tuple>) objects[0];
+
+ double[][] locationVectors = new double[tuples.size()][2];
+ List<String> features = new ArrayList();
+ features.add("lat");
+ features.add("long");
+
+ List<String> rowLabels = new ArrayList();
+
+ for(int i=0; i< tuples.size(); i++) {
+ Tuple tuple = tuples.get(i);
+ String value = tuple.getString(field);
+ String[] latLong = null;
+ if(value.contains(",")) {
+ latLong = value.split(",");
+ } else {
+ latLong = value.split(" ");
+ }
+
+ locationVectors[i][0] = Double.parseDouble(latLong[0].trim());
+ locationVectors[i][1] = Double.parseDouble(latLong[1].trim());
+ if(tuple.get("id") != null) {
+ rowLabels.add(tuple.get("id").toString());
+ }
+ }
+
+ Matrix matrix = new Matrix(locationVectors);
+ matrix.setColumnLabels(features);
+ matrix.setRowLabels(rowLabels);
+ return matrix;
+ } else {
+ throw new IOException("The termVectors function takes a single positional parameter.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8e87a10/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index 8c2cb65..ee8c1e1 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -70,7 +70,7 @@ public class TestLang extends LuceneTestCase {
"mod", "ceil", "floor", "sin", "asin", "sinh", "cos", "acos", "cosh", "tan", "atan", "tanh", "round", "sqrt",
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft", "euclidean","manhattan",
"earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue", "knnRegress", "gaussfit",
- "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores"};
+ "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "locationVectors"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b8e87a10/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 229e9eb..4bcf50d 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -310,6 +310,52 @@ public class MathExpressionTest extends SolrCloudTestCase {
assertEquals(array.get(2).intValue(), 50);
assertEquals(array.get(3).intValue(), 50);
}
+
+ @Test
+ public void testLocationFunctions() throws Exception {
+ UpdateRequest updateRequest = new UpdateRequest();
+
+ int i=0;
+ while(i<5) {
+ updateRequest.add(id, "id_"+(++i),"test_dt", getDateString("2016", "5", "1"),
+ "price_i", Integer.toString(i), "loc_p", (42.906797030808235+i)+","+(76.69455762489834+i));
+ }
+
+
+ updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ String expr = "let(echo=true," +
+ " a=search("+COLLECTIONORALIAS+", q=*:*, fl=\"id, loc_p, price_i\",rows=100, sort=\"price_i asc\"),"+
+ " b=locationVectors(a, field=loc_p)," +
+ " c=distance(array(40.7128, 74.0060), array(45.7128, 74.0060), haversineMeters()))";
+
+
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ List<List<Number>>locVectors = (List<List<Number>>)tuples.get(0).get("b");
+ System.out.println(locVectors);
+ int v=1;
+ for(List<Number> row : locVectors) {
+ double lat = row.get(0).doubleValue();
+ double lon = row.get(1).doubleValue();
+ assertEquals(lat, 42.906797030808235+v, 0);
+ assertEquals(lon, 76.69455762489834+v, 0);
+ ++v;
+ }
+
+ double distance = tuples.get(0).getDouble("c");
+ assertEquals(distance, 555975.3986718428, 1.0);
+
+ }
@Test
public void testHist() throws Exception {
[07/47] lucene-solr:jira/solr-12709: LUCENE-8484: Drop fully deleted
reader in SubReaderWrapper (#445)
Posted by ab...@apache.org.
LUCENE-8484: Drop fully deleted reader in SubReaderWrapper (#445)
Today we can only wrap readers in SubReaderWrapper but never filter them out
entirely. This causes a invariant for soft-deletes that exposes fully deleted
segments with SoftDeletesDirectoryReaderWrapper. This change drops fully
deleted readers after they are wrapped.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7223a8bf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7223a8bf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7223a8bf
Branch: refs/heads/jira/solr-12709
Commit: 7223a8bf7a8a112756364ba15c534c85436c8586
Parents: df20c4b
Author: Simon Willnauer <si...@apache.org>
Authored: Tue Sep 4 15:09:39 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Wed Sep 5 12:44:54 2018 +0200
----------------------------------------------------------------------
.../lucene/index/FilterDirectoryReader.java | 13 +++--
.../TestSoftDeletesDirectoryReaderWrapper.java | 51 ++++++++++++++++++++
2 files changed, 60 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7223a8bf/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
index 7003df5..1efd482 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -51,11 +52,15 @@ public abstract class FilterDirectoryReader extends DirectoryReader {
public static abstract class SubReaderWrapper {
private LeafReader[] wrap(List<? extends LeafReader> readers) {
- LeafReader[] wrapped = new LeafReader[readers.size()];
- for (int i = 0; i < readers.size(); i++) {
- wrapped[i] = wrap(readers.get(i));
+ List<LeafReader> wrapped = new ArrayList<>(readers.size());
+ for (LeafReader reader : readers) {
+ LeafReader wrap = wrap(reader);
+ assert wrap != null;
+ if (wrap.numDocs() > 0) {
+ wrapped.add(wrap);
+ }
}
- return wrapped;
+ return wrapped.toArray(new LeafReader[0]);
}
/** Constructor */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7223a8bf/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
index 32d0529..00e845e 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSoftDeletesDirectoryReaderWrapper.java
@@ -27,6 +27,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
@@ -34,6 +35,56 @@ import org.apache.lucene.util.LuceneTestCase;
public class TestSoftDeletesDirectoryReaderWrapper extends LuceneTestCase {
+ public void testDropFullyDeletedSegments() throws IOException {
+ IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
+ String softDeletesField = "soft_delete";
+ indexWriterConfig.setSoftDeletesField(softDeletesField);
+ indexWriterConfig.setMergePolicy(new SoftDeletesRetentionMergePolicy(softDeletesField, MatchAllDocsQuery::new,
+ NoMergePolicy.INSTANCE));
+ try (Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, indexWriterConfig)) {
+
+ Document doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.YES));
+ doc.add(new StringField("version", "1", Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit();
+ doc = new Document();
+ doc.add(new StringField("id", "2", Field.Store.YES));
+ doc.add(new StringField("version", "1", Field.Store.YES));
+ writer.addDocument(doc);
+ writer.commit();
+
+ try (DirectoryReader reader = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(dir), softDeletesField)) {
+ assertEquals(2, reader.leaves().size());
+ assertEquals(2, reader.numDocs());
+ assertEquals(2, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ }
+ writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField(softDeletesField, 1));
+ writer.commit();
+ try (DirectoryReader reader = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(writer), softDeletesField)) {
+ assertEquals(1, reader.numDocs());
+ assertEquals(1, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ assertEquals(1, reader.leaves().size());
+ }
+ try (DirectoryReader reader = new SoftDeletesDirectoryReaderWrapper(DirectoryReader.open(dir), softDeletesField)) {
+ assertEquals(1, reader.numDocs());
+ assertEquals(1, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ assertEquals(1, reader.leaves().size());
+ }
+
+ try (DirectoryReader reader = DirectoryReader.open(dir)) {
+ assertEquals(2, reader.numDocs());
+ assertEquals(2, reader.maxDoc());
+ assertEquals(0, reader.numDeletedDocs());
+ assertEquals(2, reader.leaves().size());
+ }
+ }
+ }
+
public void testReuseUnchangedLeafReader() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
[12/47] lucene-solr:jira/solr-12709: Fix 'including' typo in
cloud-screens.adoc file.
Posted by ab...@apache.org.
Fix 'including' typo in cloud-screens.adoc file.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f385f02e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f385f02e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f385f02e
Branch: refs/heads/jira/solr-12709
Commit: f385f02e4b9aed7a4f0685a7f4005ffddbfde163
Parents: 719d922
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Sep 5 17:46:47 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Sep 5 17:46:47 2018 +0100
----------------------------------------------------------------------
solr/solr-ref-guide/src/cloud-screens.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f385f02e/solr/solr-ref-guide/src/cloud-screens.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/cloud-screens.adoc b/solr/solr-ref-guide/src/cloud-screens.adoc
index 28160dd..6e7c5a0 100644
--- a/solr/solr-ref-guide/src/cloud-screens.adoc
+++ b/solr/solr-ref-guide/src/cloud-screens.adoc
@@ -43,7 +43,7 @@ image::images/cloud-screens/cloud-tree.png[image,width=487,height=250]
As an aid to debugging, the data shown in the "Tree" view can be exported locally using the following command `bin/solr zk ls -r /`
== ZK Status View
-The "ZK Status" view gives an overview over the Zookeepers used by Solr. It lists whether running in `standalone` or `ensemble` mode, shows how many zookeepers are configured, and then displays a table listing detailed monitoring status for each of the zookeepers, inlcuding who is the leader, configuration parameters and more.
+The "ZK Status" view gives an overview over the Zookeepers used by Solr. It lists whether running in `standalone` or `ensemble` mode, shows how many zookeepers are configured, and then displays a table listing detailed monitoring status for each of the zookeepers, including who is the leader, configuration parameters and more.
image::images/cloud-screens/cloud-zkstatus.png[image,width=512,height=509]
[47/47] lucene-solr:jira/solr-12709: SOLR-12709: Clean up statics.
Posted by ab...@apache.org.
SOLR-12709: Clean up statics.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8837c2d6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8837c2d6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8837c2d6
Branch: refs/heads/jira/solr-12709
Commit: 8837c2d69c9d79a7d42bab0ae8a37cad9342a032
Parents: 4a6dfda
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Sep 17 11:43:18 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Sep 17 11:43:18 2018 +0200
----------------------------------------------------------------------
.../solr/cloud/autoscaling/sim/TestSimAutoScaling.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8837c2d6/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
index 6cbb132..4b0b790 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
@@ -15,6 +15,7 @@ import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.util.LogLevel;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@@ -34,8 +35,8 @@ public class TestSimAutoScaling extends SimSolrCloudTestCase {
private static final int SPEED = 500;
private static final int NUM_NODES = 200;
- private static final long BATCH_SIZE = 2000000;
- private static final long NUM_BATCHES = 500000;
+ private static final long BATCH_SIZE = 200000;
+ private static final long NUM_BATCHES = 5000000;
private static final long ABOVE_SIZE = 20000000;
@@ -50,6 +51,11 @@ public class TestSimAutoScaling extends SimSolrCloudTestCase {
cluster.simSetUseSystemCollection(false);
}
+ @AfterClass
+ public static void tearDownCluster() throws Exception {
+ solrClient = null;
+ }
+
@Test
public void testScaleUp() throws Exception {
String collectionName = "testScaleUp_collection";
[19/47] lucene-solr:jira/solr-12709: SOLR-12733:
SolrMetricReporterTest failure
Posted by ab...@apache.org.
SOLR-12733: SolrMetricReporterTest failure
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/37375ae6
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/37375ae6
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/37375ae6
Branch: refs/heads/jira/solr-12709
Commit: 37375ae6008a069ce1362e287dfd55bed0a27f7f
Parents: b4a1548
Author: Erick <er...@apache.org>
Authored: Wed Sep 5 17:04:31 2018 -0700
Committer: Erick <er...@apache.org>
Committed: Wed Sep 5 17:04:31 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../src/test/org/apache/solr/metrics/SolrMetricReporterTest.java | 4 ++--
solr/core/src/test/org/apache/solr/util/DateMathParserTest.java | 4 ++--
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37375ae6/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6bb1299..aa0698c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -313,6 +313,8 @@ Bug Fixes
* SOLR-12728: RequestLoggingTest fails on occasion, not reproducible (Erick Erickson)
+* SOLR-12733: SolrMetricReporterTest failure (Erick Erickson, David Smiley)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37375ae6/solr/core/src/test/org/apache/solr/metrics/SolrMetricReporterTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrMetricReporterTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrMetricReporterTest.java
index f3359cc..7d255a3 100644
--- a/solr/core/src/test/org/apache/solr/metrics/SolrMetricReporterTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/SolrMetricReporterTest.java
@@ -20,15 +20,15 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Random;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
+import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.core.PluginInfo;
import org.apache.solr.metrics.reporters.MockMetricReporter;
import org.apache.solr.schema.FieldType;
import org.junit.Test;
-public class SolrMetricReporterTest extends LuceneTestCase {
+public class SolrMetricReporterTest extends SolrTestCaseJ4 {
@Test
public void testInit() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/37375ae6/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java b/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java
index 8cc417b..f5d3de2 100644
--- a/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java
+++ b/solr/core/src/test/org/apache/solr/util/DateMathParserTest.java
@@ -26,14 +26,14 @@ import java.util.Locale;
import java.util.Map;
import java.util.TimeZone;
-import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.SolrTestCaseJ4;
import static org.apache.solr.util.DateMathParser.UTC;
/**
* Tests that the functions in DateMathParser
*/
-public class DateMathParserTest extends LuceneTestCase {
+public class DateMathParserTest extends SolrTestCaseJ4 {
/**
* A formatter for specifying every last nuance of a Date for easy
[22/47] lucene-solr:jira/solr-12709: SOLR-12745: Wrong header levels
on BasicAuth refGuide page
Posted by ab...@apache.org.
SOLR-12745: Wrong header levels on BasicAuth refGuide page
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/285b743a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/285b743a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/285b743a
Branch: refs/heads/jira/solr-12709
Commit: 285b743a8bff96e3f436f40bcc86f3529a0d8951
Parents: 922295a
Author: Jan Høydahl <ja...@apache.org>
Authored: Thu Sep 6 10:31:18 2018 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Thu Sep 6 10:31:18 2018 +0200
----------------------------------------------------------------------
solr/solr-ref-guide/src/basic-authentication-plugin.adoc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/285b743a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc
index 4a5ff82..88e8a0c 100644
--- a/solr/solr-ref-guide/src/basic-authentication-plugin.adoc
+++ b/solr/solr-ref-guide/src/basic-authentication-plugin.adoc
@@ -164,7 +164,7 @@ curl --user solr:SolrRocks http://localhost:8983/api/cluster/security/authentica
====
--
-=== Using Basic Auth with SolrJ
+== Using Basic Auth with SolrJ
In SolrJ, the basic authentication credentials need to be set for each request as in this example:
@@ -184,7 +184,7 @@ req.setBasicAuthCredentials(userName, password);
QueryResponse rsp = req.process(solrClient);
----
-=== Using the Solr Control Script with Basic Auth
+== Using the Solr Control Script with Basic Auth
Add the following line to the `solr.in.sh` or `solr.in.cmd` file. This example tells the `bin/solr` command line to to use "basic" as the type of authentication, and to pass credentials with the user-name "solr" and password "SolrRocks":
[06/47] lucene-solr:jira/solr-12709: SOLR-11990: Wait for all nodes
to be live before running tests.
Posted by ab...@apache.org.
SOLR-11990: Wait for all nodes to be live before running tests.
A few tests have failed on jenkins where the very first call to Solr to set the policy fails because no live nodes were found. This commit adds a 30 second (max) wait for nodes to be registered live in ZK before attempting to run any test.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/df20c4b9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/df20c4b9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/df20c4b9
Branch: refs/heads/jira/solr-12709
Commit: df20c4b9e6f9ff3408c7eaf9ceea7a1656a72ff8
Parents: b6ee0ed
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Sep 5 16:07:10 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Sep 5 16:07:10 2018 +0530
----------------------------------------------------------------------
solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/df20c4b9/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java b/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
index 208b869..b822a24 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
@@ -111,6 +111,7 @@ public class TestWithCollection extends SolrCloudTestCase {
cluster.stopJettySolrRunner(i - 1);
}
}
+ cluster.waitForAllNodes(30);
}
private void deleteChildrenRecursively(String path) throws Exception {
[45/47] lucene-solr:jira/solr-12709: Merge branch 'master' into
jira/solr-12709
Posted by ab...@apache.org.
Merge branch 'master' into jira/solr-12709
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c4eb2f05
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c4eb2f05
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c4eb2f05
Branch: refs/heads/jira/solr-12709
Commit: c4eb2f05f0cd71050dcf1c355bf4db52431944f2
Parents: 4a9176b 3b62f23
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Sat Sep 8 10:47:09 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Sat Sep 8 10:47:09 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 7 +
lucene/MIGRATE.txt | 5 +
.../apache/lucene/collation/package-info.java | 21 +-
lucene/analysis/icu/src/java/overview.html | 21 +-
.../lucene/analysis/ko/dict/UserDictionary.java | 6 +-
lucene/common-build.xml | 3 +-
.../lucene/index/FilterDirectoryReader.java | 14 +-
.../SoftDeletesDirectoryReaderWrapper.java | 14 +
.../java/org/apache/lucene/index/Sorter.java | 27 -
.../lucene/search/Boolean2ScorerSupplier.java | 4 +-
.../org/apache/lucene/search/BooleanScorer.java | 14 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../apache/lucene/search/CachingCollector.java | 16 +-
.../org/apache/lucene/search/FakeScorer.java | 62 -
.../apache/lucene/search/MatchAllDocsQuery.java | 2 +-
.../org/apache/lucene/search/ScoreAndDoc.java | 35 +
.../java/org/apache/lucene/search/Scorer.java | 7 +-
.../org/apache/lucene/search/SortRescorer.java | 10 +-
.../org/apache/lucene/search/TermQuery.java | 2 +-
.../lucene/store/ByteBuffersIndexInput.java | 2 +-
lucene/core/src/java/overview.html | 23 +-
.../src/test/org/apache/lucene/TestDemo.java | 69 +-
.../TestSoftDeletesDirectoryReaderWrapper.java | 51 +
.../lucene/search/MultiCollectorTest.java | 20 +-
.../search/TestBoolean2ScorerSupplier.java | 80 +-
.../org/apache/lucene/search/TestBooleanOr.java | 2 +-
.../apache/lucene/search/TestBooleanScorer.java | 2 +-
.../lucene/search/TestCachingCollector.java | 23 +-
.../lucene/search/TestConjunctionDISI.java | 50 +-
.../lucene/search/TestConstantScoreQuery.java | 4 +-
.../search/TestMaxScoreSumPropagator.java | 34 +-
.../lucene/search/TestMultiCollector.java | 16 +-
.../apache/lucene/search/TestQueryRescorer.java | 2 +-
.../lucene/search/TestTopDocsCollector.java | 22 +-
.../lucene/search/TestTopFieldCollector.java | 2 +-
.../apache/lucene/expressions/FakeScorer.java | 53 -
.../lucene/facet/DrillSidewaysScorer.java | 24 +-
.../search/grouping/BlockGroupingCollector.java | 19 +-
.../lucene/search/grouping/FakeScorer.java | 52 -
lucene/ivy-versions.properties | 2 +-
.../apache/lucene/search/join/FakeScorer.java | 52 -
.../randomizedtesting-runner-2.6.0.jar.sha1 | 1 -
.../randomizedtesting-runner-2.6.4.jar.sha1 | 1 +
.../queries/function/FunctionRangeQuery.java | 4 +-
.../lucene/queries/function/FunctionValues.java | 15 +-
.../lucene/queries/function/ValueSource.java | 26 +-
.../queries/function/ValueSourceScorer.java | 8 +-
.../docvalues/DocTermsIndexDocValues.java | 5 +-
.../function/docvalues/DoubleDocValues.java | 11 +-
.../function/docvalues/IntDocValues.java | 5 +-
.../function/docvalues/LongDocValues.java | 5 +-
.../function/valuesource/EnumFieldSource.java | 5 +-
.../document/TestLatLonPolygonShapeQueries.java | 2 +
.../Geo3dShapeWGS84ModelRectRelationTest.java | 2 +-
solr/CHANGES.txt | 35 +-
.../org/apache/solr/ltr/TestLTROnSolrCloud.java | 3 +-
solr/core/ivy.xml | 2 +-
.../cloud/OverseerConfigSetMessageHandler.java | 30 +-
.../org/apache/solr/cloud/RecoveryStrategy.java | 8 +-
.../cloud/api/collections/DeleteNodeCmd.java | 4 -
.../api/collections/MaintainRoutedAliasCmd.java | 7 +-
.../cloud/api/collections/TimeRoutedAlias.java | 19 +
.../cloud/autoscaling/ComputePlanAction.java | 38 +-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 31 +-
.../src/java/org/apache/solr/core/SolrCore.java | 4 +-
.../solr/handler/HaversineMetersEvaluator.java | 59 +
.../org/apache/solr/handler/IndexFetcher.java | 2 +-
.../solr/handler/SolrDefaultStreamFactory.java | 1 +
.../solr/handler/admin/ConfigSetsHandler.java | 20 +-
.../PhrasesIdentificationComponent.java | 1129 ++++++++++++++++++
.../solr/handler/component/QueryComponent.java | 31 +-
.../response/transform/ChildDocTransformer.java | 18 +-
.../transform/ChildDocTransformerFactory.java | 30 +-
.../solr/response/transform/DocTransformer.java | 23 +
.../transform/RawValueTransformerFactory.java | 20 +-
.../solr/search/CollapsingQParserPlugin.java | 38 +-
.../src/java/org/apache/solr/search/Filter.java | 2 +-
.../apache/solr/search/FunctionRangeQuery.java | 12 +-
.../search/function/ValueSourceRangeFilter.java | 9 +-
.../java/org/apache/solr/update/PeerSync.java | 2 +-
.../apache/solr/update/PeerSyncWithLeader.java | 6 +-
.../DistributedUpdateProcessorFactory.java | 2 +-
.../TimeRoutedAliasUpdateProcessor.java | 355 ++++--
solr/core/src/test-files/log4j2.xml | 12 +-
.../conf/schema-phrases-identification.xml | 97 ++
.../conf/solrconfig-phrases-identification.xml | 53 +
.../cloud/AssignBackwardCompatibilityTest.java | 1 +
.../org/apache/solr/cloud/DeleteShardTest.java | 2 +-
.../cloud/LeaderElectionIntegrationTest.java | 3 +-
.../solr/cloud/LeaderVoteWaitTimeoutTest.java | 2 +
.../apache/solr/cloud/MoveReplicaHDFSTest.java | 3 +-
.../apache/solr/cloud/OverseerRolesTest.java | 1 +
.../apache/solr/cloud/TestCloudConsistency.java | 1 +
...TestCloudPhrasesIdentificationComponent.java | 200 ++++
.../apache/solr/cloud/TestCloudRecovery.java | 2 +-
.../solr/cloud/TestClusterProperties.java | 25 +-
.../apache/solr/cloud/TestConfigSetsAPI.java | 14 +-
.../TestStressCloudBlindAtomicUpdates.java | 2 +-
.../apache/solr/cloud/TestWithCollection.java | 1 +
.../CollectionsAPIAsyncDistributedZkTest.java | 3 +-
.../AutoAddReplicasPlanActionTest.java | 2 +-
.../autoscaling/ComputePlanActionTest.java | 65 +-
.../autoscaling/ExecutePlanActionTest.java | 2 +-
.../MetricTriggerIntegrationTest.java | 3 +-
.../autoscaling/SearchRateTriggerTest.java | 3 +-
.../sim/TestSimExecutePlanAction.java | 2 +-
.../sim/TestSimGenericDistributedQueue.java | 6 +-
.../autoscaling/sim/TestSimLargeCluster.java | 2 +-
.../sim/TestSimTriggerIntegration.java | 8 +-
.../solr/cloud/cdcr/CdcrBootstrapTest.java | 4 +-
.../apache/solr/cloud/hdfs/StressHdfsTest.java | 1 +
.../apache/solr/handler/RequestLoggingTest.java | 27 +-
.../admin/ZookeeperStatusHandlerTest.java | 1 +
.../component/DistributedMLTComponentTest.java | 3 +-
.../PhrasesIdentificationComponentTest.java | 796 ++++++++++++
.../org/apache/solr/logging/TestLogWatcher.java | 79 +-
.../solr/metrics/SolrMetricReporterTest.java | 4 +-
.../reporters/solr/SolrCloudReportersTest.java | 2 +-
.../metrics/rrd/SolrRrdBackendFactoryTest.java | 1 +
.../transform/TestChildDocTransformer.java | 51 +-
.../solr/rest/TestManagedResourceStorage.java | 3 +-
.../solr/schema/SchemaApiFailureTest.java | 2 +-
.../apache/solr/search/TestRankQueryPlugin.java | 32 +-
.../apache/solr/search/TestStressRecovery.java | 3 +
.../solr/search/stats/TestDistribIDF.java | 3 +-
.../hadoop/TestDelegationWithHadoopAuth.java | 3 +-
.../solr/servlet/HttpSolrCallGetCoreTest.java | 3 +-
.../solr/store/hdfs/HdfsDirectoryTest.java | 7 +-
.../TestDocTermOrdsUninvertLimit.java | 2 +-
.../solr/update/TestInPlaceUpdatesDistrib.java | 3 +-
.../TimeRoutedAliasUpdateProcessorTest.java | 237 +++-
.../TrackingUpdateProcessorFactory.java | 136 +--
.../apache/solr/util/DateMathParserTest.java | 4 +-
solr/licenses/junit4-ant-2.6.0.jar.sha1 | 1 -
solr/licenses/junit4-ant-2.6.4.jar.sha1 | 1 +
.../randomizedtesting-runner-2.6.0.jar.sha1 | 1 -
.../randomizedtesting-runner-2.6.4.jar.sha1 | 1 +
solr/server/resources/log4j2-console.xml | 8 +-
solr/server/resources/log4j2.xml | 30 +-
.../src/basic-authentication-plugin.adoc | 4 +-
solr/solr-ref-guide/src/cloud-screens.adoc | 2 +-
solr/solr-ref-guide/src/collections-api.adoc | 24 +-
solr/solr-ref-guide/src/configsets-api.adoc | 2 +-
.../src/distributed-requests.adoc | 4 +-
solr/solr-ref-guide/src/docvalues.adoc | 10 +-
solr/solr-ref-guide/src/format-of-solr-xml.adoc | 4 +-
solr/solr-ref-guide/src/machine-learning.adoc | 242 +++-
.../src/solrcloud-autoscaling-triggers.adoc | 354 +++---
.../src/stream-decorator-reference.adoc | 23 +-
.../src/stream-source-reference.adoc | 32 +-
.../src/transforming-result-documents.adoc | 60 +-
...store-data-with-the-data-import-handler.adoc | 25 +-
.../org/apache/solr/client/solrj/io/Lang.java | 1 +
.../solrj/io/eval/KnnRegressionEvaluator.java | 3 +
.../solrj/io/eval/LatLonVectorsEvaluator.java | 115 ++
.../solrj/io/stream/TimeSeriesStream.java | 2 +
.../solrj/request/CollectionAdminRequest.java | 10 +
.../solrj/request/ConfigSetAdminRequest.java | 5 +-
.../solr/common/cloud/ClusterProperties.java | 23 +-
.../apispec/cluster.configs.Commands.json | 2 +-
.../resources/apispec/collections.Commands.json | 4 +
.../solr/client/solrj/SolrExampleTests.java | 4 +-
.../solrj/embedded/LargeVolumeJettyTest.java | 3 +-
.../client/solrj/impl/CloudSolrClientTest.java | 2 +-
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/graph/GraphExpressionTest.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 46 +
.../solrj/io/stream/StreamExpressionTest.java | 18 +-
.../client/solrj/io/stream/StreamingTest.java | 2 +
.../request/TestConfigSetAdminRequest.java | 2 -
170 files changed, 4563 insertions(+), 1339 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c4eb2f05/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c4eb2f05/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c4eb2f05/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
[10/47] lucene-solr:jira/solr-12709: SOLR-11863: Add knnRegress to
RefGuide
Posted by ab...@apache.org.
SOLR-11863: Add knnRegress to RefGuide
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0113adeb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0113adeb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0113adeb
Branch: refs/heads/jira/solr-12709
Commit: 0113adebceac2e5605afcaf2c3e43f935da4c0c5
Parents: e4f256b
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Sep 5 11:19:54 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Sep 5 11:20:30 2018 -0400
----------------------------------------------------------------------
solr/solr-ref-guide/src/machine-learning.adoc | 241 ++++++++++++++++++-
.../solrj/io/eval/KnnRegressionEvaluator.java | 3 +
2 files changed, 233 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0113adeb/solr/solr-ref-guide/src/machine-learning.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/machine-learning.adoc b/solr/solr-ref-guide/src/machine-learning.adoc
index ce8e91f..ae781bb 100644
--- a/solr/solr-ref-guide/src/machine-learning.adoc
+++ b/solr/solr-ref-guide/src/machine-learning.adoc
@@ -171,17 +171,21 @@ This expression returns the following response:
}
----
-== Distance Measures
+== Distance and Distance Measures
-The `distance` function computes a distance measure for two
+The `distance` function computes the distance for two
numeric arrays or a *distance matrix* for the columns of a matrix.
-There are four distance measures currently supported:
+There are four distance measure functions that return a function
+that performs the actual distance calculation:
-* euclidean (default)
-* manhattan
-* canberra
-* earthMovers
+* euclidean() (default)
+* manhattan()
+* canberra()
+* earthMovers()
+
+The distance measure functions can be used with all machine learning functions
+that support different distance measures.
Below is an example for computing euclidean distance for
two numeric arrays:
@@ -213,6 +217,35 @@ This expression returns the following response:
}
----
+Below the distance is calculated using *Manahattan* distance.
+
+[source,text]
+----
+let(a=array(20, 30, 40, 50),
+ b=array(21, 29, 41, 49),
+ c=distance(a, b, manhattan()))
+----
+
+This expression returns the following response:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "c": 4
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 1
+ }
+ ]
+ }
+}
+----
+
+
Below is an example for computing a distance matrix for columns
of a matrix:
@@ -603,13 +636,13 @@ This expression returns the following response:
}
----
-== K-nearest Neighbor (knn)
+== K-nearest Neighbor (KNN)
The `knn` function searches the rows of a matrix for the
K-nearest neighbors of a search vector. The `knn` function
returns a *matrix* of the K-nearest neighbors. The `knn` function
-has a *named parameter* called *distance* which specifies the distance measure.
-There are four distance measures currently supported:
+supports changing of the distance measure by providing one of the
+four distance measure functions as the fourth parameter:
* euclidean (Default)
* manhattan
@@ -677,4 +710,190 @@ This expression returns the following response:
]
}
}
-----
\ No newline at end of file
+----
+
+== KNN Regression
+
+KNN regression is a non-linear, multi-variate regression method. Knn regression is a lazy learning
+technique which means it does not fit a model to the training set in advance. Instead the
+entire training set of observations and outcomes are held in memory and predictions are made
+by averaging the outcomes of the k-nearest neighbors.
+
+The `knnRegress` function prepares the training set for use with the `predict` function.
+
+Below is an example of the `knnRegress` function. In this example 10000 random samples
+are taken each containing the variables *filesize_d*, *service_d* and *response_d*. The pairs of
+*filesize_d* and *service_d* will be use to predict the value of *response_d*.
+
+Notice that `kknRegress` simply returns a tuple describing the regression inputs.
+
+[source,text]
+----
+let(samples=random(collection1, q="*:*", rows="10000", fl="filesize_d, service_d, response_d"),
+ filesizes=col(samples, filesize_d),
+ serviceLevels=col(samples, service_d),
+ outcomes=col(samples, response_d),
+ observations=transpose(matrix(filesizes, serviceLevels)),
+ lazyModel=knnRegress(observations, outcomes , 5))
+----
+
+This expression returns the following response:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "lazyModel": {
+ "features": 2,
+ "robust": false,
+ "distance": "EuclideanDistance",
+ "observations": 10000,
+ "scale": false,
+ "k": 5
+ }
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 170
+ }
+ ]
+ }
+}
+----
+
+=== Prediction and Residuals
+
+The output of knnRegress can be used with the `predict` function like other regression models.
+In the example below the `predict` function is used to predict results for the original training
+data. The sumSq of the residuals is then calculated.
+
+[source,text]
+----
+let(samples=random(collection1, q="*:*", rows="10000", fl="filesize_d, service_d, response_d"),
+ filesizes=col(samples, filesize_d),
+ serviceLevels=col(samples, service_d),
+ outcomes=col(samples, response_d),
+ observations=transpose(matrix(filesizes, serviceLevels)),
+ lazyModel=knnRegress(observations, outcomes , 5),
+ predictions=predict(lazyModel, observations),
+ residuals=ebeSubtract(outcomes, predictions),
+ sumSqErr=sumSq(residuals))
+----
+
+This expression returns the following response:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "sumSqErr": 1920290.1204126712
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 3796
+ }
+ ]
+ }
+}
+----
+
+=== Setting Feature Scaling
+
+If the features in the observation matrix are not in the same scale then the larger features
+will carry more weight in the distance calculation then the smaller features. This can greatly
+impact the accuracy of the prediction. The `knnRegress` function has a *scale* parameter which
+can be set to *true* to automatically scale the features in the same range.
+
+Notice that when feature scaling is turned on the sumSqErr in the output is much lower.
+This shows how much more accurate the predictions are when feature scaling is turned on in
+this particular example. This is because the *filesize_d* feature is significantly larger then
+the *service_d* feature.
+
+[source,text]
+----
+let(samples=random(collection1, q="*:*", rows="10000", fl="filesize_d, service_d, response_d"),
+ filesizes=col(samples, filesize_d),
+ serviceLevels=col(samples, service_d),
+ outcomes=col(samples, response_d),
+ observations=transpose(matrix(filesizes, serviceLevels)),
+ lazyModel=knnRegress(observations, outcomes , 5, scale=true),
+ predictions=predict(lazyModel, observations),
+ residuals=ebeSubtract(outcomes, predictions),
+ sumSqErr=sumSq(residuals))
+----
+
+This expression returns the following response:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "sumSqErr": 4076.794951120683
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 3790
+ }
+ ]
+ }
+}
+----
+
+
+=== Setting Robust Regression
+
+The default prediction approach is to take the *mean* of the outcomes of the k-nearest
+neighbors. If the outcomes contain outliers the *mean* value can be skewed. Setting
+the *robust* parameter to true will take the *median* outcome of the k-nearest neighbors.
+This provides a regression prediction that is robust to outliers.
+
+
+=== Setting the Distance Measure
+
+The distance measure can be changed for the k-nearest neighbor search by adding distance measure
+function to the `knnRegress` parameters. Below is an example using manhattan distance.
+
+[source,text]
+----
+let(samples=random(collection1, q="*:*", rows="10000", fl="filesize_d, service_d, response_d"),
+ filesizes=col(samples, filesize_d),
+ serviceLevels=col(samples, service_d),
+ outcomes=col(samples, response_d),
+ observations=transpose(matrix(filesizes, serviceLevels)),
+ lazyModel=knnRegress(observations, outcomes, 5, manhattan(), scale=true),
+ predictions=predict(lazyModel, observations),
+ residuals=ebeSubtract(outcomes, predictions),
+ sumSqErr=sumSq(residuals))
+----
+
+This expression returns the following response:
+
+[source,json]
+----
+{
+ "result-set": {
+ "docs": [
+ {
+ "sumSqErr": 4761.221942288098
+ },
+ {
+ "EOF": true,
+ "RESPONSE_TIME": 3571
+ }
+ ]
+ }
+}
+----
+
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0113adeb/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
index e6f6d80..e298f45 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/KnnRegressionEvaluator.java
@@ -86,6 +86,7 @@ public class KnnRegressionEvaluator extends RecursiveObjectEvaluator implements
if(values.length == 4) {
if(values[3] instanceof DistanceMeasure) {
distanceMeasure = (DistanceMeasure) values[3];
+ } else {
throw new IOException("The fourth parameter for knnRegress should be a distance measure. ");
}
}
@@ -100,6 +101,8 @@ public class KnnRegressionEvaluator extends RecursiveObjectEvaluator implements
map.put("observations", observations.getRowCount());
map.put("features", observations.getColumnCount());
map.put("distance", distanceMeasure.getClass().getSimpleName());
+ map.put("robust", robust);
+ map.put("scale", scale);
return new KnnRegressionTuple(observations, outcomeData, k, distanceMeasure, map, scale, robust);
}
[09/47] lucene-solr:jira/solr-12709: SOLR-12722: [child] transformer
now processes 'fl'
Posted by ab...@apache.org.
SOLR-12722: [child] transformer now processes 'fl'
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e4f256be
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e4f256be
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e4f256be
Branch: refs/heads/jira/solr-12709
Commit: e4f256be15ca44f12a4aecb32c13d1ab2617cc00
Parents: c8b47e2
Author: David Smiley <ds...@apache.org>
Authored: Wed Sep 5 10:50:14 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Wed Sep 5 10:50:14 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 4 ++
.../response/transform/ChildDocTransformer.java | 18 +++++--
.../transform/ChildDocTransformerFactory.java | 30 +++++++++++-
.../solr/response/transform/DocTransformer.java | 23 +++++++++
.../transform/RawValueTransformerFactory.java | 20 +-------
.../transform/TestChildDocTransformer.java | 51 +++++++++++++++++---
.../src/transforming-result-documents.adoc | 1 +
.../solr/client/solrj/SolrExampleTests.java | 4 +-
8 files changed, 116 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index c0c6dbd..6bb1299 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -204,6 +204,10 @@ New Features
* SOLR-12716: NodeLostTrigger should support deleting replicas from lost nodes by setting preferredOperation=deletenode.
(shalin)
+* SOLR-12722: The [child] transformer now takes an 'fl' param to specify which fields to return. It will evaluate
+ doc transformers if present. In 7.5 a missing 'fl' defaults to the current behavior of all fields, but in 8.0
+ defaults to the top/request "fl". (Moshe Bla, David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
index bffbaf2..2628f75 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformer.java
@@ -54,17 +54,16 @@ class ChildDocTransformer extends DocTransformer {
private final DocSet childDocSet;
private final int limit;
private final boolean isNestedSchema;
+ private final SolrReturnFields childReturnFields;
- //TODO ought to be provided/configurable
- private final SolrReturnFields childReturnFields = new SolrReturnFields();
-
- ChildDocTransformer(String name, BitSetProducer parentsFilter,
- DocSet childDocSet, boolean isNestedSchema, int limit) {
+ ChildDocTransformer(String name, BitSetProducer parentsFilter, DocSet childDocSet,
+ SolrReturnFields returnFields, boolean isNestedSchema, int limit) {
this.name = name;
this.parentsFilter = parentsFilter;
this.childDocSet = childDocSet;
this.limit = limit;
this.isNestedSchema = isNestedSchema;
+ this.childReturnFields = returnFields!=null? returnFields: new SolrReturnFields();
}
@Override
@@ -73,6 +72,9 @@ class ChildDocTransformer extends DocTransformer {
}
@Override
+ public boolean needsSolrIndexSearcher() { return true; }
+
+ @Override
public void transform(SolrDocument rootDoc, int rootDocId) {
// note: this algorithm works if both if we have have _nest_path_ and also if we don't!
@@ -132,6 +134,12 @@ class ChildDocTransformer extends DocTransformer {
// load the doc
SolrDocument doc = searcher.getDocFetcher().solrDoc(docId, childReturnFields);
+ if(childReturnFields.getTransformer() != null) {
+ if(childReturnFields.getTransformer().context == null) {
+ childReturnFields.getTransformer().setContext(context);
+ }
+ childReturnFields.getTransformer().transform(doc, docId);
+ }
if (isAncestor) {
// if this path has pending child docs, add them.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java
index 2478c48..82be49d 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/ChildDocTransformerFactory.java
@@ -33,6 +33,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.search.SyntaxError;
import static org.apache.solr.schema.IndexSchema.NEST_PATH_FIELD_NAME;
@@ -57,12 +58,28 @@ public class ChildDocTransformerFactory extends TransformerFactory {
static final char PATH_SEP_CHAR = '/';
static final char NUM_SEP_CHAR = '#';
+ private static final ThreadLocal<Boolean> recursionCheckThreadLocal = ThreadLocal.withInitial(() -> Boolean.FALSE);
private static final BooleanQuery rootFilter = new BooleanQuery.Builder()
.add(new BooleanClause(new MatchAllDocsQuery(), BooleanClause.Occur.MUST))
.add(new BooleanClause(new DocValuesFieldExistsQuery(NEST_PATH_FIELD_NAME), BooleanClause.Occur.MUST_NOT)).build();
@Override
public DocTransformer create(String field, SolrParams params, SolrQueryRequest req) {
+ if(recursionCheckThreadLocal.get()) {
+ // this is a recursive call by SolrReturnFields, see ChildDocTransformerFactory#createChildDocTransformer
+ return new DocTransformer.NoopFieldTransformer();
+ } else {
+ try {
+ // transformer is yet to be initialized in this thread, create it
+ recursionCheckThreadLocal.set(true);
+ return createChildDocTransformer(field, params, req);
+ } finally {
+ recursionCheckThreadLocal.set(false);
+ }
+ }
+ }
+
+ private DocTransformer createChildDocTransformer(String field, SolrParams params, SolrQueryRequest req) {
SchemaField uniqueKeyField = req.getSchema().getUniqueKeyField();
if (uniqueKeyField == null) {
throw new SolrException( ErrorCode.BAD_REQUEST,
@@ -106,9 +123,20 @@ public class ChildDocTransformerFactory extends TransformerFactory {
}
}
+ String childReturnFields = params.get("fl");
+ SolrReturnFields childSolrReturnFields;
+ if(childReturnFields != null) {
+ childSolrReturnFields = new SolrReturnFields(childReturnFields, req);
+ } else if(req.getSchema().getDefaultLuceneMatchVersion().major < 8) {
+ // ensure backwards for versions prior to SOLR 8
+ childSolrReturnFields = new SolrReturnFields();
+ } else {
+ childSolrReturnFields = new SolrReturnFields(req);
+ }
+
int limit = params.getInt( "limit", 10 );
- return new ChildDocTransformer(field, parentsFilter, childDocSet, buildHierarchy, limit);
+ return new ChildDocTransformer(field, parentsFilter, childDocSet, childSolrReturnFields, buildHierarchy, limit);
}
private static Query parseQuery(String qstr, SolrQueryRequest req, String param) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
index 7665acd..f1f6bf3 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/DocTransformer.java
@@ -114,4 +114,27 @@ public abstract class DocTransformer {
public String toString() {
return getName();
}
+
+ /**
+ * Trivial Impl that ensure that the specified field is requested as an "extra" field,
+ * but then does nothing during the transformation phase.
+ */
+ public static final class NoopFieldTransformer extends DocTransformer {
+ final String field;
+
+ public NoopFieldTransformer() {
+ this.field = null;
+ }
+
+ public NoopFieldTransformer(String field ) {
+ this.field = field;
+ }
+ public String getName() { return "noop"; }
+ public String[] getExtraRequestFields() {
+ return this.field==null? null: new String[] { field };
+ }
+ public void transform(SolrDocument doc, int docid) {
+ // No-Op
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
index 25fe56c..55216e5 100644
--- a/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
+++ b/solr/core/src/java/org/apache/solr/response/transform/RawValueTransformerFactory.java
@@ -84,28 +84,10 @@ public class RawValueTransformerFactory extends TransformerFactory
if (field.equals(display)) {
// we have to ensure the field is returned
- return new NoopFieldTransformer(field);
+ return new DocTransformer.NoopFieldTransformer(field);
}
return new RenameFieldTransformer( field, display, false );
}
-
- /**
- * Trivial Impl that ensure that the specified field is requested as an "extra" field,
- * but then does nothing during the transformation phase.
- */
- private static final class NoopFieldTransformer extends DocTransformer {
- final String field;
- public NoopFieldTransformer(String field ) {
- this.field = field;
- }
- public String getName() { return "noop"; }
- public String[] getExtraRequestFields() {
- return new String[] { field };
- }
- public void transform(SolrDocument doc, int docid) {
- // No-Op
- }
- }
static class RawTransformer extends DocTransformer
{
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/core/src/test/org/apache/solr/response/transform/TestChildDocTransformer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/transform/TestChildDocTransformer.java b/solr/core/src/test/org/apache/solr/response/transform/TestChildDocTransformer.java
index 385ca4f..85b476a 100644
--- a/solr/core/src/test/org/apache/solr/response/transform/TestChildDocTransformer.java
+++ b/solr/core/src/test/org/apache/solr/response/transform/TestChildDocTransformer.java
@@ -16,9 +16,15 @@
*/
package org.apache.solr.response.transform;
+import java.util.Collection;
+import java.util.Iterator;
+
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.BasicResultContext;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -61,6 +67,7 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
testSubQueryJSON();
testChildDocNonStoredDVFields();
+ testChildReturnFields();
}
private void testChildDoctransformerXML() throws Exception {
@@ -91,10 +98,10 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
"fl", "*,[child parentFilter=\"subject:parentDocument\"]"), test1);
assertQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
- "fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
+ "fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
assertQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
- "fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
+ "fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
}
private void testSubQueryXML() {
@@ -212,10 +219,10 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
"fl", "*,[child parentFilter=\"subject:parentDocument\"]"), test1);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
- "fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
+ "fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
- "fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=3]"), test3);
+ "fl", "id, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=3]"), test3);
}
private void testChildDocNonStoredDVFields() throws Exception {
@@ -243,11 +250,39 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
"fl", "*,[child parentFilter=\"subject:parentDocument\"]"), test1);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
- "fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
+ "fl", "intDvoDefault, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:foo\"]"), test2);
assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
- "fl", "subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
+ "fl", "intDvoDefault, subject,[child parentFilter=\"subject:parentDocument\" childFilter=\"title:bar\" limit=2]"), test3);
+
+ }
+
+ private void testChildReturnFields() throws Exception {
+ assertJQ(req("q", "*:*", "fq", "subject:\"parentDocument\" ",
+ "fl", "*,[child parentFilter=\"subject:parentDocument\" fl=\"intDvoDefault,child_fl:[value v='child_fl_test']\"]"),
+ "/response/docs/[0]/intDefault==42",
+ "/response/docs/[0]/_childDocuments_/[0]/intDvoDefault==42",
+ "/response/docs/[0]/_childDocuments_/[0]/child_fl=='child_fl_test'");
+
+ try(SolrQueryRequest req = req("q", "*:*", "fq", "subject:\"parentDocument\" ",
+ "fl", "intDefault,[child parentFilter=\"subject:parentDocument\" fl=\"intDvoDefault, [docid]\"]")) {
+ BasicResultContext res = (BasicResultContext) h.queryAndResponse("/select", req).getResponse();
+ Iterator<SolrDocument> docsStreamer = res.getProcessedDocuments();
+ while (docsStreamer.hasNext()) {
+ SolrDocument doc = docsStreamer.next();
+ assertFalse("root docs should not contain fields specified in child return fields", doc.containsKey("intDvoDefault"));
+ assertTrue("root docs should contain fields specified in query return fields", doc.containsKey("intDefault"));
+ Collection<SolrDocument> childDocs = doc.getChildDocuments();
+ for(SolrDocument childDoc: childDocs) {
+ assertEquals("child doc should only have 2 keys", 2, childDoc.keySet().size());
+ assertTrue("child docs should contain fields specified in child return fields", childDoc.containsKey("intDvoDefault"));
+ assertEquals("child docs should contain fields specified in child return fields",
+ 42, childDoc.getFieldValue("intDvoDefault"));
+ assertTrue("child docs should contain fields specified in child return fields", childDoc.containsKey("[docid]"));
+ }
+ }
+ }
}
private void createSimpleIndex() {
@@ -339,7 +374,7 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
assertJQ(req("q", "*:*",
"sort", "id asc",
"fq", "subject:\"parentDocument\" ",
- "fl", "id,[child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
+ "fl", "id, cat, title, [child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
tests);
}
@@ -398,7 +433,7 @@ public class TestChildDocTransformer extends SolrTestCaseJ4 {
assertQ(req("q", "*:*",
"sort", "id asc",
"fq", "subject:\"parentDocument\" ",
- "fl", "id,[child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
+ "fl", "id, cat, title, [child childFilter='cat:childDocument' parentFilter=\"subject:parentDocument\"]"),
tests);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/solr-ref-guide/src/transforming-result-documents.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/transforming-result-documents.adoc b/solr/solr-ref-guide/src/transforming-result-documents.adoc
index f9fe805..df9bff4 100644
--- a/solr/solr-ref-guide/src/transforming-result-documents.adoc
+++ b/solr/solr-ref-guide/src/transforming-result-documents.adoc
@@ -137,6 +137,7 @@ When using this transformer, the `parentFilter` parameter must be specified, and
* `childFilter` - query to filter which child documents should be included, this can be particularly useful when you have multiple levels of hierarchical documents (default: all children)
* `limit` - the maximum number of child documents to be returned per parent document (default: 10)
+* `fl` - the field list which the transformer is to return (default: top level "fl"). There is a further limitation in which the fields here should be a subset of those specified by the top level fl param
=== [shard] - ShardAugmenterFactory
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e4f256be/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
index f6628da..1dabe5d 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java
@@ -1831,7 +1831,7 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
q = new SolrQuery("q", "*:*", "indent", "true");
q.setFilterQueries(parentFilter);
- q.setFields("id,[child parentFilter=\"" + parentFilter +
+ q.setFields("id, level_i, [child parentFilter=\"" + parentFilter +
"\" childFilter=\"" + childFilter +
"\" limit=\"" + maxKidCount + "\"], name");
resp = client.query(q);
@@ -1916,7 +1916,7 @@ abstract public class SolrExampleTests extends SolrExampleTestsBase
q = new SolrQuery("q", "name:" + name, "indent", "true");
}
q.setFilterQueries(parentFilter);
- q.setFields("id,[child parentFilter=\"" + parentFilter +
+ q.setFields("id, level_i, [child parentFilter=\"" + parentFilter +
"\" childFilter=\"" + childFilter +
"\" limit=\"" + maxKidCount + "\"],name");
resp = client.query(q);
[16/47] lucene-solr:jira/solr-12709: SOLR-11690: put command examples
in [source] blocks; make NOTE a CAUTION and move up to associated command
Posted by ab...@apache.org.
SOLR-11690: put command examples in [source] blocks; make NOTE a CAUTION and move up to associated command
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/25650102
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/25650102
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/25650102
Branch: refs/heads/jira/solr-12709
Commit: 256501026149a9d15675b3db885cf23b7d442156
Parents: a3aa014
Author: Cassandra Targett <ct...@apache.org>
Authored: Tue Sep 4 20:58:04 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Sep 5 12:47:08 2018 -0500
----------------------------------------------------------------------
...store-data-with-the-data-import-handler.adoc | 25 ++++++++++++--------
1 file changed, 15 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/25650102/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
index cdfee7d..d724055 100644
--- a/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
+++ b/solr/solr-ref-guide/src/uploading-structured-data-store-data-with-the-data-import-handler.adoc
@@ -143,27 +143,32 @@ http://localhost:8983/solr/dih/dataimport?command=full-import&jdbcurl=jdbc:hsqld
The database password can be encrypted if necessary to avoid plaintext passwords being exposed in unsecured files. To do this, we will replace the password in `data-config.xml` with an encrypted password. We will use the `openssl` tool for the encryption, and the encryption key will be stored in a file which is only readable to the `solr` process. Please follow these steps:
-. Create a strong encryption password and store it in a file. Then make sure it is readable only for the `solr` user. Example commands:
-
- echo -n "a-secret" > /var/solr/data/dih-encryptionkey
- chown solr:solr /var/solr/data/dih-encryptionkey
- chmod 600 /var/solr/data/dih-encryptionkey
+. Create a strong encryption password and store it in a file. Then make sure it is readable only for the `solr` user. Example commands:
++
+[source,text]
+echo -n "a-secret" > /var/solr/data/dih-encryptionkey
+chown solr:solr /var/solr/data/dih-encryptionkey
+chmod 600 /var/solr/data/dih-encryptionkey
++
+CAUTION: Note that we use the `-n` argument to `echo` to avoid including a newline character at the end of the password. If you use another method to generate the encrypted password, make sure to avoid newlines as well.
. Encrypt the JDBC database password using `openssl` as follows:
++
+[source,text]
+echo -n "my-jdbc-password" | openssl enc -aes-128-cbc -a -salt -md md5 -pass file:/var/solr/data/dih-encryptionkey
++
+The output of the command will be a long string such as `U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o=`. You will use this as `password` in your `data-config.xml` file.
- echo -n "my-jdbc-password" | openssl enc -aes-128-cbc -a -salt -md md5 -pass file:/var/solr/data/dih-encryptionkey
-
-.. The output of the command will be a long string such as `U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o=`. You will use this as `password` in your `data-config.xml` file.
. In your `data-config.xml`, you'll add the `password` and `encryptKeyFile` parameters to the `<datasource>` configuration, as in this example:
+
[source,xml]
<dataSource driver="org.hsqldb.jdbcDriver"
url="jdbc:hsqldb:./example-DIH/hsqldb/ex"
- user="sa"
+ user="sa"
password="U2FsdGVkX18QMjY0yfCqlfBMvAB4d3XkwY96L7gfO2o="
encryptKeyFile="/var/solr/data/dih-encryptionkey" />
-NOTE: Note that we use the `-n` argument to `echo` to avoid including a newline character at the end of the password. If you use another method to generate the encrypted password, make sure to avoid newlines as well.
+
== DataImportHandler Commands
[42/47] lucene-solr:jira/solr-12709: SOLR-11943: Change location...
to latlon...
Posted by ab...@apache.org.
SOLR-11943: Change location... to latlon...
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f5ce384f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f5ce384f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f5ce384f
Branch: refs/heads/jira/solr-12709
Commit: f5ce384fb8e0c44f833344727740d6e92753417c
Parents: 2c88922
Author: Joel Bernstein <jb...@apache.org>
Authored: Fri Sep 7 15:42:03 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Fri Sep 7 15:42:34 2018 -0400
----------------------------------------------------------------------
.../org/apache/solr/client/solrj/io/Lang.java | 2 +-
.../solrj/io/eval/LatLonVectorsEvaluator.java | 115 +++++++++++++++++++
.../solrj/io/eval/LocationVectorsEvaluator.java | 105 -----------------
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 4 +-
5 files changed, 119 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f5ce384f/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
index 9568ecb..3f24e7b 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/Lang.java
@@ -255,7 +255,7 @@ public class Lang {
.withFunctionName("removeCache", RemoveCacheEvaluator.class)
.withFunctionName("listCache", ListCacheEvaluator.class)
.withFunctionName("zscores", NormalizeEvaluator.class)
- .withFunctionName("locationVectors", LocationVectorsEvaluator.class)
+ .withFunctionName("latlonVectors", LatLonVectorsEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f5ce384f/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LatLonVectorsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LatLonVectorsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LatLonVectorsEvaluator.java
new file mode 100644
index 0000000..de8168a
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LatLonVectorsEvaluator.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+/**
+ * The LatLonVectorsEvaluator maps to the latlonVectors Math Expression. The latlonVectors expression
+ * takes a list of Tuples that contain a lat,lon point field and a named parameter called field
+ * as input. The field parameter specifies which field to parse the lat,lon vectors from in the Tuples.
+ *
+ * The latlonVectors function returns a matrix of lat,lon vectors. Each row in the matrix
+ * contains a single lat,lon pair.
+ *
+ **/
+
+public class LatLonVectorsEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ private String field;
+
+ public LatLonVectorsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+
+ List<StreamExpressionNamedParameter> namedParams = factory.getNamedOperands(expression);
+
+ for (StreamExpressionNamedParameter namedParam : namedParams) {
+ if(namedParam.getName().equals("field")) {
+ this.field = namedParam.getParameter().toString();
+ } else {
+ throw new IOException("Unexpected named parameter:" + namedParam.getName());
+ }
+ }
+
+ if(field == null) {
+ throw new IOException("The named parameter \"field\" must be set for the latlonVectors function.");
+ }
+ }
+
+ @Override
+ public Object doWork(Object... objects) throws IOException {
+
+ if (objects.length == 1) {
+ //Just docs
+ if(!(objects[0] instanceof List)) {
+ throw new IOException("The latlonVectors function expects a list of Tuples as a parameter.");
+ } else {
+ List list = (List)objects[0];
+ if(list.size() > 0) {
+ Object o = list.get(0);
+ if(!(o instanceof Tuple)) {
+ throw new IOException("The latlonVectors function expects a list of Tuples as a parameter.");
+ }
+ } else {
+ throw new IOException("Empty list was passed as a parameter to termVectors function.");
+ }
+ }
+
+ List<Tuple> tuples = (List<Tuple>) objects[0];
+
+ double[][] locationVectors = new double[tuples.size()][2];
+ List<String> features = new ArrayList();
+ features.add("lat");
+ features.add("lon");
+
+ List<String> rowLabels = new ArrayList();
+
+ for(int i=0; i< tuples.size(); i++) {
+ Tuple tuple = tuples.get(i);
+ String value = tuple.getString(field);
+ String[] latLong = null;
+ if(value.contains(",")) {
+ latLong = value.split(",");
+ } else {
+ latLong = value.split(" ");
+ }
+
+ locationVectors[i][0] = Double.parseDouble(latLong[0].trim());
+ locationVectors[i][1] = Double.parseDouble(latLong[1].trim());
+ if(tuple.get("id") != null) {
+ rowLabels.add(tuple.get("id").toString());
+ }
+ }
+
+ Matrix matrix = new Matrix(locationVectors);
+ matrix.setColumnLabels(features);
+ matrix.setRowLabels(rowLabels);
+ return matrix;
+ } else {
+ throw new IOException("The latlonVectors function takes a single positional parameter.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f5ce384f/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java
deleted file mode 100644
index 0c1ba99..0000000
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/LocationVectorsEvaluator.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.client.solrj.io.eval;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-
-import org.apache.solr.client.solrj.io.Tuple;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
-import org.apache.solr.client.solrj.io.stream.expr.StreamExpressionNamedParameter;
-import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
-
-public class LocationVectorsEvaluator extends RecursiveObjectEvaluator implements ManyValueWorker {
- protected static final long serialVersionUID = 1L;
-
- private String field;
-
- public LocationVectorsEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
- super(expression, factory);
-
- List<StreamExpressionNamedParameter> namedParams = factory.getNamedOperands(expression);
-
- for (StreamExpressionNamedParameter namedParam : namedParams) {
- if(namedParam.getName().equals("field")) {
- this.field = namedParam.getParameter().toString();
- } else {
- throw new IOException("Unexpected named parameter:" + namedParam.getName());
- }
- }
-
- if(field == null) {
- throw new IOException("The named parameter \"field\" must be set for the locationVectors function.");
- }
- }
-
- @Override
- public Object doWork(Object... objects) throws IOException {
-
- if (objects.length == 1) {
- //Just docs
- if(!(objects[0] instanceof List)) {
- throw new IOException("The locationVectors function expects a list of Tuples as a parameter.");
- } else {
- List list = (List)objects[0];
- if(list.size() > 0) {
- Object o = list.get(0);
- if(!(o instanceof Tuple)) {
- throw new IOException("The locationVectors function expects a list of Tuples as a parameter.");
- }
- } else {
- throw new IOException("Empty list was passed as a parameter to termVectors function.");
- }
- }
-
- List<Tuple> tuples = (List<Tuple>) objects[0];
-
- double[][] locationVectors = new double[tuples.size()][2];
- List<String> features = new ArrayList();
- features.add("lat");
- features.add("long");
-
- List<String> rowLabels = new ArrayList();
-
- for(int i=0; i< tuples.size(); i++) {
- Tuple tuple = tuples.get(i);
- String value = tuple.getString(field);
- String[] latLong = null;
- if(value.contains(",")) {
- latLong = value.split(",");
- } else {
- latLong = value.split(" ");
- }
-
- locationVectors[i][0] = Double.parseDouble(latLong[0].trim());
- locationVectors[i][1] = Double.parseDouble(latLong[1].trim());
- if(tuple.get("id") != null) {
- rowLabels.add(tuple.get("id").toString());
- }
- }
-
- Matrix matrix = new Matrix(locationVectors);
- matrix.setColumnLabels(features);
- matrix.setRowLabels(rowLabels);
- return matrix;
- } else {
- throw new IOException("The termVectors function takes a single positional parameter.");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f5ce384f/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
index ee8c1e1..12ff2e9 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/TestLang.java
@@ -70,7 +70,7 @@ public class TestLang extends LuceneTestCase {
"mod", "ceil", "floor", "sin", "asin", "sinh", "cos", "acos", "cosh", "tan", "atan", "tanh", "round", "sqrt",
"cbrt", "coalesce", "uuid", "if", "convert", "valueAt", "memset", "fft", "ifft", "euclidean","manhattan",
"earthMovers", "canberra", "chebyshev", "ones", "zeros", "setValue", "getValue", "knnRegress", "gaussfit",
- "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "locationVectors"};
+ "outliers", "stream", "getCache", "putCache", "listCache", "removeCache", "zscores", "latlonVectors"};
@Test
public void testLang() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f5ce384f/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
index 4bcf50d..137add6 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java
@@ -312,7 +312,7 @@ public class MathExpressionTest extends SolrCloudTestCase {
}
@Test
- public void testLocationFunctions() throws Exception {
+ public void testLatlonFunctions() throws Exception {
UpdateRequest updateRequest = new UpdateRequest();
int i=0;
@@ -326,7 +326,7 @@ public class MathExpressionTest extends SolrCloudTestCase {
String expr = "let(echo=true," +
" a=search("+COLLECTIONORALIAS+", q=*:*, fl=\"id, loc_p, price_i\",rows=100, sort=\"price_i asc\"),"+
- " b=locationVectors(a, field=loc_p)," +
+ " b=latlonVectors(a, field=loc_p)," +
" c=distance(array(40.7128, 74.0060), array(45.7128, 74.0060), haversineMeters()))";
[30/47] lucene-solr:jira/solr-12709: SOLR-11943: Update CHANGES.txt
Posted by ab...@apache.org.
SOLR-11943: Update CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c684773e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c684773e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c684773e
Branch: refs/heads/jira/solr-12709
Commit: c684773e8df0c12eb490b53e41eedb5de0686b1e
Parents: b8e87a1
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Sep 6 14:23:53 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Sep 6 14:23:53 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c684773e/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3d947c7..a36be87 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -211,6 +211,8 @@ New Features
* SOLR-9418: Added a new (experimental) PhrasesIdentificationComponent for identifying potential phrases
in query input based on overlapping shingles in the index. (Akash Mehta, Trey Grainger, hossman)
+* SOLR-11943: Add machine learning functions for location data (Joel Bernstein)
+
Bug Fixes
----------------------
[44/47] lucene-solr:jira/solr-12709: SOLR-12055: Enable async logging
by default - rollback
Posted by ab...@apache.org.
SOLR-12055: Enable async logging by default - rollback
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3b62f23f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3b62f23f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3b62f23f
Branch: refs/heads/jira/solr-12709
Commit: 3b62f23f72ed826d363b81826be9caf0a2edbd1b
Parents: 8f49892
Author: Erick Erickson <Er...@gmail.com>
Authored: Fri Sep 7 22:51:50 2018 -0700
Committer: Erick Erickson <Er...@gmail.com>
Committed: Fri Sep 7 22:51:50 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 9 --
solr/core/src/test-files/log4j2.xml | 12 +--
.../apache/solr/handler/RequestLoggingTest.java | 27 ++----
.../org/apache/solr/logging/TestLogWatcher.java | 91 +++++++++-----------
solr/server/resources/log4j2-console.xml | 8 +-
solr/server/resources/log4j2.xml | 30 +++----
6 files changed, 74 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b62f23f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 7d30c0c..797acfc 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -100,11 +100,6 @@ Upgrade Notes
WINDOWS USERS: This JIRA corrects a bug in the start scripts that put example logs under ...\server, solr.log is
now under ...\example. (Erick Erickson)
-* SOLR-12055: Enable async logging by default. This change improves throughput for logging. This opens
- up a small window where log messages could possibly be lost. If this is unacceptable, switching back to synchronous
- logging can be done by changing the log4j2.xml files, no internal Solr code changed to make async logging the default.
- (Erick Erickson)
-
New Features
----------------------
@@ -322,12 +317,8 @@ Bug Fixes
* SOLR-12704: Guard AddSchemaFieldsUpdateProcessorFactory against null field names and field values.
(Steve Rowe, Varun Thacker)
-* SOLR-12728: RequestLoggingTest fails on occasion, not reproducible (Erick Erickson)
-
* SOLR-12733: SolrMetricReporterTest failure (Erick Erickson, David Smiley)
-* SOLR-12732: TestLogWatcher failure on Jenkins (Erick Erickson)
-
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b62f23f/solr/core/src/test-files/log4j2.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/log4j2.xml b/solr/core/src/test-files/log4j2.xml
index 5447843..7d0ebf7 100644
--- a/solr/core/src/test-files/log4j2.xml
+++ b/solr/core/src/test-files/log4j2.xml
@@ -27,13 +27,13 @@
</Console>
</Appenders>
<Loggers>
- <AsyncLogger name="org.apache.zookeeper" level="WARN"/>
- <AsyncLogger name="org.apache.hadoop" level="WARN"/>
- <AsyncLogger name="org.apache.directory" level="WARN"/>
- <AsyncLogger name="org.apache.solr.hadoop" level="INFO"/>
+ <Logger name="org.apache.zookeeper" level="WARN"/>
+ <Logger name="org.apache.hadoop" level="WARN"/>
+ <Logger name="org.apache.directory" level="WARN"/>
+ <Logger name="org.apache.solr.hadoop" level="INFO"/>
- <AsyncRoot level="INFO">
+ <Root level="INFO">
<AppenderRef ref="STDERR"/>
- </AsyncRoot>
+ </Root>
</Loggers>
</Configuration>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b62f23f/solr/core/src/test/org/apache/solr/handler/RequestLoggingTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/RequestLoggingTest.java b/solr/core/src/test/org/apache/solr/handler/RequestLoggingTest.java
index ae08e9a..e203e80 100644
--- a/solr/core/src/test/org/apache/solr/handler/RequestLoggingTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/RequestLoggingTest.java
@@ -48,15 +48,17 @@ public class RequestLoggingTest extends SolrTestCaseJ4 {
@Before
public void setupAppender() {
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
+ LoggerConfig config = ctx.getConfiguration().getLoggerConfig("RequestLoggingTest");
writer = new StringWriter();
appender = WriterAppender.createAppender(
PatternLayout
.newBuilder()
.withPattern("%-5p [%t]: %m%n")
- .build(),
+ .build(),
null, writer, "RequestLoggingTest", false, true);
appender.start();
+
}
@Test
@@ -73,7 +75,7 @@ public class RequestLoggingTest extends SolrTestCaseJ4 {
public void testLogBeforeExecute(Logger logger) {
Level level = logger.getLevel();
-
+
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
LoggerConfig config = ctx.getConfiguration().getLoggerConfig(logger.getName());
config.setLevel(Level.DEBUG);
@@ -82,23 +84,10 @@ public class RequestLoggingTest extends SolrTestCaseJ4 {
try {
assertQ(req("q", "*:*"));
- Matcher matcher = null;
- boolean foundDebugMsg = false;
- String output = "";
- for (int msgIdx = 0; msgIdx < 100; ++msgIdx) {
- output = writer.toString();
- matcher = Pattern.compile("DEBUG.*q=\\*:\\*.*").matcher(output);
- if (matcher.find()) {
- foundDebugMsg = true;
- break;
- }
- try {
- Thread.sleep(10);
- } catch (InterruptedException ie) {
- ;
- }
- }
- assertTrue("Should have found debug-level message. Found " + output, foundDebugMsg);
+
+ String output = writer.toString();
+ Matcher matcher = Pattern.compile("DEBUG.*q=\\*:\\*.*").matcher(output);
+ assertTrue(matcher.find());
final String group = matcher.group();
final String msg = "Should not have post query information";
assertFalse(msg, group.contains("hits"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b62f23f/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java b/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
index a93a11b..6f7987f 100644
--- a/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
+++ b/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.logging;
-import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.junit.Before;
@@ -27,63 +26,55 @@ import org.slf4j.LoggerFactory;
import java.lang.invoke.MethodHandles;
import java.util.concurrent.atomic.AtomicBoolean;
-public class TestLogWatcher extends SolrTestCaseJ4 {
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestLogWatcher {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private LogWatcherConfig config;
@Before
- public void before() {
- config = new LogWatcherConfig(true, null, "INFO", 1);
+ public void setUp() {
+ config = new LogWatcherConfig(true, null, null, 50);
}
- // Create several log watchers and ensure that new messages go to the new watcher.
@Test
public void testLog4jWatcher() {
- LogWatcher watcher;
- int lim = random().nextInt(3) + 2;
- for (int idx = 0; idx < lim; ++idx) {
- String msg = "This is a test message: " + idx;
- watcher = LogWatcher.newRegisteredLogWatcher(config, null);
-
- // First ensure there's nothing in the new watcher.
-
- // Every time you put a message in the queue, you wait for it to come out _before_ creating
- // a new watcher so it should be fine.
- if (looper(watcher, null) == false) {
- fail("There should be no messages when a new watcher finally gets registered! In loop: " + idx);
- }
-
- // Now log a message and ensure that the new watcher sees it.
- log.warn(msg);
-
- // Loop to give the logger time to process the async message and notify the new watcher.
- if (looper(watcher, msg) == false) {
- fail("Should have found message " + msg + ". In loop: " + idx);
- }
- }
+ LogWatcher watcher = LogWatcher.newRegisteredLogWatcher(config, null);
+
+ assertEquals(watcher.getLastEvent(), -1);
+
+ log.warn("This is a test message");
+
+ assertTrue(watcher.getLastEvent() > -1);
+
+ SolrDocumentList events = watcher.getHistory(-1, new AtomicBoolean());
+ assertEquals(events.size(), 1);
+
+ SolrDocument event = events.get(0);
+ assertEquals(event.get("logger"), "org.apache.solr.logging.TestLogWatcher");
+ assertEquals(event.get("message"), "This is a test message");
+
}
- private boolean looper(LogWatcher watcher, String msg) {
- // In local testing this loop usually succeeds 1-2 tries.
- boolean success = false;
- boolean testingNew = msg == null;
- for (int msgIdx = 0; msgIdx < 100 && success == false; ++msgIdx) {
- if (testingNew) { // check that there are no entries registered for the watcher
- success = watcher.getLastEvent() == -1;
- } else { // check that the expected message is there.
- // Returns an empty (but non-null) list even if there are no messages yet.
- SolrDocumentList events = watcher.getHistory(-1, new AtomicBoolean());
- for (SolrDocument doc : events) {
- if (doc.get("message").equals(msg)) {
- success = true;
- }
- }
- }
- try {
- Thread.sleep(10);
- } catch (InterruptedException ie) {
- ;
- }
- }
- return success;
+
+ // This seems weird to do the same thing twice, this is valid. We need to test whether listeners are replaced....
+ @Test
+ public void testLog4jWatcherRepeat() {
+ LogWatcher watcher = LogWatcher.newRegisteredLogWatcher(config, null);
+
+ assertEquals(watcher.getLastEvent(), -1);
+
+ log.warn("This is a test message");
+
+ assertTrue(watcher.getLastEvent() > -1);
+
+ SolrDocumentList events = watcher.getHistory(-1, new AtomicBoolean());
+ assertEquals(events.size(), 1);
+
+ SolrDocument event = events.get(0);
+ assertEquals(event.get("logger"), "org.apache.solr.logging.TestLogWatcher");
+ assertEquals(event.get("message"), "This is a test message");
+
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b62f23f/solr/server/resources/log4j2-console.xml
----------------------------------------------------------------------
diff --git a/solr/server/resources/log4j2-console.xml b/solr/server/resources/log4j2-console.xml
index 698227b..f32f4c1 100644
--- a/solr/server/resources/log4j2-console.xml
+++ b/solr/server/resources/log4j2-console.xml
@@ -29,11 +29,11 @@
</Console>
</Appenders>
<Loggers>
- <AsyncLogger name="org.apache.zookeeper" level="WARN"/>
- <AsyncLogger name="org.apache.hadoop" level="WARN"/>
+ <Logger name="org.apache.zookeeper" level="WARN"/>
+ <Logger name="org.apache.hadoop" level="WARN"/>
- <AsyncRoot level="INFO">
+ <Root level="INFO">
<AppenderRef ref="STDERR"/>
- </AsyncRoot>
+ </Root>
</Loggers>
</Configuration>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b62f23f/solr/server/resources/log4j2.xml
----------------------------------------------------------------------
diff --git a/solr/server/resources/log4j2.xml b/solr/server/resources/log4j2.xml
index d9c05d8..17bcf4c 100644
--- a/solr/server/resources/log4j2.xml
+++ b/solr/server/resources/log4j2.xml
@@ -27,8 +27,8 @@
</PatternLayout>
</Console>
- <RollingRandomAccessFile
- name="MainLogFile"
+ <RollingFile
+ name="RollingFile"
fileName="${sys:solr.log.dir}/solr.log"
filePattern="${sys:solr.log.dir}/solr.log.%i" >
<PatternLayout>
@@ -41,10 +41,10 @@
<SizeBasedTriggeringPolicy size="32 MB"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- </RollingRandomAccessFile>
+ </RollingFile>
- <RollingRandomAccessFile
- name="SlowLogFile"
+ <RollingFile
+ name="SlowFile"
fileName="${sys:solr.log.dir}/solr_slow_requests.log"
filePattern="${sys:solr.log.dir}/solr_slow_requests.log.%i" >
<PatternLayout>
@@ -57,20 +57,20 @@
<SizeBasedTriggeringPolicy size="32 MB"/>
</Policies>
<DefaultRolloverStrategy max="10"/>
- </RollingRandomAccessFile>
+ </RollingFile>
</Appenders>
<Loggers>
- <AsyncLogger name="org.apache.hadoop" level="warn"/>
- <AsyncLogger name="org.apache.solr.update.LoggingInfoStream" level="off"/>
- <AsyncLogger name="org.apache.zookeeper" level="warn"/>
- <AsyncLogger name="org.apache.solr.core.SolrCore.SlowRequest" level="info" additivity="false">
- <AppenderRef ref="SlowLogFile"/>
- </AsyncLogger>
+ <Logger name="org.apache.hadoop" level="warn"/>
+ <Logger name="org.apache.solr.update.LoggingInfoStream" level="off"/>
+ <Logger name="org.apache.zookeeper" level="warn"/>
+ <Logger name="org.apache.solr.core.SolrCore.SlowRequest" level="info" additivity="false">
+ <AppenderRef ref="SlowFile"/>
+ </Logger>
- <AsyncRoot level="info">
- <AppenderRef ref="MainLogFile"/>
+ <Root level="info">
+ <AppenderRef ref="RollingFile"/>
<AppenderRef ref="STDOUT"/>
- </AsyncRoot>
+ </Root>
</Loggers>
</Configuration>
[26/47] lucene-solr:jira/solr-12709: SOLR-12716: Move common params
to top of page; insert links to common param section for each trigger;
improve consistency
Posted by ab...@apache.org.
SOLR-12716: Move common params to top of page; insert links to common param section for each trigger; improve consistency
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cac589b8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cac589b8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cac589b8
Branch: refs/heads/jira/solr-12709
Commit: cac589b803c518a388366a506a0067254e5b6c22
Parents: 00ce9e0
Author: Cassandra Targett <ct...@apache.org>
Authored: Thu Sep 6 11:58:51 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu Sep 6 11:58:51 2018 -0500
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-triggers.adoc | 320 +++++++++++--------
1 file changed, 181 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cac589b8/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index 9c8aac5..0b41e21 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -1,4 +1,5 @@
= SolrCloud Autoscaling Triggers
+:page-tocclass: right
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -16,31 +17,27 @@
// specific language governing permissions and limitations
// under the License.
-Triggers are used in autoscaling to watch for cluster events such as nodes joining, leaving, search rate or any other metric breaching a threshold.
-
-In the future other cluster, node, and replica events that are important from the
-point of view of cluster performance will also have available triggers.
+Triggers are used in autoscaling to watch for cluster events such as nodes joining or leaving, search rate, index rate, on a schedule, or any other metric breaching a threshold.
Trigger implementations verify the state of resources that they monitor. When they detect a
-change that merits attention they generate events, which are then queued and processed by configured
-`TriggerAction` implementations. This usually involves computing and executing a plan to manage the new cluster
-resources (e.g., move replicas). Solr provides predefined implementations of triggers for specific event types.
+change that merits attention they generate _events_, which are then queued and processed by configured
+`TriggerAction` implementations. This usually involves computing and executing a plan to do something (e.g., move replicas). Solr provides predefined implementations of triggers for <<Event Types,specific event types>>.
-Triggers execute on the node that runs `Overseer`. They are scheduled to run periodically, at a default interval of 1 second between each execution (not every execution produces events).
+Triggers execute on the node that runs `Overseer`. They are scheduled to run periodically, at a default interval of 1 second between each execution (although it's important to note that not every execution of a trigger produces events).
== Event Types
Currently the following event types (and corresponding trigger implementations) are defined:
-* `nodeAdded`: generated when a node joins the cluster
-* `nodeLost`: generated when a node leaves the cluster
-* `metric`: generated when the configured metric crosses a configured lower or upper threshold value
+* `nodeAdded`: generated when a node joins the cluster. See <<Node Added Trigger>>.
+* `nodeLost`: generated when a node leaves the cluster. See <<Node Lost Trigger>> and <<Auto Add Replicas Trigger>>.
+* `metric`: generated when the configured metric crosses a configured lower or upper threshold value. See <<Metric Trigger>>.
* `indexSize`: generated when a shard size (defined as index size in bytes or number of documents)
-exceeds upper or lower threshold values
-* `searchRate`: generated when the search rate exceeds configured upper or lower thresholds
-* `scheduled`: generated according to a scheduled time period such as every 24 hours, etc
+exceeds upper or lower threshold values. See <<Index Size Trigger>>.
+* `searchRate`: generated when the search rate exceeds configured upper or lower thresholds. See <<Search Rate Trigger>>.
+* `scheduled`: generated according to a scheduled time period such as every 24 hours, etc. See <<Scheduled Trigger>>.
-Events are not necessarily generated immediately after the corresponding state change occurred - the
-maximum rate of events is controlled by the `waitFor` configuration parameter (see below).
+Events are not necessarily generated immediately after the corresponding state change occurred; the
+maximum rate of events is controlled by the `waitFor` configuration parameter (see <<Trigger Configuration>> below for more explanation).
The following properties are common to all event types:
@@ -57,12 +54,82 @@ generated, which may significantly differ due to the rate limits set by `waitFor
`properties`:: (map, optional) Any additional properties. Currently includes e.g., `nodeNames` property that
indicates the nodes that were lost or added.
-== Node Added Trigger
+== Trigger Configuration
+Trigger configurations are managed using the <<solrcloud-autoscaling-api.adoc#write-api,Autoscaling Write API>> with the commands `<<solrcloud-autoscaling-api.adoc#create-update-trigger,set-trigger>>`, `<<solrcloud-autoscaling-api.adoc#remove-trigger,remove-trigger>>`,
+`suspend-trigger`, and `resume-trigger`.
+
+=== Trigger Properties
+
+Trigger configuration consists of the following properties:
+
+`name`:: (string, required) A unique trigger configuration name.
+
+`event`:: (string, required) One of the predefined event types (`nodeAdded` or `nodeLost`).
+
+`actions`:: (list of action configs, optional) An ordered list of actions to execute when event is fired.
+
+`waitFor`:: (string, optional) The time to wait between generating new events, as an integer number immediately
+followed by unit symbol, one of `s` (seconds), `m` (minutes), or `h` (hours). Default is `0s`. A condition must
+persist at least for the `waitFor` period to generate an event.
+
+`enabled`:: (boolean, optional) When `true` the trigger is enabled. Default is `true`.
+
+Additional implementation-specific properties may be provided, as described in the sections for individual triggers below.
+
+=== Action Properties
+
+Action configuration consists of the following properties:
+
+`name`:: (string, required) A unique name of the action configuration.
+
+`class`:: (string, required) The action implementation class.
+
+Additional implementation-specific properties may be provided, as described in the sections for individual triggers below.
+
+If the `actions` configuration is omitted, then by default, the `ComputePlanAction` and the `ExecutePlanAction` are automatically added to the trigger configuration.
+
+=== Example Trigger Configuration
+
+This simple example shows the configuration for adding (or updating) a trigger for `nodeAdded` events.
+
+[source,json]
+----
+{
+ "set-trigger": {
+ "name" : "node_added_trigger",
+ "event" : "nodeAdded",
+ "waitFor" : "1s",
+ "enabled" : true,
+ "actions" : [
+ {
+ "name" : "compute_plan",
+ "class": "solr.ComputePlanAction"
+ },
+ {
+ "name" : "custom_action",
+ "class": "com.example.CustomAction"
+ },
+ {
+ "name" : "execute_plan",
+ "class": "solr.ExecutePlanAction"
+ }
+ ]
+ }
+}
+----
+
+This trigger configuration will compute and execute a plan to allocate the resources available on the new node. A custom action could also be used to possibly modify the plan.
+
+== Available Triggers
+
+As described earlier, there are several triggers available to watch for events.
+
+=== Node Added Trigger
The `NodeAddedTrigger` generates `nodeAdded` events when a node joins the cluster. It can be used to either move replicas
from other nodes to the new node or to add new replicas.
-Apart from the parameters described at <<#trigger-configuration, Trigger Configuration>>, this trigger supports the following configuration:
+In addition to the parameters described at <<Trigger Configuration>>, this trigger supports one more parameter:
`preferredOperation`:: (string, optional, defaults to `movereplica`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from other nodes to the added node. The only other supported value is `addreplica` which adds more replicas of the existing collections on the new node.
@@ -91,12 +158,12 @@ Apart from the parameters described at <<#trigger-configuration, Trigger Configu
}
----
-== Node Lost Trigger
+=== Node Lost Trigger
The `NodeLostTrigger` generates `nodeLost` events when a node leaves the cluster. It can be used to either move replicas
that were hosted by the lost node to other nodes or to delete them from the cluster.
-Apart from the parameters described at <<#trigger-configuration, Trigger Configuration>>, this trigger supports the following configuration:
+In addition to the parameters described at <<Trigger Configuration>>, this trigger supports the one more parameter:
`preferredOperation`:: (string, optional, defaults to `MOVEREPLICA`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from the lost nodes to the other nodes in the cluster. The only other supported value is `DELETENODE` which deletes all information about replicas that were hosted by the lost node.
@@ -125,9 +192,9 @@ Apart from the parameters described at <<#trigger-configuration, Trigger Configu
}
----
-TIP: It is recommended that the value of `waitFor` configuration for node lost trigger be larger than a minute so that large full garbage collection pauses do not cause this trigger to generate events and needlessly move or delete replicas in the cluster.
+TIP: It is recommended that the value of `waitFor` configuration for the node lost trigger be larger than 1 minute so that large full garbage collection pauses do not cause this trigger to generate events and needlessly move or delete replicas in the cluster.
-== Auto Add Replicas Trigger
+=== Auto Add Replicas Trigger
When a collection has the parameter `autoAddReplicas` set to true then a trigger configuration named `.auto_add_replicas` is automatically created to watch for nodes going away. This trigger produces `nodeLost` events,
which are then processed by configured actions (usually resulting in computing and executing a plan
@@ -135,32 +202,39 @@ to add replicas on the live nodes to maintain the expected replication factor).
Refer to the section <<solrcloud-autoscaling-auto-add-replicas.adoc#solrcloud-autoscaling-auto-add-replicas, Autoscaling Automatically Adding Replicas>> to learn more about how the `.autoAddReplicas` trigger works.
-This trigger supports one parameter, which is defined in the `<solrcloud>` section of `solr.xml`:
+In addition to the parameters described at <<Trigger Configuration>>, this trigger supports one parameter, which is defined in the `<solrcloud>` section of `solr.xml`:
`autoReplicaFailoverWaitAfterExpiration`::
The minimum time in milliseconds to wait for initiating replacement of a replica after first noticing it not being live. This is important to prevent false positives while stopping or starting the cluster. The default is `120000` (2 minutes). The value provided for this parameter is used as the value for the `waitFor` parameter in the `.auto_add_replicas` trigger.
TIP: See <<format-of-solr-xml.adoc#the-solrcloud-element,The <solrcloud> Element>> for more details about how to work with `solr.xml`.
-== Metric Trigger
+=== Metric Trigger
The metric trigger can be used to monitor any metric exposed by the <<metrics-reporting.adoc#metrics-reporting,Metrics API>>. It supports lower and upper threshold configurations as well as optional filters to limit operation to specific collection, shards, and nodes.
-This trigger supports the following configuration:
+In addition to the parameters described at <<Trigger Configuration>>, this trigger supports the following parameters:
-`metric`:: (string, required) The metric property name to be watched in the format metric:__group__:__prefix__, e.g., `metric:solr.node:CONTAINER.fs.coreRoot.usableSpace`.
+`metric`::
+(string, required) The metric property name to be watched in the format metric:__group__:__prefix__, e.g., `metric:solr.node:CONTAINER.fs.coreRoot.usableSpace`.
-`below`:: (double, optional) The lower threshold for the metric value. The trigger produces a metric breached event if the metric's value falls below this value.
+`below`::
+(double, optional) The lower threshold for the metric value. The trigger produces a metric breached event if the metric's value falls below this value.
-`above`:: (double, optional) The upper threshold for the metric value. The trigger produces a metric breached event if the metric's value crosses above this value.
+`above`::
+(double, optional) The upper threshold for the metric value. The trigger produces a metric breached event if the metric's value crosses above this value.
-`collection`:: (string, optional) The collection used to limit the nodes on which the given metric is watched. When the metric is breached, trigger actions will limit operations to this collection only.
+`collection`::
+(string, optional) The collection used to limit the nodes on which the given metric is watched. When the metric is breached, trigger actions will limit operations to this collection only.
-`shard`:: (string, optional) The shard used to limit the nodes on which the given metric is watched. When the metric is breached, trigger actions will limit operations to this shard only.
+`shard`::
+(string, optional) The shard used to limit the nodes on which the given metric is watched. When the metric is breached, trigger actions will limit operations to this shard only.
-`node`:: (string, optional) The node on which the given metric is watched. Trigger actions will operate on this node only.
+`node`::
+(string, optional) The node on which the given metric is watched. Trigger actions will operate on this node only.
-`preferredOperation`:: (string, optional, defaults to `MOVEREPLICA`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from the hot node to others. The only other supported value is `ADDREPLICA` which adds more replicas if the metric is breached.
+`preferredOperation`::
+(string, optional, defaults to `MOVEREPLICA`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from the hot node to others. The only other supported value is `ADDREPLICA` which adds more replicas if the metric is breached.
.Example: a metric trigger that fires when total usable space on a node having replicas of "mycollection" falls below 100GB
[source,json]
@@ -177,7 +251,7 @@ This trigger supports the following configuration:
}
----
-== Index Size Trigger
+=== Index Size Trigger
This trigger can be used for monitoring the size of collection shards, measured either by the
number of documents in a shard or the physical size of the shard's index in bytes.
@@ -193,34 +267,41 @@ that operation is not yet implemented (see https://issues.apache.org/jira/browse
Additionally, monitoring can be restricted to a list of collections; by default
all collections are monitored.
-This trigger supports the following configuration parameters (all thresholds are exclusive):
+In addition to the parameters described at <<Trigger Configuration>>, this trigger supports the following configuration parameters (all thresholds are exclusive):
-`aboveBytes`:: upper threshold in bytes. This value is compared to the `INDEX.sizeInBytes` metric.
+`aboveBytes`::
+A upper threshold in bytes. This value is compared to the `INDEX.sizeInBytes` metric.
-`belowBytes`:: lower threshold in bytes. Note that this value should be at least 2x smaller than
+`belowBytes`::
+A lower threshold in bytes. Note that this value should be at least 2x smaller than
`aboveBytes`
-`aboveDocs`:: upper threshold expressed as the number of documents. This value is compared with `SEARCHER.searcher.numDocs` metric.
+`aboveDocs`::
+An upper threshold expressed as the number of documents. This value is compared with `SEARCHER.searcher.numDocs` metric.
+
NOTE: Due to the way Lucene indexes work, a shard may exceed the `aboveBytes` threshold
even if the number of documents is relatively small, because replaced and deleted documents keep
occupying disk space until they are actually removed during Lucene index merging.
-`belowDocs`:: lower threshold expressed as the number of documents.
+`belowDocs`::
+A lower threshold expressed as the number of documents.
-`aboveOp`:: operation to request when an upper threshold is exceeded. If not specified the
+`aboveOp`::
+The operation to request when an upper threshold is exceeded. If not specified the
default value is `SPLITSHARD`.
-`belowOp`:: operation to request when a lower threshold is exceeded. If not specified
+`belowOp`::
+The operation to request when a lower threshold is exceeded. If not specified
the default value is `MERGESHARDS` (but see the note above).
-`collections`:: comma-separated list of collection names that this trigger should monitor. If not
+`collections`::
+A comma-separated list of collection names that this trigger should monitor. If not
specified or empty all collections are monitored.
Events generated by this trigger contain additional details about the shards
that exceeded thresholds and the types of violations (upper / lower bounds, bytes / docs metrics).
-.Example:
+.Example: Index Size Trigger
This configuration specifies an index size trigger that monitors collections "test1" and "test2",
with both bytes (1GB) and number of docs (1 million) upper limits, and a custom `belowOp`
operation `NONE` (which still can be monitored and acted upon by an appropriate trigger listener):
@@ -253,7 +334,7 @@ operation `NONE` (which still can be monitored and acted upon by an appropriate
}
----
-== Search Rate Trigger
+=== Search Rate Trigger
The search rate trigger can be used for monitoring search rates in a selected
collection (1-min average rate by default), and request that either replicas be moved from
@@ -269,82 +350,101 @@ This method was chosen to avoid generating false events when a simple client kee
to a single specific replica (because adding or removing other replicas can't solve this situation,
only proper load balancing can - either by using `CloudSolrClient` or another load-balancing client).
-Note: this trigger calculates node-level cumulative rates using per-replica rates reported by
+This trigger calculates node-level cumulative rates using per-replica rates reported by
replicas that are part of monitored collections / shards on each node. This means that it may report
some nodes as "cold" (underutilized) because it ignores other, perhaps more active, replicas
belonging to other collections. Also, nodes that don't host any of the monitored replicas or
those that are explicitly excluded by `node` config property won't be reported at all.
-Note 2: special care should be taken when configuring `waitFor` property. By default the trigger
-monitors a 1-min average search rate of a replica. Changes to the number of replicas that should in turn
+.Calculating `waitFor`
+[CAUTION]
+====
+Special care should be taken when configuring the `waitFor` property. By default the trigger
+monitors a 1-minute average search rate of a replica. Changes to the number of replicas that should in turn
change per-replica search rates may be requested and executed relatively quickly if the
-`waitFor` is set to comparable values of 1 min or shorter. However, the metric value, being a
-moving average, will always lag behind the new "momentary" rate after the changes. This in turn means that
-the monitored metric may not change sufficiently enough to prevent the
-trigger from firing again (because it will continue to measure the average rate as still violating
-the threshold for some time after the change was executed). As a result the trigger may keep
+`waitFor` is set to comparable values of 1 min or shorter.
+
+However, the metric value, being a moving average, will always lag behind the new "momentary" rate after the changes. This in turn means that the monitored metric may not change sufficiently enough to prevent the
+trigger from firing again, because it will continue to measure the average rate as still violating
+the threshold for some time after the change was executed. As a result the trigger may keep
requesting that even more replicas be added (or removed) and thus it may "overshoot" the optimal number of replicas.
+
For this reason it's recommended to always set `waitFor` to values several
-times longer than the time constant of the used metric. For example, with the default 1-min average the
-`waitFor` should be set to at least `2m` or more.
+times longer than the time constant of the used metric. For example, with the default 1-minute average the
+`waitFor` should be set to at least `2m` (2 minutes) or more.
+====
-This trigger supports the following configuration properties:
+In addition to the parameters described at <<Trigger Configuration>>, this trigger supports the following configuration properties:
-`collections`:: (string, optional) comma-separated list of collection names to monitor, or any collection if empty / not set.
+`collections`::
+(string, optional) A comma-separated list of collection names to monitor, or any collection if empty or not set.
-`shard`:: (string, optional) shard name within the collection (requires `collections` to be set to exactly one name), or any shard if empty.
+`shard`::
+(string, optional) A shard name within the collection (requires `collections` to be set to exactly one name), or any shard if empty.
-`node`:: (string, optional) node name to monitor, or any if empty.
+`node`::
+(string, optional) A node name to monitor, or any if empty.
-`metric`:: (string, optional) metric name that represents the search rate
-(default is `QUERY./select.requestTimes:1minRate`). This name has to identify a single numeric
-metric value, and it may use the colon syntax for selecting one property of a complex metric. This value
-is collected from all replicas for a shard, and then an arithmetic average is calculated per shard
-to determine shard-level violations.
+`metric`::
+(string, optional) A metric name that represents the search rate. The default is `QUERY./select.requestTimes:1minRate`. This name has to identify a single numeric metric value, and it may use the colon syntax for selecting one property of
+a complex metric. This value is collected from all replicas for a shard, and then an arithmetic average is calculated
+per shard to determine shard-level violations.
-`maxOps`:: (integer, optional) maximum number of add replica / delete replica operations
-requested in a single autoscaling event. The default value is 3 and it helps to smooth out
+`maxOps`::
+(integer, optional) The maximum number of `ADDREPLICA` or `DELETEREPLICA` operations
+requested in a single autoscaling event. The default value is `3` and it helps to smooth out
the changes to the number of replicas during periods of large search rate fluctuations.
-`minReplicas`:: (integer, optional) minimum acceptable number of searchable replicas (i.e., replicas other
-than `PULL` type). The trigger will not generate any DELETEREPLICA requests when the number of
-searchable replicas in a shard reaches this threshold. When this value is not set (the default)
+`minReplicas`::
+(integer, optional) The minimum acceptable number of searchable replicas (i.e., replicas other
+than `PULL` type). The trigger will not generate any `DELETEREPLICA` requests when the number of
+searchable replicas in a shard reaches this threshold.
++
+When this value is not set (the default)
the `replicationFactor` property of the collection is used, and if that property is not set then
-the value is set to 1. Note also that shard leaders are never deleted.
+the value is set to `1`. Note also that shard leaders are never deleted.
-`aboveRate`:: (float) the upper bound for the request rate metric value. At least one of
+`aboveRate`::
+(float) The upper bound for the request rate metric value. At least one of
`aboveRate` or `belowRate` must be set.
-`belowRate`:: (float) the lower bound for the request rate metric value. At least one of
+`belowRate`::
+(float) The lower bound for the request rate metric value. At least one of
`aboveRate` or `belowRate` must be set.
-`aboveNodeRate`:: (float) the upper bound for the total request rate metric value per node. If not
+`aboveNodeRate`::
+(float) The upper bound for the total request rate metric value per node. If not
set then cumulative per-node rates will be ignored.
-`belowNodeRate`:: (float) the lower bound for the total request rate metric value per node. If not
+`belowNodeRate`::
+(float) The lower bound for the total request rate metric value per node. If not
set then cumulative per-node rates will be ignored.
-`aboveOp`:: (string, optional) collection action to request when the upper threshold for a shard is
+`aboveOp`::
+(string, optional) A collection action to request when the upper threshold for a shard is
exceeded. Default action is `ADDREPLICA` and the trigger will request from 1 up to `maxOps` operations
per shard per event, proportionally to how much the rate is exceeded. This property can be set to 'NONE'
to effectively disable the action but still report it to the listeners.
-`aboveNodeOp`:: (string, optional) collection action to request when the upper threshold for a node (`aboveNodeRate`) is exceeded.
+`aboveNodeOp`::
+(string, optional) The collection action to request when the upper threshold for a node (`aboveNodeRate`) is exceeded.
Default action is `MOVEREPLICA`, and the trigger will request 1 replica operation per hot node per event.
If both `aboveOp` and `aboveNodeOp` operations are to be requested then `aboveNodeOp` operations are
always requested first, and only if no `aboveOp` (shard level) operations are to be requested (because `aboveOp`
operations will change node-level rates anyway). This property can be set to 'NONE' to effectively disable
the action but still report it to the listeners.
-`belowOp`:: (string, optional) collection action to request when the lower threshold for a shard is
+`belowOp`::
+(string, optional) The collection action to request when the lower threshold for a shard is
exceeded. Default action is `DELETEREPLICA`, and the trigger will request at most `maxOps` replicas
to be deleted from eligible cold shards. This property can be set to 'NONE'
to effectively disable the action but still report it to the listeners.
-`belowNodeOp`:: action to request when the lower threshold for a node (`belowNodeRate`) is exceeded.
+`belowNodeOp`::
+(string, optional) The action to request when the lower threshold for a node (`belowNodeRate`) is exceeded.
Default action is null (not set) and the condition is ignored, because in many cases the
trigger will monitor only some selected resources (replicas from selected
-collections / shards) so setting this by default to e.g., `DELETENODE` could interfere with
+collections or shards) so setting this by default to e.g., `DELETENODE` could interfere with
these non-monitored resources. The trigger will request 1 operation per cold node per event.
If both `belowOp` and `belowNodeOp` operations are requested then `belowOp` operations are
always requested first.
@@ -355,6 +455,7 @@ average request rate of "/select" handler exceeds 100 requests/sec, and the cond
for over 20 minutes. If the rate falls below 0.01 and persists for 20 min the trigger will
request not only replica deletions (leaving at most 1 replica per shard) but also it may
request node deletion.
+
[source,json]
----
{
@@ -384,11 +485,11 @@ request node deletion.
}
----
-== Scheduled Trigger
+=== Scheduled Trigger
The Scheduled trigger generates events according to a fixed rate schedule.
-The trigger supports the following configuration:
+In addition to the parameters described at <<Trigger Configuration>>, this trigger supports the following configuration:
`startTime`::
(string, required) The start date/time of the schedule. This should either be a DateMath string e.g., 'NOW', or be an ISO-8601 date time string (the same standard used during search and indexing in Solr, which defaults to UTC), or be specified without the trailing 'Z' accompanied with the `timeZone` parameter. For example, each of the following values are acceptable:
@@ -410,65 +511,6 @@ The trigger supports the following configuration:
This trigger applies the `every` date math expression on the `startTime` or the last event time to derive the next scheduled time and if current time is greater than next scheduled time but within `graceTime` then an event is generated.
-Apart from the common event properties described in the Event Types section, the trigger adds an additional `actualEventTime` event property which has the actual event time as opposed to the scheduled time.
+Apart from the common event properties described in the <<Event Types>> section, the trigger adds an additional `actualEventTime` event property which has the actual event time as opposed to the scheduled time.
For example, if the scheduled time was `2018-01-31T15:30:00Z` and grace time was `+15MINUTES` then an event may be fired at `2018-01-31T15:45:00Z`. Such an event will have `eventTime` as `2018-01-31T15:30:00Z`, the scheduled time, but the `actualEventTime` property will have a value of `2018-01-31T15:45:00Z`, the actual time.
-
-== Trigger Configuration
-Trigger configurations are managed using the Autoscaling Write API and the commands `set-trigger`, `remove-trigger`,
-`suspend-trigger`, and `resume-trigger`.
-
-Trigger configuration consists of the following properties:
-
-`name`:: (string, required) A unique trigger configuration name.
-
-`event`:: (string, required) One of the predefined event types (`nodeAdded` or `nodeLost`).
-
-`actions`:: (list of action configs, optional) An ordered list of actions to execute when event is fired.
-
-`waitFor`:: (string, optional) The time to wait between generating new events, as an integer number immediately
-followed by unit symbol, one of `s` (seconds), `m` (minutes), or `h` (hours). Default is `0s`. A condition must
-persist at least for the `waitFor` period to generate an event.
-
-`enabled`:: (boolean, optional) When `true` the trigger is enabled. Default is `true`.
-
-Additional implementation-specific properties may be provided.
-
-Action configuration consists of the following properties:
-
-`name`:: (string, required) A unique name of the action configuration.
-
-`class`:: (string, required) The action implementation class.
-
-Additional implementation-specific properties may be provided
-
-If the `actions` configuration is omitted, then by default, the `ComputePlanAction` and the `ExecutePlanAction` are automatically added to the trigger configuration.
-
-.Example: adding or updating a trigger for `nodeAdded` events
-[source,json]
-----
-{
- "set-trigger": {
- "name" : "node_added_trigger",
- "event" : "nodeAdded",
- "waitFor" : "1s",
- "enabled" : true,
- "actions" : [
- {
- "name" : "compute_plan",
- "class": "solr.ComputePlanAction"
- },
- {
- "name" : "custom_action",
- "class": "com.example.CustomAction"
- },
- {
- "name" : "execute_plan",
- "class": "solr.ExecutePlanAction"
- }
- ]
- }
-}
-----
-
-This trigger configuration will compute and execute a plan to allocate the resources available on the new node. A custom action is also used to possibly modify the plan.
[36/47] lucene-solr:jira/solr-12709: SOLR-12357: TRA
preemptiveCreateMath option. Simplified test utility
TrackingUpdateProcessorFactory. Reverted some attempts the TRA used to make
in avoiding overseer communication (too complicated). Closes #433
Posted by ab...@apache.org.
SOLR-12357: TRA preemptiveCreateMath option.
Simplified test utility TrackingUpdateProcessorFactory.
Reverted some attempts the TRA used to make in avoiding overseer communication (too complicated).
Closes #433
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/21d130c3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/21d130c3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/21d130c3
Branch: refs/heads/jira/solr-12709
Commit: 21d130c3edf8bfb21a3428fc95e5b67d6be757e7
Parents: 9e04375
Author: David Smiley <ds...@apache.org>
Authored: Thu Sep 6 23:38:44 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Thu Sep 6 23:38:44 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../api/collections/MaintainRoutedAliasCmd.java | 7 +-
.../cloud/api/collections/TimeRoutedAlias.java | 19 +
.../DistributedUpdateProcessorFactory.java | 2 +-
.../TimeRoutedAliasUpdateProcessor.java | 355 ++++++++++++-------
.../TimeRoutedAliasUpdateProcessorTest.java | 237 +++++++++++--
.../TrackingUpdateProcessorFactory.java | 136 ++-----
solr/solr-ref-guide/src/collections-api.adoc | 21 ++
.../solrj/request/CollectionAdminRequest.java | 10 +
.../resources/apispec/collections.Commands.json | 4 +
10 files changed, 546 insertions(+), 248 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 78bfb33..e5596e1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -216,6 +216,9 @@ New Features
* SOLR-12612: Cluster properties restriction of known keys only is relaxed, and now unknown properties starting with "ext."
will be allowed. This allows custom to plugins set their own cluster properties. (Jeffery Yuan via Tomás Fernández Löbbe)
+* SOLR-12357: Time Routed Aliases now have a preemptiveCreateMath option to preemptively and asynchronously create the
+ next collection in advance as new data gets within this time window of the end. (Gus Heck, David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java
index 083571c..e5c5de6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MaintainRoutedAliasCmd.java
@@ -75,7 +75,12 @@ public class MaintainRoutedAliasCmd implements OverseerCollectionMessageHandler.
this.ocmh = ocmh;
}
- /** Invokes this command from the client. If there's a problem it will throw an exception. */
+ /**
+ * Invokes this command from the client. If there's a problem it will throw an exception.
+ * Please note that is important to never add async to this invocation. This method must
+ * block (up to the standard OCP timeout) to prevent large batches of add's from sending a message
+ * to the overseer for every document added in TimeRoutedAliasUpdateProcessor.
+ */
public static NamedList remoteInvoke(CollectionsHandler collHandler, String aliasName, String mostRecentCollName)
throws Exception {
final String operation = CollectionParams.CollectionAction.MAINTAINROUTEDALIAS.toLower();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
index 312e736..1fb3d9e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/TimeRoutedAlias.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud.api.collections;
+import java.text.ParseException;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
@@ -63,6 +64,7 @@ public class TimeRoutedAlias {
public static final String ROUTER_START = ROUTER_PREFIX + "start";
public static final String ROUTER_INTERVAL = ROUTER_PREFIX + "interval";
public static final String ROUTER_MAX_FUTURE = ROUTER_PREFIX + "maxFutureMs";
+ public static final String ROUTER_PREEMPTIVE_CREATE_MATH = ROUTER_PREFIX + "preemptiveCreateMath";
public static final String ROUTER_AUTO_DELETE_AGE = ROUTER_PREFIX + "autoDeleteAge";
public static final String CREATE_COLLECTION_PREFIX = "create-collection.";
// plus TZ and NAME
@@ -84,6 +86,7 @@ public class TimeRoutedAlias {
public static final List<String> OPTIONAL_ROUTER_PARAMS = Collections.unmodifiableList(Arrays.asList(
ROUTER_MAX_FUTURE,
ROUTER_AUTO_DELETE_AGE,
+ ROUTER_PREEMPTIVE_CREATE_MATH,
TZ)); // kinda special
static Predicate<String> PARAM_IS_PROP =
@@ -126,6 +129,7 @@ public class TimeRoutedAlias {
private final String routeField;
private final String intervalMath; // ex: +1DAY
private final long maxFutureMs;
+ private final String preemptiveCreateMath;
private final String autoDeleteAgeMath; // ex: /DAY-30DAYS *optional*
private final TimeZone timeZone;
@@ -141,6 +145,9 @@ public class TimeRoutedAlias {
//optional:
maxFutureMs = params.getLong(ROUTER_MAX_FUTURE, TimeUnit.MINUTES.toMillis(10));
+ // the date math configured is an interval to be subtracted from the most recent collection's time stamp
+ String pcmTmp = params.get(ROUTER_PREEMPTIVE_CREATE_MATH);
+ preemptiveCreateMath = pcmTmp != null ? (pcmTmp.startsWith("-") ? pcmTmp : "-" + pcmTmp) : null;
autoDeleteAgeMath = params.get(ROUTER_AUTO_DELETE_AGE); // no default
timeZone = TimeZoneUtils.parseTimezone(aliasMetadata.get(CommonParams.TZ));
@@ -167,6 +174,13 @@ public class TimeRoutedAlias {
throw new SolrException(BAD_REQUEST, "bad " + TimeRoutedAlias.ROUTER_AUTO_DELETE_AGE + ", " + e, e);
}
}
+ if (preemptiveCreateMath != null) {
+ try {
+ new DateMathParser().parseMath(preemptiveCreateMath);
+ } catch (ParseException e) {
+ throw new SolrException(BAD_REQUEST, "Invalid date math for preemptiveCreateMath:" + preemptiveCreateMath);
+ }
+ }
if (maxFutureMs < 0) {
throw new SolrException(BAD_REQUEST, ROUTER_MAX_FUTURE + " must be >= 0");
@@ -189,6 +203,10 @@ public class TimeRoutedAlias {
return maxFutureMs;
}
+ public String getPreemptiveCreateWindow() {
+ return preemptiveCreateMath;
+ }
+
public String getAutoDeleteAgeMath() {
return autoDeleteAgeMath;
}
@@ -204,6 +222,7 @@ public class TimeRoutedAlias {
.add("routeField", routeField)
.add("intervalMath", intervalMath)
.add("maxFutureMs", maxFutureMs)
+ .add("preemptiveCreateMath", preemptiveCreateMath)
.add("autoDeleteAgeMath", autoDeleteAgeMath)
.add("timeZone", timeZone)
.toString();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
index 1930c08..9f2a0ba 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessorFactory.java
@@ -50,7 +50,7 @@ public class DistributedUpdateProcessorFactory
public UpdateRequestProcessor getInstance(SolrQueryRequest req,
SolrQueryResponse rsp, UpdateRequestProcessor next) {
// note: will sometimes return DURP (no overhead) instead of wrapping
- return TimeRoutedAliasUpdateProcessor.wrap(req, rsp,
+ return TimeRoutedAliasUpdateProcessor.wrap(req,
new DistributedUpdateProcessor(req, rsp, next));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
index d9d1da1..cc1ddb8 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessor.java
@@ -19,17 +19,15 @@ package org.apache.solr.update.processor;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.text.ParseException;
import java.time.Instant;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutorService;
import java.util.stream.Collectors;
-import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.cloud.api.collections.MaintainRoutedAliasCmd;
import org.apache.solr.cloud.api.collections.TimeRoutedAlias;
@@ -47,19 +45,24 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.update.CommitUpdateCommand;
import org.apache.solr.update.DeleteUpdateCommand;
import org.apache.solr.update.SolrCmdDistributor;
import org.apache.solr.update.processor.DistributedUpdateProcessor.DistribPhase;
+import org.apache.solr.util.DateMathParser;
+import org.apache.solr.util.DefaultSolrThreadFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+import static org.apache.solr.common.util.ExecutorUtil.newMDCAwareSingleThreadExecutor;
import static org.apache.solr.update.processor.DistributedUpdateProcessor.DISTRIB_FROM;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.CreationType.ASYNC_PREEMPTIVE;
+import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.CreationType.NONE;
+import static org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessor.CreationType.SYNCHRONOUS;
/**
* Distributes update requests to a rolling series of collections partitioned by a timestamp field. Issues
@@ -73,33 +76,31 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor
* @since 7.2.0
*/
public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
- //TODO do we make this more generic to others who want to partition collections using something else?
-
- public static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM; // param
+ //TODO do we make this more generic to others who want to partition collections using something else besides time?
+ private static final String ALIAS_DISTRIB_UPDATE_PARAM = "alias." + DISTRIB_UPDATE_PARAM; // param
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- // To avoid needless/redundant concurrent communication with the Overseer from this JVM, we
- // maintain a Semaphore from an alias name keyed ConcurrentHashMap.
- // Alternatively a Lock or CountDownLatch could have been used but they didn't seem
- // to make it any easier.
- private static ConcurrentHashMap<String, Semaphore> aliasToSemaphoreMap = new ConcurrentHashMap<>(4);
+ // refs to std infrastructure
+ private final SolrQueryRequest req;
+ private final SolrCmdDistributor cmdDistrib;
+ private final CollectionsHandler collHandler;
+ private final ZkController zkController;
+ // Stuff specific to this class
private final String thisCollection;
-
private final TimeRoutedAlias timeRoutedAlias;
-
- private final ZkController zkController;
- private final SolrCmdDistributor cmdDistrib;
- private final CollectionsHandler collHandler;
private final SolrParams outParamsToLeader;
- private final CloudDescriptor cloudDesc;
+ // These two fields may be updated within the calling thread during processing but should
+ // never be updated by any async creation thread.
private List<Map.Entry<Instant, String>> parsedCollectionsDesc; // k=timestamp (start), v=collection. Sorted descending
private Aliases parsedCollectionsAliases; // a cached reference to the source of what we parse into parsedCollectionsDesc
- private SolrQueryRequest req;
- public static UpdateRequestProcessor wrap(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ // This will be updated out in async creation threads see preemptiveAsync(Runnable r) for details
+ private volatile ExecutorService preemptiveCreationExecutor;
+
+ public static UpdateRequestProcessor wrap(SolrQueryRequest req, UpdateRequestProcessor next) {
//TODO get from "Collection property"
final String aliasName = req.getCore().getCoreDescriptor()
.getCoreProperty(TimeRoutedAlias.ROUTED_ALIAS_NAME_CORE_PROP, null);
@@ -113,18 +114,17 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
// if shardDistribPhase is not NONE, then the phase is after the scope of this URP
return next;
} else {
- return new TimeRoutedAliasUpdateProcessor(req, rsp, next, aliasName, aliasDistribPhase);
+ return new TimeRoutedAliasUpdateProcessor(req, next, aliasName, aliasDistribPhase);
}
}
- protected TimeRoutedAliasUpdateProcessor(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next,
- String aliasName,
- DistribPhase aliasDistribPhase) {
+ private TimeRoutedAliasUpdateProcessor(SolrQueryRequest req, UpdateRequestProcessor next,
+ String aliasName,
+ DistribPhase aliasDistribPhase) {
super(next);
assert aliasDistribPhase == DistribPhase.NONE;
final SolrCore core = req.getCore();
- cloudDesc = core.getCoreDescriptor().getCloudDescriptor();
- this.thisCollection = cloudDesc.getCollectionName();
+ this.thisCollection = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
this.req = req;
CoreContainer cc = core.getCoreContainer();
zkController = cc.getZkController();
@@ -164,71 +164,141 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
@Override
public void processAdd(AddUpdateCommand cmd) throws IOException {
- SolrInputDocument solrInputDocument = cmd.getSolrInputDocument();
- final Object routeValue = solrInputDocument.getFieldValue(timeRoutedAlias.getRouteField());
- final Instant routeTimestamp = parseRouteKey(routeValue);
+ final Instant docTimestamp =
+ parseRouteKey(cmd.getSolrInputDocument().getFieldValue(timeRoutedAlias.getRouteField()));
+
+ // TODO: maybe in some cases the user would want to ignore/warn instead?
+ if (docTimestamp.isAfter(Instant.now().plusMillis(timeRoutedAlias.getMaxFutureMs()))) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "The document's time routed key of " + docTimestamp + " is too far in the future given " +
+ TimeRoutedAlias.ROUTER_MAX_FUTURE + "=" + timeRoutedAlias.getMaxFutureMs());
+ }
+ // to avoid potential for race conditions, this next method should not get called again unless
+ // we have created a collection synchronously
updateParsedCollectionAliases();
- String targetCollection;
- do { // typically we don't loop; it's only when we need to create a collection
- targetCollection = findTargetCollectionGivenTimestamp(routeTimestamp);
-
- if (targetCollection == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "Doc " + cmd.getPrintableId() + " couldn't be routed with " + timeRoutedAlias.getRouteField() + "=" + routeTimestamp);
- }
-
- // Note: the following rule is tempting but not necessary and is not compatible with
- // only using this URP when the alias distrib phase is NONE; otherwise a doc may be routed to from a non-recent
- // collection to the most recent only to then go there directly instead of realizing a new collection is needed.
- // // If it's going to some other collection (not "this") then break to just send it there
- // if (!thisCollection.equals(targetCollection)) {
- // break;
- // }
- // Also tempting but not compatible: check that we're the leader, if not then break
-
- // If the doc goes to the most recent collection then do some checks below, otherwise break the loop.
- final Instant mostRecentCollTimestamp = parsedCollectionsDesc.get(0).getKey();
- final String mostRecentCollName = parsedCollectionsDesc.get(0).getValue();
- if (!mostRecentCollName.equals(targetCollection)) {
- break;
- }
-
- // Check the doc isn't too far in the future
- final Instant maxFutureTime = Instant.now().plusMillis(timeRoutedAlias.getMaxFutureMs());
- if (routeTimestamp.isAfter(maxFutureTime)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
- "The document's time routed key of " + routeValue + " is too far in the future given " +
- TimeRoutedAlias.ROUTER_MAX_FUTURE + "=" + timeRoutedAlias.getMaxFutureMs());
- }
-
- // Create a new collection?
- final Instant nextCollTimestamp = timeRoutedAlias.computeNextCollTimestamp(mostRecentCollTimestamp);
- if (routeTimestamp.isBefore(nextCollTimestamp)) {
- break; // thus we don't need another collection
- }
- createCollectionAfter(mostRecentCollName); // *should* throw if fails for some reason but...
- final boolean updated = updateParsedCollectionAliases();
- if (!updated) { // thus we didn't make progress...
- // this is not expected, even in known failure cases, but we check just in case
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "We need to create a new time routed collection but for unknown reasons were unable to do so.");
- }
- // then retry the loop ...
- } while(true);
- assert targetCollection != null;
+ String targetCollection = createCollectionsIfRequired(docTimestamp, cmd);
if (thisCollection.equals(targetCollection)) {
// pass on through; we've reached the right collection
super.processAdd(cmd);
} else {
// send to the right collection
- SolrCmdDistributor.Node targetLeaderNode = routeDocToSlice(targetCollection, solrInputDocument);
+ SolrCmdDistributor.Node targetLeaderNode = routeDocToSlice(targetCollection, cmd.getSolrInputDocument());
cmdDistrib.distribAdd(cmd, Collections.singletonList(targetLeaderNode), new ModifiableSolrParams(outParamsToLeader));
}
}
+ /**
+ * Create any required collections and return the name of the collection to which the current document should be sent.
+ *
+ * @param docTimestamp the date for the document taken from the field specified in the TRA config
+ * @param cmd The initial calculated destination collection.
+ * @return The name of the proper destination collection for the document which may or may not be a
+ * newly created collection
+ */
+ private String createCollectionsIfRequired(Instant docTimestamp, AddUpdateCommand cmd) {
+ // Even though it is possible that multiple requests hit this code in the 1-2 sec that
+ // it takes to create a collection, it's an established anti-pattern to feed data with a very large number
+ // of client connections. This in mind, we only guard against spamming the overseer within a batch of
+ // updates. We are intentionally tolerating a low level of redundant requests in favor of simpler code. Most
+ // super-sized installations with many update clients will likely be multi-tenant and multiple tenants
+ // probably don't write to the same alias. As such, we have deferred any solution to the "many clients causing
+ // collection creation simultaneously" problem until such time as someone actually has that problem in a
+ // real world use case that isn't just an anti-pattern.
+ Map.Entry<Instant, String> candidateCollectionDesc = findCandidateGivenTimestamp(docTimestamp, cmd.getPrintableId());
+ String candidateCollectionName = candidateCollectionDesc.getValue();
+ try {
+ switch (typeOfCreationRequired(docTimestamp, candidateCollectionDesc.getKey())) {
+ case SYNCHRONOUS:
+ // This next line blocks until all collections required by the current document have been created
+ return createAllRequiredCollections(docTimestamp, cmd.getPrintableId(), candidateCollectionDesc);
+ case ASYNC_PREEMPTIVE:
+ if (preemptiveCreationExecutor == null) {
+ // It's important not to add code between here and the prior call to findCandidateGivenTimestamp()
+ // in processAdd() that invokes updateParsedCollectionAliases(). Doing so would update parsedCollectionsDesc
+ // and create a race condition. We are relying on the fact that get(0) is returning the head of the parsed
+ // collections that existed when candidateCollectionDesc was created. If this class updates it's notion of
+ // parsedCollectionsDesc since candidateCollectionDesc was chosen, we could create collection n+2
+ // instead of collection n+1.
+ String mostRecentCollName = this.parsedCollectionsDesc.get(0).getValue();
+
+ // This line does not block and the document can be added immediately
+ preemptiveAsync(() -> createNextCollection(mostRecentCollName));
+ }
+ return candidateCollectionName;
+ case NONE:
+ return candidateCollectionName; // could use fall through, but fall through is fiddly for later editors.
+ default:
+ throw unknownCreateType();
+ }
+ // do nothing if creationType == NONE
+ } catch (SolrException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ private void preemptiveAsync(Runnable r) {
+ // Note: creating an executor and throwing it away is slightly expensive, but this is only likely to happen
+ // once per hour/day/week (depending on time slice size for the TRA). If the executor were retained, it
+ // would need to be shut down in a close hook to avoid test failures due to thread leaks in tests which is slightly
+ // more complicated from a code maintenance and readability stand point. An executor must used instead of a
+ // thread to ensure we pick up the proper MDC logging stuff from ExecutorUtil.
+ DefaultSolrThreadFactory threadFactory = new DefaultSolrThreadFactory("TRA-preemptive-creation");
+ preemptiveCreationExecutor = newMDCAwareSingleThreadExecutor(threadFactory);
+ preemptiveCreationExecutor.execute(() -> {
+ r.run();
+ preemptiveCreationExecutor.shutdown();
+ preemptiveCreationExecutor = null;
+ });
+ }
+
+ /**
+ * Determine if the a new collection will be required based on the document timestamp. Passing null for
+ * preemptiveCreateInterval tells you if the document is beyond all existing collections with a response of
+ * {@link CreationType#NONE} or {@link CreationType#SYNCHRONOUS}, and passing a valid date math for
+ * preemptiveCreateMath additionally distinguishes the case where the document is close enough to the end of
+ * the TRA to trigger preemptive creation but not beyond all existing collections with a value of
+ * {@link CreationType#ASYNC_PREEMPTIVE}.
+ *
+ * @param docTimeStamp The timestamp from the document
+ * @param targetCollectionTimestamp The timestamp for the presently selected destination collection
+ * @return a {@code CreationType} indicating if and how to create a collection
+ */
+ private CreationType typeOfCreationRequired(Instant docTimeStamp, Instant targetCollectionTimestamp) {
+ final Instant nextCollTimestamp = timeRoutedAlias.computeNextCollTimestamp(targetCollectionTimestamp);
+
+ if (!docTimeStamp.isBefore(nextCollTimestamp)) {
+ // current document is destined for a collection that doesn't exist, must create the destination
+ // to proceed with this add command
+ return SYNCHRONOUS;
+ }
+
+ if (isNotBlank(timeRoutedAlias.getPreemptiveCreateWindow())) {
+ Instant preemptNextColCreateTime =
+ calcPreemptNextColCreateTime(timeRoutedAlias.getPreemptiveCreateWindow(), nextCollTimestamp);
+ if (!docTimeStamp.isBefore(preemptNextColCreateTime)) {
+ return ASYNC_PREEMPTIVE;
+ }
+ }
+
+ return NONE;
+ }
+
+ private Instant calcPreemptNextColCreateTime(String preemptiveCreateMath, Instant nextCollTimestamp) {
+ DateMathParser dateMathParser = new DateMathParser();
+ dateMathParser.setNow(Date.from(nextCollTimestamp));
+ try {
+ return dateMathParser.parseMath(preemptiveCreateMath).toInstant();
+ } catch (ParseException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid Preemptive Create Window Math:'" + preemptiveCreateMath + '\'', e);
+ }
+ }
+
private Instant parseRouteKey(Object routeKey) {
final Instant docTimestamp;
if (routeKey instanceof Instant) {
@@ -261,61 +331,43 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
return false;
}
- /** Given the route key, finds the collection. Returns null if too old to go in last one. */
- private String findTargetCollectionGivenTimestamp(Instant docTimestamp) {
+ /**
+ * Given the route key, finds the correct collection or returns the most recent collection if the doc
+ * is in the future. Future docs will potentially cause creation of a collection that does not yet exist
+ * or an error if they exceed the maxFutureMs setting.
+ *
+ * @throws SolrException if the doc is too old to be stored in the TRA
+ */
+ private Map.Entry<Instant, String> findCandidateGivenTimestamp(Instant docTimestamp, String printableId) {
// Lookup targetCollection given route key. Iterates in reverse chronological order.
// We're O(N) here but N should be small, the loop is fast, and usually looking for 1st.
for (Map.Entry<Instant, String> entry : parsedCollectionsDesc) {
Instant colStartTime = entry.getKey();
if (!docTimestamp.isBefore(colStartTime)) { // i.e. docTimeStamp is >= the colStartTime
- return entry.getValue(); //found it
+ return entry; //found it
}
}
- return null; //not found
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Doc " + printableId + " couldn't be routed with " + timeRoutedAlias.getRouteField() + "=" + docTimestamp);
}
- private void createCollectionAfter(String mostRecentCollName) {
+ private void createNextCollection(String mostRecentCollName) {
// Invoke ROUTEDALIAS_CREATECOLL (in the Overseer, locked by alias name). It will create the collection
// and update the alias contingent on the most recent collection name being the same as
// what we think so here, otherwise it will return (without error).
-
- // (see docs on aliasToSemaphoreMap)
- final Semaphore semaphore = aliasToSemaphoreMap.computeIfAbsent(getAliasName(), n -> new Semaphore(1));
- if (semaphore.tryAcquire()) {
- try {
- MaintainRoutedAliasCmd.remoteInvoke(collHandler, getAliasName(), mostRecentCollName);
- // we don't care about the response. It's possible no collection was created because
- // of a race and that's okay... we'll ultimately retry any way.
-
- // Ensure our view of the aliases has updated. If we didn't do this, our zkStateReader might
- // not yet know about the new alias (thus won't see the newly added collection to it), and we might think
- // we failed.
- zkController.getZkStateReader().aliasesManager.update();
- } catch (RuntimeException e) {
- throw e;
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } finally {
- semaphore.release(); // to signal we're done to anyone waiting on it
- }
-
- } else {
- // Failed to acquire permit because another URP instance on this JVM is creating a collection.
- // So wait till it's available
- log.debug("Collection creation is already in progress so we'll wait then try again.");
- try {
- if (semaphore.tryAcquire(DEFAULT_COLLECTION_OP_TIMEOUT, TimeUnit.MILLISECONDS)) {
- semaphore.release(); // we don't actually want a permit so give it back
- // return to continue...
- } else {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Waited too long for another update thread to be done with collection creation.");
- }
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Interrupted waiting on collection creation.", e); // if we were interrupted, give up.
- }
+ try {
+ MaintainRoutedAliasCmd.remoteInvoke(collHandler, getAliasName(), mostRecentCollName);
+ // we don't care about the response. It's possible no collection was created because
+ // of a race and that's okay... we'll ultimately retry any way.
+
+ // Ensure our view of the aliases has updated. If we didn't do this, our zkStateReader might
+ // not yet know about the new alias (thus won't see the newly added collection to it), and we might think
+ // we failed.
+ zkController.getZkStateReader().aliasesManager.update();
+ } catch (RuntimeException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
}
}
@@ -404,4 +456,61 @@ public class TimeRoutedAliasUpdateProcessor extends UpdateRequestProcessor {
collection, slice.getName(), DistributedUpdateProcessor.MAX_RETRIES_ON_FORWARD_DEAULT);
}
+
+ /**
+ * Create as many collections as required. This method loops to allow for the possibility that the docTimestamp
+ * requires more than one collection to be created. Since multiple threads may be invoking maintain on separate
+ * requests to the same alias, we must pass in the name of the collection that this thread believes to be the most
+ * recent collection. This assumption is checked when the command is executed in the overseer. When this method
+ * finds that all collections required have been created it returns the (possibly new) most recent collection.
+ * The return value is ignored by the calling code in the async preemptive case.
+ *
+ * @param docTimestamp the timestamp from the document that determines routing
+ * @param printableId an identifier for the add command used in error messages
+ * @param targetCollectionDesc the descriptor for the presently selected collection which should also be
+ * the most recent collection in all cases where this method is invoked.
+ * @return The latest collection, including collections created during maintenance
+ */
+ private String createAllRequiredCollections( Instant docTimestamp, String printableId,
+ Map.Entry<Instant, String> targetCollectionDesc) {
+ do {
+ switch(typeOfCreationRequired(docTimestamp, targetCollectionDesc.getKey())) {
+ case NONE:
+ return targetCollectionDesc.getValue(); // we don't need another collection
+ case ASYNC_PREEMPTIVE:
+ // can happen when preemptive interval is longer than one time slice
+ String mostRecentCollName = this.parsedCollectionsDesc.get(0).getValue();
+ preemptiveAsync(() -> createNextCollection(mostRecentCollName));
+ return targetCollectionDesc.getValue();
+ case SYNCHRONOUS:
+ createNextCollection(targetCollectionDesc.getValue()); // *should* throw if fails for some reason but...
+ if (!updateParsedCollectionAliases()) { // thus we didn't make progress...
+ // this is not expected, even in known failure cases, but we check just in case
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "We need to create a new time routed collection but for unknown reasons were unable to do so.");
+ }
+ // then retry the loop ... have to do find again in case other requests also added collections
+ // that were made visible when we called updateParsedCollectionAliases()
+ targetCollectionDesc = findCandidateGivenTimestamp(docTimestamp, printableId);
+ break;
+ default:
+ throw unknownCreateType();
+
+ }
+ } while (true);
+ }
+
+ private SolrException unknownCreateType() {
+ return new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown creation type while adding " +
+ "document to a Time Routed Alias! This is a bug caused when a creation type has been added but " +
+ "not all code has been updated to handle it.");
+ }
+
+ enum CreationType {
+ NONE,
+ ASYNC_PREEMPTIVE,
+ SYNCHRONOUS
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
index 048cf5b..5c9fc94 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TimeRoutedAliasUpdateProcessorTest.java
@@ -18,6 +18,7 @@
package org.apache.solr.update.processor;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
@@ -33,7 +34,6 @@ import java.util.concurrent.Future;
import java.util.stream.Collectors;
import org.apache.lucene.util.IOUtils;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -54,6 +54,7 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
@@ -69,15 +70,20 @@ import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- static final String configName = "timeConfig";
- static final String alias = "myalias";
- static final String timeField = "timestamp_dt";
- static final String intField = "integer_i";
+ private static final String configName = "timeConfig";
+ private static final String alias = "myalias";
+ private static final String timeField = "timestamp_dt";
+ private static final String intField = "integer_i";
- static SolrClient solrClient;
+ private static CloudSolrClient solrClient;
private int lastDocId = 0;
private int numDocsDeletedOrFailed = 0;
@@ -88,13 +94,11 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
}
@Before
- public void doBefore() throws Exception {
+ public void doBefore() {
solrClient = getCloudSolrClient(cluster);
//log this to help debug potential causes of problems
- System.out.println("SolrClient: " + solrClient);
- if (solrClient instanceof CloudSolrClient) {
- System.out.println(((CloudSolrClient) solrClient).getClusterStateProvider());
- }
+ log.info("SolrClient: {}", solrClient);
+ log.info("ClusterStateProvider {}",solrClient.getClusterStateProvider());
}
@After
@@ -231,10 +235,9 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
" 'add-updateprocessor' : {" +
" 'name':'tolerant', 'class':'solr.TolerantUpdateProcessorFactory'" +
" }," +
- // if other tracking tests are written, add another TUPF with a unique group name, don't re-use this one!
// See TrackingUpdateProcessorFactory javadocs for details...
" 'add-updateprocessor' : {" +
- " 'name':'tracking-testSliceRouting', 'class':'solr.TrackingUpdateProcessorFactory', 'group':'testSliceRouting'" +
+ " 'name':'tracking-testSliceRouting', 'class':'solr.TrackingUpdateProcessorFactory', 'group':'" + getTrackUpdatesGroupName() + "'" +
" }," +
" 'add-updateprocessor' : {" + // for testing
" 'name':'inc', 'class':'" + IncrementURPFactory.class.getName() + "'," +
@@ -265,8 +268,8 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
* @throws Exception when it blows up unexpectedly :)
*/
@Slow
- @Nightly
@Test
+ @LogLevel("org.apache.solr.update.processor.TrackingUpdateProcessorFactory=DEBUG")
public void testSliceRouting() throws Exception {
String configName = TimeRoutedAliasUpdateProcessorTest.configName + getTestName();
createConfigSet(configName);
@@ -294,8 +297,10 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
// leader randomly and not causing a failure if the code is broken, but as a whole this test will therefore only have
// about a 3.6% false positive rate (0.33^3). If that's not good enough, add more docs or more replicas per shard :).
+ final String trackGroupName = getTrackUpdatesGroupName();
+ final List<UpdateCommand> updateCommands;
try {
- TrackingUpdateProcessorFactory.startRecording(getTestName());
+ TrackingUpdateProcessorFactory.startRecording(trackGroupName);
// cause some collections to be created
@@ -306,7 +311,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
sdoc("id", "4", "timestamp_dt", "2017-10-23T00:00:00Z")),
params));
} finally {
- TrackingUpdateProcessorFactory.stopRecording(getTestName());
+ updateCommands = TrackingUpdateProcessorFactory.stopRecording(trackGroupName);
}
try (CloudSolrClient cloudSolrClient = getCloudSolrClient(cluster)) {
@@ -315,7 +320,6 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
Set<String> leaders = getLeaderCoreNames(clusterStateProvider.getClusterState());
assertEquals("should have " + 3 * numShards + " leaders, " + numShards + " per collection", 3 * numShards, leaders.size());
- List<UpdateCommand> updateCommands = TrackingUpdateProcessorFactory.commandsForGroup(getTestName());
assertEquals(3, updateCommands.size());
for (UpdateCommand updateCommand : updateCommands) {
String node = (String) updateCommand.getReq().getContext().get(TrackingUpdateProcessorFactory.REQUEST_NODE);
@@ -324,6 +328,179 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
}
}
+ /** @see TrackingUpdateProcessorFactory */
+ private String getTrackUpdatesGroupName() {
+ return getTestName();
+ }
+
+ @Test
+ @Slow
+ public void testPreemptiveCreation() throws Exception {
+ String configName = TimeRoutedAliasUpdateProcessorTest.configName + getTestName();
+ createConfigSet(configName);
+
+ final int numShards = 1 ;
+ final int numReplicas = 1 ;
+ CollectionAdminRequest.createTimeRoutedAlias(alias, "2017-10-23T00:00:00Z", "+1DAY", timeField,
+ CollectionAdminRequest.createCollection("_unused_", configName, numShards, numReplicas)
+ .setMaxShardsPerNode(numReplicas)).setPreemptiveCreateWindow("3HOUR")
+ .process(solrClient);
+
+ // cause some collections to be created
+ assertUpdateResponse(solrClient.add(alias,
+ sdoc("id","1","timestamp_dt", "2017-10-25T00:00:00Z")
+ ));
+ assertUpdateResponse(solrClient.commit(alias));
+
+ // wait for all the collections to exist...
+ waitCol("2017-10-23", numShards); // This one should have already existed from the alias creation
+ waitCol("2017-10-24", numShards); // Create 1
+ waitCol("2017-10-25", numShards); // Create 2nd synchronously (ensure this is not broken)
+
+ // normal update, nothing special, no collection creation required.
+ List<String> cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
+ assertEquals(3,cols.size());
+
+ assertNumDocs("2017-10-23", 0);
+ assertNumDocs("2017-10-24", 0);
+ assertNumDocs("2017-10-25", 1);
+
+ // cause some collections to be created
+
+ ModifiableSolrParams params = params();
+ assertUpdateResponse(add(alias, Arrays.asList(
+ sdoc("id", "2", "timestamp_dt", "2017-10-24T00:00:00Z"),
+ sdoc("id", "3", "timestamp_dt", "2017-10-25T00:00:00Z"),
+ sdoc("id", "4", "timestamp_dt", "2017-10-23T00:00:00Z"),
+ sdoc("id", "5", "timestamp_dt", "2017-10-25T23:00:00Z")), // should cause preemptive creation
+ params));
+ assertUpdateResponse(solrClient.commit(alias));
+
+ cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
+ assertEquals(3,cols.size());
+ assertNumDocs("2017-10-23", 1);
+ assertNumDocs("2017-10-24", 1);
+ assertNumDocs("2017-10-25", 3);
+
+ assertUpdateResponse(add(alias, Collections.singletonList(
+ sdoc("id", "6", "timestamp_dt", "2017-10-25T23:01:00Z")), // might cause duplicate preemptive creation
+ params));
+ assertUpdateResponse(solrClient.commit(alias));
+
+ waitCol("2017-10-26", numShards);
+ cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
+ assertEquals(4,cols.size());
+ assertNumDocs("2017-10-23", 1);
+ assertNumDocs("2017-10-24", 1);
+ assertNumDocs("2017-10-25", 4);
+ assertNumDocs("2017-10-26", 0);
+
+ // now test with pre-create window longer than time slice, and forcing multiple creations.
+ CollectionAdminRequest.setAliasProperty(alias)
+ .addProperty(TimeRoutedAlias.ROUTER_PREEMPTIVE_CREATE_MATH, "3DAY").process(solrClient);
+
+ assertUpdateResponse(add(alias, Collections.singletonList(
+ sdoc("id", "7", "timestamp_dt", "2017-10-25T23:01:00Z")), // should cause preemptive creation now
+ params));
+ assertUpdateResponse(solrClient.commit(alias));
+ waitCol("2017-10-27", numShards);
+
+ cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
+ assertEquals(5,cols.size()); // only one created in async case
+ assertNumDocs("2017-10-23", 1);
+ assertNumDocs("2017-10-24", 1);
+ assertNumDocs("2017-10-25", 5);
+ assertNumDocs("2017-10-26", 0);
+ assertNumDocs("2017-10-27", 0);
+
+ assertUpdateResponse(add(alias, Collections.singletonList(
+ sdoc("id", "8", "timestamp_dt", "2017-10-25T23:01:00Z")), // should cause preemptive creation now
+ params));
+ assertUpdateResponse(solrClient.commit(alias));
+ waitCol("2017-10-27", numShards);
+ waitCol("2017-10-28", numShards);
+
+ cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
+ assertEquals(6,cols.size()); // Subsequent documents continue to create up to limit
+ assertNumDocs("2017-10-23", 1);
+ assertNumDocs("2017-10-24", 1);
+ assertNumDocs("2017-10-25", 6);
+ assertNumDocs("2017-10-26", 0);
+ assertNumDocs("2017-10-27", 0);
+ assertNumDocs("2017-10-28", 0);
+
+ QueryResponse resp;
+ resp = solrClient.query(alias, params(
+ "q", "*:*",
+ "rows", "10"));
+ assertEquals(8, resp.getResults().getNumFound());
+
+ assertUpdateResponse(add(alias, Arrays.asList(
+ sdoc("id", "9", "timestamp_dt", "2017-10-27T23:01:00Z"), // should cause preemptive creation
+
+ // If these are not ignored properly this test will fail during cleanup with a message about router.name being
+ // required. This happens because the test finishes while overseer threads are still trying to invoke maintain
+ // after the @After method has deleted collections and emptied out the aliases.... this leaves the maintain
+ // command cloning alias properties Aliases.EMPTY and thus not getting a value from router.name
+ // (normally router.name == 'time') The check for non-blank router.name happens to be the first validation.
+ // There is a small chance this could slip through without a fail occasionally, but it was 100% with just one
+ // of these.
+ sdoc("id", "10", "timestamp_dt", "2017-10-28T23:01:00Z"), // should be ignored due to in progress creation
+ sdoc("id", "11", "timestamp_dt", "2017-10-28T23:02:00Z"), // should be ignored due to in progress creation
+ sdoc("id", "12", "timestamp_dt", "2017-10-28T23:03:00Z")), // should be ignored due to in progress creation
+ params));
+ assertUpdateResponse(solrClient.commit(alias));
+ waitCol("2017-10-29", numShards);
+
+ cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
+ assertEquals(7,cols.size());
+ assertNumDocs("2017-10-23", 1);
+ assertNumDocs("2017-10-24", 1);
+ assertNumDocs("2017-10-25", 6);
+ assertNumDocs("2017-10-26", 0);
+ assertNumDocs("2017-10-27", 1);
+ assertNumDocs("2017-10-28", 3); // should get through even though preemptive creation ignored it.
+ assertNumDocs("2017-10-29", 0);
+
+ resp = solrClient.query(alias, params(
+ "q", "*:*",
+ "rows", "0"));
+ assertEquals(12, resp.getResults().getNumFound());
+
+ // Sych creation with an interval longer than the time slice for the alias..
+ assertUpdateResponse(add(alias, Collections.singletonList(
+ sdoc("id", "13", "timestamp_dt", "2017-10-30T23:03:00Z")), // lucky?
+ params));
+ assertUpdateResponse(solrClient.commit(alias));
+ waitCol("2017-10-30", numShards);
+ waitCol("2017-10-31", numShards); // spooky! async case arising in middle of sync creation!!
+
+ cols = new CollectionAdminRequest.ListAliases().process(solrClient).getAliasesAsLists().get(alias);
+ assertEquals(9,cols.size());
+ assertNumDocs("2017-10-23", 1);
+ assertNumDocs("2017-10-24", 1);
+ assertNumDocs("2017-10-25", 6);
+ assertNumDocs("2017-10-26", 0);
+ assertNumDocs("2017-10-27", 1);
+ assertNumDocs("2017-10-28", 3); // should get through even though preemptive creation ignored it.
+ assertNumDocs("2017-10-29", 0);
+ assertNumDocs("2017-10-30", 1);
+ assertNumDocs("2017-10-31", 0);
+
+ resp = solrClient.query(alias, params(
+ "q", "*:*",
+ "rows", "0"));
+ assertEquals(13, resp.getResults().getNumFound());
+
+ }
+
+ private void assertNumDocs(final String datePart, int expected) throws SolrServerException, IOException {
+ QueryResponse resp = solrClient.query(alias + "_" + datePart, params(
+ "q", "*:*",
+ "rows", "10"));
+ assertEquals(expected, resp.getResults().getNumFound());
+ }
+
private Set<String> getLeaderCoreNames(ClusterState clusterState) {
Set<String> leaders = new TreeSet<>(); // sorted just to make it easier to read when debugging...
List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
@@ -344,9 +521,28 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
return leaders;
}
- private void waitCol(final String datePart, int slices) {
- waitForState("waiting for collections to be created",alias + "_" + datePart,
- (liveNodes, collectionState) -> collectionState.getActiveSlices().size() == slices);
+ private void waitCol(final String datePart, int slices) throws InterruptedException {
+ // collection to exist
+ String collection = alias + "_" + datePart;
+ waitForState("waiting for collections to be created", collection,
+ (liveNodes, collectionState) -> {
+ if (collectionState == null) {
+ // per predicate javadoc, this is what we get if the collection doesn't exist at all.
+ return false;
+ }
+ Collection<Slice> activeSlices = collectionState.getActiveSlices();
+ int size = activeSlices.size();
+ return size == slices;
+ });
+ // and alias to be aware of collection
+ long start = System.nanoTime(); // mumble mumble precommit mumble mumble...
+ while (!cluster.getSolrClient().getZkStateReader().getAliases().getCollectionAliasListMap().get(alias).contains(collection)) {
+ if (NANOSECONDS.toSeconds(System.nanoTime() - start) > 10) {
+ fail("took over 10 seconds after collection creation to update aliases");
+ } else {
+ Thread.sleep(500);
+ }
+ }
}
private void testFailedDocument(Instant timestamp, String errorMsg) throws SolrServerException, IOException {
@@ -484,6 +680,7 @@ public class TimeRoutedAliasUpdateProcessorTest extends SolrCloudTestCase {
}
/** Adds the docs to Solr via {@link #solrClient} with the params */
+ @SuppressWarnings("SameParameterValue")
private static UpdateResponse add(String collection, Collection<SolrInputDocument> docs, SolrParams params) throws SolrServerException, IOException {
UpdateRequest req = new UpdateRequest();
if (params != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/core/src/test/org/apache/solr/update/processor/TrackingUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TrackingUpdateProcessorFactory.java b/solr/core/src/test/org/apache/solr/update/processor/TrackingUpdateProcessorFactory.java
index 3e8f53b..06a72ea 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TrackingUpdateProcessorFactory.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TrackingUpdateProcessorFactory.java
@@ -16,18 +16,17 @@
*/
package org.apache.solr.update.processor;
-import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.mina.util.ConcurrentHashSet;
-import org.apache.solr.common.SolrException;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
+import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -42,113 +41,60 @@ import org.slf4j.LoggerFactory;
/**
* This Factory is similar to {@link RecordingUpdateProcessorFactory}, but with the goal of
- * tracking requests across multiple collections/shards/replicas in a CloudSolrTestCase.
+ * tracking requests across multiple collections/shards/replicas in a {@link SolrCloudTestCase}.
* It can optionally save references to the commands it receives inm a single global
* Map<String,BlockingQueue> keys in the map are arbitrary, but the intention is that tests
* generate a key that is unique to that test, and configure the factory with the key as "group name"
* to avoid cross talk between tests. Tests can poll for requests from a group to observe that the expected
* commands are executed. By default, this factory does nothing except return the "next"
- * processor from the chain unless it's told to {@link #startRecording()} in which case all factories
- * with the same group will begin recording. It is critical that tests utilizing this
- * processor call {@link #close()} on at least one group member after the test finishes. The requests associated with
- * the commands are also provided with a
+ * processor from the chain unless it's told to {@link #startRecording(String)} in which case all factories
+ * with the same group will begin recording.
*
* This class is only for unit test purposes and should not be used in any production capacity. It presumes all nodes
- * exist within the same JVM (i. e. MiniSolrCloudCluster).
+ * exist within the same JVM (i.e. {@link MiniSolrCloudCluster}).
*/
public final class TrackingUpdateProcessorFactory
- extends UpdateRequestProcessorFactory implements Closeable {
+ extends UpdateRequestProcessorFactory {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final String REQUEST_COUNT = "TrackingUpdateProcessorRequestCount";
public static final String REQUEST_NODE = "TrackingUpdateProcessorRequestNode";
- private final static Map<String,Set<TrackingUpdateProcessorFactory>> groupMembership = new ConcurrentHashMap<>();
- private final static Map<String,AtomicInteger> groupSerialNums = new ConcurrentHashMap<>();
-
/**
* The map of group queues containing commands that were recorded
* @see #startRecording
*/
- private final static Map<String, List<UpdateCommand>> commandQueueMap = new ConcurrentHashMap<>();
+ private final static Map<String, List<UpdateCommand>> groupToCommands = new ConcurrentHashMap<>();
- private static final Object memoryConsistency = new Object();
+ private String group = "default";
- private volatile boolean recording = false;
+ public static void startRecording(String group) {
+ final List<UpdateCommand> updateCommands = groupToCommands.get(group);
+ assert updateCommands == null || updateCommands.isEmpty();
- private String group = "default";
+ List<UpdateCommand> existing = groupToCommands.put(group, Collections.synchronizedList(new ArrayList<>()));
+ assert existing == null : "Test cross-talk?";
+ }
/**
- * Get a copy of the queue for the group.
*
* @param group the name of the group to fetch
- * @return A cloned queue containing the same elements as the queue held in commandQueueMap
+ * @return A cloned queue containing the same elements as the queue held in groupToCommands
*/
- public static ArrayList<UpdateCommand> commandsForGroup(String group) {
- synchronized (memoryConsistency) {
- return new ArrayList<>(commandQueueMap.get(group));
- }
- }
-
- public static void startRecording(String group) {
- synchronized (memoryConsistency) {
- Set<TrackingUpdateProcessorFactory> trackingUpdateProcessorFactories = groupMembership.get(group);
- if (trackingUpdateProcessorFactories == null || trackingUpdateProcessorFactories.isEmpty()) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "There are no trackingUpdateProcessors for group " + group);
- }
- for (TrackingUpdateProcessorFactory trackingUpdateProcessorFactory : trackingUpdateProcessorFactories) {
- trackingUpdateProcessorFactory.startRecording();
- }
- }
- }
- public static void stopRecording(String group) {
- synchronized (memoryConsistency) {
- Set<TrackingUpdateProcessorFactory> trackingUpdateProcessorFactories = groupMembership.get(group);
- if (trackingUpdateProcessorFactories == null || trackingUpdateProcessorFactories.isEmpty()) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "There are no trackingUpdateProcessors for group "
- + group + " available groups are:" + groupMembership.keySet());
- }
- for (TrackingUpdateProcessorFactory trackingUpdateProcessorFactory : trackingUpdateProcessorFactories) {
- trackingUpdateProcessorFactory.stopRecording();
- }
- }
+ public static List<UpdateCommand> stopRecording(String group) {
+ List<UpdateCommand> commands = groupToCommands.remove(group);
+ return Arrays.asList(commands.toArray(new UpdateCommand[0])); // safe copy. input list is synchronized
}
@Override
public void init(NamedList args) {
if (args != null && args.indexOf("group",0) >= 0) {
group = (String) args.get("group");
+ log.debug("Init URP, group '{}'", group);
} else {
log.warn("TrackingUpdateProcessorFactory initialized without group configuration, using 'default' but this group is shared" +
"across the entire VM and guaranteed to have unpredictable behavior if used by more than one test");
}
- // compute if absent to avoid replacing in the case of multiple "default"
- commandQueueMap.computeIfAbsent(group, s -> new ArrayList<>());
- groupMembership.computeIfAbsent(group,s-> new ConcurrentHashSet<>());
- groupSerialNums.computeIfAbsent(group,s-> new AtomicInteger(0));
-
- groupMembership.get(group).add(this);
- }
-
- /**
- * @see #stopRecording
- * @see #commandQueueMap
- */
- public synchronized void startRecording() {
- Set<TrackingUpdateProcessorFactory> facts = groupMembership.get(group);
- // facts being null is a bug, all instances should have a group.
- for (TrackingUpdateProcessorFactory fact : facts) {
- fact.recording = true;
- }
- }
-
- /** @see #startRecording */
- public synchronized void stopRecording() {
- Set<TrackingUpdateProcessorFactory> factories = groupMembership.get(group);
- // facts being null is a bug, all instances should have a group.
- for (TrackingUpdateProcessorFactory fact : factories) {
- fact.recording = false;
- }
}
@Override
@@ -156,35 +102,26 @@ public final class TrackingUpdateProcessorFactory
public synchronized UpdateRequestProcessor getInstance(SolrQueryRequest req,
SolrQueryResponse rsp,
UpdateRequestProcessor next ) {
- return recording ? new RecordingUpdateRequestProcessor(group, next) : next;
- }
-
- @Override
- public void close() {
- commandQueueMap.remove(group);
- groupMembership.get(group).clear();
+ final List<UpdateCommand> commands = groupToCommands.get(group);
+ return commands == null ? next : new RecordingUpdateRequestProcessor(commands, next);
}
private static final class RecordingUpdateRequestProcessor
- extends UpdateRequestProcessor {
+ extends UpdateRequestProcessor {
- private String group;
+ private final List<UpdateCommand> groupCommands;
- public RecordingUpdateRequestProcessor(String group,
- UpdateRequestProcessor next) {
+ RecordingUpdateRequestProcessor(List<UpdateCommand> groupCommands, UpdateRequestProcessor next) {
super(next);
- this.group = group;
+ this.groupCommands = groupCommands;
}
private void record(UpdateCommand cmd) {
- synchronized (memoryConsistency) {
- String coreName = cmd.getReq().getCore().getName();
- Map<Object, Object> context = cmd.getReq().getContext();
- context.put(REQUEST_COUNT, groupSerialNums.get(group).incrementAndGet());
- context.put(REQUEST_NODE, coreName);
- List<UpdateCommand> commands = commandQueueMap.get(group);
- commands.add(cmd.clone()); // important because cmd.clear() will be called
- }
+ groupCommands.add(cmd.clone()); // important because cmd.clear() will be called
+
+ Map<Object, Object> context = cmd.getReq().getContext();
+ context.put(REQUEST_COUNT, groupCommands.size());
+ context.put(REQUEST_NODE, cmd.getReq().getCore().getName());
}
@Override
@@ -212,13 +149,6 @@ public final class TrackingUpdateProcessorFactory
record(cmd);
super.processRollback(cmd);
}
-
-
- @Override
- protected void doClose() {
- super.doClose();
- groupMembership.get(group).remove(this);
- }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/solr-ref-guide/src/collections-api.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/collections-api.adoc b/solr/solr-ref-guide/src/collections-api.adoc
index 43825e8..e4230dd 100644
--- a/solr/solr-ref-guide/src/collections-api.adoc
+++ b/solr/solr-ref-guide/src/collections-api.adoc
@@ -623,6 +623,27 @@ without error. If there was no limit, than an erroneous value could trigger man
+
The default is 10 minutes.
+`router.preemptiveCreateMath`::
+A date math expression that results in early creation of new collections.
++
+If a document arrives with a timestamp that is after the end time of the most recent collection minus this
+interval, then the next (and only the next) collection will be created asynchronously. Without this setting, collections are created
+synchronously when required by the document time stamp and thus block the flow of documents until the collection
+is created (possibly several seconds). Preemptive creation reduces these hiccups. If set to enough time (perhaps
+an hour or more) then if there are problems creating a collection, this window of time might be enough to take
+corrective action. However after a successful preemptive creation, the collection is consuming resources without
+being used, and new documents will tend to be routed through it only to be routed elsewhere. Also, note that
+router.autoDeleteAge is currently evaluated relative to the date of a newly created collection, and so you may
+want to increase the delete age by the preemptive window amount so that the oldest collection isn't deleted too
+soon. Note that it has to be possible to subtract the interval specified from a date, so if prepending a
+minus sign creates invalid date math, this will cause an error. Also note that a document that is itself
+destined for a collection that does not exist will still trigger synchronous creation up to that destination collection
+but will not trigger additional async preemptive creation. Only one type of collection creation can happen
+per document.
+Example: `90MINUTES`.
++
+This property is blank by default indicating just-in-time, synchronous creation of new collections.
+
`router.autoDeleteAge`::
A date math expression that results in the oldest collections getting deleted automatically.
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 9667f37..b400688 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -1508,6 +1508,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
public static final String ROUTER_START = "router.start";
public static final String ROUTER_INTERVAL = "router.interval";
public static final String ROUTER_MAX_FUTURE = "router.maxFutureMs";
+ public static final String ROUTER_PREEMPTIVE_CREATE_WINDOW = "router.preemptiveCreateMath";
private final String aliasName;
private final String routerField;
@@ -1516,6 +1517,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
//Optional:
private TimeZone tz;
private Integer maxFutureMs;
+ private String preemptiveCreateMath;
private final Create createCollTemplate;
@@ -1540,6 +1542,11 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return this;
}
+ public CreateTimeRoutedAlias setPreemptiveCreateWindow(String preemptiveCreateMath) {
+ this.preemptiveCreateMath = preemptiveCreateMath;
+ return this;
+ }
+
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
@@ -1554,6 +1561,9 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
if (maxFutureMs != null) {
params.add(ROUTER_MAX_FUTURE, ""+maxFutureMs);
}
+ if (preemptiveCreateMath != null) {
+ params.add(ROUTER_PREEMPTIVE_CREATE_WINDOW, preemptiveCreateMath);
+ }
// merge the above with collectionParams. Above takes precedence.
ModifiableSolrParams createCollParams = new ModifiableSolrParams(); // output target
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/21d130c3/solr/solrj/src/resources/apispec/collections.Commands.json
----------------------------------------------------------------------
diff --git a/solr/solrj/src/resources/apispec/collections.Commands.json b/solr/solrj/src/resources/apispec/collections.Commands.json
index ed55a1e..dc8e251 100644
--- a/solr/solrj/src/resources/apispec/collections.Commands.json
+++ b/solr/solrj/src/resources/apispec/collections.Commands.json
@@ -168,6 +168,10 @@
"type": "integer",
"description":"How many milliseconds into the future to accept document. Documents with a value in router.field that is greater than now() + maxFutureMs will be rejected to avoid provisioning too much resources."
},
+ "preemptiveCreateMath":{
+ "type": "string",
+ "description": "If a document arrives with a timestamp that is after the end time of the most recent collection minus this interval, then the next collection will be created asynchronously. Without this setting, collections are created synchronously when required by the document time stamp and thus block the flow of documents until the collection is created (possibly several seconds). Preemptive creation reduces these hiccups. If set to enough time (perhaps an hour or more) then if there are problems creating a collection, this window of time might be enough to take corrective action. However after a successful preemptive creation, the collection is consuming resources without being used, and new documents will tend to be routed through it only to be routed elsewhere. Also, note that router.autoDeleteAge is currently evaluated relative to the date of a newly created collection, and so you may want to increase the delete age by the preemptive window amount so that the
oldest collection isn't deleted too soon."
+ },
"autoDeleteAge": {
"type": "string",
"description": "A date math expressions yielding a time in the past. Collections covering a period of time entirely before this age will be automatically deleted."
[04/47] lucene-solr:jira/solr-12709: LUCENE-8485: Update
randomizedtesting to version 2.6.4.
Posted by ab...@apache.org.
LUCENE-8485: Update randomizedtesting to version 2.6.4.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6be01e2a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6be01e2a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6be01e2a
Branch: refs/heads/jira/solr-12709
Commit: 6be01e2ade0ad51ec6171dae3902d8747f7fd4b5
Parents: 97ccbc7
Author: Dawid Weiss <dw...@apache.org>
Authored: Wed Sep 5 11:51:02 2018 +0200
Committer: Dawid Weiss <dw...@apache.org>
Committed: Wed Sep 5 11:51:02 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 ++
lucene/common-build.xml | 3 +--
lucene/ivy-versions.properties | 2 +-
lucene/licenses/randomizedtesting-runner-2.6.0.jar.sha1 | 1 -
lucene/licenses/randomizedtesting-runner-2.6.4.jar.sha1 | 1 +
solr/licenses/junit4-ant-2.6.0.jar.sha1 | 1 -
solr/licenses/junit4-ant-2.6.4.jar.sha1 | 1 +
solr/licenses/randomizedtesting-runner-2.6.0.jar.sha1 | 1 -
solr/licenses/randomizedtesting-runner-2.6.4.jar.sha1 | 1 +
9 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 40d1c2e..f344b82 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -298,6 +298,8 @@ Improvements
Other:
+* LUCENE-8485: Update randomizedtesting to version 2.6.4. (Dawid Weiss)
+
* LUCENE-8366: Upgrade to ICU 62.1. Emoji handling now uses Unicode 11's
Extended_Pictographic property. (Robert Muir)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/lucene/common-build.xml
----------------------------------------------------------------------
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index ac4c504..68e17da 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -1148,7 +1148,6 @@
<sysproperty key="java.io.tmpdir" value="./temp" />
<!-- Restrict access to certain Java features and install security manager: -->
- <sysproperty key="junit4.tempDir" file="@{workDir}/temp" />
<sysproperty key="common.dir" file="${common.dir}" />
<sysproperty key="clover.db.dir" file="${clover.db.dir}" />
<syspropertyset>
@@ -2643,7 +2642,7 @@ The following arguments can be provided to ant to alter its behaviour and target
<junit4:pickseed property="pitest.seed" />
- <property name="pitest.sysprops" value="-Dversion=${version},-Dtest.seed=${pitest.seed},-Djava.security.manager=org.apache.lucene.util.TestSecurityManager,-Djava.security.policy=${tests.policy},-Djava.io.tmpdir=${tests.workDir},-Djunit4.childvm.cwd=${tests.workDir},-Djunit4.tempDir=${tests.workDir}" />
+ <property name="pitest.sysprops" value="-Dversion=${version},-Dtest.seed=${pitest.seed},-Djava.security.manager=org.apache.lucene.util.TestSecurityManager,-Djava.security.policy=${tests.policy},-Djava.io.tmpdir=${tests.workDir},-Djunit4.childvm.cwd=${tests.workDir}" />
<pitest
classPath="pitest.classpath"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index 5ba50c1..f53544e 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -5,7 +5,7 @@
/antlr/antlr = 2.7.7
/com.adobe.xmp/xmpcore = 5.1.3
-com.carrotsearch.randomizedtesting.version = 2.6.0
+com.carrotsearch.randomizedtesting.version = 2.6.4
/com.carrotsearch.randomizedtesting/junit4-ant = ${com.carrotsearch.randomizedtesting.version}
/com.carrotsearch.randomizedtesting/randomizedtesting-runner = ${com.carrotsearch.randomizedtesting.version}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/lucene/licenses/randomizedtesting-runner-2.6.0.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/randomizedtesting-runner-2.6.0.jar.sha1 b/lucene/licenses/randomizedtesting-runner-2.6.0.jar.sha1
deleted file mode 100644
index c7baccb..0000000
--- a/lucene/licenses/randomizedtesting-runner-2.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3787b306dc666f6fa181b1900fb6b941e2b38790
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/lucene/licenses/randomizedtesting-runner-2.6.4.jar.sha1
----------------------------------------------------------------------
diff --git a/lucene/licenses/randomizedtesting-runner-2.6.4.jar.sha1 b/lucene/licenses/randomizedtesting-runner-2.6.4.jar.sha1
new file mode 100644
index 0000000..83634d9
--- /dev/null
+++ b/lucene/licenses/randomizedtesting-runner-2.6.4.jar.sha1
@@ -0,0 +1 @@
+37bc4cf1f458d4718892d33ab77378e38d4877d8
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/solr/licenses/junit4-ant-2.6.0.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/junit4-ant-2.6.0.jar.sha1 b/solr/licenses/junit4-ant-2.6.0.jar.sha1
deleted file mode 100644
index 9902b34..0000000
--- a/solr/licenses/junit4-ant-2.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7512f4c5fb6dc9106a712c390c39e87708148456
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/solr/licenses/junit4-ant-2.6.4.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/junit4-ant-2.6.4.jar.sha1 b/solr/licenses/junit4-ant-2.6.4.jar.sha1
new file mode 100644
index 0000000..203d888
--- /dev/null
+++ b/solr/licenses/junit4-ant-2.6.4.jar.sha1
@@ -0,0 +1 @@
+b0a50c6dce5060f8f9040e072b77f7cd85242b3c
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/solr/licenses/randomizedtesting-runner-2.6.0.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/randomizedtesting-runner-2.6.0.jar.sha1 b/solr/licenses/randomizedtesting-runner-2.6.0.jar.sha1
deleted file mode 100644
index c7baccb..0000000
--- a/solr/licenses/randomizedtesting-runner-2.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3787b306dc666f6fa181b1900fb6b941e2b38790
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6be01e2a/solr/licenses/randomizedtesting-runner-2.6.4.jar.sha1
----------------------------------------------------------------------
diff --git a/solr/licenses/randomizedtesting-runner-2.6.4.jar.sha1 b/solr/licenses/randomizedtesting-runner-2.6.4.jar.sha1
new file mode 100644
index 0000000..83634d9
--- /dev/null
+++ b/solr/licenses/randomizedtesting-runner-2.6.4.jar.sha1
@@ -0,0 +1 @@
+37bc4cf1f458d4718892d33ab77378e38d4877d8
[37/47] lucene-solr:jira/solr-12709: SOLR-12028: BadApple and
AwaitsFix annotations usage
Posted by ab...@apache.org.
SOLR-12028: BadApple and AwaitsFix annotations usage
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0dc66c23
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0dc66c23
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0dc66c23
Branch: refs/heads/jira/solr-12709
Commit: 0dc66c236d5f61caad96e36454b4b15fbde35720
Parents: 21d130c
Author: Erick Erickson <Er...@gmail.com>
Authored: Thu Sep 6 20:43:51 2018 -0700
Committer: Erick Erickson <Er...@gmail.com>
Committed: Thu Sep 6 20:43:51 2018 -0700
----------------------------------------------------------------------
.../lucene/document/TestLatLonPolygonShapeQueries.java | 2 ++
.../ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java | 3 +--
.../apache/solr/cloud/AssignBackwardCompatibilityTest.java | 1 +
.../core/src/test/org/apache/solr/cloud/DeleteShardTest.java | 2 +-
.../org/apache/solr/cloud/LeaderElectionIntegrationTest.java | 3 +--
.../org/apache/solr/cloud/LeaderVoteWaitTimeoutTest.java | 2 ++
.../src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java | 3 ++-
.../src/test/org/apache/solr/cloud/OverseerRolesTest.java | 1 +
.../src/test/org/apache/solr/cloud/TestCloudConsistency.java | 1 +
.../src/test/org/apache/solr/cloud/TestCloudRecovery.java | 2 +-
.../apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java | 2 +-
.../collections/CollectionsAPIAsyncDistributedZkTest.java | 3 +--
.../apache/solr/cloud/autoscaling/ComputePlanActionTest.java | 2 +-
.../solr/cloud/autoscaling/MetricTriggerIntegrationTest.java | 3 +--
.../apache/solr/cloud/autoscaling/SearchRateTriggerTest.java | 3 +--
.../autoscaling/sim/TestSimGenericDistributedQueue.java | 6 ++----
.../solr/cloud/autoscaling/sim/TestSimLargeCluster.java | 2 +-
.../cloud/autoscaling/sim/TestSimTriggerIntegration.java | 8 ++++----
.../test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java | 4 ++--
.../src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java | 1 +
.../solr/handler/admin/ZookeeperStatusHandlerTest.java | 1 +
.../solr/handler/component/DistributedMLTComponentTest.java | 3 +--
.../solr/metrics/reporters/solr/SolrCloudReportersTest.java | 2 +-
.../apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java | 1 +
.../org/apache/solr/rest/TestManagedResourceStorage.java | 3 +--
.../test/org/apache/solr/schema/SchemaApiFailureTest.java | 2 +-
.../src/test/org/apache/solr/search/TestStressRecovery.java | 3 +++
.../test/org/apache/solr/search/stats/TestDistribIDF.java | 3 +--
.../solr/security/hadoop/TestDelegationWithHadoopAuth.java | 3 +--
.../org/apache/solr/servlet/HttpSolrCallGetCoreTest.java | 3 +--
.../solr/uninverting/TestDocTermOrdsUninvertLimit.java | 2 +-
.../org/apache/solr/update/TestInPlaceUpdatesDistrib.java | 3 +--
.../solr/client/solrj/embedded/LargeVolumeJettyTest.java | 3 +--
.../apache/solr/client/solrj/impl/CloudSolrClientTest.java | 2 +-
.../solr/client/solrj/io/graph/GraphExpressionTest.java | 2 +-
.../solr/client/solrj/io/stream/StreamExpressionTest.java | 2 +-
.../apache/solr/client/solrj/io/stream/StreamingTest.java | 2 ++
37 files changed, 48 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
index 9b844ab..ce76a82 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
@@ -23,8 +23,10 @@ import org.apache.lucene.geo.Polygon;
import org.apache.lucene.geo.Polygon2D;
import org.apache.lucene.geo.Tessellator;
import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.util.LuceneTestCase;
/** random bounding box and polygon query tests for random indexed {@link Polygon} types */
+@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
protected final PolygonValidator VALIDATOR = new PolygonValidator();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
----------------------------------------------------------------------
diff --git a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
index 26f36fd..b153a71 100644
--- a/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
+++ b/solr/contrib/ltr/src/test/org/apache/solr/ltr/TestLTROnSolrCloud.java
@@ -19,7 +19,6 @@ import java.io.File;
import java.util.SortedMap;
import org.apache.commons.io.FileUtils;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -74,7 +73,7 @@ public class TestLTROnSolrCloud extends TestRerankBase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testSimpleQuery() throws Exception {
// will randomly pick a configuration with [1..5] shards and [1..3] replicas
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java b/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
index 74cf252..3a131a8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AssignBackwardCompatibilityTest.java
@@ -57,6 +57,7 @@ public class AssignBackwardCompatibilityTest extends SolrCloudTestCase {
@Test
//05-Jul-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void test() throws IOException, SolrServerException, KeeperException, InterruptedException {
Set<String> coreNames = new HashSet<>();
Set<String> coreNodeNames = new HashSet<>();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java
index 293e65b..92abd56 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteShardTest.java
@@ -102,7 +102,7 @@ public class DeleteShardTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void testDirectoryCleanupAfterDeleteShard() throws InterruptedException, IOException, SolrServerException {
final String collection = "deleteshard_test";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
index fa08381..29156cc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionIntegrationTest.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -62,7 +61,7 @@ public class LeaderElectionIntegrationTest extends SolrCloudTestCase {
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testSimpleSliceLeaderElection() throws Exception {
String collection = "collection1";
createCollection(collection);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/LeaderVoteWaitTimeoutTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderVoteWaitTimeoutTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderVoteWaitTimeoutTest.java
index 01ad51c..c1e9901 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LeaderVoteWaitTimeoutTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LeaderVoteWaitTimeoutTest.java
@@ -97,6 +97,7 @@ public class LeaderVoteWaitTimeoutTest extends SolrCloudTestCase {
@Test
//28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void basicTest() throws Exception {
final String collectionName = "basicTest";
CollectionAdminRequest.createCollection(collectionName, 1, 1)
@@ -134,6 +135,7 @@ public class LeaderVoteWaitTimeoutTest extends SolrCloudTestCase {
@Test
//commented 2-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testMostInSyncReplicasCanWinElection() throws Exception {
final String collectionName = "collection1";
CollectionAdminRequest.createCollection(collectionName, 1, 3)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
index 4786671..557810f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaHDFSTest.java
@@ -68,7 +68,8 @@ public class MoveReplicaHDFSTest extends MoveReplicaTest {
//2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
//commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
//commented 23-AUG-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testNormalFailedMove() throws Exception {
inPlaceMove = false;
testFailedMove();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
index 4ec445e..83569eb 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
@@ -86,6 +86,7 @@ public class OverseerRolesTest extends SolrCloudTestCase {
@Test
//commented 2-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testOverseerRole() throws Exception {
logOverseerState();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
index 6eea5b8..dac1c91 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudConsistency.java
@@ -90,6 +90,7 @@ public class TestCloudConsistency extends SolrCloudTestCase {
@Test
//commented 2-Aug-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testOutOfSyncReplicasCannotBecomeLeader() throws Exception {
testOutOfSyncReplicasCannotBecomeLeader(false);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
index 955b4b8..7f822a3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery.java
@@ -87,7 +87,7 @@ public class TestCloudRecovery extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
public void leaderRecoverFromLogOnStartupTest() throws Exception {
AtomicInteger countReplayLog = new AtomicInteger(0);
DirectUpdateHandler2.commitOnClose = false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java
index 1b4e876..ae1161d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java
@@ -220,7 +220,7 @@ public class TestStressCloudBlindAtomicUpdates extends SolrCloudTestCase {
checkField(field);
}
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Jul-2018
public void test_dv_idx() throws Exception {
String field = "long_dv_idx";
checkExpectedSchemaField(map("name", field,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
index 28ee350..0cd0456 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIAsyncDistributedZkTest.java
@@ -24,7 +24,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrClient;
@@ -191,7 +190,7 @@ public class CollectionsAPIAsyncDistributedZkTest extends SolrCloudTestCase {
.processAndWait(client, MAX_TIMEOUT_SECONDS);
assertSame("DeleteCollection did not complete", RequestStatusState.COMPLETED, state);
}
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testAsyncIdRaceCondition() throws Exception {
SolrClient[] clients = new SolrClient[cluster.getJettySolrRunners().size()];
int j = 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 42c72ec..2eaec83 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -169,7 +169,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
@Test
//28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testNodeLost() throws Exception {
// let's start a node so that we have at least two
JettySolrRunner runner = cluster.startJettySolrRunner();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
index 7131357..81cac33 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
@@ -27,7 +27,6 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
@@ -82,7 +81,7 @@ public class MetricTriggerIntegrationTest extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testMetricTrigger() throws Exception {
cluster.waitForAllNodes(5);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
index 0e9f4fb..c39dec8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
@@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit;
import com.codahale.metrics.MetricRegistry;
import com.google.common.util.concurrent.AtomicDouble;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.cloud.NodeStateProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
@@ -85,7 +84,7 @@ public class SearchRateTriggerTest extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18
public void testTrigger() throws Exception {
JettySolrRunner targetNode = cluster.getJettySolrRunner(0);
SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
index 25e36f3..436542a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimGenericDistributedQueue.java
@@ -16,24 +16,22 @@
*/
package org.apache.solr.cloud.autoscaling.sim;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.cloud.DistributedQueue;
import org.apache.solr.client.solrj.cloud.DistribStateManager;
/**
*
*/
-@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-02-26
+// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-02-26
public class TestSimGenericDistributedQueue extends TestSimDistributedQueue {
DistribStateManager stateManager = new SimDistribStateManager();
@Override
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
protected DistributedQueue makeDistributedQueue(String dqZNode) throws Exception {
return new GenericDistributedQueue(stateManager, dqZNode);
}
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void testDistributedQueue() throws Exception {
super.testDistributedQueue();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
index 825d390..4a0c362 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimLargeCluster.java
@@ -389,7 +389,7 @@ public class TestSimLargeCluster extends SimSolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2018-06-18
public void testNodeLost() throws Exception {
doTestNodeLost(waitForSeconds, 5000, 0);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index 81952af..dded18c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -286,7 +286,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
@Test
// commented 20-July-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void testNodeLostTriggerRestoreState() throws Exception {
// for this test we want to update the trigger so we must assert that the actions were created twice
TestSimTriggerIntegration.actionInitCalled = new CountDownLatch(2);
@@ -460,7 +460,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
public void testNodeLostTrigger() throws Exception {
SolrClient solrClient = cluster.simGetSolrClient();
String setTriggerCommand = "{" +
@@ -634,7 +634,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
public static long eventQueueActionWait = 5000;
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 16-Apr-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 16-Apr-2018
public void testEventQueue() throws Exception {
waitForSeconds = 1;
SolrClient solrClient = cluster.simGetSolrClient();
@@ -687,7 +687,7 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //2018-03-10
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") //2018-03-10
public void testEventFromRestoredState() throws Exception {
SolrClient solrClient = cluster.simGetSolrClient();
String setTriggerCommand = "{" +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
index 652f602..4ba32a0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/cdcr/CdcrBootstrapTest.java
@@ -22,7 +22,6 @@ import java.lang.invoke.MethodHandles;
import java.util.LinkedHashMap;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
@@ -58,7 +57,7 @@ public class CdcrBootstrapTest extends SolrTestCaseJ4 {
* call returns the same version as the last update indexed on the source.
*/
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testConvertClusterToCdcrAndBootstrap() throws Exception {
// start the target first so that we know its zkhost
MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));
@@ -240,6 +239,7 @@ public class CdcrBootstrapTest extends SolrTestCaseJ4 {
}
// 29-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
@Test
public void testBootstrapWithContinousIndexingOnSourceCluster() throws Exception {
// start the target first so that we know its zkhost
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
index bbd83a6..d8ee98d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/StressHdfsTest.java
@@ -61,6 +61,7 @@ import java.util.concurrent.TimeUnit;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
+@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public class StressHdfsTest extends BasicDistributedZkTest {
private static final String DELETE_DATA_DIR_COLLECTION = "delete_data_dir";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
index 7ec8bf2..def06d9 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/ZookeeperStatusHandlerTest.java
@@ -63,6 +63,7 @@ public class ZookeeperStatusHandlerTest extends SolrCloudTestCase {
NOTE: We do not currently test with multiple zookeepers, but the only difference is that there are multiple "details" objects and mode is "ensemble"...
*/
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void monitorZookeeper() throws IOException, SolrServerException, InterruptedException, ExecutionException, TimeoutException {
URL baseUrl = cluster.getJettySolrRunner(0).getBaseUrl();
HttpSolrClient solr = new HttpSolrClient.Builder(baseUrl.toString()).build();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java
index 4b07775..d8cdd2a 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/DistributedMLTComponentTest.java
@@ -19,7 +19,6 @@ package org.apache.solr.handler.component;
import java.util.HashMap;
import java.util.Map;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.BaseDistributedSearchTestCase;
@@ -76,7 +75,7 @@ public class DistributedMLTComponentTest extends BaseDistributedSearchTestCase {
@Test
@ShardsFixed(num = 3)
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void test() throws Exception {
del("*:*");
index(id, "1", "lowerfilt", "toyota", "lowerfilt1", "x");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
index 8b8b828..e395fac 100644
--- a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
@@ -59,7 +59,7 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testExplicitConfiguration() throws Exception {
String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr-solrreporter.xml"), "UTF-8");
configureCluster(2)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
index 8cc660e..fb1f55d 100644
--- a/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/rrd/SolrRrdBackendFactoryTest.java
@@ -80,6 +80,7 @@ public class SolrRrdBackendFactoryTest extends SolrTestCaseJ4 {
@Test
//commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testBasic() throws Exception {
long startTime = 1000000000;
RrdDb db = new RrdDb(createDef(startTime), factory);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java b/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java
index 4b7fac0..d537cf3 100644
--- a/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java
+++ b/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java
@@ -23,7 +23,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.cloud.AbstractZkTestCase;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -39,7 +38,7 @@ import org.junit.Test;
* Depends on ZK for testing ZooKeeper backed storage logic.
*/
@Slow
-@LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-6443")
+// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-6443")
public class TestManagedResourceStorage extends AbstractZkTestCase {
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java b/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java
index bc865ad..95cd2a7 100644
--- a/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/SchemaApiFailureTest.java
@@ -45,7 +45,7 @@ public class SchemaApiFailureTest extends SolrCloudTestCase {
}
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018
+ // commented 4-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018
public void testAddTheSameFieldTwice() throws Exception {
CloudSolrClient client = cluster.getSolrClient();
SchemaRequest.Update fieldAddition = new SchemaRequest.AddField
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
index e627d6e..b43c8aa 100644
--- a/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
+++ b/solr/core/src/test/org/apache/solr/search/TestStressRecovery.java
@@ -18,6 +18,7 @@ package org.apache.solr.search;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.LuceneTestCase;
import org.noggit.ObjectBuilder;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.update.UpdateHandler;
@@ -43,6 +44,7 @@ import java.util.concurrent.atomic.AtomicLong;
import static org.apache.solr.core.SolrCore.verbose;
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
+@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public class TestStressRecovery extends TestRTGBase {
@BeforeClass
@@ -60,6 +62,7 @@ public class TestStressRecovery extends TestRTGBase {
// and tests the ability to buffer updates and apply them later
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testStressRecovery() throws Exception {
assumeFalse("FIXME: This test is horribly slow sometimes on Windows!", Constants.WINDOWS);
clearIndex();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java b/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
index e748744..be5ede0 100644
--- a/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
+++ b/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
@@ -19,7 +19,6 @@ package org.apache.solr.search.stats;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
@@ -142,7 +141,7 @@ public class TestDistribIDF extends SolrTestCaseJ4 {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testMultiCollectionQuery() throws Exception {
// collection1 and collection2 are collections which have distributed idf enabled
// collection1_local and collection2_local don't have distributed idf available
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java b/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java
index beb6f19..5672b29 100644
--- a/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java
+++ b/solr/core/src/test/org/apache/solr/security/hadoop/TestDelegationWithHadoopAuth.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
import org.apache.hadoop.util.Time;
import org.apache.http.HttpStatus;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -315,7 +314,7 @@ public class TestDelegationWithHadoopAuth extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testDelegationTokenRenew() throws Exception {
// test with specifying renewer
verifyDelegationTokenRenew(USER_1, USER_1);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallGetCoreTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallGetCoreTest.java b/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallGetCoreTest.java
index eb67221..4f94388 100644
--- a/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallGetCoreTest.java
+++ b/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallGetCoreTest.java
@@ -25,7 +25,6 @@ import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
@@ -35,7 +34,7 @@ import org.eclipse.jetty.server.Response;
import org.junit.BeforeClass;
import org.junit.Test;
-@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public class HttpSolrCallGetCoreTest extends SolrCloudTestCase {
private static final String COLLECTION = "collection1";
private static final int NUM_SHARD = 3;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java
index 427cd6e..0caad54 100644
--- a/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java
+++ b/solr/core/src/test/org/apache/solr/uninverting/TestDocTermOrdsUninvertLimit.java
@@ -38,7 +38,7 @@ public class TestDocTermOrdsUninvertLimit extends LuceneTestCase {
* New limit is 2^31, which is not very realistic to unit-test. */
@SuppressWarnings({"ConstantConditions", "PointlessBooleanExpression"})
@Nightly
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
+// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public void testTriggerUnInvertLimit() throws IOException {
final boolean SHOULD_TRIGGER = false; // Set this to true to use the test with the old implementation
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
index 2441671..4f51ca3 100644
--- a/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
+++ b/solr/core/src/test/org/apache/solr/update/TestInPlaceUpdatesDistrib.java
@@ -32,7 +32,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.NoMergePolicy;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrClient;
@@ -121,7 +120,7 @@ public class TestInPlaceUpdatesDistrib extends AbstractFullDistribZkTestBase {
@ShardsFixed(num = 3)
@SuppressWarnings("unchecked")
//28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void test() throws Exception {
waitForRecoveriesToFinish(true);
mapReplicasToClients();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java
index 89f937a..ed6ab1f 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/embedded/LargeVolumeJettyTest.java
@@ -16,11 +16,10 @@
*/
package org.apache.solr.client.solrj.embedded;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.LargeVolumeTestBase;
import org.junit.BeforeClass;
-@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+// commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public class LargeVolumeJettyTest extends LargeVolumeTestBase {
@BeforeClass
public static void beforeTest() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
index 283d5a0..fefe868 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
@@ -395,7 +395,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
* limits the distributed query to locally hosted shards only
*/
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void preferLocalShardsTest() throws Exception {
String collectionName = "localShardsTestColl";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java
index f14003c..2294d71 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java
@@ -97,7 +97,7 @@ public class GraphExpressionTest extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testShortestPathStream() throws Exception {
new UpdateRequest()
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index b9c6153..add34cb 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -1551,7 +1551,7 @@ public class StreamExpressionTest extends SolrCloudTestCase {
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testParallelTopicStream() throws Exception {
Assume.assumeTrue(!useAlias);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dc66c23/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
index c444b6c..8f21100 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamingTest.java
@@ -1961,6 +1961,7 @@ public void testParallelRankStream() throws Exception {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testZeroParallelReducerStream() throws Exception {
new UpdateRequest()
@@ -2119,6 +2120,7 @@ public void testParallelRankStream() throws Exception {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 6-Sep-2018
public void testParallelMergeStream() throws Exception {
new UpdateRequest()
[28/47] lucene-solr:jira/solr-12709: SOLR-9418: Added a new
(experimental) PhrasesIdentificationComponent for identifying potential
phrases in query input based on overlapping shingles in the index
Posted by ab...@apache.org.
SOLR-9418: Added a new (experimental) PhrasesIdentificationComponent for identifying potential phrases in query input based on overlapping shingles in the index
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/597bd5db
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/597bd5db
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/597bd5db
Branch: refs/heads/jira/solr-12709
Commit: 597bd5db77465e1282ebf722264423d631861596
Parents: cac589b
Author: Chris Hostetter <ho...@apache.org>
Authored: Thu Sep 6 10:50:56 2018 -0700
Committer: Chris Hostetter <ho...@apache.org>
Committed: Thu Sep 6 10:50:56 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 5 +-
.../PhrasesIdentificationComponent.java | 1129 ++++++++++++++++++
.../conf/schema-phrases-identification.xml | 97 ++
.../conf/solrconfig-phrases-identification.xml | 53 +
...TestCloudPhrasesIdentificationComponent.java | 200 ++++
.../PhrasesIdentificationComponentTest.java | 796 ++++++++++++
6 files changed, 2279 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/597bd5db/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 19db81e..3d947c7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -1,4 +1,4 @@
- Apache Solr Release Notes
+ Apache Solr Release Notes
Introduction
------------
@@ -208,6 +208,9 @@ New Features
doc transformers if present. In 7.5 a missing 'fl' defaults to the current behavior of all fields, but in 8.0
defaults to the top/request "fl". (Moshe Bla, David Smiley)
+* SOLR-9418: Added a new (experimental) PhrasesIdentificationComponent for identifying potential phrases
+ in query input based on overlapping shingles in the index. (Akash Mehta, Trey Grainger, hossman)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/597bd5db/solr/core/src/java/org/apache/solr/handler/component/PhrasesIdentificationComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/PhrasesIdentificationComponent.java b/solr/core/src/java/org/apache/solr/handler/component/PhrasesIdentificationComponent.java
new file mode 100644
index 0000000..bac5a4c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/component/PhrasesIdentificationComponent.java
@@ -0,0 +1,1129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.LongSummaryStatistics;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.stream.Collectors;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.shingle.ShingleFilter;
+import org.apache.lucene.analysis.shingle.ShingleFilterFactory;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CharsRefBuilder;
+
+import org.apache.solr.analysis.TokenizerChain;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.util.SolrPluginUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * A component that can be used in isolation, or in conjunction with {@link QueryComponent} to identify
+ * & score "phrases" found in the input string, based on shingles in indexed fields.
+ *
+ * <p>
+ * The most common way to use this component is in conjunction with field that use
+ * {@link ShingleFilterFactory} on both the <code>index</code> and <code>query</code> analyzers.
+ * An example field type configuration would be something like this...
+ * </p>
+ * <pre class="prettyprint">
+ * <fieldType name="phrases" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer type="index">
+ * <tokenizer class="solr.StandardTokenizerFactory"/>
+ * <filter class="solr.LowerCaseFilterFactory"/>
+ * <filter class="solr.ShingleFilterFactory" minShingleSize="2" maxShingleSize="3" outputUnigrams="true"/>
+ * </analyzer>
+ * <analyzer type="query">
+ * <tokenizer class="solr.StandardTokenizerFactory"/>
+ * <filter class="solr.LowerCaseFilterFactory"/>
+ * <filter class="solr.ShingleFilterFactory" minShingleSize="2" maxShingleSize="7" outputUnigramsIfNoShingles="true" outputUnigrams="true"/>
+ * </analyzer>
+ * </fieldType>
+ * </pre>
+ * <p>
+ * ...where the <code>query</code> analyzer's <code>maxShingleSize="7"</code> determines the maximum
+ * possible phrase length that can be hueristically deduced, the <code>index</code> analyzer's
+ * <code>maxShingleSize="3"</code> determines the accuracy of phrases identified. The large the
+ * indexed <code>maxShingleSize</code> the higher the accuracy. Both analyzers must include
+ * <code>minShingleSize="2" outputUnigrams="true"</code>.
+ * </p>
+ * <p>
+ * With a field type like this, one or more fields can be specified (with weights) via a
+ * <code>phrases.fields</code> param to request that this component identify possible phrases in the
+ * input <code>q</code> param, or an alternative <code>phrases.q</code> override param. The identified
+ * phrases will include their scores relative each field specified, as well an overal weighted score based
+ * on the field weights provided by the client. Higher score values indicate a greater confidence in the
+ * Phrase.
+ * </p>
+ *
+ * <p>
+ * <b>NOTE:</b> In a distributed request, this component uses a single phase (piggy backing on the
+ * {@link ShardRequest#PURPOSE_GET_TOP_IDS} generated by {@link QueryComponent} if it is in use) to
+ * collect all field & shingle stats. No "refinement" requests are used.
+ * </p>
+ *
+ * @lucene.experimental
+ */
+public class PhrasesIdentificationComponent extends SearchComponent {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ /** The only shard purpose that will cause this component to do work & return data during shard req */
+ public static final int SHARD_PURPOSE = ShardRequest.PURPOSE_GET_TOP_IDS;
+
+ /** Name, also used as a request param to identify whether the user query concerns this component */
+ public static final String COMPONENT_NAME = "phrases";
+
+ // TODO: ideally these should live in a commons.params class?
+ public static final String PHRASE_INPUT = "phrases.q";
+ public static final String PHRASE_FIELDS = "phrases.fields";
+ public static final String PHRASE_ANALYSIS_FIELD = "phrases.analysis.field";
+ public static final String PHRASE_SUMMARY_PRE = "phrases.pre";
+ public static final String PHRASE_SUMMARY_POST = "phrases.post";
+ public static final String PHRASE_INDEX_MAXLEN = "phrases.maxlength.index";
+ public static final String PHRASE_QUERY_MAXLEN = "phrases.maxlength.query";
+
+ @Override
+ public void prepare(ResponseBuilder rb) throws IOException {
+ final SolrParams params = rb.req.getParams();
+ if (!params.getBool(COMPONENT_NAME, false)) {
+ return;
+ }
+ if (params.getBool(ShardParams.IS_SHARD, false)) {
+ // only one stage/purpose where we should do any work on a shard
+ if (0 == (SHARD_PURPOSE & params.getInt(ShardParams.SHARDS_PURPOSE, 0))) {
+ return;
+ }
+ }
+
+ // if we're still here, then we should parse & validate our input,
+ // putting it in the request context so our process method knows it should do work
+ rb.req.getContext().put(this.getClass(), PhrasesContextData.parseAndValidateRequest(rb.req));
+ }
+
+ @Override
+ public int distributedProcess(ResponseBuilder rb) {
+ final PhrasesContextData contextData = (PhrasesContextData) rb.req.getContext().get(this.getClass());
+ if (null == contextData) {
+ // if prepare didn't give us anything to work with, then we should do nothing
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+ if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) {
+ return ResponseBuilder.STAGE_EXECUTE_QUERY;
+
+ } else if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
+ // if we're being used in conjunction with QueryComponent, it should have already created
+ // (in this staged) the only ShardRequest we need...
+ for (ShardRequest sreq : rb.outgoing) {
+ if (0 != (SHARD_PURPOSE & sreq.purpose) ) {
+ return ResponseBuilder.STAGE_GET_FIELDS;
+ }
+ }
+ // ...if we can't find it, then evidently we're being used in isolation,
+ // and we need to create our own ShardRequest...
+ ShardRequest sreq = new ShardRequest();
+ sreq.purpose = SHARD_PURPOSE;
+ sreq.params = new ModifiableSolrParams(rb.req.getParams());
+ sreq.params.remove(ShardParams.SHARDS);
+ rb.addRequest(this, sreq);
+ return ResponseBuilder.STAGE_GET_FIELDS;
+
+ } else if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
+ // NOTE: we don't do any actual work in this stage, but we need to ensure that even if
+ // we are being used in isolation w/o QueryComponent that SearchHandler "tracks" a STAGE_GET_FIELDS
+ // so that finishStage(STAGE_GET_FIELDS) is called on us and we can add our merged results
+ // (w/o needing extra code paths for merging phrase results when QueryComponent is/is not used)
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+ return ResponseBuilder.STAGE_DONE;
+ }
+
+ @Override
+ public void finishStage(ResponseBuilder rb) {
+ // NOTE: we don't do this after STAGE_EXECUTE_QUERY because if we're also being used with
+ // QueryComponent, we don't want to add our results to the response until *after*
+ // QueryComponent adds the main DocList
+
+ final PhrasesContextData contextData = (PhrasesContextData) rb.req.getContext().get(this.getClass());
+ if (null == contextData || rb.stage != ResponseBuilder.STAGE_GET_FIELDS) {
+ // if prepare didn't give us anything to work with, or this isn't our stage, then do nothing
+ return;
+ }
+
+ // sanity check: the shard requests we use/piggy-back on should only hapen once per shard,
+ // but let's future proof ourselves against the possibility that some shards might get/respond
+ // to the same request "purpose" multiple times...
+ final BitSet shardsHandled = new BitSet(rb.shards.length);
+
+ // Collect Shard responses
+ for (ShardRequest sreq : rb.finished) {
+ if (0 != (sreq.purpose & SHARD_PURPOSE)) {
+ for (ShardResponse shardRsp : sreq.responses) {
+ final int shardNum = rb.getShardNum(shardRsp.getShard());
+ if (! shardsHandled.get(shardNum)) {
+ shardsHandled.set(shardNum);
+ // shards.tolerant=true can cause nulls on exceptions/errors
+ // if we don't get phrases/stats from a shard, just ignore that shard
+ final SolrResponse rsp = shardRsp.getSolrResponse();
+ if (null == rsp) continue;
+ final NamedList<Object> top = rsp.getResponse();
+ if (null == top) continue;
+ final NamedList<Object> phrasesWrapper = (NamedList<Object>) top.get("phrases");
+ if (null == phrasesWrapper) continue;
+ final List<NamedList<Object>> shardPhrases = (List<NamedList<Object>>) phrasesWrapper.get("_all");
+ if (null == shardPhrases) continue;
+
+ Phrase.populateStats(contextData.allPhrases, shardPhrases);
+ }
+ }
+ }
+ }
+ scoreAndAddResultsToResponse(rb, contextData);
+ }
+
+
+ @Override
+ public void process(ResponseBuilder rb) throws IOException {
+ final PhrasesContextData contextData = (PhrasesContextData) rb.req.getContext().get(this.getClass());
+ if (null == contextData) {
+ // if prepare didn't give us anything to work with, then we should do nothing
+ return;
+ }
+
+ // regardless of single node / shard, we need local stats...
+ Phrase.populateStats(contextData.allPhrases, contextData.fieldWeights.keySet(), rb.req.getSearcher());
+
+ if ( rb.req.getParams().getBool(ShardParams.IS_SHARD, false) ) {
+ // shard request, return stats for all phrases (in original order)
+ SimpleOrderedMap<Object> output = new SimpleOrderedMap<>();
+ output.add("_all", Phrase.formatShardResponse(contextData.allPhrases));
+ // TODO: might want to add numDocs() & getSumTotalTermFreq(f)/getDocCount(f) stats from each field...
+ // so that we can sum/merge them for use in scoring?
+ rb.rsp.add("phrases", output);
+ } else {
+ // full single node request...
+ scoreAndAddResultsToResponse(rb, contextData);
+ }
+ }
+
+ /**
+ * Helper method (suitable for both single node & distributed coordinator node) to
+ * score, sort, and format the end user response once all phrases have been populated with stats.
+ */
+ private void scoreAndAddResultsToResponse(final ResponseBuilder rb, final PhrasesContextData contextData) {
+ assert null != contextData : "Should not be called if no phrase data to use";
+ if (null == contextData) {
+ // if prepare didn't give us anything to work with, then we should do nothing
+ return;
+ }
+
+ SimpleOrderedMap<Object> output = new SimpleOrderedMap<>();
+ rb.rsp.add("phrases", output);
+ output.add("input", contextData.rawInput);
+
+ if (0 == contextData.allPhrases.size()) {
+ // w/o any phrases, the summary is just the input again...
+ output.add("summary", contextData.rawInput);
+ output.add("details", Collections.<Object>emptyList());
+ return;
+ }
+
+ Phrase.populateScores(contextData);
+ final int maxPosition = contextData.allPhrases.get(contextData.allPhrases.size()-1).getPositionEnd();
+
+ final List<Phrase> validScoringPhrasesSorted = contextData.allPhrases.stream()
+ // TODO: ideally this cut off of "0.0" should be a request option...
+ // so users can tune how aggresive/conservative they want to be in finding phrases
+ // but for that to be useful, we need:
+ // - more hard & fast documentation about the "range" of scores that may be returned
+ // - "useful" scores for single words
+ .filter(p -> 0.0D < p.getTotalScore())
+ .sorted(Comparator.comparing((p -> p.getTotalScore()), Collections.reverseOrder()))
+ .collect(Collectors.toList());
+
+ // we want to return only high scoring phrases that don't overlap w/higher scoring phrase
+ final BitSet positionsCovered = new BitSet(maxPosition+1);
+ final List<Phrase> results = new ArrayList<>(maxPosition);
+ for (Phrase phrase : validScoringPhrasesSorted) {
+ final BitSet phrasePositions = phrase.getPositionsBitSet();
+
+ if (! phrasePositions.intersects(positionsCovered)) {
+ // we can use this phrase, record it...
+ positionsCovered.or(phrasePositions);
+ results.add(phrase);
+ } // else: overlaps higher scoring position(s), skip this phrase
+
+ if (positionsCovered.cardinality() == maxPosition+1) {
+ // all positions are covered, so we can bail out and skip the rest
+ break;
+ }
+ }
+
+ // a "quick summary" of the suggested parsing
+ output.add("summary", contextData.summarize(results));
+ // useful user level info on every (high scoring) phrase found (in current, descending score, order)
+ output.add("details", results.stream()
+ .map(p -> p.getDetails()).collect(Collectors.toList()));
+ }
+
+ @Override
+ public String getDescription() {
+ return "Phrases Identification Component";
+ }
+
+ /**
+ * Simple container for all request options and data this component needs to store in the Request Context
+ * @lucene.internal
+ */
+ public static final class PhrasesContextData {
+
+ public final String rawInput;
+ public final int maxIndexedPositionLength;
+ public final int maxQueryPositionLength;
+ public final Map<String,Double> fieldWeights;
+ public final SchemaField analysisField;
+ public final List<Phrase> allPhrases;
+ public final String summaryPre;
+ public final String summaryPost;
+
+ // TODO: add an option to bias field weights based on sumTTF of the fields
+ // (easy enough to "sum the sums" across multiple shards before scoring)
+
+ /**
+ * Parses the params included in this request, throwing appropriate user level
+ * Exceptions for invalid input, and returning a <code>PhrasesContextData</code>
+ * suitable for use in this request.
+ */
+ public static PhrasesContextData parseAndValidateRequest(final SolrQueryRequest req) throws SolrException {
+ return new PhrasesContextData(req);
+ }
+ private PhrasesContextData(final SolrQueryRequest req) throws SolrException {
+ final SolrParams params = req.getParams();
+
+ this.rawInput = params.get(PHRASE_INPUT, params.get(CommonParams.Q));
+ if (null == this.rawInput) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "phrase identification requires a query string or "
+ + PHRASE_INPUT + " param override");
+ }
+
+ { // field weights & analysis field...
+
+ SchemaField tmpAnalysisField = null;
+ Map<String,Double> tmpWeights = new TreeMap<>();
+
+ final String analysisFieldName = params.get(PHRASE_ANALYSIS_FIELD);
+ if (null != analysisFieldName) {
+ tmpAnalysisField = req.getSchema().getFieldOrNull(analysisFieldName);
+ if (null == tmpAnalysisField) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ PHRASE_ANALYSIS_FIELD + " param specifies a field name that does not exist: " +
+ analysisFieldName);
+ }
+ }
+
+ final Map<String,Float> rawFields = SolrPluginUtils.parseFieldBoosts(params.getParams(PHRASE_FIELDS));
+ if (rawFields.isEmpty()) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ PHRASE_FIELDS + " param must specify a (weighted) list of fields " +
+ "to evaluate for phrase identification");
+ }
+
+ for (Map.Entry<String,Float> entry : rawFields.entrySet()) {
+ final SchemaField field = req.getSchema().getFieldOrNull(entry.getKey());
+ if (null == field) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ PHRASE_FIELDS + " param contains a field name that does not exist: " +
+ entry.getKey());
+ }
+ if (null == tmpAnalysisField) {
+ tmpAnalysisField = field;
+ }
+ if ( null == analysisFieldName ) {
+ if (! field.getType().equals(tmpAnalysisField.getType())) {
+ throw new SolrException
+ (ErrorCode.BAD_REQUEST,
+ "All fields specified in " + PHRASE_FIELDS + " must have the same fieldType, " +
+ "or the advanced " + PHRASE_ANALYSIS_FIELD + " option must specify an override");
+ }
+ }
+ // if a weight isn't specified, assume "1.0"
+ final double weight = null == entry.getValue() ? 1.0D : entry.getValue();
+ if (weight < 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ PHRASE_FIELDS + " param must use non-negative weight value for field " + field.getName());
+ }
+ tmpWeights.put(entry.getKey(), weight);
+ }
+ assert null != tmpAnalysisField;
+
+ this.analysisField = tmpAnalysisField;
+ this.fieldWeights = Collections.unmodifiableMap(tmpWeights);
+ }
+
+ { // index/query max phrase sizes...
+ final FieldType ft = analysisField.getType();
+ this.maxIndexedPositionLength = req.getParams().getInt(PHRASE_INDEX_MAXLEN,
+ getMaxShingleSize(ft.getIndexAnalyzer()));
+ if (this.maxIndexedPositionLength < 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Unable to determine max position length of indexed phrases using " +
+ "index analyzer for analysis field: " + analysisField.getName() +
+ " and no override detected using param: " + PHRASE_INDEX_MAXLEN);
+ }
+ this.maxQueryPositionLength = req.getParams().getInt(PHRASE_QUERY_MAXLEN,
+ getMaxShingleSize(ft.getQueryAnalyzer()));
+ if (this.maxQueryPositionLength < 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Unable to determine max position length of query phrases using " +
+ "query analyzer for analysis field: " + analysisField.getName() +
+ " and no override detected using param: " + PHRASE_QUERY_MAXLEN);
+ }
+ if (this.maxQueryPositionLength < this.maxIndexedPositionLength) {
+ throw new SolrException
+ (ErrorCode.BAD_REQUEST,
+ "Effective value of " + PHRASE_INDEX_MAXLEN + " (either from index analyzer shingle factory, " +
+ " or expert param override) must be less then or equal to the effective value of " +
+ PHRASE_QUERY_MAXLEN + " (either from query analyzer shingle factory, or expert param override)");
+ }
+ }
+
+ this.summaryPre = params.get(PHRASE_SUMMARY_PRE, "{");
+ this.summaryPost = params.get(PHRASE_SUMMARY_POST, "}");
+
+ this.allPhrases = Phrase.extractPhrases(this.rawInput, this.analysisField,
+ this.maxIndexedPositionLength,
+ this.maxQueryPositionLength);
+
+ }
+
+ /**
+ * Given a list of phrases to be returned to the user, summarizes those phrases by decorating the
+ * original input string to indicate where the identified phrases exist, using {@link #summaryPre}
+ * and {@link #summaryPost}
+ *
+ * @param results a list of (non overlapping) Phrases that have been identified, sorted from highest scoring to lowest
+ * @return the original user input, decorated to indicate the identified phrases
+ */
+ public String summarize(final List<Phrase> results) {
+ final StringBuffer out = new StringBuffer(rawInput);
+
+ // sort by *reverse* position so we can go back to front
+ final List<Phrase> reversed = results.stream()
+ .sorted(Comparator.comparing((p -> p.getPositionStart()), Collections.reverseOrder()))
+ .collect(Collectors.toList());
+
+ for (Phrase p : reversed) {
+ out.insert(p.getOffsetEnd(), summaryPost);
+ out.insert(p.getOffsetStart(), summaryPre);
+ }
+ return out.toString();
+ }
+ }
+
+
+ /**
+ * Model the data known about a single (candidate) Phrase -- which may or may not be indexed
+ * @lucene.internal
+ */
+ public static final class Phrase {
+
+ /**
+ * Factory method for constructing a list of Phrases given the specified input and using the analyzer
+ * for the specified field. The <code>maxIndexedPositionLength</code> and
+ * <code>maxQueryPositionLength</code> provided *must* match the effective values used by
+ * respective analyzers.
+ */
+ public static List<Phrase> extractPhrases(final String input, final SchemaField analysisField,
+ final int maxIndexedPositionLength,
+ final int maxQueryPositionLength) {
+
+ // TODO: rather then requiring the query analyzer to produce the Phrases for us (assuming Shingles)
+ // we could potentially just require that it produces unigrams compatible with the unigrams in the
+ // indexed fields, and then build our own Phrases at query time -- making the maxQueryPositionLength
+ // a 100% run time configuration option.
+ // But that could be tricky given an arbitrary analyzer -- we'd have pay careful attention
+ // to positions, and we'd have to guess/assume what placeholders/fillers was used in the indexed Phrases
+ // (typically shingles)
+
+ assert maxIndexedPositionLength <= maxQueryPositionLength;
+
+ final CharsRefBuilder buffer = new CharsRefBuilder();
+ final FieldType ft = analysisField.getType();
+ final Analyzer analyzer = ft.getQueryAnalyzer();
+ final List<Phrase> results = new ArrayList<>(42);
+ try (TokenStream tokenStream = analyzer.tokenStream(analysisField.getName(), input)) {
+
+ final OffsetAttribute offsetAttr = tokenStream.addAttribute(OffsetAttribute.class);
+ final PositionIncrementAttribute posIncAttr = tokenStream.addAttribute(PositionIncrementAttribute.class);
+ final PositionLengthAttribute posLenAttr = tokenStream.addAttribute(PositionLengthAttribute.class);
+ final TermToBytesRefAttribute termAttr = tokenStream.addAttribute(TermToBytesRefAttribute.class);
+
+ int position = 0;
+ int lastPosLen = -1;
+
+ tokenStream.reset();
+ while (tokenStream.incrementToken()) {
+ final Phrase phrase = new Phrase();
+
+ final int posInc = posIncAttr.getPositionIncrement();
+ final int posLen = posLenAttr.getPositionLength();
+
+ if (0 == posInc && posLen <= lastPosLen) {
+ // This requirement of analyzers to return tokens in ascending order of length
+ // is currently neccessary for the "linking" logic below to work
+ // if people run into real world sitautions where this is problematic,
+ // we can relax this check if we also make the linking logic more complex
+ // (ie: less optimzied)
+ throw new SolrException
+ (ErrorCode.BAD_REQUEST, "Phrase identification currently requires that " +
+ "the analyzer used must produce tokens that overlap in increasing order of length. ");
+ }
+
+ position += posInc;
+ lastPosLen = posLen;
+
+ phrase.position_start = position;
+ phrase.position_end = position + posLen;
+
+ phrase.is_indexed = (posLen <= maxIndexedPositionLength);
+
+ phrase.offset_start = offsetAttr.startOffset();
+ phrase.offset_end = offsetAttr.endOffset();
+
+ // populate the subsequence directly from the raw input using the offsets,
+ // (instead of using the TermToBytesRefAttribute) so we preserve the original
+ // casing, whitespace, etc...
+ phrase.subSequence = input.subSequence(phrase.offset_start, phrase.offset_end);
+
+ if (phrase.is_indexed) {
+ // populate the bytes so we can build term queries
+ phrase.bytes = BytesRef.deepCopyOf(termAttr.getBytesRef());
+ }
+
+ results.add(phrase);
+ }
+ tokenStream.end();
+ } catch (IOException e) {
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "Analysis error extracting phrases from: " + input, e);
+ }
+
+ // fill in the relationships of each phrase
+ //
+ // NOTE: this logic currently requries that the phrases are sorted by position ascending
+ // (automatic because of how PositionIncrementAttribute works) then by length ascending
+ // (when positions are tied).
+ // We could de-optimize this code if we find that secondary ordering is too restrictive for
+ // some analyzers
+ //
+ // NOTE changes to scoring model may be allow optimize/prune down the relationships tracked,
+ // ...OR.... may require us to add/track more details about sub/parent phrases
+ //
+ for (int p = 0; p < results.size(); p++) {
+ final Phrase current = results.get(p);
+ if (! current.is_indexed) {
+ // we're not an interesting sub phrase of anything
+ continue;
+ }
+
+ // setup links from the phrase to itself if needed
+ addLinkages(current, current, maxIndexedPositionLength);
+
+ // scan backwards looking for phrases that might include us...
+ BEFORE: for (int i = p-1; 0 <= i; i--) {
+ final Phrase previous = results.get(i);
+ if (previous.position_start < (current.position_end - maxQueryPositionLength)) {
+ // we've scanned so far back nothing else is viable
+ break BEFORE;
+ }
+ // any 'previous' phrases must start where current starts or earlier,
+ // so only need to check the end...
+ if (current.position_end <= previous.position_end) {
+ addLinkages(previous, current, maxIndexedPositionLength);
+ }
+ }
+ // scan forwards looking for phrases that might include us...
+ AFTER: for (int i = p+1; i < results.size(); i++) {
+ final Phrase next = results.get(i);
+ // the only way a phrase that comes after current can include current is
+ // if they have the same start position...
+ if (current.position_start != next.position_start) {
+ // we've scanned so far forward nothing else is viable
+ break AFTER;
+ }
+ // any 'next' phrases must start where current starts, so only need to check the end...
+ if (current.position_end <= next.position_end) {
+ addLinkages(next, current, maxIndexedPositionLength);
+ }
+ }
+ }
+
+ return Collections.unmodifiableList(results);
+ }
+
+ /**
+ * Given two phrases, one of which is a super set of the other, adds the neccessary linkages
+ * needed by the scoring model
+ */
+ private static void addLinkages(final Phrase outer, final Phrase inner,
+ final int maxIndexedPositionLength) {
+
+ assert outer.position_start <= inner.position_start;
+ assert inner.position_end <= outer.position_end;
+ assert inner.is_indexed;
+
+ final int inner_len = inner.getPositionLength();
+ if (1 == inner_len) {
+ outer.individualIndexedTerms.add(inner);
+ }
+ if (maxIndexedPositionLength == inner_len
+ || (inner == outer && inner_len < maxIndexedPositionLength)) {
+ outer.largestIndexedSubPhrases.add(inner);
+ }
+ if (outer.is_indexed && inner != outer) {
+ inner.indexedSuperPhrases.add(outer);
+ }
+ }
+
+ /**
+ * Format the phrases suitable for returning in a shard response
+ * @see #populateStats(List,List)
+ */
+ public static List<NamedList<Object>> formatShardResponse(final List<Phrase> phrases) {
+ List<NamedList<Object>> results = new ArrayList<>(phrases.size());
+ for (Phrase p : phrases) {
+ NamedList<Object> data = new SimpleOrderedMap<>();
+ // quick and dirty way to validate that our shards aren't using different analyzers
+ // so the coordinating node can fail fast when mergingthe results
+ data.add("checksum", p.getChecksum());
+ if (p.is_indexed) {
+ data.add("ttf", new NamedList<Object>(p.phrase_ttf));
+ data.add("df", new NamedList<Object>(p.phrase_df));
+ }
+ data.add("conj_dc", new NamedList<Object>(p.subTerms_conjunctionCounts));
+
+ results.add(data);
+ }
+ return results;
+ }
+
+ /**
+ * Populates the phrases with (merged) stats from a remote shard
+ * @see #formatShardResponse
+ */
+ public static void populateStats(final List<Phrase> phrases, final List<NamedList<Object>> shardData) {
+ final int numPhrases = phrases.size();
+ if (shardData.size() != numPhrases) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "num phrases in shard data not consistent: " +
+ numPhrases + " vs " + shardData.size());
+ }
+ for (int i = 0; i < phrases.size(); i++) {
+ // rather then being paranoid about the expected structure, we'll just let the low level
+ // code throw an NPE / CCE / AIOOBE / etc. and wrap & rethrow later...
+ try {
+ final Phrase p = phrases.get(i);
+ final NamedList<Object> data = shardData.get(i);
+ // sanity check the correct phrase
+ if (! p.getChecksum().equals(data.get("checksum"))) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "phrase #" + i + " in shard data had invalid checksum");
+ }
+ if (p.is_indexed) {
+ for (Map.Entry<String,Long> ttf : (NamedList<Long>) data.get("ttf")) {
+ p.phrase_ttf.merge(ttf.getKey(), ttf.getValue(), Long::sum);
+ }
+ for (Map.Entry<String,Long> df : (NamedList<Long>) data.get("df")) {
+ p.phrase_df.merge(df.getKey(), df.getValue(), Long::sum);
+ }
+ }
+ for (Map.Entry<String,Long> conj_dc : (NamedList<Long>) data.get("conj_dc")) {
+ p.subTerms_conjunctionCounts.merge(conj_dc.getKey(), conj_dc.getValue(), Long::sum);
+ }
+ } catch (RuntimeException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "shard data for phrase#" + i + " not consistent", e);
+ }
+ }
+ }
+
+ /**
+ * Populates the phrases with stats from the local index for the specified fields
+ */
+ public static void populateStats(final List<Phrase> phrases, final Collection<String> fieldNames,
+ final SolrIndexSearcher searcher) throws IOException {
+ final IndexReader reader = searcher.getIndexReader();
+ for (String field : fieldNames) {
+ for (Phrase phrase : phrases) {
+ if (phrase.is_indexed) {
+ // add stats based on this entire phrase as an indexed term
+ final Term t = new Term(field, phrase.bytes);
+ phrase.phrase_ttf.put(field, reader.totalTermFreq(t));
+ phrase.phrase_df.put(field, (long)reader.docFreq(t));
+ }
+
+ // even if our phrase is too long to be indexed whole, add stats based on the
+ // conjunction of all the individual terms in the phrase
+ List<Query> filters = new ArrayList<>(phrase.individualIndexedTerms.size());
+ for (Phrase term : phrase.individualIndexedTerms) {
+ // trust the SolrIndexSearcher to cache & intersect the individual terms so that this
+ // can be efficient regardless of how often terms are re-used multiple times in the input/phrases
+ filters.add(new TermQuery(new Term(field, term.bytes)));
+ }
+ final long count = searcher.getDocSet(filters).size();
+ phrase.subTerms_conjunctionCounts.put(field, count);
+ }
+ }
+ }
+
+ /**
+ * Uses the previously popuated stats to populate each Phrase with it's scores for the specified fields,
+ * and it's over all (weighted) total score. This is not needed on shard requests.
+ *
+ * @see #populateStats
+ * @see #getFieldScore(String)
+ * @see #getTotalScore
+ */
+ public static void populateScores(final PhrasesContextData contextData) {
+ populateScores(contextData.allPhrases, contextData.fieldWeights,
+ contextData.maxIndexedPositionLength,
+ contextData.maxQueryPositionLength);
+ }
+
+ /**
+ * Public for testing purposes
+ * @see #populateScores(PhrasesIdentificationComponent.PhrasesContextData)
+ * @lucene.internal
+ */
+ public static void populateScores(final List<Phrase> phrases, final Map<String,Double> fieldWeights,
+ final int maxIndexedPositionLength,
+ final int maxQueryPositionLength) {
+ final double total_weight = fieldWeights.values().stream().mapToDouble(Double::doubleValue).sum();
+ for (Phrase phrase : phrases) {
+ double phrase_cumulative_score = 0.0D;
+ for (Map.Entry<String,Double> entry : fieldWeights.entrySet()) {
+ final String field = entry.getKey();
+ final double weight = entry.getValue();
+ double field_score = computeFieldScore(phrase, field,
+ maxIndexedPositionLength, maxQueryPositionLength);
+ phrase.fieldScores.put(field,field_score);
+ phrase_cumulative_score += (field_score * weight);
+ }
+ phrase.total_score = (total_weight < 0 ? Double.NEGATIVE_INFINITY
+ : (phrase_cumulative_score / total_weight));
+ }
+ }
+
+ private Phrase() {
+ // No-Op
+ }
+
+ private boolean is_indexed;
+ private double total_score = -1.0D; // until we get a computed score, this is "not a phrase"
+
+ private CharSequence subSequence;
+ private BytesRef bytes;
+ private int offset_start;
+ private int offset_end;
+ private int position_start;
+ private int position_end;
+ private Integer checksum = null;
+
+ /** NOTE: Indexed phrases of length 1 are the (sole) individual terms of themselves */
+ private final List<Phrase> individualIndexedTerms = new ArrayList<>(7);
+ /**
+ * NOTE: Indexed phrases of length less then the max indexed length are the (sole)
+ * largest sub-phrases of themselves
+ */
+ private final List<Phrase> largestIndexedSubPhrases = new ArrayList<>(7);
+ /** Phrases larger then this phrase which are indexed and fully contain it */
+ private final List<Phrase> indexedSuperPhrases = new ArrayList<>(7);
+
+ // NOTE: keys are field names
+ private final Map<String,Long> subTerms_conjunctionCounts = new TreeMap<>();
+ private final Map<String,Long> phrase_ttf = new TreeMap<>();
+ private final Map<String,Long> phrase_df = new TreeMap<>();
+ private final Map<String,Double> fieldScores = new TreeMap<>();
+
+ public String toString() {
+ return "'" + subSequence + "'"
+ + "[" + offset_start + ":" + offset_end + "]"
+ + "[" + position_start + ":" + position_end + "]";
+ }
+
+ public NamedList getDetails() {
+ SimpleOrderedMap<Object> out = new SimpleOrderedMap<Object>();
+ out.add("text", subSequence);
+ out.add("offset_start", getOffsetStart());
+ out.add("offset_end", getOffsetEnd());
+ out.add("score", getTotalScore());
+ out.add("field_scores", fieldScores);
+ return out;
+ }
+
+ /**
+ * Computes & caches the checksum of this Phrase (if not already cached).
+ * needed only when merging shard data to validate no inconsistencies with the remote shards
+ */
+ private Integer getChecksum() {
+ if (null == checksum) {
+ checksum = Arrays.hashCode(new int[] { offset_start, offset_end, position_start, position_end });
+ }
+ return checksum;
+ }
+ /** The characters from the original input that corrispond with this Phrase */
+ public CharSequence getSubSequence() {
+ return subSequence;
+ }
+
+ /**
+ * Returns the list of "individual" (ie: <code>getPositionLength()==1</code> terms.
+ * NOTE: Indexed phrases of length 1 are the (sole) individual terms of themselves
+ */
+ public List<Phrase> getIndividualIndexedTerms() {
+ return individualIndexedTerms;
+ }
+ /**
+ * Returns the list of (overlapping) sub phrases that have the largest possible size based on
+ * the effective value of {@link PhrasesContextData#maxIndexedPositionLength}.
+ * NOTE: Indexed phrases of length less then the max indexed length are the (sole)
+ * largest sub-phrases of themselves.
+ */
+ public List<Phrase> getLargestIndexedSubPhrases() {
+ return largestIndexedSubPhrases;
+ }
+ /**
+ * Returns all phrases larger then this phrase, which fully include this phrase, and are indexed.
+ * NOTE: A Phrase is <em>never</em> the super phrase of itself.
+ */
+ public List<Phrase> getIndexedSuperPhrases() {
+ return indexedSuperPhrases;
+ }
+
+ /** NOTE: positions start at '1' */
+ public int getPositionStart() {
+ return position_start;
+ }
+ /** NOTE: positions start at '1' */
+ public int getPositionEnd() {
+ return position_end;
+ }
+ public int getPositionLength() {
+ return position_end - position_start;
+ }
+ /** Each set bit identifies a position filled by this Phrase */
+ public BitSet getPositionsBitSet() {
+ final BitSet result = new BitSet();
+ result.set(position_start, position_end);
+ return result;
+ }
+ public int getOffsetStart() {
+ return offset_start;
+ }
+ public int getOffsetEnd() {
+ return offset_end;
+ }
+
+ /**
+ * Returns the overall score for this Phrase. In the current implementation,
+ * the only garuntee made regarding the range of possible values is that 0 (or less) means
+ * it is not a good phrase.
+ *
+ * @return A numeric value indicating the confidence in this Phrase, higher numbers are higher confidence.
+ */
+ public double getTotalScore() {
+ return total_score;
+ }
+ /**
+ * Returns the score for this Phrase in this given field. In the current implementation,
+ * the only garuntee made regarding the range of possible values is that 0 (or less) means
+ * it is not a good phrase.
+ *
+ * @return A numeric value indicating the confidence in this Phrase for this field, higher numbers are higher confidence.
+ */
+ public double getFieldScore(String field) {
+ return fieldScores.getOrDefault(field, -1.0D);
+ }
+
+ /**
+ * Returns the number of total TTF of this (indexed) Phrase <em>as term</em> in the specified field.
+ * NOTE: behavior of calling this method is undefined unless one of the {@link #populateStats}
+ * methods has been called with this field.
+ */
+ public long getTTF(String field) {
+ if (!is_indexed) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "TTF is only available for indexed phrases");
+ }
+ return phrase_ttf.getOrDefault(field, 0L);
+ }
+ /**
+ * Returns the number of documents that contain <em>all</em> of the {@link #getIndividualIndexedTerms}
+ * that make up this Phrase, in the specified field.
+ * NOTE: behavior of calling this method is undefined unless one of the {@link #populateStats}
+ * methods has been called with this field.
+ */
+ public long getConjunctionDocCount(String field) {
+ return subTerms_conjunctionCounts.getOrDefault(field, 0L);
+ }
+ /**
+ * Returns the number of documents that contain this (indexed) Phrase <em>as term</em>
+ * in the specified field.
+ * NOTE: behavior of calling this method is undefined unless one of the {@link #populateStats}
+ * methods has been called with this field.
+ */
+ public long getDocFreq(String field) {
+ if (!is_indexed) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "DF is only available for indexed phrases");
+ }
+ return phrase_df.getOrDefault(field, 0L);
+ }
+
+ /**
+ * Uses the previously popuated stats to compute a score for the specified field.
+ *
+ * <p>
+ * The current implementation returns scores in the range of <code>[0,1]</code>, but this
+ * may change in future implementations. The only current garuntees are:
+ * </p>
+ *
+ * <ul>
+ * <li>0 (or less) means this is garunteed to not be a phrase</li>
+ * <li>larger numbers are higher confidence</li>
+ * </li>
+ *
+ * @see #populateStats
+ * @see #populateScores
+ * @see #getFieldScore(String)
+ * @return a score value
+ */
+ private static double computeFieldScore(final Phrase input,
+ final String field,
+ final int maxIndexedPositionLength,
+ final int maxQueryPositionLength) {
+ final long num_indexed_sub_phrases = input.getLargestIndexedSubPhrases().size();
+ assert 0 <= num_indexed_sub_phrases; // should be impossible
+
+ if (input.getIndividualIndexedTerms().size() < input.getPositionLength()) {
+ // there are "gaps" in our input, where individual words have not been indexed (stop words,
+ // or multivalue position gap) which means we are not a viable candidate for being a valid Phrase.
+ return -1.0D;
+ }
+
+ final long phrase_conj_count = input.getConjunctionDocCount(field);
+ // if there isn't a single document containing all the terms in our
+ // phrase, then it is 100% not a phrase
+ if (phrase_conj_count <= 0) {
+ return -1.0D;
+ }
+
+ // single words automatically score 0.0 (unless they already scored less for not existing
+ if (input.getPositionLength() <= 1) {
+ return 0.0D;
+ }
+
+ double field_score = 0.0D;
+ long max_sub_conj_count = phrase_conj_count;
+
+ // At the moment, the contribution of each "words" sub-Phrase to the field score to the input
+ // Phrase is independent of any context of "input". Depending on if/how sub-phrase scoring
+ // changes, we might consider computing the scores of all the indexed phrases first, and
+ // aching the portions of their values that are re-used when computing the scores of
+ // longer phrases?
+ //
+ // This would make the overall scoring of all phrases a lot more complicated,
+ // but could save CPU cycles?
+ // (particularly when maxIndexedPositionLength <<< maxQueryPositionLength ???)
+ //
+ // My gut says that knowing the conj_count(input) "context" should help us score the
+ // sub-phrases better, but i can't yet put my finger on why/how. maybe by comparing
+ // the conj_count(input) to the max(conj_count(parent of words)) ?
+
+ // for each of the longest indexed phrases, aka indexed sub-sequence of "words", we have...
+ for (Phrase words : input.getLargestIndexedSubPhrases()) {
+ // we're going to compute scores in range of [-1:1] to indicate the likelihood that our
+ // "words" should be used as a "phrase", based on a bayesian document categorization model,
+ // where the "words as a phrase" (aka: phrase) is our candidate category.
+ //
+ // P(words|phrase) * P(phrase) - P(words|not phrase) * P(not phrase)
+ //
+ // Where...
+ // P(words|phrase) = phrase_ttf / min(word_ttf)
+ // P(phrase) =~ phrase_docFreq / conj_count(words in phrase) *SEE NOTE BELOW*
+ // P(words|not phrase) = phrase_ttf / max(word_ttf)
+ // P(not a phrase) = 1 - P(phrase)
+ //
+ // ... BUT! ...
+ //
+ // NOTE: we're going to reduce our "P(phrase) by the max "P(phrase)" of all the (indexed)
+ // candidate phrases we are a sub-phrase of, to try to offset the inherent bias in favor
+ // of small indexed phrases -- because anytime the super-phrase exists, the sub-phrase exists
+
+
+ // IDEA: consider replacing this entire baysian model with LLR (or rootLLR)...
+ // http://mahout.apache.org/docs/0.13.0/api/docs/mahout-math/org/apache/mahout/math/stats/LogLikelihood.html
+ // ...where we compute LLR over each of the TTF of the pairs of adjacent sub-phrases of each
+ // indexed phrase and take the min|max|avg of the LLR scores.
+ //
+ // ie: for indexed shingle "quick brown fox" compute LLR(ttf("quick"), ttf("brown fox")) &
+ // LLR(ttf("quick brown"), ttf("fox")) using ttf("quick brown fox") as the co-occurance
+ // count, and sumTTF-ttf("quick")-ttf("brown")-ttf("fox") as the "something else"
+ //
+ // (we could actually compute LLR stats over TTF and DF and combine them)
+ //
+ // NOTE: Going the LLR/rootLLR route would require building a full "tree" of every (indexed)
+ // sub-phrase of every other phrase (or at least: all siblings of diff sizes that add up to
+ // an existing phrase). As well as require us to give up on a predictible "range" of
+ // legal values for scores (IIUC from the LLR docs)
+
+ final long phrase_ttf = words.getTTF(field);
+ final long phrase_df = words.getDocFreq(field);
+ final long words_conj_count = words.getConjunctionDocCount(field);
+ max_sub_conj_count = Math.max(words_conj_count, max_sub_conj_count);
+
+ final double max_wrapper_phrase_probability =
+ words.getIndexedSuperPhrases().stream()
+ .mapToDouble(p -> p.getConjunctionDocCount(field) <= 0 ?
+ // special case check -- we already know *our* conj count > 0,
+ // but we need a similar check for wrapper phrases: if <= 0, their probability is 0
+ 0.0D : ((double)p.getDocFreq(field) / p.getConjunctionDocCount(field))).max().orElse(0.0D);
+
+ final LongSummaryStatistics words_ttfs =
+ words.getIndividualIndexedTerms().stream()
+ .collect(Collectors.summarizingLong(t -> t.getTTF(field)));
+
+ final double words_phrase_prob = (phrase_ttf / (double)words_ttfs.getMin());
+ final double words_not_phrase_prob = (phrase_ttf / (double)words_ttfs.getMax());
+
+ final double phrase_prob = (phrase_conj_count / (double)words_conj_count);
+
+
+ final double phrase_score = words_phrase_prob * (phrase_prob - max_wrapper_phrase_probability);
+ final double not_phrase_score = words_not_phrase_prob * (1 - (phrase_prob - max_wrapper_phrase_probability));
+ final double words_score = phrase_score - not_phrase_score;
+
+ field_score += words_score;
+ }
+
+ // NOTE: the "scaling" factors below can "increase" negative scores (by reducing the unsigned value)
+ // when they should ideally be penalizing the scores further, but since we currently don't care
+ // about any score lower then 0, it's not worth worrying about.
+
+ // Average the accumulated score over the number of actual indexed sub-phrases that contributed
+ //
+ // NOTE: since we subsequently want to multiply the score by a fraction with num_indexed_sub_phrases
+ // in the numerator, we can skip this...
+ // SEE BELOW // field_score /= (double) num_indexed_sub_phrases;
+
+ // If we leave field_score as is, then a phrase longer then the maxIndexedPositionLength
+ // will never score higher then the highest scoring sub-phrase it has (because we've averaged them)
+ // so we scale the scores against the longest possible phrase length we're considering
+ //
+ // NOTE: We don't use num_indexed_sub_phrases in the numerator since we skipped it when
+ // averating above...
+ field_score *= ( 1.0D // SEE ABOVE // * ( (double)num_indexed_sub_phrases )
+ / (1 + maxQueryPositionLength - maxIndexedPositionLength) );
+
+ // scale the field_score based on the ratio of the conjunction docCount for the whole phrase
+ // realtive to the largest conjunction docCount of it's (largest indexed) sub phrases, to penalize
+ // the scores of very long phrases that exist very rarely relative to the how often their
+ // sub phrases exist in the index
+ field_score *= ( ((double) phrase_conj_count) / max_sub_conj_count);
+
+ return field_score;
+ }
+ }
+
+ /**
+ * Helper method, public for testing purposes only.
+ * <p>
+ * Given an analyzer, inspects it to determine if:
+ * <ul>
+ * <li>it is a {@link TokenizerChain}</li>
+ * <li>it contains exactly one instance of {@link ShingleFilterFactory}</li>
+ * </ul>
+ * <p>
+ * If these these conditions are met, then this method returns the <code>maxShingleSize</code>
+ * in effect for this analyzer, otherwise returns -1.
+ * </p>
+ *
+ * @param analyzer An analyzer inspect
+ * @return <code>maxShingleSize</code> if available
+ * @lucene.internal
+ */
+ public static int getMaxShingleSize(Analyzer analyzer) {
+ if (!TokenizerChain.class.isInstance(analyzer)) {
+ return -1;
+ }
+
+ final TokenFilterFactory[] factories = ((TokenizerChain) analyzer).getTokenFilterFactories();
+ if (0 == factories.length) {
+ return -1;
+ }
+ int result = -1;
+ for (TokenFilterFactory tff : factories) {
+ if (ShingleFilterFactory.class.isInstance(tff)) {
+ if (0 < result) {
+ // more then one shingle factory in our analyzer, which is weird, so make no assumptions...
+ return -1;
+ }
+ // would be nice if there was an easy way to just ask a factory for the effective value
+ // of an arguement...
+ final Map<String,String> args = tff.getOriginalArgs();
+ result = args.containsKey("maxShingleSize")
+ ? Integer.parseInt(args.get("maxShingleSize")) : ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE;
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/597bd5db/solr/core/src/test-files/solr/collection1/conf/schema-phrases-identification.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-phrases-identification.xml b/solr/core/src/test-files/solr/collection1/conf/schema-phrases-identification.xml
new file mode 100644
index 0000000..ab38f9f
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-phrases-identification.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<schema name="phrase-identification" version="1.6">
+ <field name="id" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+
+ <field name="title" type="text" indexed="true" stored="true" />
+ <field name="body" type="text" indexed="true" stored="true" />
+
+ <field name="multigrams_title" type="multigrams_3_7" indexed="true" stored="false" />
+ <field name="multigrams_body" type="multigrams_3_7" indexed="true" stored="false" />
+
+ <field name="multigrams_title_short" type="multigrams_3" indexed="true" stored="false" />
+ <field name="multigrams_body_short" type="multigrams_3" indexed="true" stored="false" />
+
+ <field name="multigrams_title_stop" type="multigrams_3_7_stop" indexed="true" stored="false" />
+
+ <copyField source="title" dest="multigrams_title" />
+ <copyField source="title" dest="multigrams_title_short" />
+ <copyField source="title" dest="multigrams_title_stop" />
+
+ <copyField source="body" dest="multigrams_body_short" />
+ <copyField source="body" dest="multigrams_body" />
+
+ <uniqueKey>id</uniqueKey>
+
+ <fieldType name="text" class="solr.TextField">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.StopFilterFactory"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="multigrams_3_7" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ShingleFilterFactory" minShingleSize="2" maxShingleSize="3" outputUnigrams="true"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ShingleFilterFactory" minShingleSize="2" maxShingleSize="7" outputUnigramsIfNoShingles="true" outputUnigrams="true"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="multigrams_3" class="solr.TextField" positionIncrementGap="100">
+ <!-- only one analyzer -->
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.ShingleFilterFactory" minShingleSize="2" maxShingleSize="3" outputUnigrams="true"/>
+ </analyzer>
+ </fieldType>
+
+ <fieldType name="multigrams_3_7_stop" class="solr.TextField" positionIncrementGap="100">
+ <analyzer type="index">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.StopFilterFactory"/>
+ <filter class="solr.ShingleFilterFactory" minShingleSize="2" maxShingleSize="3" outputUnigrams="true"/>
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.ASCIIFoldingFilterFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ <filter class="solr.StopFilterFactory"/>
+ <filter class="solr.ShingleFilterFactory" minShingleSize="2" maxShingleSize="7" outputUnigramsIfNoShingles="true" outputUnigrams="true"/>
+ </analyzer>
+ </fieldType>
+
+
+ <fieldType name="long" class="${solr.tests.LongFieldType}" docValues="${solr.tests.numeric.dv}" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+
+</schema>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/597bd5db/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrases-identification.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrases-identification.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrases-identification.xml
new file mode 100644
index 0000000..65ccd5e
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrases-identification.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<config>
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+
+ <searchComponent class="solr.PhrasesIdentificationComponent" name="phrases" />
+
+ <!-- our default search handler should suggest phrases in addition to doing queries if requested -->
+ <requestHandler name="/select" class="solr.SearchHandler">
+ <arr name="last-components">
+ <str>phrases</str>
+ </arr>
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <str name="df">body</str>
+ <str name="phrases.fields">multigrams_body multigrams_title^2</str>
+ </lst>
+ </requestHandler>
+
+ <!-- a custom handler should support exclusively giving phrases w/o doing a query -->
+ <requestHandler name="/phrases" class="solr.SearchHandler">
+ <arr name="components">
+ <str>phrases</str>
+ </arr>
+ <lst name="defaults">
+ <str name="echoParams">explicit</str>
+ <str name="indent">true</str>
+ <bool name="phrases">true</bool>
+ <str name="phrases.fields">multigrams_body multigrams_title^2</str>
+ </lst>
+ </requestHandler>
+
+</config>
+
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/597bd5db/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java
new file mode 100644
index 0000000..cbe1cdc
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * A very simple sanity check that Phrase Identification works across a cloud cluster
+ * using distributed term stat collection.
+ *
+ * @see org.apache.solr.handler.component.PhrasesIdentificationComponentTest
+ */
+@Slow
+public class TestCloudPhrasesIdentificationComponent extends SolrCloudTestCase {
+
+ private static final String DEBUG_LABEL = MethodHandles.lookup().lookupClass().getName();
+ private static final String COLLECTION_NAME = DEBUG_LABEL + "_collection";
+
+ /** A basic client for operations at the cloud level, default collection will be set */
+ private static CloudSolrClient CLOUD_CLIENT;
+ /** One client per node */
+ private static ArrayList<HttpSolrClient> CLIENTS = new ArrayList<>(5);
+
+ @BeforeClass
+ private static void createMiniSolrCloudCluster() throws Exception {
+
+ // multi replicas should not matter...
+ final int repFactor = usually() ? 1 : 2;
+ // ... but we definitely want to test multiple shards
+ final int numShards = TestUtil.nextInt(random(), 1, (usually() ? 2 :3));
+ final int numNodes = (numShards * repFactor);
+
+ final String configName = DEBUG_LABEL + "_config-set";
+ final Path configDir = Paths.get(TEST_HOME(), "collection1", "conf");
+
+ configureCluster(numNodes).addConfig(configName, configDir).configure();
+
+ Map<String, String> collectionProperties = new LinkedHashMap<>();
+ collectionProperties.put("config", "solrconfig-phrases-identification.xml");
+ collectionProperties.put("schema", "schema-phrases-identification.xml");
+ CollectionAdminRequest.createCollection(COLLECTION_NAME, configName, numShards, repFactor)
+ .setProperties(collectionProperties)
+ .process(cluster.getSolrClient());
+
+ CLOUD_CLIENT = cluster.getSolrClient();
+ CLOUD_CLIENT.setDefaultCollection(COLLECTION_NAME);
+
+ waitForRecoveriesToFinish(CLOUD_CLIENT);
+
+ for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
+ CLIENTS.add(getHttpSolrClient(jetty.getBaseUrl() + "/" + COLLECTION_NAME + "/"));
+ }
+
+ // index some docs...
+ CLOUD_CLIENT.add
+ (sdoc("id", "42",
+ "title","Tale of the Brown Fox: was he lazy?",
+ "body", "No. The quick brown fox was a very brown fox who liked to get into trouble."));
+ CLOUD_CLIENT.add
+ (sdoc("id", "43",
+ "title","A fable in two acts",
+ "body", "The brOwn fOx jumped. The lazy dog did not"));
+ CLOUD_CLIENT.add
+ (sdoc("id", "44",
+ "title","Why the LazY dog was lazy",
+ "body", "News flash: Lazy Dog was not actually lazy, it just seemd so compared to Fox"));
+ CLOUD_CLIENT.add
+ (sdoc("id", "45",
+ "title","Why Are We Lazy?",
+ "body", "Because we are. that's why"));
+ CLOUD_CLIENT.commit();
+ }
+
+ @AfterClass
+ private static void afterClass() throws Exception {
+ CLOUD_CLIENT.close(); CLOUD_CLIENT = null;
+ for (HttpSolrClient client : CLIENTS) {
+ client.close();
+ }
+ CLIENTS = null;
+ }
+
+ public void testBasicPhrases() throws Exception {
+ final String input = " did a Quick brown FOX perniciously jump over the lazy dog";
+ final String expected = " did a Quick {brown FOX} perniciously jump over {the lazy dog}";
+
+ // based on the documents indexed, these assertions should all pass regardless of
+ // how many shards we have, or wether the request is done via /phrases or /select...
+ for (String path : Arrays.asList("/select", "/phrases")) {
+ // ... or if we muck with "q" and use the alternative phrases.q for the bits we care about...
+ for (SolrParams p : Arrays.asList(params("q", input, "phrases", "true"),
+ params("q", "*:*", "phrases.q", input, "phrases", "true"),
+ params("q", "-*:*", "phrases.q", input, "phrases", "true"))) {
+ final QueryRequest req = new QueryRequest(p);
+ req.setPath(path);
+ final QueryResponse rsp = req.process(getRandClient(random()));
+ try {
+ NamedList<Object> phrases = (NamedList<Object>) rsp.getResponse().get("phrases");
+ assertEquals("input", input, phrases.get("input"));
+ assertEquals("summary", expected, phrases.get("summary"));
+
+ final List<NamedList<Object>> details = (List<NamedList<Object>>) phrases.get("details");
+ assertNotNull("null details", details);
+ assertEquals("num phrases found", 2, details.size());
+
+ final NamedList<Object> lazy_dog = details.get(0);
+ assertEquals("dog text", "the lazy dog", lazy_dog.get("text"));
+ assertEquals("dog score", 0.166666D, ((Double)lazy_dog.get("score")).doubleValue(), 0.000001D);
+
+ final NamedList<Object> brown_fox = details.get(1);
+ assertEquals("fox text", "brown FOX", brown_fox.get("text"));
+ assertEquals("fox score", 0.083333D, ((Double)brown_fox.get("score")).doubleValue(), 0.000001D);
+
+ } catch (AssertionError e) {
+ throw new AssertionError(e.getMessage() + " ::: " + path + " ==> " + rsp, e);
+ }
+ }
+ }
+ }
+
+ public void testEmptyInput() throws Exception {
+ // empty input shouldn't error, just produce empty results...
+ for (String input : Arrays.asList("", " ")) {
+ for (SolrParams p : Arrays.asList(params("q", "*:*", "phrases.q", input, "phrases", "true"),
+ params("q", "-*:*", "phrases.q", input, "phrases", "true"))) {
+ final QueryRequest req = new QueryRequest(p);
+ req.setPath("/phrases");
+ final QueryResponse rsp = req.process(getRandClient(random()));
+ try {
+ NamedList<Object> phrases = (NamedList<Object>) rsp.getResponse().get("phrases");
+ assertEquals("input", input, phrases.get("input"));
+ assertEquals("summary", input, phrases.get("summary"));
+
+ final List<NamedList<Object>> details = (List<NamedList<Object>>) phrases.get("details");
+ assertNotNull("null details", details);
+ assertEquals("num phrases found", 0, details.size());
+
+ } catch (AssertionError e) {
+ throw new AssertionError(e.getMessage() + " ==> " + rsp, e);
+ }
+ }
+ }
+ }
+
+ /**
+ * returns a random SolrClient -- either a CloudSolrClient, or an HttpSolrClient pointed
+ * at a node in our cluster
+ */
+ public static SolrClient getRandClient(Random rand) {
+ int numClients = CLIENTS.size();
+ int idx = TestUtil.nextInt(rand, 0, numClients);
+
+ return (idx == numClients) ? CLOUD_CLIENT : CLIENTS.get(idx);
+ }
+
+ public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception {
+ assert null != client.getDefaultCollection();
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(client.getDefaultCollection(),
+ client.getZkStateReader(),
+ true, true, 330);
+ }
+
+}
[35/47] lucene-solr:jira/solr-12709: SOLR-12732: TestLogWatcher
failure on Jenkins
Posted by ab...@apache.org.
SOLR-12732: TestLogWatcher failure on Jenkins
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9e04375d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9e04375d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9e04375d
Branch: refs/heads/jira/solr-12709
Commit: 9e04375dc193d3815e9d755514a960f902c60cd2
Parents: 98611d3
Author: Erick Erickson <Er...@gmail.com>
Authored: Thu Sep 6 19:25:33 2018 -0700
Committer: Erick Erickson <Er...@gmail.com>
Committed: Thu Sep 6 19:25:33 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/logging/TestLogWatcher.java | 40 +++++++++++++-------
2 files changed, 29 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9e04375d/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a975f06..78bfb33 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -323,6 +323,8 @@ Bug Fixes
* SOLR-12733: SolrMetricReporterTest failure (Erick Erickson, David Smiley)
+* SOLR-12732: TestLogWatcher failure on Jenkins (Erick Erickson)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9e04375d/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java b/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
index 658bc34..a93a11b 100644
--- a/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
+++ b/solr/core/src/test/org/apache/solr/logging/TestLogWatcher.java
@@ -39,37 +39,51 @@ public class TestLogWatcher extends SolrTestCaseJ4 {
// Create several log watchers and ensure that new messages go to the new watcher.
@Test
public void testLog4jWatcher() {
- LogWatcher watcher = null;
+ LogWatcher watcher;
int lim = random().nextInt(3) + 2;
for (int idx = 0; idx < lim; ++idx) {
String msg = "This is a test message: " + idx;
watcher = LogWatcher.newRegisteredLogWatcher(config, null);
// First ensure there's nothing in the new watcher.
- assertEquals(-1, watcher.getLastEvent());
+
+ // Every time you put a message in the queue, you wait for it to come out _before_ creating
+ // a new watcher so it should be fine.
+ if (looper(watcher, null) == false) {
+ fail("There should be no messages when a new watcher finally gets registered! In loop: " + idx);
+ }
// Now log a message and ensure that the new watcher sees it.
log.warn(msg);
// Loop to give the logger time to process the async message and notify the new watcher.
- boolean foundMsg = false;
- // In local testing this loop usually succeeds 1-2 tries.
- for (int msgIdx = 0; msgIdx < 100; ++msgIdx) {
+ if (looper(watcher, msg) == false) {
+ fail("Should have found message " + msg + ". In loop: " + idx);
+ }
+ }
+ }
+ private boolean looper(LogWatcher watcher, String msg) {
+ // In local testing this loop usually succeeds 1-2 tries.
+ boolean success = false;
+ boolean testingNew = msg == null;
+ for (int msgIdx = 0; msgIdx < 100 && success == false; ++msgIdx) {
+ if (testingNew) { // check that there are no entries registered for the watcher
+ success = watcher.getLastEvent() == -1;
+ } else { // check that the expected message is there.
// Returns an empty (but non-null) list even if there are no messages yet.
SolrDocumentList events = watcher.getHistory(-1, new AtomicBoolean());
for (SolrDocument doc : events) {
if (doc.get("message").equals(msg)) {
- foundMsg = true;
- break;
+ success = true;
}
}
- try {
- Thread.sleep(10);
- } catch (InterruptedException ie) {
- ;
- }
}
- assertTrue("Should have found message " + msg + " in loop: " + idx, foundMsg);
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException ie) {
+ ;
+ }
}
+ return success;
}
}
[34/47] lucene-solr:jira/solr-12709: SOLR-12749: timeseries()
expression missing sum() results for empty buckets
Posted by ab...@apache.org.
SOLR-12749: timeseries() expression missing sum() results for empty buckets
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/98611d33
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/98611d33
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/98611d33
Branch: refs/heads/jira/solr-12709
Commit: 98611d33a7f334ece5faba594120ac3398a0009d
Parents: ccd9f6f
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Sep 6 20:34:11 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Sep 6 20:34:26 2018 -0400
----------------------------------------------------------------------
.../client/solrj/io/stream/TimeSeriesStream.java | 2 ++
.../solrj/io/stream/StreamExpressionTest.java | 16 +++++++++++-----
2 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98611d33/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
index cb743e9..dcd938e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
@@ -397,6 +397,8 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
} else {
t.put(identifier, d.doubleValue());
}
+ } else {
+ t.put(identifier, 0);
}
++m;
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/98611d33/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index e5455b5..b9c6153 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -1856,7 +1856,7 @@ public class StreamExpressionTest extends SolrCloudTestCase {
updateRequest.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
String expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
- "end=\"2016-12-01T01:00:00.000Z\", " +
+ "end=\"2017-12-01T01:00:00.000Z\", " +
"gap=\"+1YEAR\", " +
"field=\"test_dt\", " +
"count(*), sum(price_f), max(price_f), min(price_f))";
@@ -1870,7 +1870,7 @@ public class StreamExpressionTest extends SolrCloudTestCase {
StreamContext context = new StreamContext();
solrStream.setStreamContext(context);
List<Tuple> tuples = getTuples(solrStream);
- assertTrue(tuples.size() == 4);
+ assertTrue(tuples.size() == 5);
assertTrue(tuples.get(0).get("test_dt").equals("2013-01-01T01:00:00Z"));
assertTrue(tuples.get(0).getLong("count(*)").equals(100L));
@@ -1896,6 +1896,12 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(tuples.get(3).getDouble("max(price_f)").equals(400D));
assertTrue(tuples.get(3).getDouble("min(price_f)").equals(400D));
+ assertTrue(tuples.get(4).get("test_dt").equals("2017-01-01T01:00:00Z"));
+ assertEquals((long)tuples.get(4).getLong("count(*)"), 0L);
+ assertEquals(tuples.get(4).getDouble("sum(price_f)"), 0D, 0);
+ assertEquals(tuples.get(4).getDouble("max(price_f)"),0D, 0);
+ assertEquals(tuples.get(4).getDouble("min(price_f)"), 0D, 0);
+
expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2013-01-01T01:00:00.000Z\", " +
"end=\"2016-12-01T01:00:00.000Z\", " +
@@ -1995,9 +2001,9 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(tuples.size() == 5);
assertTrue(tuples.get(0).get("test_dt").equals("2012-01"));
assertTrue(tuples.get(0).getLong("count(*)").equals(0L));
- assertTrue(tuples.get(0).getDouble("sum(price_f)") == null);
- assertTrue(tuples.get(0).getDouble("max(price_f)") == null);
- assertTrue(tuples.get(0).getDouble("min(price_f)") == null);
+ assertTrue(tuples.get(0).getDouble("sum(price_f)") == 0);
+ assertTrue(tuples.get(0).getDouble("max(price_f)") == 0);
+ assertTrue(tuples.get(0).getDouble("min(price_f)") == 0);
assertTrue(tuples.get(1).get("test_dt").equals("2013-01"));
assertTrue(tuples.get(1).getLong("count(*)").equals(100L));
[27/47] lucene-solr:jira/solr-12709: SOLR-9418: Added a new
(experimental) PhrasesIdentificationComponent for identifying potential
phrases in query input based on overlapping shingles in the index
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/597bd5db/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java
new file mode 100644
index 0000000..c8d9edf
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java
@@ -0,0 +1,796 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.handler.component.PhrasesIdentificationComponent;
+import org.apache.solr.handler.component.PhrasesIdentificationComponent.Phrase;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Before;
+import org.hamcrest.Description;
+import org.hamcrest.Matcher;
+import org.hamcrest.BaseMatcher;
+
+public class PhrasesIdentificationComponentTest extends SolrTestCaseJ4 {
+
+ private static final String HANDLER = "/phrases";
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig-phrases-identification.xml","schema-phrases-identification.xml");
+ }
+
+ @Before
+ public void addSomeDocs() throws Exception {
+ assertU(adoc("id", "42",
+ "title","Tale of the Brown Fox: was he lazy?",
+ "body", "No. The quick brown fox was a very brown fox who liked to get into trouble."));
+ assertU(adoc("id", "43",
+ "title","A fable in two acts",
+ "body", "The brOwn fOx jumped. The lazy dog did not"));
+ assertU(adoc("id", "44",
+ "title","Why the LazY dog was lazy",
+ "body", "News flash: Lazy Dog was not actually lazy, it just seemd so compared to Fox"));
+ assertU(adoc("id", "45",
+ "title","Why Are We Lazy?",
+ "body", "Because we are. that's why"));
+ assertU((commit()));
+ }
+
+ @After
+ public void deleteAllDocs() throws Exception {
+ assertU(delQ("*:*"));
+ assertU((commit()));
+ }
+
+ public void testWhiteBoxPhraseParsingLongInput() throws Exception {
+ final SchemaField field = h.getCore().getLatestSchema().getField("multigrams_body");
+ assertNotNull(field);
+ final List<Phrase> phrases = Phrase.extractPhrases
+ (" did a Quick brown FOX perniciously jump over the lAZy dog", field, 3, 7);
+
+ assertEquals(IntStream.rangeClosed((11-7+1), 11).sum(), // 11 words, max query phrase size is 7
+ phrases.size());
+
+ // spot check a few explicitly choosen phrases of various lengths...
+
+ { // single term, close to edge so not as many super phrases as other terms might have
+ final Phrase lazy = phrases.get(phrases.size() - 1 - 2);
+ final String debug = lazy.toString();
+
+ assertEquals(debug, "lAZy", lazy.getSubSequence());
+ assertEquals(debug, 10, lazy.getPositionStart());
+ assertEquals(debug, 11, lazy.getPositionEnd());
+ assertEquals(debug, 1, lazy.getPositionLength());
+
+ assertEquals(debug, 54, lazy.getOffsetStart());
+ assertEquals(debug, 58, lazy.getOffsetEnd());
+
+ assertEquals(debug, 1, lazy.getIndividualIndexedTerms().size());
+ assertEquals(debug, 1, lazy.getLargestIndexedSubPhrases().size());
+ assertEquals(debug, lazy, lazy.getIndividualIndexedTerms().get(0));
+ assertEquals(debug, lazy, lazy.getLargestIndexedSubPhrases().get(0));
+ assertEquals(debug, 4, lazy.getIndexedSuperPhrases().size()); // (2 each: len=2, len=3)
+ }
+ { // length 2, middle of the pack
+ final Phrase brown_fox = phrases.get((7 * 3) + 1);
+ final String debug = brown_fox.toString();
+
+ assertEquals(debug, "brown FOX", brown_fox.getSubSequence());
+ assertEquals(debug, 4, brown_fox.getPositionStart());
+ assertEquals(debug, 6, brown_fox.getPositionEnd());
+ assertEquals(debug, 2, brown_fox.getPositionLength());
+
+ assertEquals(debug, 17, brown_fox.getOffsetStart());
+ assertEquals(debug, 26, brown_fox.getOffsetEnd());
+
+ assertEquals(debug, 2, brown_fox.getIndividualIndexedTerms().size());
+ assertEquals(debug, 1, brown_fox.getLargestIndexedSubPhrases().size());
+ assertEquals(debug, brown_fox, brown_fox.getLargestIndexedSubPhrases().get(0));
+ assertEquals(debug, 2, brown_fox.getIndexedSuperPhrases().size()); // (2 @ len=3)
+
+ }
+ { // length 3 (which is the max indexed size) @ start of the string
+ final Phrase daq = phrases.get(2);
+ final String debug = daq.toString();
+
+ assertEquals(debug, "did a Quick", daq.getSubSequence());
+ assertEquals(debug, 1, daq.getPositionStart());
+ assertEquals(debug, 4, daq.getPositionEnd());
+ assertEquals(debug, 3, daq.getPositionLength());
+
+ assertEquals(debug, 1, daq.getOffsetStart());
+ assertEquals(debug, 13, daq.getOffsetEnd());
+
+ assertEquals(debug, 3, daq.getIndividualIndexedTerms().size());
+ assertEquals(debug, 1, daq.getLargestIndexedSubPhrases().size());
+ assertEquals(debug, daq, daq.getLargestIndexedSubPhrases().get(0));
+ assertEquals(debug, 0, daq.getIndexedSuperPhrases().size());
+ }
+ { // length 4 phrase (larger then the max indexed size)
+ final Phrase qbfp = phrases.get((7 * 2) + 3);
+ final String debug = qbfp.toString();
+
+ assertEquals(debug, "Quick brown FOX perniciously", qbfp.getSubSequence());
+ assertEquals(debug, 3, qbfp.getPositionStart());
+ assertEquals(debug, 7, qbfp.getPositionEnd());
+ assertEquals(debug, 4, qbfp.getPositionLength());
+
+ assertEquals(debug, 8, qbfp.getOffsetStart());
+ assertEquals(debug, 39, qbfp.getOffsetEnd());
+
+ assertEquals(debug, 4, qbfp.getIndividualIndexedTerms().size());
+ assertEquals(debug, 2, qbfp.getLargestIndexedSubPhrases().size());
+ assertEquals(debug, 0, qbfp.getIndexedSuperPhrases().size());
+ }
+
+ // some blanket assumptions about the results...
+ assertBasicSanityChecks(phrases, 11, 3, 7);
+ }
+
+ public void testWhiteBoxPhraseParsingShortInput() throws Exception {
+ // for input this short, either of these fields should be (mostly) equivilent
+ final Map<String,Integer> fields = new TreeMap<>();
+ fields.put("multigrams_body", 7);
+ fields.put("multigrams_body_short", 3);
+ for (Map.Entry<String,Integer> entry : fields.entrySet()) {
+ try {
+ final int maxQ = entry.getValue();
+ final SchemaField field = h.getCore().getLatestSchema().getField(entry.getKey());
+ assertNotNull(field);
+
+ // empty input shouldn't break anything
+ assertEquals(0, Phrase.extractPhrases(random().nextBoolean() ? "" : " ", field, 3, maxQ).size());
+
+ // input shorter them our index/query phrase sizes shouldn't break anything either....
+ final List<Phrase> phrases = Phrase.extractPhrases("brown FOX", field, 3, maxQ);
+
+ assertEquals(3, phrases.size());
+
+ { // length 2
+ final Phrase brown_fox = phrases.get(1);
+ final String debug = brown_fox.toString();
+
+ assertEquals(debug, "brown FOX", brown_fox.getSubSequence());
+ assertEquals(debug, 1, brown_fox.getPositionStart());
+ assertEquals(debug, 3, brown_fox.getPositionEnd());
+ assertEquals(debug, 2, brown_fox.getPositionLength());
+
+ assertEquals(debug, 0, brown_fox.getOffsetStart());
+ assertEquals(debug, 9, brown_fox.getOffsetEnd());
+
+ assertEquals(debug, 2, brown_fox.getIndividualIndexedTerms().size());
+ assertEquals(debug, 1, brown_fox.getLargestIndexedSubPhrases().size());
+ assertEquals(debug, brown_fox, brown_fox.getLargestIndexedSubPhrases().get(0));
+ assertEquals(debug, 0, brown_fox.getIndexedSuperPhrases().size());
+ }
+ { // length 1
+ final Phrase fox = phrases.get(2);
+ final String debug = fox.toString();
+
+ assertEquals(debug, "FOX", fox.getSubSequence());
+ assertEquals(debug, 2, fox.getPositionStart());
+ assertEquals(debug, 3, fox.getPositionEnd());
+ assertEquals(debug, 1, fox.getPositionLength());
+
+ assertEquals(debug, 6, fox.getOffsetStart());
+ assertEquals(debug, 9, fox.getOffsetEnd());
+
+ assertEquals(debug, 1, fox.getIndividualIndexedTerms().size());
+ assertEquals(debug, 1, fox.getLargestIndexedSubPhrases().size());
+ assertEquals(debug, fox, fox.getLargestIndexedSubPhrases().get(0));
+ assertEquals(debug, 1, fox.getIndexedSuperPhrases().size());
+ }
+
+ assertBasicSanityChecks(phrases, 2, 3, maxQ);
+ } catch (AssertionError e) {
+ throw new AssertionError(entry.getKey() + " => " + e.getMessage(), e);
+ }
+ }
+ }
+
+ /**
+ * Asserts some basic rules that should be enforced about all Phrases
+ * & their linkages to oher phrases
+ */
+ private void assertBasicSanityChecks(final List<Phrase> phrases,
+ final int inputPositionLength,
+ final int maxIndexedPositionLength,
+ final int maxQueryPositionLength) throws Exception {
+ assert 0 < phrases.size() : "Don't use this method if phrases might be empty";
+
+ assertEmptyStream("no phrase should be longer then "+maxQueryPositionLength+" positions",
+ phrases.stream().filter(p -> p.getPositionLength() > maxQueryPositionLength));
+
+ assertEmptyStream("no phrase should have a start offset < 0",
+ phrases.stream().filter(p -> p.getOffsetStart() < 0));
+ assertEmptyStream("no phrase should have a start position < 1",
+ phrases.stream().filter(p -> p.getPositionStart() < 1));
+
+ assertEmptyStream("If a phrase has a start offset of 0, then it must have position 1",
+ phrases.stream().filter(p -> (p.getOffsetStart() == 0)
+ && (p.getPositionStart() != 1)));
+
+ final Phrase first = phrases.get(0);
+ final Phrase last = phrases.get(phrases.size()-1);
+
+ assertEmptyStream("no phrase should have a start offset < first phrase",
+ phrases.stream().filter(p -> p.getOffsetStart() < first.getOffsetStart()));
+ assertEmptyStream("no phrase should have an end offset > last phrase",
+ phrases.stream().filter(p -> last.getOffsetEnd() < p.getOffsetEnd()));
+
+ assertEmptyStream("no phrase should have a start position < first phrase",
+ phrases.stream().filter(p -> p.getPositionStart() < first.getPositionStart()));
+ assertEmptyStream("no phrase should have an end position > last phrase",
+ phrases.stream().filter(p -> last.getPositionEnd() < p.getPositionEnd()));
+
+
+ // NOTE: stuff below this point may not be true for all analyzers (ie: stopwords)
+ // but should be valid for the analyzers used in this test...
+ // (if we expand test to cover analyzers w/stopwords, refactor this into a new method)
+
+ for (int n = 1; n <= maxQueryPositionLength; n++) {
+ final int len = n;
+ final int expected = Math.max(0, 1 + inputPositionLength - n);
+ final List<Phrase> sizeN = phrases.stream().filter(p -> p.getPositionLength() == len
+ ).collect(Collectors.toList());
+ assertEquals("Expected # phrases of size " + n + ": " + sizeN, expected, sizeN.size());
+ }
+
+ // check the quantities of sub-terms/phrases...
+ assertEmptyStream("no phrase should have num indexed terms != pos_len",
+ phrases.stream().filter
+ (p -> last.getPositionLength() != last.getIndividualIndexedTerms().size()));
+ assertEmptyStream("no phrase should have num sub-phrases != max(1, 1 + pos_len - "+maxIndexedPositionLength+")",
+ phrases.stream().filter
+ (p -> (Math.max(1, 1 + last.getPositionLength() - maxIndexedPositionLength)
+ != last.getLargestIndexedSubPhrases().size())));
+ // NOTE: indexed super phrases can be of various lengths, and differing quantities near
+ // begining/end of input so don't worry about an exact count, just check their properties (below)
+
+ // check the properties of our sub/super phrases
+ for (Phrase phrase : phrases) {
+ final String debug = phrase.toString();
+
+ assertEmptyStream(debug + " should not have any indexed terms where pos_len != 1",
+ phrase.getIndividualIndexedTerms().stream().filter
+ (term -> 1 != term.getPositionLength()));
+
+ assertEmptyStream(debug + " should not have any sub-phrases where pos_len > min(pos_len, "
+ + maxIndexedPositionLength+")",
+ phrase.getLargestIndexedSubPhrases().stream().filter
+ (inner -> (Math.min(phrase.getPositionLength(), maxIndexedPositionLength)
+ < inner.getPositionLength())));
+
+ assertEmptyStream(debug + " should not have any super-phrases where super.len <= phrase.len or "
+ + maxIndexedPositionLength + " < super.len",
+ phrase.getIndexedSuperPhrases().stream().filter
+ (outer -> (outer.getPositionLength() <= phrase.getPositionLength() ||
+ maxIndexedPositionLength < outer.getPositionLength())));
+ }
+ }
+
+ public void testWhiteboxStats() throws Exception {
+ final SchemaField analysisField = h.getCore().getLatestSchema().getField("multigrams_body");
+ assertNotNull(analysisField);
+ final String input = "BROWN fox lAzY dog xxxyyyzzz";
+
+ // a function we'll re-use on phrases generated from the above input
+ // the multiplier let's us simulate multiple shards returning the same values
+ BiConsumer<Integer,List<Phrase>> assertions = (mult, phrases) -> {
+ final Phrase brown_fox = phrases.get(1);
+ assertEquals("BROWN fox", brown_fox.getSubSequence());
+
+ assertEquals(mult * 1, brown_fox.getTTF("multigrams_title"));
+ assertEquals(mult * 1, brown_fox.getDocFreq("multigrams_title"));
+ assertEquals(mult * 1, brown_fox.getConjunctionDocCount("multigrams_title"));
+
+ assertEquals(mult * 3, brown_fox.getTTF("multigrams_body"));
+ assertEquals(mult * 2, brown_fox.getDocFreq("multigrams_body"));
+ assertEquals(mult * 2, brown_fox.getConjunctionDocCount("multigrams_body"));
+
+ final Phrase fox_lazy = phrases.get(6);
+ assertEquals("fox lAzY", fox_lazy.getSubSequence());
+
+ assertEquals(mult * 0, fox_lazy.getTTF("multigrams_title"));
+ assertEquals(mult * 0, fox_lazy.getDocFreq("multigrams_title"));
+ assertEquals(mult * 1, fox_lazy.getConjunctionDocCount("multigrams_title"));
+
+ assertEquals(mult * 0, fox_lazy.getTTF("multigrams_body"));
+ assertEquals(mult * 0, fox_lazy.getDocFreq("multigrams_body"));
+ assertEquals(mult * 2, fox_lazy.getConjunctionDocCount("multigrams_body"));
+
+ final Phrase bfld = phrases.get(3);
+ assertEquals("BROWN fox lAzY dog", bfld.getSubSequence());
+
+ expectThrows(SolrException.class, () -> { bfld.getTTF("multigrams_title"); });
+ expectThrows(SolrException.class, () -> { bfld.getDocFreq("multigrams_title"); });
+ assertEquals(mult * 0, bfld.getConjunctionDocCount("multigrams_title"));
+
+ expectThrows(SolrException.class, () -> { bfld.getTTF("multigrams_body"); });
+ expectThrows(SolrException.class, () -> { bfld.getDocFreq("multigrams_body"); });
+ assertEquals(mult * 1, bfld.getConjunctionDocCount("multigrams_body"));
+
+ final Phrase xyz = phrases.get(phrases.size()-1);
+
+ assertEquals("xxxyyyzzz", xyz.getSubSequence());
+ assertEquals(mult * 0, xyz.getTTF("multigrams_title"));
+ assertEquals(mult * 0, xyz.getDocFreq("multigrams_title"));
+ assertEquals(mult * 0, xyz.getConjunctionDocCount("multigrams_title"));
+
+ assertEquals(mult * 0, xyz.getTTF("multigrams_body"));
+ assertEquals(mult * 0, xyz.getDocFreq("multigrams_body"));
+ assertEquals(mult * 0, xyz.getConjunctionDocCount("multigrams_body"));
+ return;
+ };
+
+
+ final List<Phrase> phrasesLocal = Phrase.extractPhrases(input, analysisField, 3, 7);
+
+ // freshly parsed phrases, w/o any stats populated, all the stats should be 0
+ assertions.accept(0, phrasesLocal);
+
+ // If we populate with our index stats, we should get the basic values in our BiConsumer
+ try (SolrQueryRequest req = req()) {
+ Phrase.populateStats(phrasesLocal, Arrays.asList("multigrams_body","multigrams_title"),
+ req.getSearcher());
+ }
+ assertions.accept(1, phrasesLocal);
+
+ // likewise, if we create a new freshly parsed set of phrases, and "merge" in the previous index stats
+ // (ie: merge results from one shard) we should get the same results
+ final List<Phrase> phrasesMerged = Phrase.extractPhrases(input, analysisField, 3, 7);
+ Phrase.populateStats(phrasesMerged, Phrase.formatShardResponse(phrasesLocal));
+ assertions.accept(1, phrasesMerged);
+
+ // if we merge in a second copy of the same results (ie: two identical shards)
+ // our results should be double what we had before
+ Phrase.populateStats(phrasesMerged, Phrase.formatShardResponse(phrasesLocal));
+ assertions.accept(2, phrasesMerged);
+
+ }
+
+ public void testWhiteboxScores() throws Exception {
+ final SchemaField analysisField = h.getCore().getLatestSchema().getField("multigrams_body");
+ assertNotNull(analysisField);
+ final Map<String,Double> fieldWeights = new TreeMap<>();
+ fieldWeights.put("multigrams_title", 1.0D);
+ fieldWeights.put("multigrams_body", 0.0D); // NOTE: 0 weighting should only affect total score
+
+ final String input = "xxxyyyzzz BROWN fox why are we lAzY";
+ final List<Phrase> phrases = Phrase.extractPhrases(input, analysisField, 3, 7);
+ try (SolrQueryRequest req = req()) {
+ Phrase.populateStats(phrases, fieldWeights.keySet(), req.getSearcher());
+ }
+ Phrase.populateScores(phrases, fieldWeights, 3, 7);
+
+ // do some basic sanity checks of the field & total scores...
+
+ for (Phrase xyz : phrases.subList(0, 7)) {
+ // first 7 all start with xyz which isn't in index (in either field) so all scores should be -1
+ assertEquals(xyz.toString(), -1.0D, xyz.getTotalScore(), 0.0D);
+ assertEquals(xyz.toString(), -1.0D, xyz.getFieldScore("multigrams_title"), 0.0D);
+ assertEquals(xyz.toString(), -1.0D, xyz.getFieldScore("multigrams_body"), 0.0D);
+ }
+
+ // any individual terms (past xyz) should score 0.0 because they are all actually in the index
+ // (in both fields)
+ for (Phrase term : phrases.subList(7, phrases.size()).stream().filter
+ ((p -> 1 == p.getPositionLength())).collect(Collectors.toList())) {
+
+ assertEquals(term.toString(), 0.0D, term.getFieldScore("multigrams_title"), 0.0D);
+ assertEquals(term.toString(), 0.0D, term.getFieldScore("multigrams_body"), 0.0D);
+ assertEquals(term.toString(), 0.0D, term.getTotalScore(), 0.0D);
+ }
+
+ // "brown fox" should score positively in both fields, and overall...
+ final Phrase brown_fox = phrases.get(8);
+ assertEquals("BROWN fox", brown_fox.getSubSequence());
+ assertThat(brown_fox.toString(), brown_fox.getFieldScore("multigrams_title"), greaterThan(0.0D));
+ assertThat(brown_fox.toString(), brown_fox.getFieldScore("multigrams_body"), greaterThan(0.0D) );
+ assertThat(brown_fox.toString(), brown_fox.getTotalScore(), greaterThan(0.0D));
+
+ // "we lazy" does appear in a title value, but should score poorly given how often the terms
+ // are used in other contexts, and should score -1 against body -- but because of our weights,
+ // that shouldn't bring down the total
+ final Phrase we_lazy = phrases.get(phrases.size()-2);
+ assertEquals("we lAzY", we_lazy.getSubSequence());
+ assertEquals(we_lazy.toString(), -1.0D, we_lazy.getFieldScore("multigrams_body"), 0.0D);
+ assertThat(we_lazy.toString(), we_lazy.getFieldScore("multigrams_title"), lessThan(0.0D));
+ assertThat(we_lazy.toString(), we_lazy.getTotalScore(), lessThan(0.0D));
+ assertEquals(we_lazy.toString(), we_lazy.getFieldScore("multigrams_title"), we_lazy.getTotalScore(),
+ 0.0D);
+
+ // "why are we lazy" is longer then the max indexed phrase size & appears verbatim in a title value
+ // it should score -1 against body -- but because of our weights, that shouldn't bring down the total
+ final Phrase wawl = phrases.get(phrases.size()-7);
+ assertEquals("why are we lAzY", wawl.getSubSequence());
+ assertEquals(wawl.toString(), -1.0D, wawl.getFieldScore("multigrams_body"), 0.0D);
+ assertThat(wawl.toString(), wawl.getFieldScore("multigrams_title"), greaterThan(0.0D));
+ assertThat(wawl.toString(), wawl.getTotalScore(), greaterThan(0.0D));
+ assertEquals(wawl.toString(), wawl.getFieldScore("multigrams_title"), wawl.getTotalScore(),
+ 0.0D);
+
+ // "brown fox why are we" is longer then the max indexed phrase, and none of it's
+ // (longest) sub phrases exists in either field -- so all of it's scores should be -1
+ final Phrase bfwaw = phrases.get(11);
+ assertEquals("BROWN fox why are we", bfwaw.getSubSequence());
+ assertEquals(bfwaw.toString(), -1.0D, bfwaw.getFieldScore("multigrams_title"), 0.0D);
+ assertEquals(bfwaw.toString(), -1.0D, bfwaw.getFieldScore("multigrams_body"), 0.0D);
+ assertEquals(bfwaw.toString(), -1.0D, bfwaw.getTotalScore(), 0.0D);
+
+ }
+
+ public void testWhiteboxScorcesStopwords() throws Exception {
+ final String input = "why the lazy dog brown fox";
+ final Map<String,Double> fieldWeights = new TreeMap<>();
+ fieldWeights.put("multigrams_title", 1.0D);
+ fieldWeights.put("multigrams_title_stop", 1.0D);
+
+ { // If our analysisField uses all terms,
+ // be we also generate scores from a field that filters stopwords...
+ final SchemaField analysisField = h.getCore().getLatestSchema().getField("multigrams_title");
+ assertNotNull(analysisField);
+
+ final List<Phrase> phrases = Phrase.extractPhrases(input, analysisField, 3, 7);
+ try (SolrQueryRequest req = req()) {
+ Phrase.populateStats(phrases, fieldWeights.keySet(), req.getSearcher());
+ }
+ Phrase.populateScores(phrases, fieldWeights, 3, 7);
+
+ // phrases that span the stop word should have valid scores from the field that doesn't care
+ // about stop words, but the stopword field should reject them
+ final Phrase why_the_lazy = phrases.get(2);
+ assertEquals("why the lazy", why_the_lazy.getSubSequence());
+ assertThat(why_the_lazy.toString(), why_the_lazy.getFieldScore("multigrams_title"), greaterThan(0.0D) );
+ assertEquals(why_the_lazy.toString(), -1.0D, why_the_lazy.getFieldScore("multigrams_title_stop"), 0.0D);
+
+ final Phrase the_lazy_dog = phrases.get(8);
+ assertEquals("the lazy dog", the_lazy_dog.getSubSequence());
+ assertThat(the_lazy_dog.toString(), the_lazy_dog.getFieldScore("multigrams_title"), greaterThan(0.0D) );
+ assertEquals(the_lazy_dog.toString(), -1.0D, the_lazy_dog.getFieldScore("multigrams_title_stop"), 0.0D);
+
+ // sanity check that good scores are still possible with stopwords
+ // "brown fox" should score positively in both fields, and overall...
+ final Phrase brown_fox = phrases.get(phrases.size()-2);
+ assertEquals("brown fox", brown_fox.getSubSequence());
+ assertThat(brown_fox.toString(), brown_fox.getFieldScore("multigrams_title"), greaterThan(0.0D));
+ assertThat(brown_fox.toString(), brown_fox.getFieldScore("multigrams_title_stop"), greaterThan(0.0D) );
+ assertThat(brown_fox.toString(), brown_fox.getTotalScore(), greaterThan(0.0D));
+ }
+
+ { // now flip things: our analysisField filters stopwords,
+ // but we also generates scores from a field that doesn't know about them...
+ //
+ // (NOTE: the parser will still generate _some_ candidate phrases spaning the stop word position,
+ // but not ones that start with the stopword)
+ final SchemaField analysisField = h.getCore().getLatestSchema().getField("multigrams_title_stop");
+ assertNotNull(analysisField);
+
+ final List<Phrase> phrases = Phrase.extractPhrases(input, analysisField, 3, 7);
+ try (SolrQueryRequest req = req()) {
+ Phrase.populateStats(phrases, fieldWeights.keySet(), req.getSearcher());
+ }
+ Phrase.populateScores(phrases, fieldWeights, 3, 7);
+ assertTrue(phrases.toString(), 0 < phrases.size());
+
+ for (Phrase p : phrases) {
+ if (p.getPositionStart() <= 2 && 2 < p.getPositionEnd()) {
+ // phrases that span the stop word should have valid scores from the field that doesn't care
+ // about stop words, but the stopword field should reject them
+ assertEquals(p.toString(), -1.0D, p.getFieldScore("multigrams_title"), 0.0D);
+ assertEquals(p.toString(), -1.0D, p.getFieldScore("multigrams_title_stop"), 0.0D);
+ }
+ }
+
+ // sanity check that good scores are still possible with stopwords
+ // "brown fox" should score positively in both fields, and overall...
+ final Phrase brown_fox = phrases.get(phrases.size()-2);
+ assertEquals("brown fox", brown_fox.getSubSequence());
+ assertThat(brown_fox.toString(), brown_fox.getFieldScore("multigrams_title"), greaterThan(0.0D));
+ assertThat(brown_fox.toString(), brown_fox.getFieldScore("multigrams_title_stop"), greaterThan(0.0D) );
+ assertThat(brown_fox.toString(), brown_fox.getTotalScore(), greaterThan(0.0D));
+ }
+
+ }
+
+ public void testExpectedUserErrors() throws Exception {
+ assertQEx("empty field list should error",
+ "must specify a (weighted) list of fields",
+ req("q","foo", "phrases","true",
+ "phrases.fields", " "),
+ ErrorCode.BAD_REQUEST);
+
+ assertQEx("bogus field name should error",
+ "does not exist",
+ req("q","foo", "phrases","true",
+ "phrases.fields", "bogus1 bogus2"),
+ ErrorCode.BAD_REQUEST);
+
+ assertQEx("lack of shingles should cause error",
+ "Unable to determine max position length",
+ req("q","foo", "phrases","true",
+ "phrases.fields", "title"),
+ ErrorCode.BAD_REQUEST);
+
+ assertQEx("analyzer missmatch should cause error",
+ "must have the same fieldType",
+ req("q","foo", "phrases","true",
+ "phrases.fields", "multigrams_title multigrams_title_short"),
+ ErrorCode.BAD_REQUEST);
+
+ assertQEx("analysis field must exist",
+ "does not exist",
+ req("q","foo", "phrases","true",
+ "phrases.analysis.field", "bogus",
+ "phrases.fields", "multigrams_title multigrams_title_short"),
+ ErrorCode.BAD_REQUEST);
+
+ assertQEx("no query param should error",
+ "requires a query string",
+ req("qt", "/phrases",
+ "phrases.fields", "multigrams_title"),
+ ErrorCode.BAD_REQUEST);
+ }
+
+ public void testMaxShingleSizeHelper() throws Exception {
+ IndexSchema schema = h.getCore().getLatestSchema();
+
+ assertEquals(3, PhrasesIdentificationComponent.getMaxShingleSize
+ (schema.getFieldTypeByName("multigrams_3_7").getIndexAnalyzer()));
+ assertEquals(7, PhrasesIdentificationComponent.getMaxShingleSize
+ (schema.getFieldTypeByName("multigrams_3_7").getQueryAnalyzer()));
+
+ assertEquals(3, PhrasesIdentificationComponent.getMaxShingleSize
+ (schema.getFieldTypeByName("multigrams_3").getIndexAnalyzer()));
+ assertEquals(3, PhrasesIdentificationComponent.getMaxShingleSize
+ (schema.getFieldTypeByName("multigrams_3").getQueryAnalyzer()));
+
+ assertEquals(-1, PhrasesIdentificationComponent.getMaxShingleSize
+ (schema.getFieldTypeByName("text").getIndexAnalyzer()));
+ assertEquals(-1, PhrasesIdentificationComponent.getMaxShingleSize
+ (schema.getFieldTypeByName("text").getQueryAnalyzer()));
+
+ }
+
+ public void testSimplePhraseRequest() throws Exception {
+ final String input = " did a Quick brown FOX perniciously jump over the lazy dog";
+ final String expected = " did a Quick {brown FOX} perniciously jump over {the lazy dog}";
+
+ // should get same behavior regardless of wether we use "q" or "phrases.q"
+ for (String p : Arrays.asList("q", "phrases.q")) {
+ // basic request...
+ assertQ(req("qt", HANDLER, p, input)
+ // expect no search results...
+ , "count(//result)=0"
+
+ // just phrase info...
+ , "//lst[@name='phrases']/str[@name='input'][.='"+input+"']"
+ , "//lst[@name='phrases']/str[@name='summary'][.='"+expected+"']"
+ , "count(//lst[@name='phrases']/arr[@name='details']/lst) = 2"
+ //
+ , "//lst[@name='phrases']/arr[@name='details']/lst[1]/str[@name='text'][.='the lazy dog']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst[1]/int[@name='offset_start'][.='50']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst[1]/int[@name='offset_end'][.='62']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst[1]/double[@name='score'][number(.) > 0]"
+ //
+ , "//lst[@name='phrases']/arr[@name='details']/lst[2]/str[@name='text'][.='brown FOX']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst[2]/int[@name='offset_start'][.='17']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst[2]/int[@name='offset_end'][.='26']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst[2]/double[@name='score'][number(.) > 0]"
+ );
+
+ // empty input, empty phrases (and no error)...
+ assertQ(req("qt", HANDLER, p, "")
+ // expect no search results...
+ , "count(//result)=0"
+ // just empty phrase info for our empty input...
+ , "//lst[@name='phrases']/str[@name='input'][.='']"
+ , "//lst[@name='phrases']/str[@name='summary'][.='']"
+ , "count(//lst[@name='phrases']/arr[@name='details']) = 1"
+ , "count(//lst[@name='phrases']/arr[@name='details']/lst) = 0"
+ );
+ }
+ }
+
+ public void testSimpleSearchRequests() throws Exception {
+ final String input = "\"brown fox\"";
+
+ assertQ(req("q", input)
+ // basic search should have worked...
+ , "//result[@numFound='2']"
+ , "//result/doc/str[@name='id'][.='42']"
+ , "//result/doc/str[@name='id'][.='43']"
+ // and phrases should not be returned since they weren't requested...
+ , "0=count(//lst[@name='phrases'])"
+ );
+
+ assertQ(req("phrases", "false", "q", input)
+ // basic search should have worked...
+ , "//result[@numFound='2']"
+ , "//result/doc/str[@name='id'][.='42']"
+ , "//result/doc/str[@name='id'][.='43']"
+ // and phrases should not be returned since they were explicitly disabled...
+ , "0=count(//lst[@name='phrases'])"
+ );
+
+ // with input this short, all of these permutations of requests should produce the same output...
+ for (SolrQueryRequest req : Arrays.asList
+ ( // simple, using 3/7 defaults
+ req("phrases","true", "q", input),
+
+ // simple, using just the 3/3 'short' fields
+ req("phrases","true", "q", input,
+ "phrases.fields", "multigrams_body_short multigrams_title_short^2"),
+
+ // diff analysers, but explicit override using 3/3 "short" field...
+ req("phrases","true", "q", input,
+ "phrases.fields", "multigrams_body multigrams_title_short^2",
+ "phrases.analysis.field", "multigrams_title_short"))) {
+ assertQ(req
+ // basic search should have worked...
+ , "//result[@numFound='2']"
+ , "//result/doc/str[@name='id'][.='42']"
+ , "//result/doc/str[@name='id'][.='43']"
+
+ // and we should have gotten phrase info...
+ , "//lst[@name='phrases']/str[@name='input'][.='"+input+"']"
+ , "//lst[@name='phrases']/str[@name='summary'][.='\"{brown fox}\"']"
+ , "count(//lst[@name='phrases']/arr[@name='details']/lst)=1"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/str[@name='text'][.='brown fox']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/int[@name='offset_start'][.='1']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/int[@name='offset_end'][.='10']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/double[@name='score'][number(.) > 0]"
+ );
+ }
+
+ // override the query string to get different phrases
+ assertQ(req("phrases","true", "q", "*:*", "phrases.q", input)
+ // basic search should have found all docs...
+ , "//result[@numFound='4']"
+ // and we should have gotten phrase info for our alternative q string...
+ , "//lst[@name='phrases']/str[@name='input'][.='"+input+"']"
+ , "//lst[@name='phrases']/str[@name='summary'][.='\"{brown fox}\"']"
+ , "count(//lst[@name='phrases']/arr[@name='details']/lst)=1"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/str[@name='text'][.='brown fox']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/int[@name='offset_start'][.='1']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/int[@name='offset_end'][.='10']"
+ , "//lst[@name='phrases']/arr[@name='details']/lst/double[@name='score'][number(.) > 0]"
+ );
+
+ // empty input, empty phrases (but no error)
+ assertQ(req("phrases","true", "q", "*:*", "phrases.q", "")
+ // basic search should have found all docs...
+ , "//result[@numFound='4']"
+ // and we should have gotten (empty) phrase info for our alternative q string...
+ , "//lst[@name='phrases']/str[@name='input'][.='']"
+ , "//lst[@name='phrases']/str[@name='summary'][.='']"
+ , "count(//lst[@name='phrases']/arr[@name='details']) = 1"
+ , "count(//lst[@name='phrases']/arr[@name='details']/lst) = 0"
+ );
+ }
+
+ public void testGreyboxShardSearchRequests() throws Exception {
+ final String input = "quick brown fox ran";
+
+ final String phrase_xpath = "//lst[@name='phrases']";
+ final String all_phrase_xpath = phrase_xpath + "/arr[@name='_all']";
+
+ // phrases requested, and correct request stage / shard purpose ...
+ assertQ(req("q", input,
+ "phrases","true",
+ ShardParams.IS_SHARD, "true",
+ ShardParams.SHARDS_PURPOSE, ""+PhrasesIdentificationComponent.SHARD_PURPOSE)
+
+ // this shard request should have caused stats to be returned about all phrases...
+ , "10=count("+ all_phrase_xpath +"/lst)"
+ // "quick" ...
+ , all_phrase_xpath + "/lst[1]/lst[@name='ttf']/long[@name='multigrams_body'][.='1']"
+ , all_phrase_xpath + "/lst[1]/lst[@name='ttf']/long[@name='multigrams_title'][.='0']"
+ // ...
+ // "brown fox"
+ , all_phrase_xpath + "/lst[6]/lst[@name='ttf']/long[@name='multigrams_body'][.='3']"
+ , all_phrase_xpath + "/lst[6]/lst[@name='ttf']/long[@name='multigrams_title'][.='1']"
+ , all_phrase_xpath + "/lst[6]/lst[@name='df']/long[@name='multigrams_body'][.='2']"
+ , all_phrase_xpath + "/lst[6]/lst[@name='df']/long[@name='multigrams_title'][.='1']"
+ , all_phrase_xpath + "/lst[6]/lst[@name='conj_dc']/long[@name='multigrams_body'][.='2']"
+ , all_phrase_xpath + "/lst[6]/lst[@name='conj_dc']/long[@name='multigrams_title'][.='1']"
+
+ // but no computed "scores"...
+ , "0=count("+phrase_xpath+"//*[@name='score'])"
+ );
+
+ // phrases requested, but incorrect request stage / shard purpose ...
+ assertQ(req("q", input,
+ "phrases","true",
+ ShardParams.IS_SHARD, "true",
+ ShardParams.SHARDS_PURPOSE, ""+ShardRequest.PURPOSE_GET_FIELDS)
+ , "0=count("+ phrase_xpath +"/lst)");
+
+ // phrases disabled, regardless of request stage / shard purpose ...
+ assertTrue("sanity check failed, stage was modified in code w/o updating test",
+ PhrasesIdentificationComponent.SHARD_PURPOSE != ShardRequest.PURPOSE_GET_FIELDS);
+ assertQ(req("q", input,
+ "phrases","false",
+ ShardParams.IS_SHARD, "true",
+ ShardParams.SHARDS_PURPOSE, ""+ShardRequest.PURPOSE_GET_FIELDS)
+ , "0=count("+ phrase_xpath +"/lst)");
+ assertQ(req("q", input,
+ "phrases","false",
+ ShardParams.IS_SHARD, "true",
+ ShardParams.SHARDS_PURPOSE, ""+PhrasesIdentificationComponent.SHARD_PURPOSE)
+ , "0=count("+ phrase_xpath +"/lst)");
+ }
+
+
+
+ // ////////////////////////////////////////////////////////////////
+
+
+
+
+ /**
+ * Trivial Helper method that collects & compares to an empty List so
+ * the assertion shows the unexpected stream elements
+ */
+ public <T> void assertEmptyStream(final String msg, final Stream<? extends T> stream) {
+ assertEquals(msg,
+ Collections.emptyList(),
+ stream.collect(Collectors.toList()));
+ }
+
+ /** helper, docs for future junit/hamcrest seems to have something similar */
+ public static Matcher lessThan(double expected) {
+ return new BaseMatcher() {
+ @Override public boolean matches(Object actual) {
+ return ((Double)actual).compareTo(expected) < 0;
+ }
+ @Override public void describeTo(Description d) {
+ d.appendText("should be less than " + expected);
+ }
+ };
+ }
+ /** helper, docs for future junit/hamcrest seems to have something similar */
+ public static Matcher greaterThan(double expected) {
+ return new BaseMatcher() {
+ @Override public boolean matches(Object actual) {
+ return 0 < ((Double)actual).compareTo(expected);
+ }
+ @Override public void describeTo(Description d) {
+ d.appendText("should be greater than " + expected);
+ }
+ };
+ }
+}
[31/47] lucene-solr:jira/solr-12709: SOLR-10697: update Ref Guide for
default value changes
Posted by ab...@apache.org.
SOLR-10697: update Ref Guide for default value changes
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8caa34c4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8caa34c4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8caa34c4
Branch: refs/heads/jira/solr-12709
Commit: 8caa34c4cfe1c23beddc6861646558138adb87ad
Parents: c684773
Author: Cassandra Targett <ct...@apache.org>
Authored: Thu Sep 6 13:25:10 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu Sep 6 13:25:53 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/distributed-requests.adoc | 4 ++--
solr/solr-ref-guide/src/format-of-solr-xml.adoc | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8caa34c4/solr/solr-ref-guide/src/distributed-requests.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/distributed-requests.adoc b/solr/solr-ref-guide/src/distributed-requests.adoc
index dce45d5..842a021 100644
--- a/solr/solr-ref-guide/src/distributed-requests.adoc
+++ b/solr/solr-ref-guide/src/distributed-requests.adoc
@@ -94,10 +94,10 @@ The amount of time in ms that a socket is allowed to wait. The default is `0`, w
The amount of time in ms that is accepted for binding / connecting a socket. The default is `0`, where the operating system's default will be used.
`maxConnectionsPerHost`::
-The maximum number of concurrent connections that is made to each individual shard in a distributed search. The default is `20`.
+The maximum number of concurrent connections that is made to each individual shard in a distributed search. The default is `100000`.
`maxConnections`::
-The total maximum number of concurrent connections in distributed searches. The default is `10000`
+The total maximum number of concurrent connections in distributed searches. The default is `100000`
`corePoolSize`::
The retained lowest limit on the number of threads used in coordinating distributed search. The default is `0`.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8caa34c4/solr/solr-ref-guide/src/format-of-solr-xml.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/format-of-solr-xml.adoc b/solr/solr-ref-guide/src/format-of-solr-xml.adoc
index 4b3d901..2a77c95 100644
--- a/solr/solr-ref-guide/src/format-of-solr-xml.adoc
+++ b/solr/solr-ref-guide/src/format-of-solr-xml.adoc
@@ -172,10 +172,10 @@ The connection timeout for intra-cluster query and administrative requests. Defa
The URL scheme to be used in distributed search.
`maxConnectionsPerHost`::
-Maximum connections allowed per host. Defaults to `20`.
+Maximum connections allowed per host. Defaults to `100000`.
`maxConnections`::
-Maximum total connections allowed. Defaults to `10000`.
+Maximum total connections allowed. Defaults to `100000`.
`corePoolSize`::
The initial core size of the threadpool servicing requests. Default is `0`.
[02/47] lucene-solr:jira/solr-12709: LUCENE-6228: Missed refactoring
of CollapsingQParserPlugin delegating collector
Posted by ab...@apache.org.
LUCENE-6228: Missed refactoring of CollapsingQParserPlugin delegating collector
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3b1a335f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3b1a335f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3b1a335f
Branch: refs/heads/jira/solr-12709
Commit: 3b1a335fb3dfc9d4f085740d30095ff07f48f25c
Parents: b1b0963
Author: Alan Woodward <ro...@apache.org>
Authored: Wed Sep 5 08:14:18 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Wed Sep 5 08:14:39 2018 +0100
----------------------------------------------------------------------
.../org/apache/solr/search/CollapsingQParserPlugin.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3b1a335f/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 5c0848a..54664fb 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -1147,7 +1147,8 @@ public class CollapsingQParserPlugin extends QParserPlugin {
@Override public ScoreMode scoreMode() { return needsScores ? ScoreMode.COMPLETE : super.scoreMode(); }
- public void setScorer(Scorer scorer) throws IOException {
+ @Override
+ public void setScorer(Scorable scorer) throws IOException {
this.collapseStrategy.setScorer(scorer);
}
@@ -2003,7 +2004,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
private static abstract class IntFieldValueStrategy {
protected int nullPolicy;
protected IntIntHashMap cmap;
- protected Scorer scorer;
+ protected Scorable scorer;
protected FloatArrayList nullScores;
protected float nullScore;
protected float[] scores;
@@ -2083,7 +2084,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return collapsedSet;
}
- public void setScorer(Scorer scorer) throws IOException {
+ public void setScorer(Scorable scorer) throws IOException {
this.scorer = scorer;
}
@@ -2495,7 +2496,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
@Override
- public void setScorer(Scorer s) throws IOException {
+ public void setScorer(Scorable s) throws IOException {
super.setScorer(s);
this.compareState.setScorer(s);
}
[32/47] lucene-solr:jira/solr-12709: SOLR-12612: Accept custom keys
in cluster properties
Posted by ab...@apache.org.
SOLR-12612: Accept custom keys in cluster properties
Cluster properties restriction of known keys only is relaxed, and now unknown properties starting with "ext."
will be allowed. This allows custom to plugins set their own cluster properties.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0af269fb
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0af269fb
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0af269fb
Branch: refs/heads/jira/solr-12709
Commit: 0af269fb4975e404e07d6e512cfbbac206920672
Parents: 8caa34c
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Thu Sep 6 14:07:30 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Thu Sep 6 14:07:30 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../solr/cloud/TestClusterProperties.java | 25 +++++++++++++++++++-
.../solr/common/cloud/ClusterProperties.java | 23 ++++++++++++++----
3 files changed, 46 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0af269fb/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a36be87..a975f06 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -213,6 +213,9 @@ New Features
* SOLR-11943: Add machine learning functions for location data (Joel Bernstein)
+* SOLR-12612: Cluster properties restriction of known keys only is relaxed, and now unknown properties starting with "ext."
+ will be allowed. This allows custom to plugins set their own cluster properties. (Jeffery Yuan via Tomás Fernández Löbbe)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0af269fb/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java b/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java
index c5575af..c082e37 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestClusterProperties.java
@@ -18,6 +18,7 @@
package org.apache.solr.cloud;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.ZkStateReader;
import org.junit.BeforeClass;
@@ -25,14 +26,21 @@ import org.junit.Test;
public class TestClusterProperties extends SolrCloudTestCase {
+ private ClusterProperties props;
+
@BeforeClass
public static void setupCluster() throws Exception {
configureCluster(1).configure();
}
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ props = new ClusterProperties(zkClient());
+ }
+
@Test
public void testClusterProperties() throws Exception {
- ClusterProperties props = new ClusterProperties(zkClient());
assertEquals("false", props.getClusterProperty(ZkStateReader.LEGACY_CLOUD, "false"));
CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, "true").process(cluster.getSolrClient());
@@ -41,5 +49,20 @@ public class TestClusterProperties extends SolrCloudTestCase {
CollectionAdminRequest.setClusterProperty(ZkStateReader.LEGACY_CLOUD, "false").process(cluster.getSolrClient());
assertEquals("false", props.getClusterProperty(ZkStateReader.LEGACY_CLOUD, "true"));
}
+
+ @Test
+ public void testSetPluginClusterProperty() throws Exception {
+ String propertyName = ClusterProperties.EXT_PROPRTTY_PREFIX + "pluginA.propertyA";
+ CollectionAdminRequest.setClusterProperty(propertyName, "valueA")
+ .process(cluster.getSolrClient());
+ assertEquals("valueA", props.getClusterProperty(propertyName, null));
+ }
+
+ @Test(expected = SolrException.class)
+ public void testSetInvalidPluginClusterProperty() throws Exception {
+ String propertyName = "pluginA.propertyA";
+ CollectionAdminRequest.setClusterProperty(propertyName, "valueA")
+ .process(cluster.getSolrClient());
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0af269fb/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
index 446923b..2452540 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterProperties.java
@@ -42,7 +42,8 @@ import org.slf4j.LoggerFactory;
public class ClusterProperties {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
+ public static final String EXT_PROPRTTY_PREFIX = "ext.";
+
private final SolrZkClient client;
/**
@@ -119,9 +120,7 @@ public class ClusterProperties {
@SuppressWarnings("unchecked")
public void setClusterProperty(String propertyName, String propertyValue) throws IOException {
- if (!ZkStateReader.KNOWN_CLUSTER_PROPS.contains(propertyName)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Not a known cluster property " + propertyName);
- }
+ validatePropertyName(propertyName);
for (; ; ) {
Stat s = new Stat();
@@ -155,4 +154,20 @@ public class ClusterProperties {
break;
}
}
+
+ /**
+ * The propertyName should be either: <br/>
+ * 1. <code>ZkStateReader.KNOWN_CLUSTER_PROPS</code> that is used by solr itself.<br/>
+ * 2. Custom property: it can be created by third-party extensions and should start with prefix <b>"ext."</b> and it's
+ * recommended to also add prefix of plugin name or company name or package name to avoid conflict.
+ *
+ * @param propertyName The property name to validate
+ */
+ private void validatePropertyName(String propertyName) {
+ if (!ZkStateReader.KNOWN_CLUSTER_PROPS.contains(propertyName)
+ && !propertyName.startsWith(EXT_PROPRTTY_PREFIX)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Not a known cluster property or starts with prefix "
+ + EXT_PROPRTTY_PREFIX + ", propertyName: " + propertyName);
+ }
+ }
}
[05/47] lucene-solr:jira/solr-12709: SOLR-12716: NodeLostTrigger
should support deleting replicas from lost nodes by setting
preferredOperation=deletenode
Posted by ab...@apache.org.
SOLR-12716: NodeLostTrigger should support deleting replicas from lost nodes by setting preferredOperation=deletenode
This commit adds support for preferredOperation configuration for NodeLostTrigger. The ComputePlanAction now creates DeleteNodeSuggester for each lost node serially when preferredOperation=deletenode. A new section for node lost trigger with exampls is added to the ref guide.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b6ee0ed5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b6ee0ed5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b6ee0ed5
Branch: refs/heads/jira/solr-12709
Commit: b6ee0ed5d5ff8890b941c6519896386b5091a82b
Parents: 6be01e2
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Sep 5 15:40:10 2018 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Sep 5 15:40:10 2018 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../cloud/api/collections/DeleteNodeCmd.java | 4 --
.../cloud/autoscaling/ComputePlanAction.java | 38 +++++++++---
.../solr/cloud/autoscaling/NodeLostTrigger.java | 31 +++++++++-
.../AutoAddReplicasPlanActionTest.java | 2 +-
.../autoscaling/ComputePlanActionTest.java | 63 +++++++++++++++++++-
.../autoscaling/ExecutePlanActionTest.java | 2 +-
.../sim/TestSimExecutePlanAction.java | 2 +-
.../src/solrcloud-autoscaling-triggers.adoc | 38 +++++++++++-
9 files changed, 164 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 56f5668..c0c6dbd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -201,6 +201,9 @@ New Features
* SOLR-11861: When creating a configSet via the API, the "baseConfigSet" parameter now defaults to "_default".
(Amrit Sarkar, David Smiley)
+* SOLR-12716: NodeLostTrigger should support deleting replicas from lost nodes by setting preferredOperation=deletenode.
+ (shalin)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java
index ab4dc0c..5f6e29c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteNodeCmd.java
@@ -25,7 +25,6 @@ import java.util.Locale;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@@ -55,9 +54,6 @@ public class DeleteNodeCmd implements OverseerCollectionMessageHandler.Cmd {
public void call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
ocmh.checkRequired(message, "node");
String node = message.getStr("node");
- if (!state.liveNodesContain(node)) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source Node: " + node + " is not live");
- }
List<ZkNodeProps> sourceReplicas = ReplaceNodeCmd.getReplicasOfNode(node, state);
List<String> singleReplicas = verifyReplicaAvailability(sourceReplicas, state);
if (!singleReplicas.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 3fd0d34..6bad63d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -48,6 +48,8 @@ import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.cloud.autoscaling.TriggerEvent.NODE_NAMES;
+
/**
* This class is responsible for using the configured policy and preferences
* with the hints provided by the trigger event to compute the required cluster operations.
@@ -206,8 +208,29 @@ public class ComputePlanAction extends TriggerActionBase {
suggester = getNodeAddedSuggester(cloudManager, session, event);
break;
case NODELOST:
- suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA)
- .hint(Suggester.Hint.SRC_NODE, event.getProperty(TriggerEvent.NODE_NAMES));
+ String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
+ CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);
+ switch (action) {
+ case MOVEREPLICA:
+ suggester = session.getSuggester(action)
+ .hint(Suggester.Hint.SRC_NODE, event.getProperty(NODE_NAMES));
+ break;
+ case DELETENODE:
+ int start = (Integer)event.getProperty(START, 0);
+ List<String> srcNodes = (List<String>) event.getProperty(NODE_NAMES);
+ if (srcNodes.isEmpty() || start >= srcNodes.size()) {
+ return NoneSuggester.get(session);
+ }
+ String sourceNode = srcNodes.get(start);
+ suggester = session.getSuggester(action)
+ .hint(Suggester.Hint.SRC_NODE, Collections.singletonList(sourceNode));
+ event.getProperties().put(START, ++start);
+ break;
+ case NONE:
+ return NoneSuggester.get(session);
+ default:
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unsupported preferredOperation: " + action.toLower() + " specified for node lost trigger");
+ }
break;
case SEARCHRATE:
case METRIC:
@@ -227,16 +250,15 @@ public class ComputePlanAction extends TriggerActionBase {
suggester = suggester.hint(e.getKey(), e.getValue());
}
suggester = suggester.forceOperation(true);
- start++;
- event.getProperties().put(START, start);
+ event.getProperties().put(START, ++start);
break;
case SCHEDULED:
- String preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
- CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);
+ preferredOp = (String) event.getProperty(AutoScalingParams.PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
+ action = CollectionParams.CollectionAction.get(preferredOp);
suggester = session.getSuggester(action);
break;
default:
- throw new UnsupportedOperationException("No support for events other than nodeAdded, nodeLost, searchRate, metric and indexSize. Received: " + event.getEventType());
+ throw new UnsupportedOperationException("No support for events other than nodeAdded, nodeLost, searchRate, metric, scheduled and indexSize. Received: " + event.getEventType());
}
return suggester;
}
@@ -246,7 +268,7 @@ public class ComputePlanAction extends TriggerActionBase {
CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);
Suggester suggester = session.getSuggester(action)
- .hint(Suggester.Hint.TARGET_NODE, event.getProperty(TriggerEvent.NODE_NAMES));
+ .hint(Suggester.Hint.TARGET_NODE, event.getProperty(NODE_NAMES));
switch (action) {
case ADDREPLICA:
// add all collection/shard pairs and let policy engine figure out which one
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index bc283e9..ddb4913 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -24,17 +24,23 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.core.SolrResourceLoader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.common.params.AutoScalingParams.PREFERRED_OP;
+
/**
* Trigger for the {@link TriggerEventType#NODELOST} event
*/
@@ -45,8 +51,11 @@ public class NodeLostTrigger extends TriggerBase {
private Map<String, Long> nodeNameVsTimeRemoved = new HashMap<>();
+ private String preferredOp;
+
public NodeLostTrigger(String name) {
super(TriggerEventType.NODELOST, name);
+ TriggerUtils.validProperties(validProperties, PREFERRED_OP);
}
@Override
@@ -73,6 +82,23 @@ public class NodeLostTrigger extends TriggerBase {
}
@Override
+ public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map<String, Object> properties) throws TriggerValidationException {
+ super.configure(loader, cloudManager, properties);
+ preferredOp = (String) properties.getOrDefault(PREFERRED_OP, CollectionParams.CollectionAction.MOVEREPLICA.toLower());
+ preferredOp = preferredOp.toLowerCase(Locale.ROOT);
+ // verify
+ CollectionParams.CollectionAction action = CollectionParams.CollectionAction.get(preferredOp);
+ switch (action) {
+ case MOVEREPLICA:
+ case DELETENODE:
+ case NONE:
+ break;
+ default:
+ throw new TriggerValidationException("Unsupported preferredOperation=" + preferredOp + " specified for node lost trigger");
+ }
+ }
+
+ @Override
public void restoreState(AutoScaling.Trigger old) {
assert old.isClosed();
if (old instanceof NodeLostTrigger) {
@@ -154,7 +180,7 @@ public class NodeLostTrigger extends TriggerBase {
if (!nodeNames.isEmpty()) {
if (processor != null) {
log.debug("NodeLostTrigger firing registered processor for lost nodes: {}", nodeNames);
- if (processor.process(new NodeLostEvent(getEventType(), getName(), times, nodeNames))) {
+ if (processor.process(new NodeLostEvent(getEventType(), getName(), times, nodeNames, preferredOp))) {
// remove from tracking set only if the fire was accepted
nodeNames.forEach(n -> {
nodeNameVsTimeRemoved.remove(n);
@@ -191,11 +217,12 @@ public class NodeLostTrigger extends TriggerBase {
public static class NodeLostEvent extends TriggerEvent {
- public NodeLostEvent(TriggerEventType eventType, String source, List<Long> times, List<String> nodeNames) {
+ public NodeLostEvent(TriggerEventType eventType, String source, List<Long> times, List<String> nodeNames, String preferredOp) {
// use the oldest time as the time of the event
super(eventType, source, times.get(0), null);
properties.put(NODE_NAMES, nodeNames);
properties.put(EVENT_TIMES, times);
+ properties.put(PREFERRED_OP, preferredOp);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
index c315713..1e25014 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
@@ -154,7 +154,7 @@ public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
@SuppressForbidden(reason = "Needs currentTimeMillis to create unique id")
private List<SolrRequest> getOperations(JettySolrRunner actionJetty, String lostNodeName) throws Exception {
try (AutoAddReplicasPlanAction action = new AutoAddReplicasPlanAction()) {
- TriggerEvent lostNode = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST, ".auto_add_replicas", Collections.singletonList(System.currentTimeMillis()), Collections.singletonList(lostNodeName));
+ TriggerEvent lostNode = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST, ".auto_add_replicas", Collections.singletonList(System.currentTimeMillis()), Collections.singletonList(lostNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
ActionContext context = new ActionContext(actionJetty.getCoreContainer().getZkController().getSolrCloudManager(), null, new HashMap<>());
action.process(lostNode, context);
List<SolrRequest> operations = (List) context.getProperty("operations");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index cd56f42..42c72ec 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -540,7 +540,6 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
NamedList<Object> response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
- // the default policy limits 1 replica per node, we need more right now
String setClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," +
@@ -612,4 +611,66 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
assertEquals(numShards, affectedShards.size());
assertEquals(numCollections * numShards, affectedCollShards.size());
}
+
+ @Test
+ public void testNodeLostTriggerWithDeleteNodePreferredOp() throws Exception {
+ String collectionNamePrefix = "testNodeLostTriggerWithDeleteNodePreferredOp";
+ int numCollections = 1 + random().nextInt(3), numShards = 1 + random().nextInt(3);
+
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '1s'," +
+ "'enabled' : true," +
+ "'" + AutoScalingParams.PREFERRED_OP + "':'deletenode'," +
+ "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
+ "{'name':'execute_plan','class':'solr.ExecutePlanAction'}" +
+ "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String setClusterPolicyCommand = "{" +
+ " 'set-cluster-policy': [" +
+ " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole':'overseer', 'replica':0}" +
+ " ]" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ // cache the node name because it won't be available once the node is shutdown
+ String newNodeName = newNode.getNodeName();
+
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0",
+ "conf", numShards, 2);
+ create.process(solrClient);
+
+ waitForState("Timed out waiting for replicas of new collection to be active",
+ collectionNamePrefix + "_0", (liveNodes, collectionState) ->
+ collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes)));
+
+ cluster.stopJettySolrRunner(newNode);
+ assertTrue(triggerFiredLatch.await(30, TimeUnit.SECONDS));
+ assertTrue(fired.get());
+ Map actionContext = actionContextPropsRef.get();
+ List operations = (List) actionContext.get("operations");
+ assertNotNull(operations);
+ assertEquals(1, operations.size());
+ for (Object operation : operations) {
+ assertTrue(operation instanceof CollectionAdminRequest.DeleteNode);
+ CollectionAdminRequest.DeleteNode deleteNode = (CollectionAdminRequest.DeleteNode) operation;
+ SolrParams deleteNodeParams = deleteNode.getParams();
+ assertEquals(newNodeName, deleteNodeParams.get("node"));
+ }
+
+ waitForState("Timed out waiting for all shards to have only 1 replica",
+ collectionNamePrefix + "_0", clusterShape(numShards, 1));
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
index b26da52..c876557 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
@@ -148,7 +148,7 @@ public class ExecutePlanActionTest extends SolrCloudTestCase {
List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST,
"mock_trigger_name", Collections.singletonList(TimeSource.CURRENT_TIME.getTimeNs()),
- Collections.singletonList(sourceNodeName));
+ Collections.singletonList(sourceNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
ActionContext actionContext = new ActionContext(survivor.getCoreContainer().getZkController().getSolrCloudManager(), null,
new HashMap<>(Collections.singletonMap("operations", operations)));
action.process(nodeLostEvent, actionContext);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java
index d0d08fd..e593695 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimExecutePlanAction.java
@@ -137,7 +137,7 @@ public class TestSimExecutePlanAction extends SimSolrCloudTestCase {
List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST,
"mock_trigger_name", Collections.singletonList(TimeSource.CURRENT_TIME.getTimeNs()),
- Collections.singletonList(sourceNodeName));
+ Collections.singletonList(sourceNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
ActionContext actionContext = new ActionContext(cluster, null,
new HashMap<>(Collections.singletonMap("operations", operations)));
action.process(nodeLostEvent, actionContext);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b6ee0ed5/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index e5906c0..83a3f87 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -86,11 +86,47 @@ Apart from the parameters described at <<#trigger-configuration, Trigger Configu
"name": "node_added_trigger",
"event": "nodeAdded",
"waitFor": "5s",
- "preferredOperation": "addreplica"
+ "preferredOperation": "ADDREPLICA"
}
}
----
+== Node Lost Trigger
+
+The `NodeLostTrigger` generates `nodeLost` events when a node leaves the cluster. It can be used to either move replicas
+that were hosted by the lost node to other nodes or to delete them from the cluster.
+
+Apart from the parameters described at <<#trigger-configuration, Trigger Configuration>>, this trigger supports the following configuration:
+
+`preferredOperation`:: (string, optional, defaults to `MOVEREPLICA`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from the lost nodes to the other nodes in the cluster. The only other supported value is `DELETENODE` which deletes all information about replicas that were hosted by the lost node.
+
+.Example: Node Lost Trigger to move replicas to new node
+[source,json]
+----
+{
+ "set-trigger": {
+ "name": "node_lost_trigger",
+ "event": "nodeLost",
+ "waitFor": "120s"
+ }
+}
+----
+
+.Example: Node Lost Trigger to delete replicas
+[source,json]
+----
+{
+ "set-trigger": {
+ "name": "node_lost_trigger",
+ "event": "nodeLost",
+ "waitFor": "120s",
+ "preferredOperation": "DELETENODE"
+ }
+}
+----
+
+TIP: It is recommended that the value of `waitFor` configuration for node lost trigger be larger than a minute so that large full garbage collection pauses do not cause this trigger to generate events and needlessly move or delete replicas in the cluster.
+
== Auto Add Replicas Trigger
When a collection has the parameter `autoAddReplicas` set to true then a trigger configuration named `.auto_add_replicas` is automatically created to watch for nodes going away. This trigger produces `nodeLost` events,
[11/47] lucene-solr:jira/solr-12709: SOLR-11863: Fix RefGuide typos
Posted by ab...@apache.org.
SOLR-11863: Fix RefGuide typos
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/719d922c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/719d922c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/719d922c
Branch: refs/heads/jira/solr-12709
Commit: 719d922cbc8acb7594f1f7862c917cbc70fda341
Parents: 0113ade
Author: Joel Bernstein <jb...@apache.org>
Authored: Wed Sep 5 11:30:27 2018 -0400
Committer: Joel Bernstein <jb...@apache.org>
Committed: Wed Sep 5 11:30:27 2018 -0400
----------------------------------------------------------------------
solr/solr-ref-guide/src/machine-learning.adoc | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/719d922c/solr/solr-ref-guide/src/machine-learning.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/machine-learning.adoc b/solr/solr-ref-guide/src/machine-learning.adoc
index ae781bb..ca0ae74 100644
--- a/solr/solr-ref-guide/src/machine-learning.adoc
+++ b/solr/solr-ref-guide/src/machine-learning.adoc
@@ -723,9 +723,9 @@ The `knnRegress` function prepares the training set for use with the `predict` f
Below is an example of the `knnRegress` function. In this example 10000 random samples
are taken each containing the variables *filesize_d*, *service_d* and *response_d*. The pairs of
-*filesize_d* and *service_d* will be use to predict the value of *response_d*.
+*filesize_d* and *service_d* will be used to predict the value of *response_d*.
-Notice that `kknRegress` simply returns a tuple describing the regression inputs.
+Notice that `knnRegress` returns a tuple describing the regression inputs.
[source,text]
----
@@ -765,7 +765,7 @@ This expression returns the following response:
=== Prediction and Residuals
-The output of knnRegress can be used with the `predict` function like other regression models.
+The output of `knnRegress` can be used with the `predict` function like other regression models.
In the example below the `predict` function is used to predict results for the original training
data. The sumSq of the residuals is then calculated.
@@ -808,6 +808,7 @@ will carry more weight in the distance calculation then the smaller features. Th
impact the accuracy of the prediction. The `knnRegress` function has a *scale* parameter which
can be set to *true* to automatically scale the features in the same range.
+The example below shows `knnRegress` with feature scaling turned on.
Notice that when feature scaling is turned on the sumSqErr in the output is much lower.
This shows how much more accurate the predictions are when feature scaling is turned on in
this particular example. This is because the *filesize_d* feature is significantly larger then
@@ -856,7 +857,7 @@ This provides a regression prediction that is robust to outliers.
=== Setting the Distance Measure
-The distance measure can be changed for the k-nearest neighbor search by adding distance measure
+The distance measure can be changed for the k-nearest neighbor search by adding a distance measure
function to the `knnRegress` parameters. Below is an example using manhattan distance.
[source,text]
[39/47] lucene-solr:jira/solr-12709: LUCENE-8483: Scorer cannot have
a null Weight
Posted by ab...@apache.org.
LUCENE-8483: Scorer cannot have a null Weight
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/66c671ea
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/66c671ea
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/66c671ea
Branch: refs/heads/jira/solr-12709
Commit: 66c671ea80f81596dad6d2e7745328f31f530cb8
Parents: 0dc66c2
Author: Alan Woodward <ro...@apache.org>
Authored: Tue Sep 4 13:24:32 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Fri Sep 7 11:41:47 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +
lucene/MIGRATE.txt | 5 ++
.../java/org/apache/lucene/index/Sorter.java | 27 -------
.../lucene/search/Boolean2ScorerSupplier.java | 4 +-
.../org/apache/lucene/search/BooleanScorer.java | 14 ++--
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../apache/lucene/search/CachingCollector.java | 16 +---
.../org/apache/lucene/search/FakeScorer.java | 62 ---------------
.../apache/lucene/search/MatchAllDocsQuery.java | 2 +-
.../org/apache/lucene/search/ScoreAndDoc.java | 35 +++++++++
.../java/org/apache/lucene/search/Scorer.java | 7 +-
.../org/apache/lucene/search/SortRescorer.java | 10 +--
.../lucene/search/MultiCollectorTest.java | 20 ++---
.../search/TestBoolean2ScorerSupplier.java | 80 ++++++++++++++------
.../org/apache/lucene/search/TestBooleanOr.java | 2 +-
.../apache/lucene/search/TestBooleanScorer.java | 2 +-
.../lucene/search/TestCachingCollector.java | 23 ++----
.../lucene/search/TestConjunctionDISI.java | 50 +++++++++---
.../lucene/search/TestConstantScoreQuery.java | 4 +-
.../search/TestMaxScoreSumPropagator.java | 34 ++++++++-
.../lucene/search/TestMultiCollector.java | 16 +---
.../apache/lucene/search/TestQueryRescorer.java | 2 +-
.../lucene/search/TestTopDocsCollector.java | 22 +-----
.../lucene/search/TestTopFieldCollector.java | 2 +-
.../apache/lucene/expressions/FakeScorer.java | 53 -------------
.../lucene/facet/DrillSidewaysScorer.java | 24 +-----
.../search/grouping/BlockGroupingCollector.java | 19 ++++-
.../lucene/search/grouping/FakeScorer.java | 52 -------------
.../apache/lucene/search/join/FakeScorer.java | 52 -------------
.../queries/function/FunctionRangeQuery.java | 4 +-
.../lucene/queries/function/FunctionValues.java | 15 ++--
.../lucene/queries/function/ValueSource.java | 26 ++-----
.../queries/function/ValueSourceScorer.java | 8 +-
.../docvalues/DocTermsIndexDocValues.java | 5 +-
.../function/docvalues/DoubleDocValues.java | 11 +--
.../function/docvalues/IntDocValues.java | 5 +-
.../function/docvalues/LongDocValues.java | 5 +-
.../function/valuesource/EnumFieldSource.java | 5 +-
.../solr/handler/component/QueryComponent.java | 31 +-------
.../solr/search/CollapsingQParserPlugin.java | 29 ++-----
.../src/java/org/apache/solr/search/Filter.java | 2 +-
.../apache/solr/search/FunctionRangeQuery.java | 12 ++-
.../search/function/ValueSourceRangeFilter.java | 9 ++-
.../apache/solr/search/TestRankQueryPlugin.java | 32 +-------
44 files changed, 309 insertions(+), 533 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f344b82..07163b3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -76,6 +76,8 @@ API Changes
* LUCENE-8475: Deprecated constants have been removed from RamUsageEstimator.
(Dimitrios Athanasiou)
+* LUCENE-8483: Scorers may no longer take null as a Weight (Alan Woodward)
+
Changes in Runtime Behavior
* LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index f09751e..90ed4fc 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -113,3 +113,8 @@ Scorer has a number of methods that should never be called from Collectors, for
those that advance the underlying iterators. To hide these, LeafCollector.setScorer()
now takes a Scorable, an abstract class that Scorers can extend, with methods
docId() and score() (LUCENE-6228)
+
+## Scorers must have non-null Weights ##
+
+If a custom Scorer implementation does not have an associated Weight, it can probably
+be replaced with a Scorable instead.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/index/Sorter.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/Sorter.java b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
index c47f9a1..a3718c2 100644
--- a/lucene/core/src/java/org/apache/lucene/index/Sorter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/Sorter.java
@@ -20,9 +20,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Comparator;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
-import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSelector;
@@ -445,30 +443,5 @@ final class Sorter {
public String toString() {
return getID();
}
-
- static final Scorer FAKESCORER = new Scorer(null) {
-
- float score;
- int doc = -1;
-
- @Override
- public int docID() {
- return doc;
- }
-
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public float score() throws IOException {
- return score;
- }
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
- };
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
index a13547f..e50eec4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Boolean2ScorerSupplier.java
@@ -30,13 +30,13 @@ import org.apache.lucene.search.BooleanClause.Occur;
final class Boolean2ScorerSupplier extends ScorerSupplier {
- private final BooleanWeight weight;
+ private final Weight weight;
private final Map<BooleanClause.Occur, Collection<ScorerSupplier>> subs;
private final ScoreMode scoreMode;
private final int minShouldMatch;
private long cost = -1;
- Boolean2ScorerSupplier(BooleanWeight weight,
+ Boolean2ScorerSupplier(Weight weight,
Map<Occur, Collection<ScorerSupplier>> subs,
ScoreMode scoreMode, int minShouldMatch) {
if (minShouldMatch < 0) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
index a66305b..6ec17ba 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
@@ -119,7 +119,7 @@ final class BooleanScorer extends BulkScorer {
final BulkScorerAndDoc[] leads;
final HeadPriorityQueue head;
final TailPriorityQueue tail;
- final FakeScorer fakeScorer = new FakeScorer();
+ final ScoreAndDoc scoreAndDoc = new ScoreAndDoc();
final int minShouldMatch;
final long cost;
@@ -178,12 +178,12 @@ final class BooleanScorer extends BulkScorer {
}
private void scoreDocument(LeafCollector collector, int base, int i) throws IOException {
- final FakeScorer fakeScorer = this.fakeScorer;
+ final ScoreAndDoc scoreAndDoc = this.scoreAndDoc;
final Bucket bucket = buckets[i];
if (bucket.freq >= minShouldMatch) {
- fakeScorer.score = (float) bucket.score;
+ scoreAndDoc.score = (float) bucket.score;
final int doc = base | i;
- fakeScorer.doc = doc;
+ scoreAndDoc.doc = doc;
collector.collect(doc);
}
bucket.freq = 0;
@@ -276,7 +276,7 @@ final class BooleanScorer extends BulkScorer {
bulkScorer.score(collector, acceptDocs, windowMin, end);
// reset the scorer that should be used for the general case
- collector.setScorer(fakeScorer);
+ collector.setScorer(scoreAndDoc);
}
private BulkScorerAndDoc scoreWindow(BulkScorerAndDoc top, LeafCollector collector,
@@ -307,8 +307,8 @@ final class BooleanScorer extends BulkScorer {
@Override
public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
- fakeScorer.doc = -1;
- collector.setScorer(fakeScorer);
+ scoreAndDoc.doc = -1;
+ collector.setScorer(scoreAndDoc);
BulkScorerAndDoc top = advance(min);
while (top.next < max) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
index d58b246..6298513 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanWeight.java
@@ -160,7 +160,7 @@ final class BooleanWeight extends Weight {
@Override
public int score(final LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
final LeafCollector noScoreCollector = new LeafCollector() {
- FakeScorer fake = new FakeScorer();
+ ScoreAndDoc fake = new ScoreAndDoc();
@Override
public void setScorer(Scorable scorer) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java b/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
index f2da14b..273ece4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
+++ b/lucene/core/src/java/org/apache/lucene/search/CachingCollector.java
@@ -47,7 +47,7 @@ public abstract class CachingCollector extends FilterCollector {
private static final int INITIAL_ARRAY_SIZE = 128;
- private static final class CachedScorer extends Scorer {
+ private static final class CachedScorable extends Scorable {
// NOTE: these members are package-private b/c that way accessing them from
// the outer class does not incur access check by the JVM. The same
@@ -56,22 +56,10 @@ public abstract class CachingCollector extends FilterCollector {
int doc;
float score;
- private CachedScorer() { super(null); }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
@Override
public final float score() { return score; }
@Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-
- @Override
public int docID() {
return doc;
}
@@ -188,7 +176,7 @@ public abstract class CachingCollector extends FilterCollector {
final int[] docs = this.docs.get(i);
final float[] scores = this.scores.get(i);
assert docs.length == scores.length;
- final CachedScorer scorer = new CachedScorer();
+ final CachedScorable scorer = new CachedScorable();
collector.setScorer(scorer);
for (int j = 0; j < docs.length; ++j) {
scorer.doc = docs[j];
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java b/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java
deleted file mode 100644
index 271833a..0000000
--- a/lucene/core/src/java/org/apache/lucene/search/FakeScorer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search;
-
-
-import java.io.IOException;
-import java.util.Collection;
-
-/** Used by {@link BulkScorer}s that need to pass a {@link
- * Scorer} to {@link LeafCollector#setScorer}. */
-final class FakeScorer extends Scorer {
- float score;
- int doc = -1;
-
- public FakeScorer() {
- super(null);
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public float score() {
- return score;
- }
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Weight getWeight() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Collection<ChildScorable> getChildren() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
index 89b2997..7094bdf 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
@@ -53,7 +53,7 @@ public final class MatchAllDocsQuery extends Query {
@Override
public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
max = Math.min(max, maxDoc);
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
scorer.score = score;
collector.setScorer(scorer);
for (int doc = min; doc < max; ++doc) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/ScoreAndDoc.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ScoreAndDoc.java b/lucene/core/src/java/org/apache/lucene/search/ScoreAndDoc.java
new file mode 100644
index 0000000..f9610f8
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/ScoreAndDoc.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+
+/** Used by {@link BulkScorer}s that need to pass a {@link
+ * Scorable} to {@link LeafCollector#setScorer}. */
+final class ScoreAndDoc extends Scorable {
+ float score;
+ int doc = -1;
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public float score() {
+ return score;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/Scorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/Scorer.java b/lucene/core/src/java/org/apache/lucene/search/Scorer.java
index aef2168..47baefb 100644
--- a/lucene/core/src/java/org/apache/lucene/search/Scorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/Scorer.java
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
import java.io.IOException;
+import java.util.Objects;
/**
* Expert: Common scoring functionality for different types of queries.
@@ -38,8 +39,8 @@ import java.io.IOException;
* with these scores.
*/
public abstract class Scorer extends Scorable {
- /** the Scorer's parent Weight. in some cases this may be null */
- // TODO can we clean this up?
+
+ /** the Scorer's parent Weight */
protected final Weight weight;
/**
@@ -47,7 +48,7 @@ public abstract class Scorer extends Scorable {
* @param weight The scorers <code>Weight</code>.
*/
protected Scorer(Weight weight) {
- this.weight = weight;
+ this.weight = Objects.requireNonNull(weight);
}
/** returns parent Weight
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java
index ff8f46a..982813d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/SortRescorer.java
@@ -59,7 +59,7 @@ public class SortRescorer extends Rescorer {
int docBase = 0;
LeafCollector leafCollector = null;
- FakeScorer fakeScorer = new FakeScorer();
+ ScoreAndDoc scoreAndDoc = new ScoreAndDoc();
while (hitUpto < hits.length) {
ScoreDoc hit = hits[hitUpto];
@@ -74,14 +74,14 @@ public class SortRescorer extends Rescorer {
if (readerContext != null) {
// We advanced to another segment:
leafCollector = collector.getLeafCollector(readerContext);
- leafCollector.setScorer(fakeScorer);
+ leafCollector.setScorer(scoreAndDoc);
docBase = readerContext.docBase;
}
- fakeScorer.score = hit.score;
- fakeScorer.doc = docID - docBase;
+ scoreAndDoc.score = hit.score;
+ scoreAndDoc.doc = docID - docBase;
- leafCollector.collect(fakeScorer.doc);
+ leafCollector.collect(scoreAndDoc.doc);
hitUpto++;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java b/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java
index de86924..c3b4f42 100644
--- a/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java
+++ b/lucene/core/src/test/org/apache/lucene/search/MultiCollectorTest.java
@@ -70,7 +70,7 @@ public class MultiCollectorTest extends LuceneTestCase {
final LeafCollector ac = c.getLeafCollector(null);
ac.collect(1);
c.getLeafCollector(null);
- c.getLeafCollector(null).setScorer(new FakeScorer());
+ c.getLeafCollector(null).setScorer(new ScoreAndDoc());
}
@Test
@@ -92,7 +92,7 @@ public class MultiCollectorTest extends LuceneTestCase {
LeafCollector ac = c.getLeafCollector(null);
ac.collect(1);
ac = c.getLeafCollector(null);
- ac.setScorer(new FakeScorer());
+ ac.setScorer(new ScoreAndDoc());
for (DummyCollector dc : dcs) {
assertTrue(dc.collectCalled);
@@ -142,23 +142,23 @@ public class MultiCollectorTest extends LuceneTestCase {
final LeafReaderContext ctx = reader.leaves().get(0);
expectThrows(AssertionError.class, () -> {
- collector(ScoreMode.COMPLETE_NO_SCORES, ScoreCachingWrappingScorer.class).getLeafCollector(ctx).setScorer(new FakeScorer());
+ collector(ScoreMode.COMPLETE_NO_SCORES, ScoreCachingWrappingScorer.class).getLeafCollector(ctx).setScorer(new ScoreAndDoc());
});
// no collector needs scores => no caching
- Collector c1 = collector(ScoreMode.COMPLETE_NO_SCORES, FakeScorer.class);
- Collector c2 = collector(ScoreMode.COMPLETE_NO_SCORES, FakeScorer.class);
- MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new FakeScorer());
+ Collector c1 = collector(ScoreMode.COMPLETE_NO_SCORES, ScoreAndDoc.class);
+ Collector c2 = collector(ScoreMode.COMPLETE_NO_SCORES, ScoreAndDoc.class);
+ MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());
// only one collector needs scores => no caching
- c1 = collector(ScoreMode.COMPLETE, FakeScorer.class);
- c2 = collector(ScoreMode.COMPLETE_NO_SCORES, FakeScorer.class);
- MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new FakeScorer());
+ c1 = collector(ScoreMode.COMPLETE, ScoreAndDoc.class);
+ c2 = collector(ScoreMode.COMPLETE_NO_SCORES, ScoreAndDoc.class);
+ MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());
// several collectors need scores => caching
c1 = collector(ScoreMode.COMPLETE, ScoreCachingWrappingScorer.class);
c2 = collector(ScoreMode.COMPLETE, ScoreCachingWrappingScorer.class);
- MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new FakeScorer());
+ MultiCollector.wrap(c1, c2).getLeafCollector(ctx).setScorer(new ScoreAndDoc());
reader.close();
dir.close();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestBoolean2ScorerSupplier.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2ScorerSupplier.java b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2ScorerSupplier.java
index 3118fa8..ea2fd4c 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBoolean2ScorerSupplier.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBoolean2ScorerSupplier.java
@@ -22,7 +22,10 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.EnumMap;
import java.util.Map;
+import java.util.Set;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -31,12 +34,39 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks;
public class TestBoolean2ScorerSupplier extends LuceneTestCase {
+ private static class FakeWeight extends Weight {
+
+ FakeWeight() {
+ super(new MatchNoDocsQuery());
+ }
+
+ @Override
+ public void extractTerms(Set<Term> terms) {
+
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ return null;
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return false;
+ }
+ }
+
private static class FakeScorer extends Scorer {
private final DocIdSetIterator it;
FakeScorer(long cost) {
- super(null);
+ super(new FakeWeight());
this.it = DocIdSetIterator.all(Math.toIntExact(cost));
}
@@ -124,17 +154,17 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
}
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(42));
- ScorerSupplier s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
+ ScorerSupplier s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
assertEquals(42, s.cost());
assertEquals(42, s.get(random().nextInt(100)).iterator().cost());
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12));
- s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
+ s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
assertEquals(42 + 12, s.cost());
assertEquals(42 + 12, s.get(random().nextInt(100)).iterator().cost());
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(20));
- s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
+ s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0);
assertEquals(42 + 12 + 20, s.cost());
assertEquals(42 + 12 + 20, s.get(random().nextInt(100)).iterator().cost());
}
@@ -147,26 +177,26 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(42));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12));
- ScorerSupplier s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 1);
+ ScorerSupplier s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 1);
assertEquals(42 + 12, s.cost());
assertEquals(42 + 12, s.get(random().nextInt(100)).iterator().cost());
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(20));
- s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 1);
+ s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 1);
assertEquals(42 + 12 + 20, s.cost());
assertEquals(42 + 12 + 20, s.get(random().nextInt(100)).iterator().cost());
- s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2);
+ s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2);
assertEquals(12 + 20, s.cost());
assertEquals(12 + 20, s.get(random().nextInt(100)).iterator().cost());
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(30));
- s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 1);
+ s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 1);
assertEquals(42 + 12 + 20 + 30, s.cost());
assertEquals(42 + 12 + 20 + 30, s.get(random().nextInt(100)).iterator().cost());
- s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2);
+ s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2);
assertEquals(12 + 20 + 30, s.cost());
assertEquals(12 + 20 + 30, s.get(random().nextInt(100)).iterator().cost());
- s = new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 3);
+ s = new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 3);
assertEquals(12 + 20, s.cost());
assertEquals(12 + 20, s.get(random().nextInt(100)).iterator().cost());
}
@@ -201,7 +231,7 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
continue;
}
int minShouldMatch = numShoulds == 0 ? 0 : TestUtil.nextInt(random(), 0, numShoulds - 1);
- Boolean2ScorerSupplier supplier = new Boolean2ScorerSupplier(null,
+ Boolean2ScorerSupplier supplier = new Boolean2ScorerSupplier(new FakeWeight(),
subs, scoreMode, minShouldMatch);
long cost1 = supplier.cost();
long cost2 = supplier.get(Long.MAX_VALUE).iterator().cost();
@@ -226,7 +256,7 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
// If the clauses are less costly than the lead cost, the min cost is the new lead cost
subs.get(RandomPicks.randomFrom(random(), Arrays.asList(Occur.FILTER, Occur.MUST))).add(new FakeScorerSupplier(42, 12));
subs.get(RandomPicks.randomFrom(random(), Arrays.asList(Occur.FILTER, Occur.MUST))).add(new FakeScorerSupplier(12, 12));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(Long.MAX_VALUE); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(Long.MAX_VALUE); // triggers assertions as a side-effect
subs = new EnumMap<>(Occur.class);
for (Occur occur : Occur.values()) {
@@ -236,7 +266,7 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
// If the lead cost is less that the clauses' cost, then we don't modify it
subs.get(RandomPicks.randomFrom(random(), Arrays.asList(Occur.FILTER, Occur.MUST))).add(new FakeScorerSupplier(42, 7));
subs.get(RandomPicks.randomFrom(random(), Arrays.asList(Occur.FILTER, Occur.MUST))).add(new FakeScorerSupplier(12, 7));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(7); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(7); // triggers assertions as a side-effect
}
public void testDisjunctionLeadCost() throws IOException {
@@ -246,12 +276,12 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
}
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(42, 54));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12, 54));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(100); // triggers assertions as a side-effect
subs.get(Occur.SHOULD).clear();
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(42, 20));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12, 20));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(20); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(20); // triggers assertions as a side-effect
}
public void testDisjunctionWithMinShouldMatchLeadCost() throws IOException {
@@ -265,7 +295,7 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(50, 42));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12, 42));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(30, 42));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2).get(100); // triggers assertions as a side-effect
subs = new EnumMap<>(Occur.class);
for (Occur occur : Occur.values()) {
@@ -276,7 +306,7 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(42, 20));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12, 20));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(30, 20));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2).get(20); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2).get(20); // triggers assertions as a side-effect
subs = new EnumMap<>(Occur.class);
for (Occur occur : Occur.values()) {
@@ -287,7 +317,7 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12, 62));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(30, 62));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(20, 62));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 2).get(100); // triggers assertions as a side-effect
subs = new EnumMap<>(Occur.class);
for (Occur occur : Occur.values()) {
@@ -298,7 +328,7 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(12, 32));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(30, 32));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(20, 32));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 3).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 3).get(100); // triggers assertions as a side-effect
}
public void testProhibitedLeadCost() throws IOException {
@@ -310,19 +340,19 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
// The MUST_NOT clause is called with the same lead cost as the MUST clause
subs.get(Occur.MUST).add(new FakeScorerSupplier(42, 42));
subs.get(Occur.MUST_NOT).add(new FakeScorerSupplier(30, 42));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(100); // triggers assertions as a side-effect
subs.get(Occur.MUST).clear();
subs.get(Occur.MUST_NOT).clear();
subs.get(Occur.MUST).add(new FakeScorerSupplier(42, 42));
subs.get(Occur.MUST_NOT).add(new FakeScorerSupplier(80, 42));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(100); // triggers assertions as a side-effect
subs.get(Occur.MUST).clear();
subs.get(Occur.MUST_NOT).clear();
subs.get(Occur.MUST).add(new FakeScorerSupplier(42, 20));
subs.get(Occur.MUST_NOT).add(new FakeScorerSupplier(30, 20));
- new Boolean2ScorerSupplier(null, subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(20); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, RandomPicks.randomFrom(random(), ScoreMode.values()), 0).get(20); // triggers assertions as a side-effect
}
public void testMixedLeadCost() throws IOException {
@@ -334,19 +364,19 @@ public class TestBoolean2ScorerSupplier extends LuceneTestCase {
// The SHOULD clause is always called with the same lead cost as the MUST clause
subs.get(Occur.MUST).add(new FakeScorerSupplier(42, 42));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(30, 42));
- new Boolean2ScorerSupplier(null, subs, ScoreMode.COMPLETE, 0).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, ScoreMode.COMPLETE, 0).get(100); // triggers assertions as a side-effect
subs.get(Occur.MUST).clear();
subs.get(Occur.SHOULD).clear();
subs.get(Occur.MUST).add(new FakeScorerSupplier(42, 42));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(80, 42));
- new Boolean2ScorerSupplier(null, subs, ScoreMode.COMPLETE, 0).get(100); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, ScoreMode.COMPLETE, 0).get(100); // triggers assertions as a side-effect
subs.get(Occur.MUST).clear();
subs.get(Occur.SHOULD).clear();
subs.get(Occur.MUST).add(new FakeScorerSupplier(42, 20));
subs.get(Occur.SHOULD).add(new FakeScorerSupplier(80, 20));
- new Boolean2ScorerSupplier(null, subs, ScoreMode.COMPLETE, 0).get(20); // triggers assertions as a side-effect
+ new Boolean2ScorerSupplier(new FakeWeight(), subs, ScoreMode.COMPLETE, 0).get(20); // triggers assertions as a side-effect
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java
index 7b12500..16477a7 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java
@@ -221,7 +221,7 @@ public class TestBooleanOr extends LuceneTestCase {
private static BulkScorer scorer(int... matches) {
return new BulkScorer() {
- final FakeScorer scorer = new FakeScorer();
+ final ScoreAndDoc scorer = new ScoreAndDoc();
int i = 0;
@Override
public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
index 86733a4..75fdd01 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java
@@ -105,7 +105,7 @@ public class TestBooleanScorer extends LuceneTestCase {
@Override
public int score(LeafCollector collector, Bits acceptDocs, int min, int max) throws IOException {
assert min == 0;
- collector.setScorer(new FakeScorer());
+ collector.setScorer(new ScoreAndDoc());
collector.collect(0);
return DocIdSetIterator.NO_MORE_DOCS;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java
index 12136b5..877c496 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java
@@ -25,25 +25,14 @@ public class TestCachingCollector extends LuceneTestCase {
private static final double ONE_BYTE = 1.0 / (1024 * 1024); // 1 byte out of MB
- private static class MockScorer extends Scorer {
+ private static class MockScorable extends Scorable {
- private MockScorer() {
- super((Weight) null);
- }
-
- @Override
- public float score() throws IOException { return 0; }
-
@Override
- public float getMaxScore(int upTo) throws IOException { return 0; }
+ public float score() { return 0; }
@Override
public int docID() { return 0; }
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
}
private static class NoOpCollector extends SimpleCollector {
@@ -62,7 +51,7 @@ public class TestCachingCollector extends LuceneTestCase {
for (boolean cacheScores : new boolean[] { false, true }) {
CachingCollector cc = CachingCollector.create(new NoOpCollector(), cacheScores, 1.0);
LeafCollector acc = cc.getLeafCollector(null);
- acc.setScorer(new MockScorer());
+ acc.setScorer(new MockScorable());
// collect 1000 docs
for (int i = 0; i < 1000; i++) {
@@ -90,7 +79,7 @@ public class TestCachingCollector extends LuceneTestCase {
public void testIllegalStateOnReplay() throws Exception {
CachingCollector cc = CachingCollector.create(new NoOpCollector(), true, 50 * ONE_BYTE);
LeafCollector acc = cc.getLeafCollector(null);
- acc.setScorer(new MockScorer());
+ acc.setScorer(new MockScorable());
// collect 130 docs, this should be enough for triggering cache abort.
for (int i = 0; i < 130; i++) {
@@ -115,7 +104,7 @@ public class TestCachingCollector extends LuceneTestCase {
CachingCollector cc = CachingCollector.create(new NoOpCollector(),
cacheScores, bytesPerDoc * ONE_BYTE * numDocs);
LeafCollector acc = cc.getLeafCollector(null);
- acc.setScorer(new MockScorer());
+ acc.setScorer(new MockScorable());
for (int i = 0; i < numDocs; i++) acc.collect(i);
assertTrue(cc.isCached());
@@ -130,7 +119,7 @@ public class TestCachingCollector extends LuceneTestCase {
// create w/ null wrapped collector, and test that the methods work
CachingCollector cc = CachingCollector.create(cacheScores, 50 * ONE_BYTE);
LeafCollector acc = cc.getLeafCollector(null);
- acc.setScorer(new MockScorer());
+ acc.setScorer(new MockScorable());
acc.collect(0);
assertTrue(cc.isCached());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java
index 083ac24..c10b78e 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctionDISI.java
@@ -22,7 +22,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
@@ -82,6 +85,33 @@ public class TestConjunctionDISI extends LuceneTestCase {
return scorer(TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator), twoPhaseIterator);
}
+ private static class FakeWeight extends Weight {
+
+ protected FakeWeight() {
+ super(new MatchNoDocsQuery());
+ }
+
+ @Override
+ public void extractTerms(Set<Term> terms) {
+
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ return null;
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return false;
+ }
+ }
+
/**
* Create a {@link Scorer} that wraps the given {@link DocIdSetIterator}. It
* also accepts a {@link TwoPhaseIterator} view, which is exposed in
@@ -91,7 +121,7 @@ public class TestConjunctionDISI extends LuceneTestCase {
* advantage of the {@link TwoPhaseIterator} view.
*/
private static Scorer scorer(DocIdSetIterator it, TwoPhaseIterator twoPhaseIterator) {
- return new Scorer(null) {
+ return new Scorer(new FakeWeight()) {
@Override
public DocIdSetIterator iterator() {
@@ -204,12 +234,12 @@ public class TestConjunctionDISI extends LuceneTestCase {
case 0:
// simple iterator
sets[i] = set;
- iterators[i] = new ConstantScoreScorer(null, 0f, anonymizeIterator(new BitDocIdSet(set).iterator()));
+ iterators[i] = new ConstantScoreScorer(new FakeWeight(), 0f, anonymizeIterator(new BitDocIdSet(set).iterator()));
break;
case 1:
// bitSet iterator
sets[i] = set;
- iterators[i] = new ConstantScoreScorer(null, 0f, new BitDocIdSet(set).iterator());
+ iterators[i] = new ConstantScoreScorer(new FakeWeight(), 0f, new BitDocIdSet(set).iterator());
break;
default:
// scorer with approximation
@@ -240,7 +270,7 @@ public class TestConjunctionDISI extends LuceneTestCase {
if (random().nextBoolean()) {
// simple iterator
sets[i] = set;
- iterators[i] = new ConstantScoreScorer(null, 0f, new BitDocIdSet(set).iterator());
+ iterators[i] = new ConstantScoreScorer(new FakeWeight(), 0f, new BitDocIdSet(set).iterator());
} else {
// scorer with approximation
final FixedBitSet confirmed = clearRandomBits(set);
@@ -276,12 +306,12 @@ public class TestConjunctionDISI extends LuceneTestCase {
case 0:
// simple iterator
sets[i] = set;
- newIterator = new ConstantScoreScorer(null, 0f, anonymizeIterator(new BitDocIdSet(set).iterator()));
+ newIterator = new ConstantScoreScorer(new FakeWeight(), 0f, anonymizeIterator(new BitDocIdSet(set).iterator()));
break;
case 1:
// bitSet iterator
sets[i] = set;
- newIterator = new ConstantScoreScorer(null, 0f, new BitDocIdSet(set).iterator());
+ newIterator = new ConstantScoreScorer(new FakeWeight(), 0f, new BitDocIdSet(set).iterator());
break;
default:
// scorer with approximation
@@ -322,7 +352,7 @@ public class TestConjunctionDISI extends LuceneTestCase {
if (random().nextBoolean()) {
// simple iterator
sets[i] = set;
- scorers.add(new ConstantScoreScorer(null, 0f, new BitDocIdSet(set).iterator()));
+ scorers.add(new ConstantScoreScorer(new FakeWeight(), 0f, new BitDocIdSet(set).iterator()));
} else {
// scorer with approximation
final FixedBitSet confirmed = clearRandomBits(set);
@@ -340,9 +370,9 @@ public class TestConjunctionDISI extends LuceneTestCase {
List<Scorer> subIterators = scorers.subList(subSeqStart, subSeqEnd);
Scorer subConjunction;
if (wrapWithScorer) {
- subConjunction = new ConjunctionScorer(null, subIterators, Collections.emptyList());
+ subConjunction = new ConjunctionScorer(new FakeWeight(), subIterators, Collections.emptyList());
} else {
- subConjunction = new ConstantScoreScorer(null, 0f, ConjunctionDISI.intersectScorers(subIterators));
+ subConjunction = new ConstantScoreScorer(new FakeWeight(), 0f, ConjunctionDISI.intersectScorers(subIterators));
}
scorers.set(subSeqStart, subConjunction);
int toRemove = subSeqEnd - subSeqStart - 1;
@@ -352,7 +382,7 @@ public class TestConjunctionDISI extends LuceneTestCase {
}
if (scorers.size() == 1) {
// ConjunctionDISI needs two iterators
- scorers.add(new ConstantScoreScorer(null, 0f, DocIdSetIterator.all(maxDoc)));
+ scorers.add(new ConstantScoreScorer(new FakeWeight(), 0f, DocIdSetIterator.all(maxDoc)));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
index 2035f9e..3e15070 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
@@ -54,7 +54,7 @@ public class TestConstantScoreQuery extends LuceneTestCase {
QueryUtils.checkUnequal(q1, new TermQuery(new Term("a", "b")));
}
- private void checkHits(IndexSearcher searcher, Query q, final float expectedScore, final Class<? extends Scorer> innerScorerClass) throws IOException {
+ private void checkHits(IndexSearcher searcher, Query q, final float expectedScore, final Class<? extends Scorable> innerScorerClass) throws IOException {
final int[] count = new int[1];
searcher.search(q, new SimpleCollector() {
private Scorable scorer;
@@ -131,7 +131,7 @@ public class TestConstantScoreQuery extends LuceneTestCase {
checkHits(searcher, csq2, csq2.getBoost(), TermScorer.class);
// for the combined BQ, the scorer should always be BooleanScorer's BucketScorer, because our scorer supports out-of order collection!
- final Class<FakeScorer> bucketScorerClass = FakeScorer.class;
+ final Class<ScoreAndDoc> bucketScorerClass = ScoreAndDoc.class;
checkHits(searcher, csqbq, csqbq.getBoost(), bucketScorerClass);
} finally {
IOUtils.close(reader, directory);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java
index d072233..96a34a5 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java
@@ -22,7 +22,10 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.Set;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -30,13 +33,40 @@ import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
public class TestMaxScoreSumPropagator extends LuceneTestCase {
+ private static class FakeWeight extends Weight {
+
+ FakeWeight() {
+ super(new MatchNoDocsQuery());
+ }
+
+ @Override
+ public void extractTerms(Set<Term> terms) {
+
+ }
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ return null;
+ }
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return false;
+ }
+ }
+
private static class FakeScorer extends Scorer {
final float maxScore;
float minCompetitiveScore;
- FakeScorer(float maxScore) {
- super(null);
+ FakeScorer(float maxScore) throws IOException {
+ super(new FakeWeight());
this.maxScore = maxScore;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
index f189821..dda314b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMultiCollector.java
@@ -134,7 +134,7 @@ public class TestMultiCollector extends LuceneTestCase {
collector1 = new TerminateAfterCollector(collector1, 1);
collector2 = new TerminateAfterCollector(collector2, 2);
- Scorer scorer = new FakeScorer();
+ Scorable scorer = new ScoreAndDoc();
List<Collector> collectors = Arrays.asList(collector1, collector2);
Collections.shuffle(collectors, random());
@@ -172,28 +172,18 @@ public class TestMultiCollector extends LuceneTestCase {
IndexReader reader = DirectoryReader.open(w);
w.close();
- Scorer scorer = new Scorer(null) {
+ Scorable scorer = new Scorable() {
@Override
public int docID() {
throw new UnsupportedOperationException();
}
@Override
- public float score() throws IOException {
+ public float score() {
return 0;
}
@Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
public void setMinCompetitiveScore(float minScore) {
throw new AssertionError();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
index 8bbfeb5..2e95653 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
@@ -429,7 +429,7 @@ public class TestQueryRescorer extends LuceneTestCase {
@Override
public Scorer scorer(final LeafReaderContext context) throws IOException {
- return new Scorer(null) {
+ return new Scorer(this) {
int docID = -1;
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
index caba25f..0506310 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTopDocsCollector.java
@@ -192,15 +192,11 @@ public class TestTopDocsCollector extends LuceneTestCase {
}
}
- private static class FakeScorer extends Scorer {
+ private static class ScoreAndDoc extends Scorable {
int doc = -1;
float score;
Float minCompetitiveScore = null;
- FakeScorer() {
- super(null);
- }
-
@Override
public void setMinCompetitiveScore(float minCompetitiveScore) {
this.minCompetitiveScore = minCompetitiveScore;
@@ -215,16 +211,6 @@ public class TestTopDocsCollector extends LuceneTestCase {
public float score() throws IOException {
return score;
}
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
}
public void testSetMinCompetitiveScore() throws Exception {
@@ -240,7 +226,7 @@ public class TestTopDocsCollector extends LuceneTestCase {
w.close();
TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1);
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
leafCollector.setScorer(scorer);
@@ -269,7 +255,7 @@ public class TestTopDocsCollector extends LuceneTestCase {
assertEquals(Math.nextUp(2f), scorer.minCompetitiveScore, 0f);
// Make sure the min score is set on scorers on new segments
- scorer = new FakeScorer();
+ scorer = new ScoreAndDoc();
leafCollector = collector.getLeafCollector(reader.leaves().get(1));
leafCollector.setScorer(scorer);
assertEquals(Math.nextUp(2f), scorer.minCompetitiveScore, 0f);
@@ -302,7 +288,7 @@ public class TestTopDocsCollector extends LuceneTestCase {
for (int totalHitsThreshold = 1; totalHitsThreshold < 20; ++ totalHitsThreshold) {
TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, totalHitsThreshold);
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
leafCollector.setScorer(scorer);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
index 36a1838..3a86449 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestTopFieldCollector.java
@@ -149,7 +149,7 @@ public class TestTopFieldCollector extends LuceneTestCase {
for (int totalHitsThreshold = 1; totalHitsThreshold < 20; ++ totalHitsThreshold) {
for (FieldDoc after : new FieldDoc[] { null, new FieldDoc(4, Float.NaN, new Object[] { 2L })}) {
TopFieldCollector collector = TopFieldCollector.create(sort, 2, after, totalHitsThreshold);
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
LeafCollector leafCollector1 = collector.getLeafCollector(reader.leaves().get(0));
leafCollector1.setScorer(scorer);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/expressions/src/java/org/apache/lucene/expressions/FakeScorer.java
----------------------------------------------------------------------
diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/FakeScorer.java b/lucene/expressions/src/java/org/apache/lucene/expressions/FakeScorer.java
deleted file mode 100644
index 026c789..0000000
--- a/lucene/expressions/src/java/org/apache/lucene/expressions/FakeScorer.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.expressions;
-
-
-import java.io.IOException;
-
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Scorer;
-
-class FakeScorer extends Scorer {
-
- float score;
- int doc = -1;
-
- FakeScorer() {
- super(null);
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public float score() throws IOException {
- return score;
- }
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
----------------------------------------------------------------------
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
index e1e5b55..ddace0d 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java
@@ -26,9 +26,9 @@ import org.apache.lucene.search.BulkScorer;
import org.apache.lucene.search.Collector;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.LeafCollector;
+import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
-import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
@@ -82,7 +82,7 @@ class DrillSidewaysScorer extends BulkScorer {
// System.out.println("\nscore: reader=" + context.reader());
//}
//System.out.println("score r=" + context.reader());
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
collector.setScorer(scorer);
if (drillDownCollector != null) {
drillDownLeafCollector = drillDownCollector.getLeafCollector(context);
@@ -580,21 +580,12 @@ class DrillSidewaysScorer extends BulkScorer {
sidewaysCollector.collect(collectDocID);
}
- private final class FakeScorer extends Scorer {
-
- public FakeScorer() {
- super(null);
- }
+ private final class ScoreAndDoc extends Scorable {
@Override
public int docID() {
return collectDocID;
}
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException("FakeScorer doesn't support nextDoc()");
- }
@Override
public float score() {
@@ -602,19 +593,10 @@ class DrillSidewaysScorer extends BulkScorer {
}
@Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-
- @Override
public Collection<ChildScorable> getChildren() {
return Collections.singletonList(new ChildScorable(baseScorer, "MUST"));
}
- @Override
- public Weight getWeight() {
- throw new UnsupportedOperationException();
- }
}
static class DocsAndCost {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
index d5b3555..23601ca 100644
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
+++ b/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
@@ -286,7 +286,7 @@ public class BlockGroupingCollector extends SimpleCollector {
}
int totalGroupedHitCount = 0;
- final FakeScorer fakeScorer = new FakeScorer();
+ final ScoreAndDoc fakeScorer = new ScoreAndDoc();
float maxScore = Float.MIN_VALUE;
@@ -494,4 +494,21 @@ public class BlockGroupingCollector extends SimpleCollector {
public ScoreMode scoreMode() {
return needsScores ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES;
}
+
+ private static class ScoreAndDoc extends Scorable {
+
+ float score;
+ int doc = -1;
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public float score() {
+ return score;
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/grouping/src/java/org/apache/lucene/search/grouping/FakeScorer.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/java/org/apache/lucene/search/grouping/FakeScorer.java b/lucene/grouping/src/java/org/apache/lucene/search/grouping/FakeScorer.java
deleted file mode 100644
index b46662d..0000000
--- a/lucene/grouping/src/java/org/apache/lucene/search/grouping/FakeScorer.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.grouping;
-
-import java.io.IOException;
-
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Scorer;
-
-class FakeScorer extends Scorer {
-
- float score;
- int doc = -1;
-
- FakeScorer() {
- super(null);
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public float score() throws IOException {
- return score;
- }
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/join/src/java/org/apache/lucene/search/join/FakeScorer.java
----------------------------------------------------------------------
diff --git a/lucene/join/src/java/org/apache/lucene/search/join/FakeScorer.java b/lucene/join/src/java/org/apache/lucene/search/join/FakeScorer.java
deleted file mode 100644
index 125ce88..0000000
--- a/lucene/join/src/java/org/apache/lucene/search/join/FakeScorer.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.join;
-
-import java.io.IOException;
-
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Scorer;
-
-class FakeScorer extends Scorer {
-
- float score;
- int doc = -1;
-
- FakeScorer() {
- super(null);
- }
-
- @Override
- public int docID() {
- return doc;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public float score() throws IOException {
- return score;
- }
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java
index 2d55bae..315f650 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionRangeQuery.java
@@ -34,7 +34,7 @@ import org.apache.lucene.search.Weight;
* range. The score is the float value. This can be a slow query if run by itself since it must visit all docs;
* ideally it's combined with other queries.
* It's mostly a wrapper around
- * {@link FunctionValues#getRangeScorer(LeafReaderContext, String, String, boolean, boolean)}.
+ * {@link FunctionValues#getRangeScorer(Weight, LeafReaderContext, String, String, boolean, boolean)}.
*
* A similar class is {@code org.apache.lucene.search.DocValuesRangeQuery} in the sandbox module. That one is
* constant scoring.
@@ -152,7 +152,7 @@ public class FunctionRangeQuery extends Query {
public ValueSourceScorer scorer(LeafReaderContext context) throws IOException {
FunctionValues functionValues = valueSource.getValues(vsContext, context);
// getRangeScorer takes String args and parses them. Weird.
- return functionValues.getRangeScorer(context, lowerVal, upperVal, includeLower, includeUpper);
+ return functionValues.getRangeScorer(this, context, lowerVal, upperVal, includeLower, includeUpper);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
index 9e73b4b..76dda25 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/FunctionValues.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueFloat;
@@ -144,8 +145,8 @@ public abstract class FunctionValues {
* Yields a {@link Scorer} that matches all documents,
* and that which produces scores equal to {@link #floatVal(int)}.
*/
- public ValueSourceScorer getScorer(LeafReaderContext readerContext) {
- return new ValueSourceScorer(readerContext, this) {
+ public ValueSourceScorer getScorer(Weight weight, LeafReaderContext readerContext) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) {
return true;
@@ -161,7 +162,7 @@ public abstract class FunctionValues {
// because it needs different behavior depending on the type of fields. There is also
// a setup cost - parsing and normalizing params, and doing a binary search on the StringIndex.
// TODO: change "reader" to LeafReaderContext
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) throws IOException {
+ public ValueSourceScorer getRangeScorer(Weight weight, LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) throws IOException {
float lower;
float upper;
@@ -180,7 +181,7 @@ public abstract class FunctionValues {
final float u = upper;
if (includeLower && includeUpper) {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
@@ -190,7 +191,7 @@ public abstract class FunctionValues {
};
}
else if (includeLower && !includeUpper) {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
@@ -200,7 +201,7 @@ public abstract class FunctionValues {
};
}
else if (!includeLower && includeUpper) {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
@@ -210,7 +211,7 @@ public abstract class FunctionValues {
};
}
else {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
index 5290dcc..209fb15 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java
@@ -23,7 +23,6 @@ import java.util.Map;
import java.util.Objects;
import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Explanation;
@@ -32,6 +31,7 @@ import org.apache.lucene.search.FieldComparatorSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LongValues;
import org.apache.lucene.search.LongValuesSource;
+import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.SimpleFieldComparator;
import org.apache.lucene.search.SortField;
@@ -87,34 +87,20 @@ public abstract class ValueSource {
return context;
}
- private static class FakeScorer extends Scorer {
+ private static class ScoreAndDoc extends Scorable {
int current = -1;
float score = 0;
- FakeScorer() {
- super(null);
- }
-
@Override
public int docID() {
return current;
}
@Override
- public float score() throws IOException {
+ public float score() {
return score;
}
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
}
/**
@@ -135,7 +121,7 @@ public abstract class ValueSource {
@Override
public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
Map context = new IdentityHashMap<>();
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
context.put("scorer", scorer);
final FunctionValues fv = in.getValues(context, ctx);
return new LongValues() {
@@ -211,7 +197,7 @@ public abstract class ValueSource {
@Override
public DoubleValues getValues(LeafReaderContext ctx, DoubleValues scores) throws IOException {
Map context = new HashMap<>();
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
context.put("scorer", scorer);
context.put("searcher", searcher);
FunctionValues fv = in.getValues(context, ctx);
@@ -248,7 +234,7 @@ public abstract class ValueSource {
@Override
public Explanation explain(LeafReaderContext ctx, int docId, Explanation scoreExplanation) throws IOException {
Map context = new HashMap<>();
- FakeScorer scorer = new FakeScorer();
+ ScoreAndDoc scorer = new ScoreAndDoc();
scorer.score = scoreExplanation.getValue().floatValue();
context.put("scorer", scorer);
context.put("searcher", searcher);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
index 509f454..b2270be 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java
@@ -22,13 +22,13 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
/**
* {@link Scorer} which returns the result of {@link FunctionValues#floatVal(int)} as
* the score for a document, and which filters out documents that don't match {@link #matches(int)}.
* This Scorer has a {@link TwoPhaseIterator}. This is similar to {@link FunctionQuery},
- * but this one has no {@link org.apache.lucene.search.Weight} normalization factors/multipliers
- * and that one doesn't filter either.
+ * with an added filter.
* <p>
* Note: If the scores are needed, then the underlying value will probably be
* fetched/computed twice -- once to filter and next to return the score. If that's non-trivial then
@@ -43,8 +43,8 @@ public abstract class ValueSourceScorer extends Scorer {
private final TwoPhaseIterator twoPhaseIterator;
private final DocIdSetIterator disi;
- protected ValueSourceScorer(LeafReaderContext readerContext, FunctionValues values) {
- super(null);//no weight
+ protected ValueSourceScorer(Weight weight, LeafReaderContext readerContext, FunctionValues values) {
+ super(weight);
this.values = values;
final DocIdSetIterator approximation = DocIdSetIterator.all(readerContext.reader().maxDoc()); // no approximation!
this.twoPhaseIterator = new TwoPhaseIterator(approximation) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
index e855e27..1cc79a4 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexDocValues.java
@@ -24,6 +24,7 @@ import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.CharsRefBuilder;
@@ -115,7 +116,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
public abstract Object objectVal(int doc) throws IOException; // force subclasses to override
@Override
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) throws IOException {
+ public ValueSourceScorer getRangeScorer(Weight weight, LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) throws IOException {
// TODO: are lowerVal and upperVal in indexed form or not?
lowerVal = lowerVal == null ? null : toTerm(lowerVal);
upperVal = upperVal == null ? null : toTerm(upperVal);
@@ -143,7 +144,7 @@ public abstract class DocTermsIndexDocValues extends FunctionValues {
final int ll = lower;
final int uu = upper;
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
final SortedDocValues values = readerContext.reader().getSortedDocValues(field);
private int lastDocID;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
index 646db9c..321c05c 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleDocValues.java
@@ -22,6 +22,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueDouble;
@@ -85,7 +86,7 @@ public abstract class DoubleDocValues extends FunctionValues {
}
@Override
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
+ public ValueSourceScorer getRangeScorer(Weight weight, LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
double lower,upper;
if (lowerVal==null) {
@@ -105,7 +106,7 @@ public abstract class DoubleDocValues extends FunctionValues {
if (includeLower && includeUpper) {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
@@ -115,7 +116,7 @@ public abstract class DoubleDocValues extends FunctionValues {
};
}
else if (includeLower && !includeUpper) {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
@@ -125,7 +126,7 @@ public abstract class DoubleDocValues extends FunctionValues {
};
}
else if (!includeLower && includeUpper) {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
@@ -135,7 +136,7 @@ public abstract class DoubleDocValues extends FunctionValues {
};
}
else {
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
index 2c40fb7..1b98666 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntDocValues.java
@@ -22,6 +22,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
@@ -80,7 +81,7 @@ public abstract class IntDocValues extends FunctionValues {
}
@Override
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
+ public ValueSourceScorer getRangeScorer(Weight weight, LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
int lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
@@ -102,7 +103,7 @@ public abstract class IntDocValues extends FunctionValues {
final int ll = lower;
final int uu = upper;
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
index d36afa0..3637bb2 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongDocValues.java
@@ -22,6 +22,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueLong;
@@ -89,7 +90,7 @@ public abstract class LongDocValues extends FunctionValues {
}
@Override
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
+ public ValueSourceScorer getRangeScorer(Weight weight, LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
long lower,upper;
// instead of using separate comparison functions, adjust the endpoints.
@@ -111,7 +112,7 @@ public abstract class LongDocValues extends FunctionValues {
final long ll = lower;
final long uu = upper;
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
[38/47] lucene-solr:jira/solr-12709: LUCENE-8483: Scorer cannot have
a null Weight
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
index 2667660..21dbfab 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/EnumFieldSource.java
@@ -25,6 +25,7 @@ import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.queries.function.docvalues.IntDocValues;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.mutable.MutableValue;
import org.apache.lucene.util.mutable.MutableValueInt;
@@ -135,7 +136,7 @@ public class EnumFieldSource extends FieldCacheSource {
}
@Override
- public ValueSourceScorer getRangeScorer(LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
+ public ValueSourceScorer getRangeScorer(Weight weight, LeafReaderContext readerContext, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) {
Integer lower = stringValueToIntValue(lowerVal);
Integer upper = stringValueToIntValue(upperVal);
@@ -156,7 +157,7 @@ public class EnumFieldSource extends FieldCacheSource {
final int ll = lower;
final int uu = upper;
- return new ValueSourceScorer(readerContext, this) {
+ return new ValueSourceScorer(weight, readerContext, this) {
@Override
public boolean matches(int doc) throws IOException {
if (!exists(doc)) return false;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index 023aed6..e937370 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -35,15 +35,13 @@ import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.search.Weight;
import org.apache.lucene.search.grouping.GroupDocs;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.search.grouping.TopGroups;
@@ -469,7 +467,7 @@ public class QueryComponent extends SearchComponent
}
doc -= currentLeaf.docBase; // adjust for what segment this is in
- leafComparator.setScorer(new FakeScorer(doc, score));
+ leafComparator.setScorer(new ScoreAndDoc(doc, score));
leafComparator.copy(0, doc);
Object val = comparator.value(0);
if (null != ft) val = ft.marshalSortValue(val);
@@ -1461,12 +1459,11 @@ public class QueryComponent extends SearchComponent
*
* TODO: when SOLR-5595 is fixed, this wont be needed, as we dont need to recompute sort values here from the comparator
*/
- protected static class FakeScorer extends Scorer {
+ protected static class ScoreAndDoc extends Scorable {
final int docid;
final float score;
- FakeScorer(int docid, float score) {
- super(null);
+ ScoreAndDoc(int docid, float score) {
this.docid = docid;
this.score = score;
}
@@ -1480,25 +1477,5 @@ public class QueryComponent extends SearchComponent
public float score() throws IOException {
return score;
}
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Weight getWeight() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Collection<ChildScorable> getChildren() {
- throw new UnsupportedOperationException();
- }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 54664fb..08867db 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -56,7 +56,6 @@ import org.apache.lucene.search.LeafFieldComparator;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.ArrayUtil;
@@ -434,36 +433,18 @@ public class CollapsingQParserPlugin extends QParserPlugin {
}
- private static class DummyScorer extends Scorer {
+ private static class ScoreAndDoc extends Scorable {
public float score;
public int docId;
- public DummyScorer() {
- super(null);
- }
-
public float score() {
return score;
}
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return Float.POSITIVE_INFINITY;
- }
-
- public int freq() {
- return 0;
- }
-
public int docID() {
return docId;
}
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
}
@@ -647,7 +628,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
- DummyScorer dummy = new DummyScorer();
+ ScoreAndDoc dummy = new ScoreAndDoc();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = new BitSetIterator(collapsedSet, 0L); // cost is not useful here
int docId = -1;
@@ -850,7 +831,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.field);
int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
- DummyScorer dummy = new DummyScorer();
+ ScoreAndDoc dummy = new ScoreAndDoc();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = new BitSetIterator(collapsedSet, 0L); // cost is not useful here
int globalDoc = -1;
@@ -1022,7 +1003,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
- DummyScorer dummy = new DummyScorer();
+ ScoreAndDoc dummy = new ScoreAndDoc();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here
int globalDoc = -1;
@@ -1181,7 +1162,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
this.collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField);
int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
- DummyScorer dummy = new DummyScorer();
+ ScoreAndDoc dummy = new ScoreAndDoc();
leafDelegate.setScorer(dummy);
DocIdSetIterator it = new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here
int globalDoc = -1;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/solr/core/src/java/org/apache/solr/search/Filter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/Filter.java b/solr/core/src/java/org/apache/solr/search/Filter.java
index 3af83e2..847ef46 100644
--- a/solr/core/src/java/org/apache/solr/search/Filter.java
+++ b/solr/core/src/java/org/apache/solr/search/Filter.java
@@ -89,7 +89,7 @@ public abstract class Filter extends Query {
//
@Override
- public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) {
return new Weight(this) {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java b/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java
index c5c6205..fdcdfc3 100644
--- a/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java
+++ b/solr/core/src/java/org/apache/solr/search/FunctionRangeQuery.java
@@ -24,10 +24,13 @@ import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.ValueSourceScorer;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Weight;
import org.apache.solr.search.function.ValueSourceRangeFilter;
// This class works as either a normal constant score query, or as a PostFilter using a collector
public class FunctionRangeQuery extends SolrConstantScoreQuery implements PostFilter {
+
final ValueSourceRangeFilter rangeFilt;
public FunctionRangeQuery(ValueSourceRangeFilter filter) {
@@ -39,16 +42,19 @@ public class FunctionRangeQuery extends SolrConstantScoreQuery implements PostFi
@Override
public DelegatingCollector getFilterCollector(IndexSearcher searcher) {
Map fcontext = ValueSource.newContext(searcher);
- return new FunctionRangeCollector(fcontext);
+ Weight weight = rangeFilt.createWeight(searcher, ScoreMode.COMPLETE, 1);
+ return new FunctionRangeCollector(fcontext, weight);
}
class FunctionRangeCollector extends DelegatingCollector {
final Map fcontext;
+ final Weight weight;
ValueSourceScorer scorer;
int maxdoc;
- public FunctionRangeCollector(Map fcontext) {
+ public FunctionRangeCollector(Map fcontext, Weight weight) {
this.fcontext = fcontext;
+ this.weight = weight;
}
@Override
@@ -64,7 +70,7 @@ public class FunctionRangeQuery extends SolrConstantScoreQuery implements PostFi
super.doSetNextReader(context);
maxdoc = context.reader().maxDoc();
FunctionValues dv = rangeFilt.getValueSource().getValues(fcontext, context);
- scorer = dv.getRangeScorer(context, rangeFilt.getLowerVal(), rangeFilt.getUpperVal(), rangeFilt.isIncludeLower(), rangeFilt.isIncludeUpper());
+ scorer = dv.getRangeScorer(weight, context, rangeFilt.getLowerVal(), rangeFilt.getUpperVal(), rangeFilt.isIncludeLower(), rangeFilt.isIncludeUpper());
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java
index 1fce97e..9f919ed 100644
--- a/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java
+++ b/solr/core/src/java/org/apache/solr/search/function/ValueSourceRangeFilter.java
@@ -21,7 +21,9 @@ import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Weight;
import org.apache.lucene.util.Bits;
import org.apache.solr.search.BitsFilteredDocIdSet;
import org.apache.solr.search.SolrFilter;
@@ -77,10 +79,13 @@ public class ValueSourceRangeFilter extends SolrFilter {
@Override
public DocIdSet getDocIdSet(final Map context, final LeafReaderContext readerContext, Bits acceptDocs) throws IOException {
- return BitsFilteredDocIdSet.wrap(new DocIdSet() {
+ // NB the IndexSearcher parameter here can be null because Filter Weights don't
+ // actually use it.
+ Weight weight = createWeight(null, ScoreMode.COMPLETE, 1);
+ return BitsFilteredDocIdSet.wrap(new DocIdSet() {
@Override
public DocIdSetIterator iterator() throws IOException {
- Scorer scorer = valueSource.getValues(context, readerContext).getRangeScorer(readerContext, lowerVal, upperVal, includeLower, includeUpper);
+ Scorer scorer = valueSource.getValues(context, readerContext).getRangeScorer(weight, readerContext, lowerVal, upperVal, includeLower, includeUpper);
return scorer == null ? null : scorer.iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66c671ea/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
index 587a889..a678110 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRankQueryPlugin.java
@@ -20,7 +20,6 @@ import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -32,7 +31,6 @@ import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.ReaderUtil;
-import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LeafCollector;
@@ -41,7 +39,6 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorable;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
@@ -424,7 +421,7 @@ public class TestRankQueryPlugin extends QParserPlugin {
}
doc -= currentLeaf.docBase; // adjust for what segment this is in
- leafComparator.setScorer(new FakeScorer(doc, score));
+ leafComparator.setScorer(new ScoreAndDoc(doc, score));
leafComparator.copy(0, doc);
Object val = comparator.value(0);
if (null != ft) val = ft.marshalSortValue(val);
@@ -438,13 +435,12 @@ public class TestRankQueryPlugin extends QParserPlugin {
}
}
- private static class FakeScorer extends Scorer {
+ private static class ScoreAndDoc extends Scorable {
final int docid;
final float score;
- FakeScorer(int docid, float score) {
- super(null);
+ ScoreAndDoc(int docid, float score) {
this.docid = docid;
this.score = score;
}
@@ -455,29 +451,9 @@ public class TestRankQueryPlugin extends QParserPlugin {
}
@Override
- public float score() throws IOException {
+ public float score() {
return score;
}
-
- @Override
- public float getMaxScore(int upTo) throws IOException {
- return score;
- }
-
- @Override
- public DocIdSetIterator iterator() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Weight getWeight() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Collection<ChildScorable> getChildren() {
- throw new UnsupportedOperationException();
- }
}
public void merge(ResponseBuilder rb, ShardRequest sreq) {
[24/47] lucene-solr:jira/solr-12709: SOLR-12684: put expression names
and params in monospace
Posted by ab...@apache.org.
SOLR-12684: put expression names and params in monospace
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9c364b2d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9c364b2d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9c364b2d
Branch: refs/heads/jira/solr-12709
Commit: 9c364b2d8640e84a2fe3b7a8d8adfc20d3d53e38
Parents: 1a00655
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Sep 5 20:35:37 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu Sep 6 08:16:27 2018 -0500
----------------------------------------------------------------------
.../src/stream-decorator-reference.adoc | 23 ++++++++++----------
1 file changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9c364b2d/solr/solr-ref-guide/src/stream-decorator-reference.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/stream-decorator-reference.adoc b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
index b397192..08f1e7a 100644
--- a/solr/solr-ref-guide/src/stream-decorator-reference.adoc
+++ b/solr/solr-ref-guide/src/stream-decorator-reference.adoc
@@ -970,18 +970,20 @@ outerHashJoin(
The `parallel` function wraps a streaming expression and sends it to N worker nodes to be processed in parallel.
-The parallel function requires that the `partitionKeys` parameter be provided to the underlying searches. The `partitionKeys` parameter will partition the search results (tuples) across the worker nodes. Tuples with the same values in the partitionKeys field will be shuffled to the same worker nodes.
+The `parallel` function requires that the `partitionKeys` parameter be provided to the underlying searches. The `partitionKeys` parameter will partition the search results (tuples) across the worker nodes. Tuples with the same values in for `partitionKeys` will be shuffled to the same worker nodes.
-The parallel function maintains the sort order of the tuples returned by the worker nodes, so the sort criteria of the parallel function must incorporate the sort order of the tuples returned by the workers.
+The `parallel` function maintains the sort order of the tuples returned by the worker nodes, so the sort criteria must incorporate the sort order of the tuples returned by the workers.
-For example if you sort on year, month and day you could partition on year only as long as there was enough different years to spread the tuples around the worker nodes.
-Solr allows sorting on more than 4 fields, but you cannot specify more than 4 partitionKeys for speed tradeoffs. Also it's an overkill to specify many partitionKeys when we one or two keys could be enough to spread the tuples.
-Parallel Stream was designed when the underlying search stream will emit a lot of tuples from the collection. If the search stream only emits a small subset of the data from the collection using parallel could potentially be slower.
+For example if you sort on year, month and day you could partition on year only as long as there are enough different years to spread the tuples around the worker nodes.
+
+Solr allows sorting on more than 4 fields, but you cannot specify more than 4 partitionKeys for speed considerations. Also it's overkill to specify many `partitionKeys` when we one or two keys could be enough to spread the tuples.
+
+Parallel stream was designed when the underlying search stream will emit a lot of tuples from the collection. If the search stream only emits a small subset of the data from the collection using `parallel` could potentially be slower.
.Worker Collections
[TIP]
====
-The worker nodes can be from the same collection as the data, or they can be a different collection entirely, even one that only exists for parallel streaming expressions. A worker collection can be any SolrCloud collection that has the `/stream` handler configured. Unlike normal SolrCloud collections, worker collections don't have to hold any data. Worker collections can be empty collections that exist only to execute streaming expressions.
+The worker nodes can be from the same collection as the data, or they can be a different collection entirely, even one that only exists for `parallel` streaming expressions. A worker collection can be any SolrCloud collection that has the `/stream` handler configured. Unlike normal SolrCloud collections, worker collections don't have to hold any data. Worker collections can be empty collections that exist only to execute streaming expressions.
====
=== parallel Parameters
@@ -1009,11 +1011,11 @@ The expression above shows a `parallel` function wrapping a `reduce` function. T
.Warmup
[TIP]
====
-The parallel stream uses the hash query parser to split the data amongst the workers. It executes on all the documents and the result bitset is cached in the filterCache.
-For a parallel stream with the same number of workers and partitonKeys the first query would be slower than subsequent queries.
+The `parallel` function uses the hash query parser to split the data amongst the workers. It executes on all the documents and the result bitset is cached in the filterCache.
++
+For a `parallel` stream with the same number of workers and `partitonKeys` the first query would be slower than subsequent queries.
A trick to not pay the penalty for the first slow query would be to use a warmup query for every new searcher.
-The following is a solrconfig.xml snippet for 2 workers and "year_i" as the partionKeys.
-
+The following is a `solrconfig.xml` snippet for 2 workers and "year_i" as the `partionKeys`.
[source,text]
----
@@ -1024,7 +1026,6 @@ The following is a solrconfig.xml snippet for 2 workers and "year_i" as the part
</arr>
</listener>
----
-
====
== priority
[20/47] lucene-solr:jira/solr-12709: SOLR-12744: Improve logging
messages and verbosity around recoveries
Posted by ab...@apache.org.
SOLR-12744: Improve logging messages and verbosity around recoveries
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/89d598e9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/89d598e9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/89d598e9
Branch: refs/heads/jira/solr-12709
Commit: 89d598e9e891e87825d45aabea45e708a51ba860
Parents: 37375ae
Author: Varun Thacker <va...@apache.org>
Authored: Wed Sep 5 19:43:36 2018 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Wed Sep 5 19:56:25 2018 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../src/java/org/apache/solr/cloud/RecoveryStrategy.java | 8 ++++----
solr/core/src/java/org/apache/solr/core/SolrCore.java | 4 ++--
solr/core/src/java/org/apache/solr/handler/IndexFetcher.java | 2 +-
solr/core/src/java/org/apache/solr/update/PeerSync.java | 2 +-
.../src/java/org/apache/solr/update/PeerSyncWithLeader.java | 6 +++---
6 files changed, 13 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d598e9/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index aa0698c..19db81e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -409,6 +409,8 @@ Other Changes
* SOLR-12590: Improve Solr resource loader coverage in the ref guide.
(Steve Rowe, Cassandra Targett, Christine Poerschke)
+* SOLR-12744: Improve logging messages and verbosity around recoveries (Cao Manh Dat, Varun Thacker)
+
================== 7.4.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d598e9/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
index 24c53bd..aa87801 100644
--- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
+++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
@@ -484,11 +484,11 @@ public class RecoveryStrategy implements Runnable, Closeable {
}
if (oldIdx > 0) {
- log.info("####### Found new versions added after startup: num=[{}]", oldIdx);
- log.info("###### currentVersions=[{}]",recentVersions);
+ log.info("Found new versions added after startup: num=[{}]", oldIdx);
+ log.info("currentVersions size={} range=[{} to {}]", recentVersions.size(), recentVersions.get(0), recentVersions.get(recentVersions.size()-1));
}
-
- log.info("###### startupVersions=[{}]", startingVersions);
+
+ log.info("startupVersions size={} range=[{} to {}]", startingVersions.size(), startingVersions.get(0), startingVersions.get(startingVersions.size()-1));
} catch (Exception e) {
SolrException.log(log, "Error getting recent versions.", e);
recentVersions = new ArrayList<>(0);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d598e9/solr/core/src/java/org/apache/solr/core/SolrCore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 3a02af9..6e13039 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -691,7 +691,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
log.info(info.className);
rsBuilder = getResourceLoader().newInstance(info.className, RecoveryStrategy.Builder.class);
} else {
- log.info("solr.RecoveryStrategy.Builder");
+ log.debug("solr.RecoveryStrategy.Builder");
rsBuilder = new RecoveryStrategy.Builder();
}
if (info != null) {
@@ -1909,7 +1909,7 @@ public final class SolrCore implements SolrInfoBean, SolrMetricProducer, Closeab
f = IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
// cache fingerprint for the segment only if all the versions in the segment are included in the fingerprint
if (f.getMaxVersionEncountered() == f.getMaxInHash()) {
- log.info("Caching fingerprint for searcher:{} leafReaderContext:{} mavVersion:{}", searcher, ctx, maxVersion);
+ log.debug("Caching fingerprint for searcher:{} leafReaderContext:{} mavVersion:{}", searcher, ctx, maxVersion);
perSegmentFingerprintCache.put(cacheHelper.getKey(), f);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d598e9/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
index a34ad7a..8df0fb0 100644
--- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
+++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java
@@ -1033,7 +1033,7 @@ public class IndexFetcher {
}
filesDownloaded.add(new HashMap<>(file));
} else {
- log.info("Skipping download for {} because it already exists", file.get(NAME));
+ log.debug("Skipping download for {} because it already exists", file.get(NAME));
}
}
log.info("Bytes downloaded: {}, Bytes skipped downloading: {}", bytesDownloaded, bytesSkippedCopying);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d598e9/solr/core/src/java/org/apache/solr/update/PeerSync.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSync.java b/solr/core/src/java/org/apache/solr/update/PeerSync.java
index 6c3c578..cb7b918 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSync.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSync.java
@@ -772,7 +772,7 @@ public class PeerSync implements SolrMetricProducer {
}
if (updatesRequest.totalRequestedUpdates > nUpdates) {
- log.info("{} Failing due to needing too many updates:{}", logPrefix, nUpdates);
+ log.info("{} PeerSync will fail because number of missed updates is more than:{}", logPrefix, nUpdates);
return MissedUpdatesRequest.UNABLE_TO_SYNC;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/89d598e9/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java b/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
index ce4bc77..b485727 100644
--- a/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
+++ b/solr/core/src/java/org/apache/solr/update/PeerSyncWithLeader.java
@@ -359,9 +359,9 @@ public class PeerSyncWithLeader implements SolrMetricProducer {
try {
IndexFingerprint ourFingerprint = IndexFingerprint.getFingerprint(core, Long.MAX_VALUE);
int cmp = IndexFingerprint.compare(leaderFingerprint, ourFingerprint);
- log.info("Fingerprint comparison: {}" , cmp);
- if(cmp != 0) {
- log.info("Other fingerprint: {}, Our fingerprint: {}", leaderFingerprint , ourFingerprint);
+ log.info("Fingerprint comparison result: {}" , cmp);
+ if (cmp != 0) {
+ log.info("Leader fingerprint: {}, Our fingerprint: {}", leaderFingerprint , ourFingerprint);
}
return cmp == 0; // currently, we only check for equality...
} catch (IOException e) {
[43/47] lucene-solr:jira/solr-12709: SOLR-11943: Change scope of
commons-math3 solr-core dependency from test to compile,
for HaversineMetersEvaluator.java
Posted by ab...@apache.org.
SOLR-11943: Change scope of commons-math3 solr-core dependency from test to compile, for HaversineMetersEvaluator.java
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8f498920
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8f498920
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8f498920
Branch: refs/heads/jira/solr-12709
Commit: 8f498920bd4f2d0059031251409c812bab55404d
Parents: f5ce384
Author: Steve Rowe <sa...@apache.org>
Authored: Fri Sep 7 16:30:22 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Fri Sep 7 16:30:22 2018 -0400
----------------------------------------------------------------------
solr/core/ivy.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8f498920/solr/core/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index 5a0fd09..80aab20 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -39,7 +39,7 @@
<dependency org="com.google.guava" name="guava" rev="${/com.google.guava/guava}" conf="compile"/>
<dependency org="org.locationtech.spatial4j" name="spatial4j" rev="${/org.locationtech.spatial4j/spatial4j}" conf="compile"/>
<dependency org="org.antlr" name="antlr4-runtime" rev="${/org.antlr/antlr4-runtime}"/>
- <dependency org="org.apache.commons" name="commons-math3" rev="${/org.apache.commons/commons-math3}" conf="test"/>
+ <dependency org="org.apache.commons" name="commons-math3" rev="${/org.apache.commons/commons-math3}" conf="compile"/>
<dependency org="org.ow2.asm" name="asm" rev="${/org.ow2.asm/asm}" conf="compile"/>
<dependency org="org.ow2.asm" name="asm-commons" rev="${/org.ow2.asm/asm-commons}" conf="compile"/>
<dependency org="org.restlet.jee" name="org.restlet" rev="${/org.restlet.jee/org.restlet}" conf="compile"/>
[46/47] lucene-solr:jira/solr-12709: SOLR-12709: Fix locking in
simUpdate so that it doesn't lose documents.
Posted by ab...@apache.org.
SOLR-12709: Fix locking in simUpdate so that it doesn't lose documents.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4a6dfdac
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4a6dfdac
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4a6dfdac
Branch: refs/heads/jira/solr-12709
Commit: 4a6dfdac8d50f52bf8afe9a4fe193fbe160a981f
Parents: c4eb2f0
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Tue Sep 11 13:50:27 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Tue Sep 11 13:50:27 2018 +0200
----------------------------------------------------------------------
.../sim/SimClusterStateProvider.java | 326 ++++++++++---------
.../autoscaling/sim/TestSimAutoScaling.java | 9 +-
2 files changed, 176 insertions(+), 159 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4a6dfdac/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
index 77bcfd7..9a32599 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/SimClusterStateProvider.java
@@ -1107,11 +1107,11 @@ public class SimClusterStateProvider implements ClusterStateProvider {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Shard " + collectionName +
" / " + sliceName.get() + " has no leader and can't be split");
}
- // start counting buffered updates and
+ // start counting buffered updates
Map<String, Object> props = sliceProperties.computeIfAbsent(collectionName, c -> new ConcurrentHashMap<>())
.computeIfAbsent(sliceName.get(), ss -> new ConcurrentHashMap<>());
if (props.containsKey(BUFFERED_UPDATES)) {
- log.trace("--- SOLR-12729: Overlapping splitShard commands for {} / {}", collectionName, sliceName.get());
+ log.debug("--- SOLR-12729: Overlapping splitShard commands for {} / {}", collectionName, sliceName.get());
return;
}
props.put(BUFFERED_UPDATES, new AtomicLong());
@@ -1219,6 +1219,8 @@ public class SimClusterStateProvider implements ClusterStateProvider {
// apply buffered updates
long perShard = bufferedUpdates.get() / subSlices.size();
long remainder = bufferedUpdates.get() % subSlices.size();
+ log.debug("-- applying {} buffered docs from {} / {}, perShard={}, remainder={}", bufferedUpdates.get(),
+ collectionName, parentSlice.getName(), perShard, remainder);
for (int i = 0; i < subSlices.size(); i++) {
String sub = subSlices.get(i);
long numUpdates = perShard;
@@ -1363,7 +1365,7 @@ public class SimClusterStateProvider implements ClusterStateProvider {
log.debug("-- attempting to delete nonexistent doc " + id + " from " + s.getLeader());
continue;
}
- AtomicLong bufferedUpdates = (AtomicLong)s.getProperties().get(BUFFERED_UPDATES);
+ AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES);
if (bufferedUpdates != null) {
if (bufferedUpdates.get() > 0) {
bufferedUpdates.decrementAndGet();
@@ -1451,86 +1453,87 @@ public class SimClusterStateProvider implements ClusterStateProvider {
// XXX don't add more than 2bln docs in one request
boolean modified = false;
- Slice[] slices = coll.getActiveSlicesArr();
- if (slices.length == 0) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection without slices");
- }
- int[] perSlice = new int[slices.length];
-
- if (it != null) {
- // BULK UPDATE: simulate random doc assignment without actually calling DocRouter,
- // which adds significant overhead
-
- int totalAdded = 0;
- for (int i = 0; i < slices.length; i++) {
- Slice s = slices[i];
- long count = (long) docCount * ((long) s.getRange().max - (long) s.getRange().min) / 0x100000000L;
- perSlice[i] = (int) count;
- totalAdded += perSlice[i];
+ lock.lockInterruptibly();
+ try {
+ coll = getClusterState().getCollection(collection);
+ Slice[] slices = coll.getActiveSlicesArr();
+ if (slices.length == 0) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection without slices");
}
- // loss of precision due to integer math
- int diff = docCount - totalAdded;
- if (diff > 0) {
- // spread the remainder more or less equally
- int perRemain = diff / slices.length;
- int remainder = diff % slices.length;
- int remainderSlice = slices.length > 1 ? bulkUpdateRandom.nextInt(slices.length) : 0;
+ int[] perSlice = new int[slices.length];
+
+ if (it != null) {
+ // BULK UPDATE: simulate random doc assignment without actually calling DocRouter,
+ // which adds significant overhead
+
+ int totalAdded = 0;
for (int i = 0; i < slices.length; i++) {
- perSlice[i] += perRemain;
- if (i == remainderSlice) {
- perSlice[i] += remainder;
- }
- }
- }
- for (int i = 0; i < slices.length; i++) {
- Slice s = slices[i];
- Replica leader = s.getLeader();
- if (leader == null) {
- log.debug("-- no leader in " + s);
- continue;
+ Slice s = slices[i];
+ long count = (long) docCount * ((long) s.getRange().max - (long) s.getRange().min) / 0x100000000L;
+ perSlice[i] = (int) count;
+ totalAdded += perSlice[i];
}
- metricUpdates.computeIfAbsent(s.getName(), sh -> new HashMap<>())
- .computeIfAbsent(leader.getCoreName(), cn -> new AtomicLong())
- .addAndGet(perSlice[i]);
- modified = true;
- AtomicLong bufferedUpdates = (AtomicLong)s.getProperties().get(BUFFERED_UPDATES);
- if (bufferedUpdates != null) {
- bufferedUpdates.addAndGet(perSlice[i]);
- continue;
- }
- docUpdates.computeIfAbsent(s.getName(), sh -> new AtomicLong())
- .addAndGet(perSlice[i]);
- }
- } else {
- // SMALL UPDATE: use exact assignment via DocRouter
- for (SolrInputDocument doc : docs) {
- String id = (String) doc.getFieldValue("id");
- if (id == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Document without id: " + doc);
+ // loss of precision due to integer math
+ int diff = docCount - totalAdded;
+ if (diff > 0) {
+ // spread the remainder more or less equally
+ int perRemain = diff / slices.length;
+ int remainder = diff % slices.length;
+ int remainderSlice = slices.length > 1 ? bulkUpdateRandom.nextInt(slices.length) : 0;
+ for (int i = 0; i < slices.length; i++) {
+ perSlice[i] += perRemain;
+ if (i == remainderSlice) {
+ perSlice[i] += remainder;
+ }
+ }
}
- Slice s = coll.getRouter().getTargetSlice(id, doc, null, null, coll);
- Replica leader = s.getLeader();
- if (leader == null) {
- log.debug("-- no leader in " + s);
- continue;
+ for (int i = 0; i < slices.length; i++) {
+ Slice s = slices[i];
+ Replica leader = s.getLeader();
+ if (leader == null) {
+ log.debug("-- no leader in " + s);
+ continue;
+ }
+ metricUpdates.computeIfAbsent(s.getName(), sh -> new HashMap<>())
+ .computeIfAbsent(leader.getCoreName(), cn -> new AtomicLong())
+ .addAndGet(perSlice[i]);
+ modified = true;
+ AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES);
+ if (bufferedUpdates != null) {
+ bufferedUpdates.addAndGet(perSlice[i]);
+ continue;
+ }
+ docUpdates.computeIfAbsent(s.getName(), sh -> new AtomicLong())
+ .addAndGet(perSlice[i]);
}
- metricUpdates.computeIfAbsent(s.getName(), sh -> new HashMap<>())
- .computeIfAbsent(leader.getCoreName(), cn -> new AtomicLong())
- .incrementAndGet();
- modified = true;
- AtomicLong bufferedUpdates = (AtomicLong)s.getProperties().get(BUFFERED_UPDATES);
- if (bufferedUpdates != null) {
- bufferedUpdates.incrementAndGet();
- continue;
+ } else {
+ // SMALL UPDATE: use exact assignment via DocRouter
+ for (SolrInputDocument doc : docs) {
+ String id = (String) doc.getFieldValue("id");
+ if (id == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Document without id: " + doc);
+ }
+ Slice s = coll.getRouter().getTargetSlice(id, doc, null, null, coll);
+ Replica leader = s.getLeader();
+ if (leader == null) {
+ log.debug("-- no leader in " + s);
+ continue;
+ }
+ metricUpdates.computeIfAbsent(s.getName(), sh -> new HashMap<>())
+ .computeIfAbsent(leader.getCoreName(), cn -> new AtomicLong())
+ .incrementAndGet();
+ modified = true;
+ AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES);
+ if (bufferedUpdates != null) {
+ bufferedUpdates.incrementAndGet();
+ continue;
+ }
+ docUpdates.computeIfAbsent(s.getName(), sh -> new AtomicLong())
+ .incrementAndGet();
}
- docUpdates.computeIfAbsent(s.getName(), sh -> new AtomicLong())
- .incrementAndGet();
}
- }
- if (modified) {
- lock.lockInterruptibly();
- try {
+ if (modified) {
docUpdates.forEach((sh, count) -> {
try {
simSetShardValue(collection, sh, "SEARCHER.searcher.numDocs", count.get(), true, false);
@@ -1551,9 +1554,9 @@ public class SimClusterStateProvider implements ClusterStateProvider {
cloudManager.getMetricManager().registry(registry).counter("UPDATE./update.requests").inc(count.get());
});
});
- } finally {
- lock.unlock();
}
+ } finally {
+ lock.unlock();
}
}
return new UpdateResponse();
@@ -1837,91 +1840,104 @@ public class SimClusterStateProvider implements ClusterStateProvider {
return new ArrayList<>(colShardReplicaMap.keySet());
}
- public Map<String, Map<String, Object>> simGetCollectionStats() throws IOException {
+ public Map<String, Map<String, Object>> simGetCollectionStats() throws IOException, InterruptedException {
Map<String, Map<String, Object>> stats = new TreeMap<>();
- ClusterState state = getClusterState();
- state.forEachCollection(coll -> {
- Map<String, Object> perColl = new LinkedHashMap<>();
- stats.put(coll.getName(), perColl);
- perColl.put("shardsTotal", coll.getSlices().size());
- Map<String, AtomicInteger> shardState = new TreeMap<>();
- int noLeader = 0;
-
- SummaryStatistics docs = new SummaryStatistics();
- SummaryStatistics bytes = new SummaryStatistics();
- SummaryStatistics inactiveDocs = new SummaryStatistics();
- SummaryStatistics inactiveBytes = new SummaryStatistics();
-
- long deletedDocs = 0;
- int totalReplicas = 0;
- int activeReplicas = 0;
-
- for (Slice s : coll.getSlices()) {
- shardState.computeIfAbsent(s.getState().toString(), st -> new AtomicInteger())
- .incrementAndGet();
- totalReplicas += s.getReplicas().size();
- if (s.getState() != Slice.State.ACTIVE) {
- if (!s.getReplicas().isEmpty()) {
- ReplicaInfo ri = getReplicaInfo(s.getReplicas().iterator().next());
- if (ri != null) {
- Number numDocs = (Number)ri.getVariable("SEARCHER.searcher.numDocs");
- Number numBytes = (Number)ri.getVariable("INDEX.sizeInBytes");
- inactiveDocs.addValue(numDocs.doubleValue());
- inactiveBytes.addValue(numBytes.doubleValue());
+ lock.lockInterruptibly();
+ try {
+ collectionsStatesRef.set(null);
+ ClusterState state = getClusterState();
+ state.forEachCollection(coll -> {
+ Map<String, Object> perColl = new LinkedHashMap<>();
+ stats.put(coll.getName(), perColl);
+ perColl.put("shardsTotal", coll.getSlices().size());
+ Map<String, AtomicInteger> shardState = new TreeMap<>();
+ int noLeader = 0;
+
+ SummaryStatistics docs = new SummaryStatistics();
+ SummaryStatistics bytes = new SummaryStatistics();
+ SummaryStatistics inactiveDocs = new SummaryStatistics();
+ SummaryStatistics inactiveBytes = new SummaryStatistics();
+
+ long deletedDocs = 0;
+ long bufferedDocs = 0;
+ int totalReplicas = 0;
+ int activeReplicas = 0;
+
+ for (Slice s : coll.getSlices()) {
+ shardState.computeIfAbsent(s.getState().toString(), st -> new AtomicInteger())
+ .incrementAndGet();
+ totalReplicas += s.getReplicas().size();
+ if (s.getState() != Slice.State.ACTIVE) {
+ if (!s.getReplicas().isEmpty()) {
+ ReplicaInfo ri = getReplicaInfo(s.getReplicas().iterator().next());
+ if (ri != null) {
+ Number numDocs = (Number)ri.getVariable("SEARCHER.searcher.numDocs");
+ Number numBytes = (Number)ri.getVariable("INDEX.sizeInBytes");
+ inactiveDocs.addValue(numDocs.doubleValue());
+ inactiveBytes.addValue(numBytes.doubleValue());
+ }
}
+ continue;
}
- continue;
- }
- activeReplicas += s.getReplicas().size();
- Replica leader = s.getLeader();
- if (leader == null) {
- noLeader++;
- if (!s.getReplicas().isEmpty()) {
- leader = s.getReplicas().iterator().next();
+ AtomicLong buffered = (AtomicLong)sliceProperties.get(coll.getName()).get(s.getName()).get(BUFFERED_UPDATES);
+ if (buffered != null) {
+ bufferedDocs += buffered.get();
}
- }
- ReplicaInfo ri = null;
- if (leader != null) {
- ri = getReplicaInfo(leader);
- if (ri == null) {
- log.warn("Unknown ReplicaInfo for {}", leader);
+ activeReplicas += s.getReplicas().size();
+ Replica leader = s.getLeader();
+ if (leader == null) {
+ noLeader++;
+ if (!s.getReplicas().isEmpty()) {
+ leader = s.getReplicas().iterator().next();
+ }
}
- }
- if (ri != null) {
- Number numDocs = (Number)ri.getVariable("SEARCHER.searcher.numDocs");
- Number delDocs = (Number)ri.getVariable("SEARCHER.searcher.deleteDocs");
- Number numBytes = (Number)ri.getVariable("INDEX.sizeInBytes");
- docs.addValue(numDocs.doubleValue());
- if (delDocs != null) {
- deletedDocs += delDocs.longValue();
+ ReplicaInfo ri = null;
+ if (leader != null) {
+ ri = getReplicaInfo(leader);
+ if (ri == null) {
+ log.warn("Unknown ReplicaInfo for {}", leader);
+ }
+ }
+ if (ri != null) {
+ Number numDocs = (Number)ri.getVariable("SEARCHER.searcher.numDocs");
+ Number delDocs = (Number)ri.getVariable("SEARCHER.searcher.deleteDocs");
+ Number numBytes = (Number)ri.getVariable("INDEX.sizeInBytes");
+ docs.addValue(numDocs.doubleValue());
+ if (delDocs != null) {
+ deletedDocs += delDocs.longValue();
+ }
+ bytes.addValue(numBytes.doubleValue());
}
- bytes.addValue(numBytes.doubleValue());
}
- }
- perColl.put("shardsState", shardState);
- perColl.put(" shardsWithoutLeader", noLeader);
- perColl.put("totalReplicas", totalReplicas);
- perColl.put(" activeReplicas", activeReplicas);
- perColl.put(" inactiveReplicas", totalReplicas - activeReplicas);
- perColl.put("totalActiveDocs", String.format("%,d", (long)docs.getSum()));
- perColl.put(" maxActiveSliceDocs", String.format("%,d", (long)docs.getMax()));
- perColl.put(" minActiveSliceDocs", String.format("%,d", (long)docs.getMin()));
- perColl.put(" avgActiveSliceDocs", String.format("%,.0f", docs.getMean()));
- perColl.put("totalInactiveDocs", String.format("%,d", (long)inactiveDocs.getSum()));
- perColl.put(" maxInactiveSliceDocs", String.format("%,d", (long)inactiveDocs.getMax()));
- perColl.put(" minInactiveSliceDocs", String.format("%,d", (long)inactiveDocs.getMin()));
- perColl.put(" avgInactiveSliceDocs", String.format("%,.0f", inactiveDocs.getMean()));
- perColl.put("totalActiveBytes", String.format("%,d", (long)bytes.getSum()));
- perColl.put(" maxActiveSliceBytes", String.format("%,d", (long)bytes.getMax()));
- perColl.put(" minActiveSliceBytes", String.format("%,d", (long)bytes.getMin()));
- perColl.put(" avgActiveSliceBytes", String.format("%,.0f", bytes.getMean()));
- perColl.put("totalInactiveBytes", String.format("%,d", (long)inactiveBytes.getSum()));
- perColl.put(" maxInactiveSliceBytes", String.format("%,d", (long)inactiveBytes.getMax()));
- perColl.put(" minInactiveSliceBytes", String.format("%,d", (long)inactiveBytes.getMin()));
- perColl.put(" avgInactiveSliceBytes", String.format("%,.0f", inactiveBytes.getMean()));
- perColl.put("totalActiveDeletedDocs", String.format("%,d", deletedDocs));
- });
- return stats;
+ perColl.put("shardsState", shardState);
+ perColl.put(" shardsWithoutLeader", noLeader);
+ perColl.put("totalReplicas", totalReplicas);
+ perColl.put(" activeReplicas", activeReplicas);
+ perColl.put(" inactiveReplicas", totalReplicas - activeReplicas);
+ long totalDocs = (long)docs.getSum() + bufferedDocs;
+ perColl.put("totalActiveDocs", String.format("%,d", totalDocs));
+ perColl.put(" bufferedDocs", String.format("%,d", bufferedDocs));
+ perColl.put(" maxActiveSliceDocs", String.format("%,d", (long)docs.getMax()));
+ perColl.put(" minActiveSliceDocs", String.format("%,d", (long)docs.getMin()));
+ perColl.put(" avgActiveSliceDocs", String.format("%,.0f", docs.getMean()));
+ perColl.put("totalInactiveDocs", String.format("%,d", (long)inactiveDocs.getSum()));
+ perColl.put(" maxInactiveSliceDocs", String.format("%,d", (long)inactiveDocs.getMax()));
+ perColl.put(" minInactiveSliceDocs", String.format("%,d", (long)inactiveDocs.getMin()));
+ perColl.put(" avgInactiveSliceDocs", String.format("%,.0f", inactiveDocs.getMean()));
+ perColl.put("totalActiveBytes", String.format("%,d", (long)bytes.getSum()));
+ perColl.put(" maxActiveSliceBytes", String.format("%,d", (long)bytes.getMax()));
+ perColl.put(" minActiveSliceBytes", String.format("%,d", (long)bytes.getMin()));
+ perColl.put(" avgActiveSliceBytes", String.format("%,.0f", bytes.getMean()));
+ perColl.put("totalInactiveBytes", String.format("%,d", (long)inactiveBytes.getSum()));
+ perColl.put(" maxInactiveSliceBytes", String.format("%,d", (long)inactiveBytes.getMax()));
+ perColl.put(" minInactiveSliceBytes", String.format("%,d", (long)inactiveBytes.getMin()));
+ perColl.put(" avgInactiveSliceBytes", String.format("%,.0f", inactiveBytes.getMean()));
+ perColl.put("totalActiveDeletedDocs", String.format("%,d", deletedDocs));
+ });
+ return stats;
+ } finally {
+ lock.unlock();
+ }
}
// interface methods
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4a6dfdac/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
index 918d7ae..6cbb132 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimAutoScaling.java
@@ -26,7 +26,7 @@ import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAut
*
*/
@TimeoutSuite(millis = 48 * 3600 * 1000)
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=INFO;org.apache.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.ComputePlanAction=INFO;org.apache.solr.cloud.autoscaling.ExecutePlanAction=DEBUG;org.apache.solr.cloud.autoscaling.ScheduledTriggers=INFO")
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=INFO;org.apache.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.ComputePlanAction=INFO;org.apache.solr.cloud.autoscaling.ExecutePlanAction=DEBUG;org.apache.solr.cloud.autoscaling.ScheduledTriggers=DEBUG")
//@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=INFO;org.apache.client.solrj.cloud.autoscaling=DEBUG;org.apache.solr.cloud.CloudTestUtils=TRACE")
public class TestSimAutoScaling extends SimSolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -34,9 +34,9 @@ public class TestSimAutoScaling extends SimSolrCloudTestCase {
private static final int SPEED = 500;
private static final int NUM_NODES = 200;
- private static final long BATCH_SIZE = 20000;
- private static final long NUM_BATCHES = 10000000;
- private static final long ABOVE_SIZE = 10000000;
+ private static final long BATCH_SIZE = 2000000;
+ private static final long NUM_BATCHES = 500000;
+ private static final long ABOVE_SIZE = 20000000;
private static TimeSource timeSource;
@@ -81,6 +81,7 @@ public class TestSimAutoScaling extends SimSolrCloudTestCase {
log.info(String.format("#### Total docs so far: %,d", ((i + 1) * batchSize)));
timeSource.sleep(waitForSeconds);
}
+ timeSource.sleep(60000);
}
private void addDocs(String collection, long start, long count) throws Exception {
[14/47] lucene-solr:jira/solr-12709: Fix typo in TermQuery.equals
javadocs.
Posted by ab...@apache.org.
Fix typo in TermQuery.equals javadocs.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a3aa014d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a3aa014d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a3aa014d
Branch: refs/heads/jira/solr-12709
Commit: a3aa014d1a42b24d3593c5d221d7695a5863b438
Parents: 3f9937b
Author: Christine Poerschke <cp...@apache.org>
Authored: Wed Sep 5 17:48:37 2018 +0100
Committer: Christine Poerschke <cp...@apache.org>
Committed: Wed Sep 5 17:51:01 2018 +0100
----------------------------------------------------------------------
lucene/core/src/java/org/apache/lucene/search/TermQuery.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a3aa014d/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
index 25c77c2..1eebbce 100644
--- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
@@ -224,7 +224,7 @@ public class TermQuery extends Query {
return perReaderTermState;
}
- /** Returns true iff <code>o</code> is equal to this. */
+ /** Returns true iff <code>other</code> is equal to <code>this</code>. */
@Override
public boolean equals(Object other) {
return sameClassAs(other) &&
[15/47] lucene-solr:jira/solr-12709: SOLR-12715: change
preferredOperation values to lowercase since that's how they're entered
Posted by ab...@apache.org.
SOLR-12715: change preferredOperation values to lowercase since that's how they're entered
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e893f348
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e893f348
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e893f348
Branch: refs/heads/jira/solr-12709
Commit: e893f3480de57c74e89ef9ee396972dcb25a9958
Parents: 2565010
Author: Cassandra Targett <ct...@apache.org>
Authored: Tue Sep 4 21:16:45 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Sep 5 12:47:08 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e893f348/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index 83a3f87..9c8aac5 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -64,7 +64,7 @@ from other nodes to the new node or to add new replicas.
Apart from the parameters described at <<#trigger-configuration, Trigger Configuration>>, this trigger supports the following configuration:
-`preferredOperation`:: (string, optional, defaults to `MOVEREPLICA`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from other nodes to the added node. The only other supported value is `ADDREPLICA` which adds more replicas of the existing collections on the new node.
+`preferredOperation`:: (string, optional, defaults to `movereplica`) The operation to be performed in response to an event generated by this trigger. By default, replicas will be moved from other nodes to the added node. The only other supported value is `addreplica` which adds more replicas of the existing collections on the new node.
.Example: Node Added Trigger to move replicas to new node
[source,json]
[25/47] lucene-solr:jira/solr-12709: SOLR-12722: expand "params" ->
"parameters (plus a bunch of other things I found in unrelated transformer
examples)
Posted by ab...@apache.org.
SOLR-12722: expand "params" -> "parameters (plus a bunch of other things I found in unrelated transformer examples)
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/00ce9e06
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/00ce9e06
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/00ce9e06
Branch: refs/heads/jira/solr-12709
Commit: 00ce9e067b8797b7dd0f1014c938354a59e15024
Parents: 9c364b2
Author: Cassandra Targett <ct...@apache.org>
Authored: Thu Sep 6 08:54:37 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Thu Sep 6 08:54:37 2018 -0500
----------------------------------------------------------------------
.../src/transforming-result-documents.adoc | 61 +++++++++++++-------
1 file changed, 41 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/00ce9e06/solr/solr-ref-guide/src/transforming-result-documents.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/transforming-result-documents.adoc b/solr/solr-ref-guide/src/transforming-result-documents.adoc
index df9bff4..386a995 100644
--- a/solr/solr-ref-guide/src/transforming-result-documents.adoc
+++ b/solr/solr-ref-guide/src/transforming-result-documents.adoc
@@ -16,7 +16,7 @@
// specific language governing permissions and limitations
// under the License.
-Document Transformers can be used to modify the information returned about each documents in the results of a query.
+Document Transformers modify the information returned about documents in the results of a query.
== Using Document Transformers
@@ -67,7 +67,7 @@ The above query would produce results like the following:
...
----
-By default, values are returned as a String, but a "```t```" parameter can be specified using a value of int, float, double, or date to force a specific return type:
+By default, values are returned as a String, but a `t` parameter can be specified using a value of `int`, `float`, `double`, or `date` to force a specific return type:
[source,plain]
----
@@ -86,7 +86,7 @@ In addition to using these request parameters, you can configure additional name
</transformer>
----
-The "```value```" option forces an explicit value to always be used, while the "```defaultValue```" option provides a default that can still be overridden using the "```v```" and "```t```" local parameters.
+The `value` option forces an explicit value to always be used, while the `defaultValue` option provides a default that can still be overridden using the `v` and `t` local parameters.
=== [explain] - ExplainAugmenterFactory
@@ -98,7 +98,7 @@ Augments each document with an inline explanation of its score exactly like the
q=features:cache&fl=id,[explain style=nl]
----
-Supported values for `style` are `text`, and `html`, and `nl` which returns the information as structured data:
+Supported values for `style` are `text`, `html`, and `nl` which returns the information as structured data. Here is the output of the above request using `style=nl`:
[source,json]
----
@@ -113,7 +113,7 @@ Supported values for `style` are `text`, and `html`, and `nl` which returns the
}]}}]}}
----
-A default style can be configured by specifying an "args" parameter in your configuration:
+A default style can be configured by specifying an `args` parameter in your `solrconfig.xml` configuration:
[source,xml]
----
@@ -133,11 +133,18 @@ fl=id,[child parentFilter=doc_type:book childFilter=doc_type:chapter limit=100]
Note that this transformer can be used even though the query itself is not a <<other-parsers.adoc#block-join-query-parsers,Block Join query>>.
-When using this transformer, the `parentFilter` parameter must be specified, and works the same as in all Block Join Queries, additional optional parameters are:
+When using this transformer, the `parentFilter` parameter must be specified, and works the same as in all Block Join Queries. Additional optional parameters are:
-* `childFilter` - query to filter which child documents should be included, this can be particularly useful when you have multiple levels of hierarchical documents (default: all children)
-* `limit` - the maximum number of child documents to be returned per parent document (default: 10)
-* `fl` - the field list which the transformer is to return (default: top level "fl"). There is a further limitation in which the fields here should be a subset of those specified by the top level fl param
+`childFilter`::
+A query to filter which child documents should be included. This can be particularly useful when you have multiple levels of hierarchical documents. The default is all children.
+
+`limit`::
+The maximum number of child documents to be returned per parent document. The default is `10`.
+
+`fl`::
+The field list which the transformer is to return. The default is the top level `fl`).
++
+There is a further limitation in which the fields here should be a subset of those specified by the top level `fl` parameter.
=== [shard] - ShardAugmenterFactory
@@ -187,7 +194,7 @@ fl=id,[elevated],[excluded]&excludeIds=GB18030TEST&elevateIds=6H500F0&markExclud
=== [json] / [xml]
-These transformers replace field value containing a string representation of a valid XML or JSON structure with the actual raw XML or JSON structure rather than just the string value. Each applies only to the specific writer, such that `[json]` only applies to `wt=json` and `[xml]` only applies to `wt=xml`.
+These transformers replace a field value containing a string representation of a valid XML or JSON structure with the actual raw XML or JSON structure instead of just the string value. Each applies only to the specific writer, such that `[json]` only applies to `wt=json` and `[xml]` only applies to `wt=xml`.
[source,plain]
----
@@ -202,10 +209,11 @@ This transformer executes a separate query per transforming document passing doc
* It must be given an unique name: `fl=*,children:[subquery]`
* There might be a few of them, e.g., `fl=*,sons:[subquery],daughters:[subquery]`.
* Every `[subquery]` occurrence adds a field into a result document with the given name, the value of this field is a document list, which is a result of executing subquery using document fields as an input.
-* Subquery would use `/select` search handler by default that causes error if it is not configured. This can be changed by supplying `foo.qt` parameter.
+* Subquery will use the `/select` search handler by default, and will return an error if `/select` is not configured. This can be changed by supplying `foo.qt` parameter.
-Here is how it looks like in various formats:
+Here is how it looks like using various formats:
+.XML
[source,xml]
----
<result name="response" numFound="2" start="0">
@@ -226,6 +234,7 @@ Here is how it looks like in various formats:
...
----
+.JSON
[source,json]
----
{ "response":{
@@ -245,6 +254,7 @@ Here is how it looks like in various formats:
}}]}}
----
+.SolrJ
[source,java]
----
SolrDocumentList subResults = (SolrDocumentList)doc.getFieldValue("children");
@@ -252,8 +262,9 @@ Here is how it looks like in various formats:
==== Subquery Result Fields
-To appear in subquery document list, a field should be specified both fl parameters, in main one fl (despite the main result documents have no this field) and in subquery's one e.g., `foo.fl`.
-Of course, you can use wildcard in any or both of these parameters. For example, if field `title` should appear in categories subquery, it can be done via one of these ways.
+To appear in subquery document list, a field should be specified in both `fl` parameters: in the main `fl` (despite the main result documents have no this field), and in subquery's `fl` (e.g., `foo.fl`).
+
+Wildcards can be used in one or both of these parameters. For example, if field `title` should appear in categories subquery, it can be done via one of these ways:
[source,plain]
----
@@ -267,27 +278,37 @@ fl=...*,categories:[subquery]&categories.fl=*&categories.q=...
If a subquery is declared as `fl=*,foo:[subquery]`, subquery parameters are prefixed with the given name and period. For example:
-`q=*:*&fl=*,**foo**:[subquery]&**foo.**q=to be continued&**foo.**rows=10&**foo.**sort=id desc`
+[source,plain]
+q=*:*&fl=*,**foo**:[subquery]&**foo.**q=to be continued&**foo.**rows=10&**foo.**sort=id desc
==== Document Field as an Input for Subquery Parameters
-It's necessary to pass some document field values as a parameter for subquery. It's supported via implicit *`row.__fieldname__`* parameter, and can be (but might not only) referred via Local Parameters syntax: `q=namne:john&fl=name,id,depts:[subquery]&depts.q={!terms f=id **v=$row.dept_id**}&depts.rows=10`
+It's necessary to pass some document field values as a parameter for subquery. It's supported via an implicit *`row.__fieldname__`* parameter, and can be (but might not only) referred via local parameters syntax:
+
+[source,plain,subs="quotes"]
+q=name:john&fl=name,id,depts:[subquery]&depts.q={!terms f=id **v=$row.dept_id**}&depts.rows=10
Here departments are retrieved per every employee in search result. We can say that it's like SQL `join ON emp.dept_id=dept.id`.
Note, when a document field has multiple values they are concatenated with a comma by default. This can be changed with the local parameter `foo:[subquery separator=' ']`, this mimics *`{!terms}`* to work smoothly with it.
-To log substituted subquery request parameters, add the corresponding parameter names, as in `depts.logParamsList=q,fl,rows,**row.dept_id**`
+To log substituted subquery request parameters, add the corresponding parameter names, as in: `depts.logParamsList=q,fl,rows,**row.dept_id**`
==== Cores and Collections in SolrCloud
-Use `foo:[subquery fromIndex=departments]` to invoke subquery on another core on the same node, it's what *`{!join}`* does for non-SolrCloud mode. But in case of SolrCloud just (and only) explicitly specify its native parameters like `collection, shards` for subquery, e.g.:
+Use `foo:[subquery fromIndex=departments]` to invoke subquery on another core on the same node. This is what `{!join}` does for non-SolrCloud mode. But with SolrCloud, just (and only) explicitly specify its native parameters like `collection, shards` for subquery, e.g.:
-`q=*:*&fl=*,foo:[subquery]&foo.q=cloud&**foo.collection**=departments`
+[source,plain,subs="quotes"]
+q=\*:*&fl=\*,foo:[subquery]&foo.q=cloud&**foo.collection**=departments
[IMPORTANT]
====
-If subquery collection has a different unique key field name (let's say `foo_id` at contrast to `id` in primary collection), add the following parameters to accommodate this difference: `foo.fl=id:foo_id&foo.distrib.singlePass=true`. Otherwise you'll get `NullPoniterException` from `QueryComponent.mergeIds`.
+If subquery collection has a different unique key field name (such as `foo_id` instead of `id` in the primary collection), add the following parameters to accommodate this difference:
+
+[source,plain]
+foo.fl=id:foo_id&foo.distrib.singlePass=true
+
+Otherwise you'll get `NullPointerException` from `QueryComponent.mergeIds`.
====
[23/47] lucene-solr:jira/solr-12709: LUCENE-8468: Add
sliceDescription to the toString() of ByteBuffersIndexInput.
Posted by ab...@apache.org.
LUCENE-8468: Add sliceDescription to the toString() of ByteBuffersIndexInput.
This fixes test failures in TestLucene50CompoundFormat#testResourceNameInsideCompoundFile.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1a006556
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1a006556
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1a006556
Branch: refs/heads/jira/solr-12709
Commit: 1a006556e5999eb17d34bef1db08af0773d4e9b6
Parents: 285b743
Author: Adrien Grand <jp...@gmail.com>
Authored: Thu Sep 6 14:01:25 2018 +0200
Committer: Adrien Grand <jp...@gmail.com>
Committed: Thu Sep 6 14:01:25 2018 +0200
----------------------------------------------------------------------
.../src/java/org/apache/lucene/store/ByteBuffersIndexInput.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1a006556/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java
index 7c87d24..7b899b0 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java
@@ -63,7 +63,7 @@ public final class ByteBuffersIndexInput extends IndexInput implements RandomAcc
public ByteBuffersIndexInput slice(String sliceDescription, long offset, long length) throws IOException {
ensureOpen();
return new ByteBuffersIndexInput(in.slice(offset, length),
- "(sliced) offset=" + offset + ", length=" + length + " " + toString());
+ "(sliced) offset=" + offset + ", length=" + length + " " + toString() + " [slice=" + sliceDescription + "]");
}
@Override