You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/09/20 23:59:07 UTC
[01/29] lucene-solr:jira/http2: SOLR-12766: Log retries after the
backoff
Repository: lucene-solr
Updated Branches:
refs/heads/jira/http2 4cc995c61 -> 4867bb841
SOLR-12766: Log retries after the backoff
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a7a4f68d
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a7a4f68d
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a7a4f68d
Branch: refs/heads/jira/http2
Commit: a7a4f68da5e3aa53206eba6b52512f73b6d41394
Parents: d6641ff
Author: Tomas Fernandez Lobbe <tf...@apache.org>
Authored: Thu Sep 13 20:46:38 2018 -0700
Committer: Tomas Fernandez Lobbe <tf...@apache.org>
Committed: Thu Sep 13 20:46:38 2018 -0700
----------------------------------------------------------------------
.../apache/solr/update/SolrCmdDistributor.java | 29 +++++++++-----------
1 file changed, 13 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a7a4f68d/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index d7388f0..cb7f9cb 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@ -129,8 +129,6 @@ public class SolrCmdDistributor implements Closeable {
for (Error err : errors) {
try {
- String oldNodeUrl = err.req.node.getUrl();
-
/*
* if this is a retryable request we may want to retry, depending on the error we received and
* the number of times we have already retried
@@ -143,20 +141,6 @@ public class SolrCmdDistributor implements Closeable {
// this can happen in certain situations such as close
if (isRetry) {
err.req.retries++;
-
- if (err.req.node instanceof ForwardNode) {
- SolrException.log(SolrCmdDistributor.log, "forwarding update to "
- + oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
- + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
- } else {
- SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
- + oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
- + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
- }
resubmitList.add(err);
} else {
allErrors.add(err);
@@ -182,6 +166,19 @@ public class SolrCmdDistributor implements Closeable {
clients.clearErrors();
this.errors.clear();
for (Error err : resubmitList) {
+ if (err.req.node instanceof ForwardNode) {
+ SolrException.log(SolrCmdDistributor.log, "forwarding update to "
+ + err.req.node.getUrl() + " failed - retrying ... retries: "
+ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
+ + err.req.cmd.toString() + " params:"
+ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
+ } else {
+ SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
+ + err.req.node.getUrl() + " failed - retrying ... retries: "
+ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
+ + err.req.cmd.toString() + " params:"
+ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
+ }
submit(err.req, false);
}
[22/29] lucene-solr:jira/http2: LUCENE-8454: Set test
TestLatLonPolygonShapeQueries#testRandomMedium to AwaitFix
Posted by da...@apache.org.
LUCENE-8454: Set test TestLatLonPolygonShapeQueries#testRandomMedium to AwaitFix
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eb0fcec5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eb0fcec5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eb0fcec5
Branch: refs/heads/jira/http2
Commit: eb0fcec50303e193339f49d8f15049551912b917
Parents: d494f6b
Author: iverase <iv...@apache.org>
Authored: Thu Sep 20 08:53:51 2018 +0200
Committer: iverase <iv...@apache.org>
Committed: Thu Sep 20 08:53:51 2018 +0200
----------------------------------------------------------------------
.../apache/lucene/document/TestLatLonPolygonShapeQueries.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb0fcec5/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
index ce76a82..9825e6a 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
@@ -99,4 +99,10 @@ public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
return queryRelation == QueryRelation.INTERSECTS ? false : true;
}
}
+
+ @Override
+ @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8454")
+ public void testRandomMedium() throws Exception {
+ super.testRandomMedium();
+ }
}
[29/29] lucene-solr:jira/http2: merge with master
Posted by da...@apache.org.
merge with master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4867bb84
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4867bb84
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4867bb84
Branch: refs/heads/jira/http2
Commit: 4867bb841119c37e323928188c1c3e688d6b662c
Parents: 4cc995c d7e97fb
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Sep 21 06:58:54 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Sep 21 06:58:54 2018 +0700
----------------------------------------------------------------------
dev-tools/scripts/smokeTestRelease.py | 39 +-
lucene/CHANGES.txt | 33 +-
lucene/MIGRATE.txt | 17 +
.../analysis/core/LowerCaseTokenizer.java | 72 ---
.../core/LowerCaseTokenizerFactory.java | 75 ---
.../lucene/analysis/core/SimpleAnalyzer.java | 4 +-
.../lucene/analysis/core/StopAnalyzer.java | 6 +-
.../miscellaneous/LimitTokenCountAnalyzer.java | 2 +-
.../query/QueryAutoStopWordAnalyzer.java | 2 +-
.../shingle/ShingleAnalyzerWrapper.java | 2 +-
.../analysis/standard/ClassicAnalyzer.java | 11 +-
.../standard/UAX29URLEmailAnalyzer.java | 13 +-
.../lucene/analysis/util/CharTokenizer.java | 86 +---
...apache.lucene.analysis.util.TokenizerFactory | 1 -
.../analysis/br/TestBrazilianAnalyzer.java | 7 +-
.../lucene/analysis/core/TestAnalyzers.java | 8 -
.../analysis/custom/TestCustomAnalyzer.java | 13 +-
.../lucene/analysis/de/TestGermanAnalyzer.java | 8 +-
.../TestPerFieldAnalyzerWrapper.java | 2 +-
.../standard/TestStandardFactories.java | 16 -
.../analysis/util/TestCharTokenizers.java | 122 +----
lucene/common-build.xml | 2 +-
.../org/apache/lucene/analysis/Analyzer.java | 42 +-
.../analysis/standard/StandardAnalyzer.java | 13 +-
.../org/apache/lucene/document/LatLonPoint.java | 24 +
.../LatLonPointDistanceFeatureQuery.java | 480 +++++++++++++++++++
.../apache/lucene/index/FilterCodecReader.java | 18 +-
.../lucene/index/FilterDirectoryReader.java | 2 +-
.../apache/lucene/index/FilterLeafReader.java | 2 +-
.../lucene/analysis/TestAnalyzerWrapper.java | 58 +++
.../TestLatLonPointDistanceFeatureQuery.java | 411 ++++++++++++++++
.../org/apache/lucene/index/TestAddIndexes.java | 2 +
.../lucene/index/TestFilterCodecReader.java | 18 +
.../function/valuesource/MultiFunction.java | 4 +-
.../java/org/apache/lucene/geo/Tessellator.java | 101 ++--
.../apache/lucene/document/TestLatLonShape.java | 23 +
.../org/apache/lucene/geo/TestTessellator.java | 22 +
.../prefix/BytesRefIteratorTokenStream.java | 2 +-
.../spatial/prefix/PrefixTreeStrategy.java | 17 +
.../analyzing/AnalyzingInfixSuggester.java | 2 +-
.../suggest/analyzing/FreeTextSuggester.java | 2 +-
.../suggest/document/CompletionAnalyzer.java | 2 +-
.../analyzing/AnalyzingSuggesterTest.java | 365 +++++---------
.../suggest/analyzing/FuzzySuggesterTest.java | 159 ++----
.../analysis/CannedBinaryTokenStream.java | 2 +-
.../lucene/analysis/CannedTokenStream.java | 8 +-
solr/CHANGES.txt | 22 +-
.../extraction/solr/collection1/conf/schema.xml | 15 -
.../ExtractingRequestHandlerTest.java | 11 +-
.../cloud/api/collections/MoveReplicaCmd.java | 13 +-
.../solr/handler/component/StatsField.java | 4 +-
.../reporters/jmx/JmxMetricsReporter.java | 10 +-
.../AbstractSpatialPrefixTreeFieldType.java | 35 +-
.../apache/solr/schema/CurrencyFieldType.java | 8 +-
.../java/org/apache/solr/schema/FieldType.java | 4 +-
.../org/apache/solr/schema/IndexSchema.java | 2 +-
.../apache/solr/schema/PreAnalyzedField.java | 17 +-
.../solr/search/CollapsingQParserPlugin.java | 15 +-
.../apache/solr/update/SolrCmdDistributor.java | 6 +-
.../CloneFieldUpdateProcessorFactory.java | 4 +-
.../TolerantUpdateProcessorFactory.java | 2 +-
.../collection1/conf/schema-copyfield-test.xml | 14 -
.../solr/collection1/conf/schema-folding.xml | 3 +-
.../solr/collection1/conf/schema-hash.xml | 13 -
.../collection1/conf/schema-required-fields.xml | 14 -
.../solr/collection1/conf/schema-rest.xml | 9 +-
.../solr/collection1/conf/schema-sql.xml | 13 -
.../collection1/conf/schema-tokenizer-test.xml | 11 +-
.../test-files/solr/collection1/conf/schema.xml | 15 +-
.../solr/collection1/conf/schema12.xml | 9 +-
.../solr/collection1/conf/schema15.xml | 14 -
.../solr/collection1/conf/schemasurround.xml | 14 -
.../conf/solrconfig-update-processor-chains.xml | 2 +-
.../org/apache/solr/BasicFunctionalityTest.java | 2 +-
.../apache/solr/cloud/DeleteReplicaTest.java | 2 +-
.../solr/cloud/DocValuesNotIndexedTest.java | 3 +-
.../cloud/FullSolrCloudDistribCmdsTest.java | 3 +-
.../solr/cloud/LIRRollingUpdatesTest.java | 6 +-
.../org/apache/solr/cloud/MoveReplicaTest.java | 14 +-
.../solr/cloud/SaslZkACLProviderTest.java | 1 +
.../cloud/TestCloudJSONFacetJoinDomain.java | 8 +-
.../solr/cloud/TestCloudJSONFacetSKG.java | 4 +-
.../apache/solr/cloud/TestWithCollection.java | 1 +
.../org/apache/solr/cloud/ZkShardTermsTest.java | 1 +
.../cloud/api/collections/ShardSplitTest.java | 2 +
.../collections/TestHdfsCloudBackupRestore.java | 2 +-
.../AutoAddReplicasIntegrationTest.java | 3 +-
.../ScheduledMaintenanceTriggerTest.java | 23 +-
.../ScheduledTriggerIntegrationTest.java | 3 +-
.../SearchRateTriggerIntegrationTest.java | 1 +
.../sim/TestSimTriggerIntegration.java | 5 +-
.../FieldAnalysisRequestHandlerTest.java | 2 +-
.../org/apache/solr/handler/TestSQLHandler.java | 1 +
.../admin/MetricsHistoryHandlerTest.java | 1 +
.../PhrasesIdentificationComponentTest.java | 2 +-
.../handler/component/StatsComponentTest.java | 10 +-
.../reporters/solr/SolrCloudReportersTest.java | 3 +-
.../apache/solr/request/SimpleFacetsTest.java | 2 +-
.../schema/TestFieldCollectionResource.java | 10 +-
.../solr/rest/schema/TestFieldTypeResource.java | 3 +-
.../solr/schema/CurrencyFieldTypeTest.java | 2 +-
.../org/apache/solr/schema/TestPointFields.java | 2 +-
.../solr/schema/TestSortableTextField.java | 4 +-
.../search/CurrencyRangeFacetCloudTest.java | 2 +-
.../apache/solr/search/QueryEqualityTest.java | 2 +-
.../solr/search/TestCollapseQParserPlugin.java | 18 +-
.../org/apache/solr/search/TestRangeQuery.java | 2 +-
.../search/function/SortByFunctionTest.java | 8 +-
.../solr/util/TestMaxTokenLenTokenizer.java | 20 +-
.../src/common-query-parameters.adoc | 2 +-
solr/solr-ref-guide/src/language-analysis.adoc | 13 +-
.../src/local-parameters-in-queries.adoc | 2 +-
.../src/solrcloud-autoscaling-overview.adoc | 25 +-
...olrcloud-autoscaling-policy-preferences.adoc | 236 +++++----
.../tools/CheckLinksAndAnchors.java | 2 +-
.../solrj/request/CollectionAdminRequest.java | 5 +
.../solrj/solr/collection1/conf/schema-sql.xml | 13 -
.../solrj/solr/collection1/conf/schema.xml | 13 -
.../solr/configsets/streaming/conf/schema.xml | 11 -
.../client/solrj/impl/CloudSolrClientTest.java | 3 +-
.../solr/client/solrj/io/graph/GraphTest.java | 2 +-
.../solr/BaseDistributedSearchTestCase.java | 2 +-
122 files changed, 1816 insertions(+), 1310 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4867bb84/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 983617f,cb7f9cb..8369fcc
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@@ -159,97 -306,36 +159,95 @@@ public class SolrCmdDistributor
+ req.node.getUrl() + " retry:"
+ req.retries + " " + req.cmd + " params:" + req.uReq.getParams());
}
-
- if (isCommit) {
- // a commit using ConncurrentUpdateSolrServer is not async,
- // so we make it async to prevent commits from happening
- // serially across multiple nodes
- pending.add(completionService.submit(() -> {
- doRequest(req);
- return null;
- }));
+
+ try {
+ req.uReq.setBasePath(req.node.getUrl());
+ if (req.synchronous) {
+ NamedList rsp = client.request(req.uReq);
+ req.trackRequestResult(rsp, true);
+ pendingTasksPhaser.arriveAndDeregister();
+ } else {
+ //TODO write add cmds in single outputstream
+ client.request(req.uReq, null, new Http2SolrClient.OnComplete<NamedList>() {
+ @Override
+ public void onSuccess(NamedList result) {
+ req.trackRequestResult(result, true);
+ pendingTasksPhaser.arriveAndDeregister();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ handleAndRetry(req, t, isCommit);
+ }
+ });
+ }
+ } catch (Exception e) {
+ handleAndRetry(req, e, isCommit);
+ }
+ }
+
+ private void handleAndRetry(Req req, Throwable t, boolean isCommit) {
+ SolrException.log(log, t);
+ Error error = new Error();
+ error.t = t;
+ error.req = req;
+ if (t instanceof SolrException) {
+ error.statusCode = ((SolrException) t).code();
+ }
+ if (checkRetry(error)) {
+ submit0(req, isCommit);
} else {
- doRequest(req);
+ req.trackRequestResult(null, false);
+ allErrors.add(error);
+ pendingTasksPhaser.arriveAndDeregister();
}
}
-
- private void doRequest(final Req req) {
+
+ private boolean checkRetry(Error err) {
+ log.info("SolrCmdDistributor got error", err);
try {
- String oldNodeUrl = err.req.node.getUrl();
-
- SolrClient solrClient = clients.getSolrClient(req);
- solrClient.request(req.uReq);
- } catch (Exception e) {
- SolrException.log(log, e);
- Error error = new Error();
- error.e = e;
- error.req = req;
- if (e instanceof SolrException) {
- error.statusCode = ((SolrException) e).code();
+ /*
+ * if this is a retryable request we may want to retry, depending on the error we received and
+ * the number of times we have already retried
+ */
+ boolean isRetry = err.req.shouldRetry(err);
+ if (testing_errorHook != null) Diagnostics.call(testing_errorHook, err.t);
+
+ // this can happen in certain situations such as close
+ if (isRetry) {
+ err.req.retries++;
+
+ if (err.req.node instanceof ForwardNode) {
+ SolrException.log(SolrCmdDistributor.log, "forwarding update to "
- + oldNodeUrl + " failed - retrying ... retries: "
++ + err.req.node.getUrl() + " failed - retrying ... retries: "
+ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
+ + err.req.cmd.toString() + " params:"
+ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.t);
+ } else {
+ SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
- + oldNodeUrl + " failed - retrying ... retries: "
++ + err.req.node.getUrl() + " failed - retrying ... retries: "
+ + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
+ + err.req.cmd.toString() + " params:"
+ + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.t);
+ }
+ try {
+ Thread.sleep(retryPause); //TODO: Do we want this wait for every error?
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ log.warn(null, e);
+ }
+ return true;
+ } else {
+ return false;
}
- errors.add(error);
+ } catch (Exception e) {
+ // continue on
+ log.error("Unexpected Error while doing request retries", e);
+ // avoid infinite loop
+ return false;
}
}
-
+
public static class Req {
public Node node;
public UpdateRequest uReq;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4867bb84/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
----------------------------------------------------------------------
[05/29] lucene-solr:jira/http2: SOLR-12773: Fix typos widely spread
around codebase
Posted by da...@apache.org.
SOLR-12773: Fix typos widely spread around codebase
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b2b597b0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b2b597b0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b2b597b0
Branch: refs/heads/jira/http2
Commit: b2b597b0383eec0b1bf4059922cf367fdee2be21
Parents: 910a1c0
Author: Erick Erickson <Er...@gmail.com>
Authored: Sun Sep 16 09:46:20 2018 -0700
Committer: Erick Erickson <Er...@gmail.com>
Committed: Sun Sep 16 09:46:20 2018 -0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 2 +-
.../queries/function/valuesource/MultiFunction.java | 4 ++--
solr/CHANGES.txt | 2 +-
.../org/apache/solr/handler/component/StatsField.java | 4 ++--
.../java/org/apache/solr/schema/CurrencyFieldType.java | 8 ++++----
solr/core/src/java/org/apache/solr/schema/FieldType.java | 4 ++--
.../core/src/java/org/apache/solr/schema/IndexSchema.java | 2 +-
.../processor/CloneFieldUpdateProcessorFactory.java | 4 ++--
.../update/processor/TolerantUpdateProcessorFactory.java | 2 +-
.../conf/solrconfig-update-processor-chains.xml | 2 +-
.../src/test/org/apache/solr/BasicFunctionalityTest.java | 2 +-
.../apache/solr/cloud/TestCloudJSONFacetJoinDomain.java | 8 ++++----
.../test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java | 4 ++--
.../component/PhrasesIdentificationComponentTest.java | 2 +-
.../apache/solr/handler/component/StatsComponentTest.java | 10 +++++-----
.../test/org/apache/solr/request/SimpleFacetsTest.java | 2 +-
.../org/apache/solr/schema/CurrencyFieldTypeTest.java | 2 +-
.../src/test/org/apache/solr/schema/TestPointFields.java | 2 +-
.../org/apache/solr/schema/TestSortableTextField.java | 4 ++--
.../apache/solr/search/CurrencyRangeFacetCloudTest.java | 2 +-
.../org/apache/solr/search/TestCollapseQParserPlugin.java | 2 +-
.../src/test/org/apache/solr/search/TestRangeQuery.java | 2 +-
.../apache/solr/search/function/SortByFunctionTest.java | 8 ++++----
solr/solr-ref-guide/src/common-query-parameters.adoc | 2 +-
solr/solr-ref-guide/src/local-parameters-in-queries.adoc | 2 +-
solr/solr-ref-guide/tools/CheckLinksAndAnchors.java | 2 +-
.../org/apache/solr/BaseDistributedSearchTestCase.java | 2 +-
27 files changed, 46 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 045a1fa..6b8cb67 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -3199,7 +3199,7 @@ Other
* LUCENE-6845: SpanScorer is now merged into Spans (Alan Woodward, David Smiley)
-* LUCENE-6887: DefaultSimilarity is deprecated, use ClassicSimilarity for equivilent behavior,
+* LUCENE-6887: DefaultSimilarity is deprecated, use ClassicSimilarity for equivalent behavior,
or consider switching to BM25Similarity which will become the new default in Lucene 6.0 (hossman)
* LUCENE-6893: factor out CorePlusQueriesParser from CorePlusExtensionsParser
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
----------------------------------------------------------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
index d7db435..591aa2c 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFunction.java
@@ -73,7 +73,7 @@ public abstract class MultiFunction extends ValueSource {
}
/**
- * Equivilent the the {@code FunctionValues[]} method with the same name, but optimized for
+ * Equivalent to the {@code FunctionValues[]} method with the same name, but optimized for
* dealing with exactly 2 arguments.
*
* @return true if <em>both</em> of the specified <code>values</code>
@@ -85,7 +85,7 @@ public abstract class MultiFunction extends ValueSource {
}
/**
- * Equivilent the the {@code FunctionValues[]} method with the same name, but optimized for
+ * Equivalent to the {@code FunctionValues[]} method with the same name, but optimized for
* dealing with exactly 2 arguments.
*
* @return true if <em>either</em> of the specified <code>values</code>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3c5eb2d..8873834 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -16790,7 +16790,7 @@ the canonical format when generating dates from "Date Math" strings
(particularly as it pertains to milliseconds ending in trailing zeros).
As a result equivalent dates could not always be compared properly.
This problem is corrected in Solr 1.3, but DateField users that might
-have been affected by indexing inconsistent formats of equivilent
+have been affected by indexing inconsistent formats of equivalent
dates (ie: 1995-12-31T23:59:59Z vs 1995-12-31T23:59:59.000Z) may want
to consider reindexing to correct these inconsistencies. Users who
depend on some of the the "broken" behavior of DateField in Solr 1.2
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/java/org/apache/solr/handler/component/StatsField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java
index 1d0ba69..aa9dd35 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/StatsField.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/StatsField.java
@@ -180,7 +180,7 @@ public class StatsField {
}
/**
- * the equivilent stats if "calcdistinct" is specified
+ * the equivalent stats if "calcdistinct" is specified
* @see Stat#countDistinct
* @see Stat#distinctValues
*/
@@ -639,7 +639,7 @@ public class StatsField {
final NumberType hashableNumType = getHashableNumericType(field);
// some sane defaults
- int log2m = 13; // roughly equivilent to "cardinality='0.33'"
+ int log2m = 13; // roughly equivalent to "cardinality='0.33'"
int regwidth = 6; // with decent hash, this is plenty for all valid long hashes
if (NumberType.FLOAT.equals(hashableNumType) || NumberType.INTEGER.equals(hashableNumType)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java b/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java
index 481db59..2b3b555 100644
--- a/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/CurrencyFieldType.java
@@ -264,10 +264,10 @@ public class CurrencyFieldType extends FieldType implements SchemaAware, Resourc
* <p>
* For example: If the default Currency specified for a field is
* <code>USD</code>, then the values returned by this value source would
- * represent the equivilent number of "cents" (ie: value in dollars * 100)
+ * represent the equivalent number of "cents" (ie: value in dollars * 100)
* after converting each document's native currency to USD -- because the
* default fractional digits for <code>USD</code> is "<code>2</code>".
- * So for a document whose indexed value was currently equivilent to
+ * So for a document whose indexed value was currently equivalent to
* "<code>5.43,USD</code>" using the the exchange provider for this field,
* this ValueSource would return a value of "<code>543</code>"
* </p>
@@ -294,9 +294,9 @@ public class CurrencyFieldType extends FieldType implements SchemaAware, Resourc
* <p>
* For example: If the <code>targetCurrencyCode</code> param is set to
* <code>USD</code>, then the values returned by this value source would
- * represent the equivilent number of dollars after converting each
+ * represent the equivalent number of dollars after converting each
* document's raw value to <code>USD</code>. So for a document whose
- * indexed value was currently equivilent to "<code>5.43,USD</code>"
+ * indexed value was currently equivalent to "<code>5.43,USD</code>"
* using the the exchange provider for this field, this ValueSource would
* return a value of "<code>5.43</code>"
* </p>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/java/org/apache/solr/schema/FieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java
index 9dcca24..41eb460 100644
--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java
@@ -1250,7 +1250,7 @@ public abstract class FieldType extends FieldProperties {
/**
* The appropriate <code>SortedSetSelector.Type</code> option for this <code>MultiValueSelector</code>,
- * may be null if there is no equivilent
+ * may be null if there is no equivalent
*/
public SortedSetSelector.Type getSortedSetSelectorType() {
return sType;
@@ -1258,7 +1258,7 @@ public abstract class FieldType extends FieldProperties {
/**
* The appropriate <code>SortedNumericSelector.Type</code> option for this <code>MultiValueSelector</code>,
- * may be null if there is no equivilent
+ * may be null if there is no equivalent
*/
public SortedNumericSelector.Type getSortedNumericSelectorType() {
return nType;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
index 1f67976..f233717 100644
--- a/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
+++ b/solr/core/src/java/org/apache/solr/schema/IndexSchema.java
@@ -360,7 +360,7 @@ public class IndexSchema {
for (FieldInfo f : MultiFields.getMergedFieldInfos(reader)) {
if (f.getDocValuesType() == DocValuesType.NONE) {
// we have a field (of some kind) in the reader w/o DocValues
- // if we have an equivilent indexed=true field in the schema, trust it's uninversion type (if any)
+ // if we have an equivalent indexed=true field in the schema, trust it's uninversion type (if any)
final SchemaField sf = getFieldOrNull(f.name);
if (sf != null && sf.indexed()) {
final UninvertingReader.Type type = sf.getType().getUninversionType(sf);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
index 7357a19..f2be2a1 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/CloneFieldUpdateProcessorFactory.java
@@ -129,10 +129,10 @@ import org.slf4j.LoggerFactory;
* <code>fieldRegex</code> selector and a destination <code>pattern</code>, a "short hand" syntax
* is support for convinience: The <code>pattern</code> and <code>replacement</code> may be specified
* at the top level, omitting <code>source</code> and <code>dest</code> declarations completely, and
- * the <code>pattern</code> will be used to construct an equivilent <code>source</code> selector internally.
+ * the <code>pattern</code> will be used to construct an equivalent <code>source</code> selector internally.
* </p>
* <p>
- * For example, both of the following configurations are equivilent:
+ * For example, both of the following configurations are equivalent:
* </p>
* <pre class="prettyprint">
* <!-- full syntax -->
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
index 207ec3d..fc46ecc 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/TolerantUpdateProcessorFactory.java
@@ -136,7 +136,7 @@ public class TolerantUpdateProcessorFactory extends UpdateRequestProcessorFactor
throw new SolrException(ErrorCode.BAD_REQUEST, "'"+MAX_ERRORS_PARAM + "' must either be non-negative, or -1 to indicate 'unlimiited': " + maxErrors);
}
- // NOTE: even if 0==maxErrors, we still inject processor into chain so respones has expected header info
+ // NOTE: even if 0==maxErrors, we still inject processor into chain so responses has expected header info
return new TolerantUpdateProcessor(req, rsp, next, maxErrors, distribPhase);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml
index f22354e..01e1a5f 100644
--- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml
@@ -487,7 +487,7 @@
<str name="replacement">key_feat$1</str>
</lst>
</processor>
- <!-- equivilent sugar syntax to above, with slightly diff destination name -->
+ <!-- equivalent sugar syntax to above, with slightly diff destination name -->
<processor class="solr.processor.CloneFieldUpdateProcessorFactory">
<str name="pattern">^feat(.*)s$</str>
<str name="replacement">best_feat$1</str>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
index 18c4f69..fc69509 100644
--- a/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
+++ b/solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
@@ -878,7 +878,7 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
assertU(adoc("id", "6", "bday", "NOW+2YEARS"));
assertU(commit());
- // a ridiculoulsy long date math expression that's still equivilent to july4
+ // a ridiculoulsy long date math expression that's still equivalent to july4
final StringBuilder july4Long = new StringBuilder(july4);
final int iters = atLeast(10);
for (int i = 0; i < iters; i++) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
index 65a871ee..150a08d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
@@ -55,7 +55,7 @@ import org.slf4j.LoggerFactory;
* Tests randomized JSON Facets, sometimes using query 'join' domain transfers and/or domain 'filter' options
* </p>
* <p>
- * The results of each facet constraint count will be compared with a verification query using an equivilent filter
+ * The results of each facet constraint count will be compared with a verification query using an equivalent filter
* </p>
*
* @see TestCloudPivotFacet
@@ -422,7 +422,7 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
/**
* Given a set of (potentially nested) term facets, and a base query string, asserts that
* the actual counts returned when executing that query with those facets match the expected results
- * of filtering on the equivilent facet terms+domain
+ * of filtering on the equivalent facet terms+domain
*/
private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
Map<String,TermFacet> expected,
@@ -465,7 +465,7 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
/**
* Recursive Helper method that walks the actual facet response, comparing the counts to the expected output
- * based on the equivilent filters generated from the original TermFacet.
+ * based on the equivalent filters generated from the original TermFacet.
*/
private void assertFacetCountsAreCorrect(final AtomicInteger maxBucketsToCheck,
final Map<String,TermFacet> expected,
@@ -781,7 +781,7 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
/**
* Given some original SolrParams, returns new SolrParams where the original "q" param is wrapped
- * as needed to apply the equivilent transformation to a query as this domain would to a facet
+ * as needed to apply the equivalent transformation to a query as this domain would to a facet
*/
public SolrParams applyDomainToQuery(String safeKey, SolrParams in) {
assert null == in.get(safeKey); // shouldn't be possible if every facet uses a unique key string
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
index 9da510d..24384cc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetSKG.java
@@ -305,7 +305,7 @@ public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
/**
* Given a set of term facets, and top level query strings, asserts that
* the SKG stats for each facet term returned when executing that query with those foreground/background
- * queries match the expected results of executing the equivilent queries in isolation.
+ * queries match the expected results of executing the equivalent queries in isolation.
*
* @see #verifySKGResults
*/
@@ -355,7 +355,7 @@ public class TestCloudJSONFacetSKG extends SolrCloudTestCase {
/**
* Recursive helper method that walks the actual facet response, comparing the SKG results to
- * the expected output based on the equivilent filters generated from the original TermFacet.
+ * the expected output based on the equivalent filters generated from the original TermFacet.
*/
private void assertFacetSKGsAreCorrect(final AtomicInteger maxBucketsToCheck,
final Map<String,TermFacet> expected,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java
index c8d9edf..61e4404 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/PhrasesIdentificationComponentTest.java
@@ -161,7 +161,7 @@ public class PhrasesIdentificationComponentTest extends SolrTestCaseJ4 {
}
public void testWhiteBoxPhraseParsingShortInput() throws Exception {
- // for input this short, either of these fields should be (mostly) equivilent
+ // for input this short, either of these fields should be (mostly) equivalent
final Map<String,Integer> fields = new TreeMap<>();
fields.put("multigrams_body", 7);
fields.put("multigrams_body_short", 3);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
index d5c7472..6d8cfd4 100644
--- a/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/component/StatsComponentTest.java
@@ -211,7 +211,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
// we should be able to compute exact same stats for a field even
// when we specify it using the "field()" function, or use other
- // identify equivilent functions
+ // identify equivalent functions
for (String param : new String[] {
// bare
"{!key="+key+" ex=key_ex_tag}" + f,
@@ -543,7 +543,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
// we should be able to compute exact same stats for a field even
// when we specify it using the "field()" function, or use other
- // identify equivilent functions
+ // identify equivalent functions
for (String param : new String[] {
// bare
"{!key="+key+" ex=key_ex_tag}" + f,
@@ -628,7 +628,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
// we should be able to compute exact same stats & stats.facet for a field even
// when we specify it using the "field()" function, or use other
- // identify equivilent functions
+ // identify equivalent functions
for (String param : new String[] {
// bare
"{!key="+f+" ex=key_ex_tag}" + f,
@@ -1697,7 +1697,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
"need to note in upgrade instructions and probably adjust hueristic",
8, HLL.MAXIMUM_REGWIDTH_PARAM);
- // all of these should produce equivilent HLLOptions (Long, Double, or String using defaults)
+ // all of these should produce equivalent HLLOptions (Long, Double, or String using defaults)
SolrParams[] longDefaultParams = new SolrParams[] {
// basic usage
params("cardinality","true"),
@@ -1733,7 +1733,7 @@ public class StatsComponentTest extends SolrTestCaseJ4 {
}
- // all of these should produce equivilent HLLOptions (Int, Float, or ValueSource using defaults)
+ // all of these should produce equivalent HLLOptions (Int, Float, or ValueSource using defaults)
SolrParams[] intDefaultParams = new SolrParams[] {
// basic usage
params("cardinality","true"),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java
index 49b70b9..e3100e2 100644
--- a/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java
+++ b/solr/core/src/test/org/apache/solr/request/SimpleFacetsTest.java
@@ -3040,7 +3040,7 @@ public class SimpleFacetsTest extends SolrTestCaseJ4 {
,"*[count(//lst[@name='facet_ranges']/lst[@name='" + field + "']/lst[@name='between'])=0]"
);
- // these should have equivilent behavior (multivalued 'other' param: top level vs local)
+ // these should have equivalent behavior (multivalued 'other' param: top level vs local)
for (SolrQueryRequest req : new SolrQueryRequest[] {
req("q", "id_i1:[42 TO 47]"
,"facet","true"
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTypeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTypeTest.java b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTypeTest.java
index c50f5c5..1959b3b 100644
--- a/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTypeTest.java
+++ b/solr/core/src/test/org/apache/solr/schema/CurrencyFieldTypeTest.java
@@ -486,7 +486,7 @@ public class CurrencyFieldTypeTest extends SolrTestCaseJ4 {
clearIndex();
// NOTE: in our test conversions EUR uses an asynetric echange rate
- // these are the equivilent values when converting to: USD EUR GBP
+ // these are the equivalent values when converting to: USD EUR GBP
assertU(adoc("id", "" + 1, fieldName, "10.00,USD")); // 10.00,USD 25.00,EUR 5.00,GBP
assertU(adoc("id", "" + 2, fieldName, "15.00,EUR")); // 7.50,USD 15.00,EUR 7.50,GBP
assertU(adoc("id", "" + 3, fieldName, "6.00,GBP")); // 12.00,USD 12.00,EUR 6.00,GBP
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
index fc0d7ad..4f41f7f 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestPointFields.java
@@ -3893,7 +3893,7 @@ public class TestPointFields extends SolrTestCaseJ4 {
for (Object value : values) {
// ideally we should require that all input values be diff forms of the same logical value
- // (ie '"42"' vs 'new Integer(42)') and assert that each produces an equivilent list of IndexableField objects
+ // (ie '"42"' vs 'new Integer(42)') and assert that each produces an equivalent list of IndexableField objects
// but that doesn't seem to work -- appears not all IndexableField classes override Object.equals?
final List<IndexableField> result = callAndCheckCreateFields(fieldName, pointType, value);
assertNotNull(value + " => null", result);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/schema/TestSortableTextField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestSortableTextField.java b/solr/core/src/test/org/apache/solr/schema/TestSortableTextField.java
index d89a0d1..7128129 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestSortableTextField.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestSortableTextField.java
@@ -295,7 +295,7 @@ public class TestSortableTextField extends SolrTestCaseJ4 {
for (String searchF : Arrays.asList("whitespace_stxt", "whitespace_plain_txt",
"whitespace_max3_stxt", "whitespace_max6_stxt",
"whitespace_max0_stxt", "whitespace_maxNeg_stxt")) {
- // maxChars of 0 or neg should be equivilent to no max at all
+ // maxChars of 0 or neg should be equivalent to no max at all
for (String sortF : Arrays.asList("whitespace_stxt", "whitespace_plain_str",
"whitespace_max0_stxt", "whitespace_maxNeg_stxt")) {
@@ -484,7 +484,7 @@ public class TestSortableTextField extends SolrTestCaseJ4 {
* tests that a SortableTextField using KeywordTokenzier (w/docValues) behaves exactly the same as
* StrFields that it's copied to for quering and sorting
*/
- public void testRandomStrEquivilentBehavior() throws Exception {
+ public void testRandomStrEquivalentBehavior() throws Exception {
final List<String> test_fields = Arrays.asList("keyword_stxt", "keyword_dv_stxt",
"keyword_s_dv", "keyword_s");
// we use embedded client instead of assertQ: we want to compare the responses from multiple requests
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/search/CurrencyRangeFacetCloudTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/CurrencyRangeFacetCloudTest.java b/solr/core/src/test/org/apache/solr/search/CurrencyRangeFacetCloudTest.java
index 362940a..50ece14 100644
--- a/solr/core/src/test/org/apache/solr/search/CurrencyRangeFacetCloudTest.java
+++ b/solr/core/src/test/org/apache/solr/search/CurrencyRangeFacetCloudTest.java
@@ -51,7 +51,7 @@ public class CurrencyRangeFacetCloudTest extends SolrCloudTestCase {
private static final List<String> STR_VALS = Arrays.asList("x0", "x1", "x2");
// NOTE: in our test conversions EUR uses an asynetric echange rate
- // these are the equivilent values relative to: USD EUR GBP
+ // these are the equivalent values relative to: USD EUR GBP
private static final List<String> VALUES = Arrays.asList("10.00,USD", // 10.00,USD 25.00,EUR 5.00,GBP
"15.00,EUR", // 7.50,USD 15.00,EUR 7.50,GBP
"6.00,GBP", // 12.00,USD 12.00,EUR 6.00,GBP
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
index 01f852e..d0521d3 100644
--- a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
@@ -710,7 +710,7 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
// multiple params for picking groupHead should work as long as only one is non-null
// sort used
for (SolrParams collapse : new SolrParams[] {
- // these should all be equivilently valid
+ // these should all be equally valid
params("fq", "{!collapse field="+group+" nullPolicy=collapse sort='test_i asc'"+hint+"}"),
params("fq", "{!collapse field="+group+" nullPolicy=collapse min='' sort='test_i asc'"+hint+"}"),
params("fq", "{!collapse field="+group+" nullPolicy=collapse max='' sort='test_i asc'"+hint+"}"),
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java
index 763c8d5..e5f1887 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRangeQuery.java
@@ -243,7 +243,7 @@ public class TestRangeQuery extends SolrTestCaseJ4 {
"foo_ti","foo_tl","foo_tf","foo_td" };
// NOTE: foo_s supports ranges, but for the arrays below we are only
- // interested in fields that support *equivilent* ranges -- strings
+ // interested in fields that support *equivalent* ranges -- strings
// are not ordered the same as ints/longs, so we can't test the ranges
// for equivilence across diff fields.
//
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
index 658a257..1ddd76f 100644
--- a/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
+++ b/solr/core/src/test/org/apache/solr/search/function/SortByFunctionTest.java
@@ -159,14 +159,14 @@ public class SortByFunctionTest extends SolrTestCaseJ4 {
// then we lose precision in sorting; so lets sanity check that our JVM isn't doing something wacky
// in converstion that violates the principle of the test
- assertEquals("WTF? small longs cast to double aren't equivilent?",
+ assertEquals("WTF? small longs cast to double aren't equivalent?",
(double)A, (double)B, 0.0D);
- assertEquals("WTF? small longs cast to double aren't equivilent?",
+ assertEquals("WTF? small longs cast to double aren't equivalent?",
(double)A, (double)C, 0.0D);
- assertEquals("WTF? big longs cast to double aren't equivilent?",
+ assertEquals("WTF? big longs cast to double aren't equivalent?",
(double)Z, (double)Y, 0.0D);
- assertEquals("WTF? big longs cast to double aren't equivilent?",
+ assertEquals("WTF? big longs cast to double aren't equivalent?",
(double)Z, (double)X, 0.0D);
int docId = 0;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/solr-ref-guide/src/common-query-parameters.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/common-query-parameters.adoc b/solr/solr-ref-guide/src/common-query-parameters.adoc
index fe2956e..4994015 100644
--- a/solr/solr-ref-guide/src/common-query-parameters.adoc
+++ b/solr/solr-ref-guide/src/common-query-parameters.adoc
@@ -41,7 +41,7 @@ Solr can sort query responses according to:
* A single-valued TextField that uses an analyzer (such as the KeywordTokenizer) that produces only a single term per document. TextField does not support `docValues="true"`, but a DocValue-like structure will be built on the fly at runtime.
** *NOTE:* If you want to be able to sort on a field whose contents you want to tokenize to facilitate searching, <<copying-fields.adoc#copying-fields,use a `copyField` directive>> in the the Schema to clone the field. Then search on the field and sort on its clone.
-In the case of primitive fields, or SortableTextFields, that are `multiValued="true"` the representative value used for each doc when sorting depends on the sort direction: The minimum value in each document is used for ascending (`asc`) sorting, while the maximal value in each document is used for descending (`desc`) sorting. This default behavior is equivilent to explicitly sorting using the 2 argument `<<function-queries.adoc#field-function,field()>>` function: `sort=field(name,min) asc` and `sort=field(name,max) desc`
+In the case of primitive fields, or SortableTextFields, that are `multiValued="true"` the representative value used for each doc when sorting depends on the sort direction: The minimum value in each document is used for ascending (`asc`) sorting, while the maximal value in each document is used for descending (`desc`) sorting. This default behavior is equivalent to explicitly sorting using the 2 argument `<<function-queries.adoc#field-function,field()>>` function: `sort=field(name,min) asc` and `sort=field(name,max) desc`
The table below explains how Solr responds to various settings of the `sort` parameter.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/solr-ref-guide/src/local-parameters-in-queries.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/local-parameters-in-queries.adoc b/solr/solr-ref-guide/src/local-parameters-in-queries.adoc
index 747b436..d6bdaff 100644
--- a/solr/solr-ref-guide/src/local-parameters-in-queries.adoc
+++ b/solr/solr-ref-guide/src/local-parameters-in-queries.adoc
@@ -56,7 +56,7 @@ If no "type" is specified (either explicitly or implicitly) then the <<the-stand
`fq={!df=summary}solr rocks`
-is equivilent to:
+is equivalent to:
`fq={!type=lucene df=summary}solr rocks`
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/solr-ref-guide/tools/CheckLinksAndAnchors.java
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/tools/CheckLinksAndAnchors.java b/solr/solr-ref-guide/tools/CheckLinksAndAnchors.java
index f58739a..49fad33 100644
--- a/solr/solr-ref-guide/tools/CheckLinksAndAnchors.java
+++ b/solr/solr-ref-guide/tools/CheckLinksAndAnchors.java
@@ -75,7 +75,7 @@ import org.jsoup.select.NodeVisitor;
*
* <p>
* This tool parses the generated HTML site, looking for these situations in order to fail the build, since
- * (depending on the type of check) these situations will result in inconsistent/broken HTML, or equivilent
+ * (depending on the type of check) these situations will result in inconsistent/broken HTML, or equivalent
* problems in the generated PDF.
* </p>
* <p>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2b597b0/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
index 4a67581..e9428ca 100644
--- a/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/BaseDistributedSearchTestCase.java
@@ -491,7 +491,7 @@ public abstract class BaseDistributedSearchTestCase extends SolrTestCaseJ4 {
/**
* Indexes the document in both the control client and the specified client asserting
- * that the respones are equivilent
+ * that the responses are equivalent
*/
protected UpdateResponse indexDoc(SolrClient client, SolrParams params, SolrInputDocument... sdocs) throws IOException, SolrServerException {
UpdateResponse controlRsp = add(controlClient, params, sdocs);
[23/29] lucene-solr:jira/http2: Start the splitting after the
triggers are in place.
Posted by da...@apache.org.
Start the splitting after the triggers are in place.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/87cd5b14
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/87cd5b14
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/87cd5b14
Branch: refs/heads/jira/http2
Commit: 87cd5b14aece34e1ee84c253d670f622029cd6c5
Parents: eb0fcec
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Sep 20 10:35:42 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Sep 20 10:36:24 2018 +0200
----------------------------------------------------------------------
.../autoscaling/ScheduledMaintenanceTriggerTest.java | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/87cd5b14/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
index 56896a7..840c0bd 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
@@ -161,12 +161,6 @@ public class ScheduledMaintenanceTriggerTest extends SolrCloudTestCase {
CloudTestUtils.waitForState(cloudManager, "failed to create " + collection1, collection1,
CloudTestUtils.clusterShape(1, 1));
- CollectionAdminRequest.SplitShard split1 = CollectionAdminRequest.splitShard(collection1)
- .setShardName("shard1");
- split1.process(solrClient);
- CloudTestUtils.waitForState(cloudManager, "failed to split " + collection1, collection1,
- CloudTestUtils.clusterShape(3, 1, true, true));
-
String setListenerCommand = "{" +
"'set-listener' : " +
"{" +
@@ -197,8 +191,17 @@ public class ScheduledMaintenanceTriggerTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
+
boolean await = listenerCreated.await(10, TimeUnit.SECONDS);
assertTrue("listener not created in time", await);
+
+ CollectionAdminRequest.SplitShard split1 = CollectionAdminRequest.splitShard(collection1)
+ .setShardName("shard1");
+ split1.process(solrClient);
+ CloudTestUtils.waitForState(cloudManager, "failed to split " + collection1, collection1,
+ CloudTestUtils.clusterShape(3, 1, true, true));
+
+
await = triggerFired.await(60, TimeUnit.SECONDS);
assertTrue("cleanup action didn't run", await);
[24/29] lucene-solr:jira/http2: iSOLR-12080: Improve error handling
of MoveReplicaCmd. Improve the test stability by avoiding killing overseer.
Posted by da...@apache.org.
iSOLR-12080: Improve error handling of MoveReplicaCmd. Improve the test stability
by avoiding killing overseer.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/52bdcf6b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/52bdcf6b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/52bdcf6b
Branch: refs/heads/jira/http2
Commit: 52bdcf6bb0b7645da55124298f6d268fe8f4a77a
Parents: 87cd5b1
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Sep 20 12:36:52 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Sep 20 12:36:52 2018 +0200
----------------------------------------------------------------------
.../solr/cloud/api/collections/MoveReplicaCmd.java | 13 +++++++++++--
.../test/org/apache/solr/cloud/MoveReplicaTest.java | 14 ++++++++++++--
2 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52bdcf6b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
index 4a9bd59..2df0f77 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java
@@ -169,7 +169,12 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
removeReplicasProps.getProperties().put(CoreAdminParams.DELETE_INDEX, false);
if (async != null) removeReplicasProps.getProperties().put(ASYNC, async);
NamedList deleteResult = new NamedList();
- ocmh.deleteReplica(clusterState, removeReplicasProps, deleteResult, null);
+ try {
+ ocmh.deleteReplica(clusterState, removeReplicasProps, deleteResult, null);
+ } catch (SolrException e) {
+ // assume this failed completely so there's nothing to roll back
+ deleteResult.add("failure", e.toString());
+ }
if (deleteResult.get("failure") != null) {
String errorString = String.format(Locale.ROOT, "Failed to cleanup replica collection=%s shard=%s name=%s, failure=%s",
coll.getName(), slice.getName(), replica.getName(), deleteResult.get("failure"));
@@ -304,7 +309,11 @@ public class MoveReplicaCmd implements OverseerCollectionMessageHandler.Cmd {
REPLICA_PROP, replica.getName());
if (async != null) removeReplicasProps.getProperties().put(ASYNC, async);
NamedList deleteResult = new NamedList();
- ocmh.deleteReplica(clusterState, removeReplicasProps, deleteResult, null);
+ try {
+ ocmh.deleteReplica(clusterState, removeReplicasProps, deleteResult, null);
+ } catch (SolrException e) {
+ deleteResult.add("failure", e.toString());
+ }
if (deleteResult.get("failure") != null) {
String errorString = String.format(Locale.ROOT, "Failed to cleanup replica collection=%s shard=%s name=%s, failure=%s",
coll.getName(), slice.getName(), replica.getName(), deleteResult.get("failure"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/52bdcf6b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
index 2b3be30..8f0f0e3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java
@@ -250,13 +250,22 @@ public class MoveReplicaTest extends SolrCloudTestCase {
addDocs(coll, 100);
- Replica replica = getRandomReplica(coll, cloudClient);
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+
+ // don't kill overseer in this test
+ Replica replica;
+ int count = 10;
+ do {
+ replica = getRandomReplica(coll, cloudClient);
+ } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0);
+ assertNotNull("could not find non-overseer replica???", replica);
Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
ArrayList<String> l = new ArrayList<>(liveNodes);
Collections.shuffle(l, random());
String targetNode = null;
for (String node : liveNodes) {
- if (!replica.getNodeName().equals(node)) {
+ if (!replica.getNodeName().equals(node) && !overseerLeader.equals(node)) {
targetNode = node;
break;
}
@@ -271,6 +280,7 @@ public class MoveReplicaTest extends SolrCloudTestCase {
for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) {
cluster.stopJettySolrRunner(i);
+ break;
}
}
CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus(asyncId);
[12/29] lucene-solr:jira/http2: SOLR-12771: Improve Autoscaling
Policy and Preferences documentation
Posted by da...@apache.org.
SOLR-12771: Improve Autoscaling Policy and Preferences documentation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ca86de17
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ca86de17
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ca86de17
Branch: refs/heads/jira/http2
Commit: ca86de17841804e7a458fcd76d4b87a43aefd550
Parents: 895bff4
Author: Steve Rowe <sa...@apache.org>
Authored: Mon Sep 17 14:57:52 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Mon Sep 17 14:57:52 2018 -0400
----------------------------------------------------------------------
.../src/solrcloud-autoscaling-overview.adoc | 25 +-
...olrcloud-autoscaling-policy-preferences.adoc | 236 +++++++++++--------
2 files changed, 139 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca86de17/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
index d36d11b..518d9ee 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-overview.adoc
@@ -49,34 +49,23 @@ The selection of the node that will host the new replica is made according to th
== Cluster Preferences
-Cluster preferences, as the name suggests, apply to all cluster management operations regardless of which collection they affect.
+Cluster preferences allow you to tell Solr how to assess system load on each node. This information is used to guide selection of the node(s) on which cluster management operations will be performed.
-A preference is a set of conditions that help Solr select nodes that either maximize or minimize given metrics. For example, a preference such as `{minimize:cores}` will help Solr select nodes such that the number of cores on each node is minimized. We write cluster preferences in a way that reduces the overall load on the system. You can add more than one preferences to break ties.
+In general, when an operation increases replica counts, the *least loaded* <<solrcloud-autoscaling-policy-preferences.adoc#node-selector,qualified node>> will be chosen, and when the operation reduces replica counts, the *most loaded* qualified node will be chosen.
-The default cluster preferences consist of the above example (`{minimize:cores}`) which is to minimize the number of cores on all nodes.
+The default cluster preferences are `[{minimize:cores}]`, which tells Solr to minimize the number of cores on all nodes. In this case, the least loaded node is the one with the fewest cores.
You can learn more about preferences in the <<solrcloud-autoscaling-policy-preferences.adoc#solrcloud-autoscaling-policy-preferences,Autoscaling Cluster Preferences>> section.
== Cluster Policy
-A cluster policy is a set of conditions that a node, shard, or collection must satisfy before it can be chosen as the target of a cluster management operation. These conditions are applied across the cluster regardless of the collection being managed. For example, the condition `{"cores":"<10", "node":"#ANY"}` means that any node must have less than 10 Solr cores in total, regardless of which collection they belong to.
+A cluster policy is a set of rules that a node, shard, or collection must satisfy before it can be chosen as the target of a cluster management operation. These rules are applied across the cluster regardless of the collection being managed. For example, the rule `{"cores":"<10", "node":"#ANY"}` means that any node must have less than 10 Solr cores in total, regardless of which collection they belong to.
-There are many metrics on which the condition can be based, e.g., system load average, heap usage, free disk space, etc. The full list of supported metrics can be found in the section describing <<solrcloud-autoscaling-policy-preferences.adoc#policy-rule-attributes,Autoscaling Policy Rule Attributes>>.
+There are many metrics on which the rule can be based, e.g., system load average, heap usage, free disk space, etc. The full list of supported metrics can be found in the section describing <<solrcloud-autoscaling-policy-preferences.adoc#policy-rule-attributes,Autoscaling Policy Rule Attributes>>.
-When a node, shard, or collection does not satisfy the policy, we call it a *violation*. Solr ensures that cluster management operations minimize the number of violations.
+When a node, shard, or collection does not satisfy a policy rule, we call it a *violation*. By default, cluster management operations will fail if there is even one violation. You can allow operations to succeed in the face of a violation by marking the corresponding rule with <<solrcloud-autoscaling-policy-preferences.adoc#rule-strictness,`"strict":false`>>. When you do this, Solr ensures that cluster management operations minimize the number of violations.
-== Collection-Specific Policies
-
-A collection may need conditions in addition to those specified in the cluster policy. In such cases, we can create named policies that can be used for specific collections. Firstly, we can use the `set-policy` API to create a new policy and then specify the `policy=<policy_name>` parameter to the CREATE command of the Collection API:
-
-[source,text]
-/admin/collections?action=CREATE&name=coll1&numShards=1&replicationFactor=2&policy=policy1
-
-The above CREATE collection command will associate a policy named `policy1` with the collection named `coll1`. Only a single policy may be associated with a collection.
-
-Note that the collection-specific policy is applied *in addition to* the cluster policy, i.e., it is not an override but an augmentation. Therefore the collection will follow all conditions laid out in the cluster preferences, cluster policy, and the policy named `policy1`.
-
-You can learn more about collection-specific policies in the section <<solrcloud-autoscaling-policy-preferences.adoc#collection-specific-policy,Defining Collection-Specific Policies>>.
+Solr also supports <<solrcloud-autoscaling-policy-preferences.adoc#collection-specific-policy,collection-specific policies>>, which operate in tandem with the cluster policy.
== Triggers
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ca86de17/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
index d3b7be3..1cd72c6 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-policy-preferences.adoc
@@ -20,15 +20,15 @@
The autoscaling policy and preferences are a set of rules and sorting preferences that help Solr select the target of cluster management operations so the overall load on the cluster remains balanced.
-Solr consults the configured policy and preferences when performing <<Commands That Use Autoscaling Policy and Preferences,Collections API commands>> in all contexts: manual, e.g., using `bin/solr`; semi-automatic, via the <<solrcloud-autoscaling-api.adoc#suggestions-api,Suggestions API>> or the Admin UI's <<suggestions-screen.adoc#suggestions-screen,Suggestions Screen>>; or fully automatic, via configured <<solrcloud-autoscaling-triggers.adoc#solrcloud-autoscaling-triggers,Triggers>>.
+The configured autoscaling policy and preferences are used by <<collections-api.adoc#collections-api,Collections API commands>> in all contexts: manual, for example using `bin/solr` to create a collection; semi-automatic, via the <<solrcloud-autoscaling-api.adoc#suggestions-api,Suggestions API>> or the Admin UI's <<suggestions-screen.adoc#suggestions-screen,Suggestions Screen>>; or fully automatic, via configured <<solrcloud-autoscaling-triggers.adoc#solrcloud-autoscaling-triggers,Triggers>>.
-See the section <<Manual Collection Creation with a Policy>> for an example of how policy and preferences affect replica placement.
+See the section <<Example: Manual Collection Creation with a Policy>> for an example of how policy and preferences affect replica placement.
== Cluster Preferences Specification
A preference is a hint to Solr on how to sort nodes based on their utilization. The default cluster preference is to sort by the total number of Solr cores (or replicas) hosted by a node. Therefore, by default, when selecting a node to which to add a replica, Solr can apply the preferences and choose the node with the fewest cores.
-More than one preference can be added to break ties. For example, we may choose to use free disk space to break ties if the number of cores on two nodes are the same. The node with the higher free disk space can be chosen as the target of the cluster operation.
+More than one preference can be added to break ties. For example, we may choose to use free disk space to break ties if the number of cores on two nodes is the same. The node with the higher free disk space can be chosen as the target of the cluster operation.
Each preference takes the following form:
@@ -102,21 +102,21 @@ Policy rules can be either global or per-collection:
Global rules have three parts:
* <<Node Selector>>
-* <<Core Count Constraint>>
-* <<Rule Strictness>>
+* <<Core Count Constraint>> (`"cores": "..."`)
+* <<Rule Strictness>> (optional)
Per-collection rules have four parts:
* <<Node Selector>>
* <<Replica Selector and Rule Evaluation Context>>
-* <<Replica Count Constraint>>
-* <<Rule Strictness>>
+* <<Replica Count Constraint>> (`"replica": "..."`)
+* <<Rule Strictness>> (optional)
==== Node Selector
-Rule evaluation is restricted to node(s) matching the value of one of the following attributes: `node`, `port`, `ip_\*`, `sysprop.*`, or `diskType`. For replica/core count constraints other than `#EQUAL`, a condition specified in one of the following attributes may instead be used to select nodes: `freedisk`, `host`, `sysLoadAvg`, `heapUsage`, `nodeRole`, or `metrics.*`.
+Rule evaluation is restricted to node(s) matching the value of one of the following attributes: <<node-attribute,`node`>>, <<port-attribute,`port`>>, <<ip-attributes,`ip_\*`>>, <<sysprop-attribute,`sysprop.*`>>, or <<diskType-attribute,`diskType`>>. For replica/core count constraints other than `#EQUAL`, a condition specified in one of the following attributes may instead be used to select nodes: <<freedisk-attribute,`freedisk`>>, <<host-attribute,`host`>>, <<sysLoadAvg-attribute,`sysLoadAvg`>>, <<heapUsage-attribute,`heapUsage`>>, <<nodeRole-attribute,`nodeRole`>>, or <<metrics-attribute,`metrics.*`>>.
-Except for `node`, the attributes above cause selected nodes to be partitioned into node groups. A node group is referred to as a "bucket". Those attributes usable with the `#EQUAL` directive may define buckets either via the value `#EACH` or an array `["value1", ...]` (a subset of all possible values); in both cases, each node is placed in the bucket corresponding to the matching attribute value.
+Except for `node`, the attributes above cause selected nodes to be partitioned into node groups. A node group is referred to as a "bucket". Those attributes usable with the `#EQUAL` directive may define buckets either via the special function <<each-function,`#EACH`>> or an <<array-operator,array>> `["value1", ...]` (a subset of all possible values); in both cases, each node is placed in the bucket corresponding to the matching attribute value.
The `node` attribute always places each selected node into its own bucket, regardless of the attribute value's form (`#ANY`, `node-name`, or `["node1-name", ...]`).
@@ -126,99 +126,126 @@ Replica and core count constraints, described below, are evaluated against the t
The `cores` attribute value can be specified in one of the following forms:
-* the `#EQUAL` directive, which will cause cores to be distributed equally among the nodes specified via the rule's <<Node Selector>>.
-* a constraint on the core count on each <<Node Selector,selected node>>, specified as one of:
-** an integer value (e.g., `2`), a lower bound (e.g., `>0`), or an upper bound (e.g., `<3`)
-** a decimal value, interpreted as an acceptable range of core counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g., `1.6`: `1` or `2` is acceptable, and `2` is preferred)
-** a range of acceptable core counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g., `3-5`)
-** a percentage (e.g., `33%`), which is multiplied by the number of cores in the cluster at runtime. This value is then interpreted as described above for literal decimal values.
+* <<equal-function,`#EQUAL`>>: distribute all cores equally across all the <<Node Selector,selected nodes>>.
+* a constraint on the core count on each <<Node Selector,selected node>>; see <<Specifying Replica and Core Count Constraints>>.
==== Replica Selector and Rule Evaluation Context
-Rule evaluation can be restricted to replicas that meet any combination of the following conditions:
+Rule evaluation can be restricted to replicas that meet any combination of conditions specified with the following attributes:
-* The replica is of a shard belonging to the collection specified in the `collection` attribute value. (Not usable with per-collection policies.)
-* The replica is of a shard specified in the `shard` attribute value.
-* The replica has the replica type specified in the `type` attribute value (`NRT`, `TLOG`, or `PULL`).
+* <<collection-attribute,`collection`>>: The replica is of a shard belonging to the collection specified in the attribute value. (Not usable with <<collection-specific-policy,collection-specific policies>>.)
+* <<shard-attribute,`shard`>>: The replica is of the shard named in the attribute value.
+* <<type-attribute,`type`>>: The replica has the specified replica type (`NRT`, `TLOG`, or `PULL`).
If none of the above attributes is specified, then the rule is evaluated separately for each collection against all types of replicas of all shards.
-Specifying `#EACH` as the `shard` attribute value causes the rule to be evaluated separately for each shard of each collection.
+Specifying <<each-function,`#EACH`>> as the `shard` attribute value causes the rule to be evaluated separately for each shard of each collection.
==== Replica Count Constraint
The `replica` attribute value can be specified in one of the following forms:
-* `#ALL`: All <<Replica Selector and Rule Evaluation Context,selected replicas>> will be placed on the <<Node Selector,selected nodes>>.
-* `#EQUAL`: Distribute <<Replica Selector and Rule Evaluation Context,selected replicas>> evenly among all the <<Node Selector,selected nodes>>.
-* a constraint on the replica count on each <<Node Selector,selected node>>, specified as one of:
-** an integer value (e.g., `2`), a lower bound (e.g., `>0`), or an upper bound (e.g., `<3`)
-** a decimal value, interpreted as an acceptable range of replica counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g., `1.6`: `1` or `2` is acceptable, and `2` is preferred)
-** a range of acceptable replica counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g., `3-5`)
-** a percentage (e.g., `33%`), which is multiplied by the number of <<Replica Selector and Rule Evaluation Context,selected replicas>> at runtime. This value is then interpreted as described above for literal decimal values.
+* <<all-function,`#ALL`>>: All <<Replica Selector and Rule Evaluation Context,selected replicas>> will be placed on the <<Node Selector,selected nodes>>.
+* <<equal-function,`#EQUAL`>>: Distribute <<Replica Selector and Rule Evaluation Context,selected replicas>> equally across all the <<Node Selector,selected nodes>>.
+* a constraint on the replica count on each <<Node Selector,selected node>>; see <<Specifying Replica and Core Count Constraints>>.
-==== Rule Strictness
+==== Specifying Replica and Core Count Constraints
+
+<<Replica Count Constraint,Replica count constraints>> (`"replica":"..."`) and <<Core Count Constraint,core count constraints>> (`"cores":"..."`) allow specification of acceptable counts for replicas (cores tied to a collection) and cores (regardless of the collection to which they belong), respectively.
-By default, the rule must be satisfied, and if it can't, then no action will be taken.
+You can specify one of the following as the value of a `replica` and `cores` policy rule attribute:
-If the `strict` attribute value is specified as `false`, Solr tries to satisfy the rule on a best effort basis, but if no node can satisfy the rule then any node may be chosen.
+* an exact integer (e.g. `2`)
+* an exclusive lower integer bound (e.g. `>0`)
+* an exclusive upper integer bound (e.g. `<3`)
+* a decimal value, interpreted as an acceptable range of core counts, from the floor of the value to the ceiling of the value, with the system preferring the rounded value (e.g., `1.6`: `1` or `2` is acceptable, and `2` is preferred)
+* a <<range-operator,range>> of acceptable replica/core counts, as inclusive lower and upper integer bounds separated by a hyphen (e.g. `3-5`)
+* a percentage (e.g. `33%`), which is multiplied at runtime either by the number of <<Replica Selector and Rule Evaluation Context,selected replicas>> (for a `replica` constraint) or the number of cores in the cluster (for a `cores` constraint). This value is then interpreted as described above for a literal decimal value.
+
+NOTE: Using an exact integer value for count constraints is of limited utility, since collection or cluster changes could quickly invalidate them. For example, attempting to add a third replica to each shard of a collection on a two-node cluster with policy rule `{"replica":1, "shard":"#EACH", "node":"#ANY"}` would cause a violation, since at least one node would have to host more than one replica. Percentage rules are less brittle. Rewriting the rule as `{"replica":"50%", "shard":"#EACH", "node":"#ANY"}` eliminates the violation: `50% of 3 replicas = 1.5 replicas per node`, meaning that it's acceptable for a node to host either one or two replicas of each shard.
=== Policy Rule Attributes
-A policy rule can have the following attributes:
+==== Rule Strictness
+
+This attribute is usable in all rules:
+
+`strict`::
+An optional boolean value. The default is `true`. If true, the rule must be satisfied; if the rule is not satisfied, the resulting violation will cause the cluster management operation to fail. If false, Solr tries to satisfy the rule on a best effort basis, but if no node can satisfy the rule, the cluster management operation will not fail, and any node may be chosen. If multiple rules declared to be `strict:false` can not be satisfied by some nodes, then a node will be chosen such that the number of such violations is minimized.
+==== Global Rule Attributes
+
+[[cores-attribute]]
`cores`::
-This is a required attribute for <<Rule Types,global rules>>. It can only be used along with the `node` attribute and no other. See <<Core Count Constraint>> for possible attribute values.
+The number of cores that must exist to satisfy the rule. This is a required attribute for <<Rule Types,global policy rules>>. The <<node-attribute,`node` attribute>> must also be specified, and the only other allowed attribute is the optional <<Rule Strictness,`strict` attribute>>. See <<Core Count Constraint>> for possible attribute values.
+
+==== Per-collection Rule Attributes
+
+The following attributes are usable with <<Rule Types,per-collection policy rules>>, in addition to the attributes in the <<Node Selection Attributes>> section below:
+[[collection-attribute]]
`collection`::
The name of the collection to which the policy rule should apply. If omitted, the rule applies to all collections. This attribute is optional.
+[[shard-attribute]]
`shard`::
-The name of the shard to which the policy rule should apply. If omitted, the rule is applied for all shards in the collection. It supports a special value `#EACH` which means that the rule is applied for each shard in the collection.
+The name of the shard to which the policy rule should apply. If omitted, the rule is applied for all shards in the collection. It supports the special function <<each-function,`#EACH`>> which means that the rule is applied for each shard in the collection.
+[[type-attribute]]
`type`::
The type of the replica to which the policy rule should apply. If omitted, the rule is applied for all replica types of this collection/shard. The allowed values are `NRT`, `TLOG` and `PULL`
+[[replica-attribute]]
`replica`::
-This is a required attribute for <<Rule Types,per-collection rules>>. The number of replicas that must exist to satisfy the rule. See <<Replica Count Constraint>> for possible attribute values.
+The number of replicas that must exist to satisfy the rule. This is a required attribute for <<Rule Types,per-collection rules>>. See <<Replica Count Constraint>> for possible attribute values.
-`strict`::
-An optional boolean value. The default is `true`. If true, the rule must be satisfied. If false, Solr tries to satisfy the rule on a best effort basis but if no node can satisfy the rule then any node may be chosen.
+==== Node Selection Attributes
-One and only one of the following attributes can be specified in addition to the above attributes:
+One and only one of the following attributes can be specified in addition to the above attributes. See the <<Node Selector>> section for more information:
+[[node-attribute]]
`node`::
-The name of the node to which the rule should apply.
+The name of the node to which the rule should apply. The <<not-operator,`!` (not) operator>> or the <<array-operator,array operator>> or the <<any-function,`#ANY` function>> may be used in this attribute's value.
+[[port-attribute]]
`port`::
-The port of the node to which the rule should apply.
+The port of the node to which the rule should apply. The <<not-operator,`!` (not) operator>> or the <<array-operator,array operator>> may be used in this attribute's value.
+[[freedisk-attribute]]
`freedisk`::
-The free disk space in gigabytes of the node. This must be a positive 64-bit integer value.
+The free disk space in gigabytes of the node. This must be a positive 64-bit integer value, or a <<percentage-function,percentage>>. If a percentage is specified, either an upper or lower bound may also be specified using the `<` or `>` operators, respectively, e.g. `>50%`, `<25%`.
+[[host-attribute]]
`host`::
The host name of the node.
+[[sysLoadAvg-attribute]]
`sysLoadAvg`::
The system load average of the node as reported by the Metrics API under the key `solr.jvm/os.systemLoadAverage`. This is floating point value between 0 and 1.
+[[heapUsage-attribute]]
`heapUsage`::
The heap usage of the node as reported by the Metrics API under the key `solr.jvm/memory.heap.usage`. This is floating point value between 0 and 1.
+[[nodeRole-attribute]]
`nodeRole`::
The role of the node. The only supported value currently is `overseer`.
+[[ip-attributes]]
`ip_1, ip_2, ip_3, ip_4`::
-The least significant to most significant segments of IP address. For example, for an IP address `192.168.1.2`, `ip_1 = 2`, `ip_2 = 1`, `ip_3 = 168`, `ip_4 = 192`.
+The least significant to most significant segments of IP address. For example, for an IP address `192.168.1.2`, `"ip_1":"2", "ip_2":"1", "ip_3":"168", "ip_4":"192"`. The <<array-operator,array operator>> may be used in any of these attributes' values.
+[[sysprop-attribute]]
`sysprop.<system_property_name>`::
-Any arbitrary system property set on the node on startup.
+Any arbitrary system property set on the node on startup. The <<not-operator,`!` (not) operator>> or the <<array-operator,array operator>> may be used in this attribute's value.
+[[metrics-attribute]]
`metrics:<full-path-to-the metric>`::
Any arbitrary metric. For example, `metrics:solr.node:CONTAINER.fs.totalSpace`. Refer to the `key` parameter in the <<metrics-reporting.adoc#metrics-reporting, Metrics API>> section.
+[[diskType-attribute]]
`diskType`::
-The type of disk drive being used for Solr's `coreRootDirectory`. The only two supported values are `rotational` and `ssd`. Refer to `coreRootDirectory` parameter in the <<format-of-solr-xml.adoc#solr-xml-parameters, Solr.xml Parameters>> section.
+The type of disk drive being used for Solr's `coreRootDirectory`. The only two supported values are `rotational` and `ssd`. Refer to `coreRootDirectory` parameter in the <<format-of-solr-xml.adoc#solr-xml-parameters, Solr.xml Parameters>> section. The <<not-operator,`!` (not) operator>> or the <<array-operator,array operator>> may be used in this attribute's value.
+
Its value is fetched from the Metrics API with the key named `solr.node:CONTAINER.fs.coreRoot.spins`. The disk type is auto-detected by Lucene using various heuristics and it is not guaranteed to be correct across all platforms or operating systems. Refer to the <<taking-solr-to-production.adoc#dynamic-defaults-for-concurrentmergescheduler, Dynamic defaults for ConcurrentMergeScheduler>> section for more details.
@@ -226,154 +253,167 @@ Its value is fetched from the Metrics API with the key named `solr.node:CONTAINE
Each attribute in the policy may specify one of the following operators along with the value.
+* No operator means equality
* `<`: Less than
* `>`: Greater than
-* `!`: Not
-* Range operator `(-)`: a value such as `"3-5"` means a value between 3 to 5 (inclusive). This is only supported in the `replica` and `cores` attributes.
-* Array operator `[]`: e.g., `sysprop.zone= ["east","west","apac"]`. This is equivalent to having multiple rules with each of these values. This can be used in the following attributes:
-** `sysprop.*`
-** `port`
-** `ip_*`
-** `node`
-** `diskType`
-* None means equal
+* [[not-operator]]`!`: Not
+* [[range-operator]]Range operator `(-)`: a value such as `"3-5"` means a value between 3 to 5 (inclusive). This is only supported in the <<replica-attribute,`replica`>> and <<cores-attribute,`cores`>> attributes.
+* [[array-operator]]Array operator `[]`: e.g., `sysprop.zone= ["east","west","apac"]`. This is equivalent to having multiple rules with each of these values. This can be used in the following attributes:
+** <<node-attribute,`node`>>
+** <<sysprop-attribute,`sysprop.*`>>
+** <<port-attribute,`port`>>
+** <<ip-attributes,`ip_*`>>
+** <<diskType-attribute,`diskType`>>
==== Special Functions
This supports values calculated at the time of execution.
-* `%` : A certain percentage of the value. This is supported by the following attributes:
-** `replica`
-** `cores`
-** `freedisk`
-* `#ALL` : This is applied to the `replica` attribute only. This means all replicas that meet the rule condition.
-* `#EQUAL`: This is applied to the `replica` and `cores` attributes only. This means an equal number of replicas/cores in each bucket. The buckets can be defined using an array operator (`[]`) or `#EACH`. The buckets can be defined using the following properties:
-** `node` \<- <<Rule Types,global rules>>, i.e., with the `cores` attribute, may only specify this attribute
-** `sysprop.*`
-** `port`
-** `diskType`
-** `ip_*`
+* [[percentage-function]]`%` : A certain percentage of the value. This is supported by the following attributes:
+** <<replica-attribute,`replica`>>
+** <<cores-attribute,`cores`>>
+** <<freedisk-attribute,`freedisk`>>
+* [[any-function]]`#ANY`: Applies to the <<node-attribute,`node` attribute>> only. This means the rule applies to any node.
+* [[all-function]]`#ALL`: Applies to the <<replica-attribute,`replica` attribute>> only. This means all replicas that meet the rule condition.
+* [[each-function]]`#EACH`: Applies to the <<shard-attribute,`shard` attribute>> (meaning the rule should be evaluated separately for each shard), and to the attributes used to define the buckets for the <<equal-function,#EQUAL function>> (meaning all possible values for the bucket-defining attribute).
+* [[equal-function]]`#EQUAL`: Applies to the <<replica-attribute,`replica`>> and <<cores-attribute,`cores`>> attributes only. This means an equal number of replicas/cores in each bucket. The buckets can be defined using the below attributes with a value that can either be <<each-function,`#EACH`>> or a list specified with the <<array-operator,array operator (`[]`)>>:
+** <<node-attribute,`node`>> \<- <<Rule Types,global rules>>, i.e. those with the <<cores-attribute,`cores` attribute>>, may only specify this attribute
+** <<sysprop-attribute,`sysprop.*`>>
+** <<port-attribute,`port`>>
+** <<diskType-attribute,`diskType`>>
+** <<ip-attributes,`ip_*`>>
+
=== Examples of Policy Rules
==== Limit Replica Placement
-Do not place more than one replica of the same shard on the same node:
+Do not place more than one replica of the same shard on the same node. The rule is evaluated separately for <<each-function,each>> shard in each collection. The rule is applied to <<any-function,any>> node.
[source,json]
{"replica": "<2", "shard": "#EACH", "node": "#ANY"}
==== Limit Cores per Node
-Do not place more than 10 cores in any node. This rule can only be added to the cluster policy because it mentions the `cores` attribute that is only applicable cluster-wide.
+Do not place more than 10 cores in <<any-function,any>> node. This rule can only be added to the cluster policy because it is a <<Rule Types,global rule>>.
[source,json]
{"cores": "<10", "node": "#ANY"}
==== Place Replicas Based on Port
-Place exactly 1 replica of each shard of collection `xyz` on a node running on port `8983`
+Place exactly 1 replica of <<each-function,each>> shard of collection `xyz` on a node running on port `8983`.
[source,json]
{"replica": 1, "shard": "#EACH", "collection": "xyz", "port": "8983"}
==== Place Replicas Based on a System Property
-Place all replicas on a node with system property `availability_zone=us-east-1a`.
+Place <<all-function,all>> replicas on nodes with system property `availability_zone=us-east-1a`.
[source,json]
{"replica": "#ALL", "sysprop.availability_zone": "us-east-1a"}
-===== Use Percentage
-
-====== Example 1
+==== Use Percentage
-Place roughly a maximum of a 3rd of the replicas of a shard in a node. In the following example, the value of `replica` is computed in real time:
+Place a maximum of (roughly) a third of the replicas of <<each-function,each>> shard in <<any-function,any>> node. In the following example, the value of `replica` is computed in real time as a percentage of the replicas of <<each-function,each>> shard of each collection:
[source,json]
{"replica": "33%", "shard": "#EACH", "node": "#ANY"}
If the number of replicas in a shard is `2`, `33% of 2 = 0.66`. This means a node may have a maximum of `1` and a minimum of `0` replicas of each shard.
-It is possible to get the same effect by hard coding the value of `replica` as follows:
+It is possible to get the same effect by hard coding the value of `replica` as a decimal value:
[source,json]
{"replica": 0.66, "shard": "#EACH", "node": "#ANY"}
-or using the range operator:
+or using the <<range-operator,range operator>>:
[source,json]
{"replica": "0-1", "shard": "#EACH", "node": "#ANY"}
-====== Example 2
+==== Multiple Percentage Rules
-Distribute replicas across datacenters `east` and `west` at a `1:2` ratio:
+Distribute replicas of <<each-function,each>> shard of each collection across datacenters `east` and `west` at a `1:2` ratio:
[source,json]
+----
{"replica": "33%", "shard": "#EACH", "sysprop.zone": "east"}
{"replica": "66%", "shard": "#EACH", "sysprop.zone": "west"}
+----
-For the above rule to work, all nodes must the started with a system property called `"zone"`
+For the above rules to work, all nodes must the started with a system property called `"zone"`
==== Distribute Replicas Equally in Each Zone
+For <<each-function,each>> shard of each collection, distribute replicas equally across the `east` and `west` zones.
+
[source,json]
{"replica": "#EQUAL", "shard": "#EACH", "sysprop.zone": ["east", "west"]}
-or simply as follows
+Distribute replicas equally across <<each-function,each>> zone.
[source,json]
{"replica": "#EQUAL", "shard": "#EACH", "sysprop.zone": "#EACH"}
==== Place Replicas Based on Node Role
-Do not place any replica on a node which has the overseer role. Note that the role is added by the `addRole` collection API. It is *not* automatically the node which is currently the overseer.
+Do not place any replica on any node that has the overseer role. Note that the role is added by the `addRole` collection API. It is *not* automatically the node which is currently the overseer.
[source,json]
{"replica": 0, "nodeRole": "overseer"}
==== Place Replicas Based on Free Disk
-Place all replicas in nodes with freedisk more than 500GB.
+Place <<all-function,all>> replicas in nodes where <<freedisk-attribute,freedisk>> is greater than 500GB.
[source,json]
{"replica": "#ALL", "freedisk": ">500"}
-Keep all replicas in nodes with over `50%` freedisk
+Keep all replicas in nodes where <<freedisk-attribute,freedisk>> percentage is greater than `50%`.
+
[source,json]
{"replica": "#ALL", "freedisk": ">50%"}
==== Try to Place Replicas Based on Free Disk
-Place all replicas in nodes with freedisk more than 500GB when possible. Here we use the strict keyword to signal that this rule is to be honored on a best effort basis.
+When possible, place <<all-function,all>> replicas in nodes where <<freedisk-attribute,freedisk>> is greater than 500GB. Here we use the <<Rule Strictness,`strict`>> attribute to signal that this rule is to be honored on a best effort basis.
[source,json]
-{"replica": "#ALL", "freedisk": ">500", "strict" : false}
+{"replica": "#ALL", "freedisk": ">500", "strict": false}
-==== Try to Place All Replicas of Type TLOG on Nodes with SSD Drives
+==== Place All Replicas of Type TLOG on Nodes with SSD Drives
[source,json]
-{ "replica": "#ALL","type" : "TLOG", "diskType" : "ssd" }
+{"replica": "#ALL", "type": "TLOG", "diskType": "ssd"}
-==== Try to Place All Replicas of Type PULL on Nodes with Rotational Disk Drives
+==== Place All Replicas of Type PULL on Nodes with Rotational Disk Drives
[source,json]
-{ "replica": "#ALL", "type" : "PULL" , "diskType" : "rotational"}
+{"replica": "#ALL", "type": "PULL", "diskType": "rotational"}
[[collection-specific-policy]]
== Defining Collection-Specific Policies
-By default, the cluster policy, if it exists, is used automatically for all collections in the cluster. However, we can create named policies which can be attached to a collection at the time of its creation by specifying the policy name along with a `policy` parameter.
+By default, the cluster policy, if it exists, is used automatically for all collections in the cluster. However, we can create named policies that can be attached to a collection at the time of its creation by specifying the policy name along with a `policy` parameter.
When a collection-specific policy is used, the rules in that policy are *appended* to the rules in the cluster policy and the combination of both are used. Therefore, it is recommended that you do not add rules to collection-specific policy that conflict with the ones in the cluster policy. Doing so will disqualify all nodes in the cluster from matching all criteria and make the policy useless.
-It is possible to override conditions specified in the cluster policy using collection-specific policy. For example, if a clause `{replica:'<3', node:'#ANY'}` is present in the cluster policy and the collection-specific policy has a clause `{replica:'<4', node:'#ANY'}`, the cluster policy is ignored in favor of the collection policy.
+It is possible to override rules specified in the cluster policy using collection-specific policy. For example, if a rule `{replica:'<3', node:'#ANY'}` is present in the cluster policy and the collection-specific policy has a rule `{replica:'<4', node:'#ANY'}`, the cluster policy is ignored in favor of the collection policy.
Also, if `maxShardsPerNode` is specified during the time of collection creation, then both `maxShardsPerNode` and the policy rules must be satisfied.
Some attributes such as `cores` can only be used in the cluster policy. See the section <<Policy Rule Attributes>> for details.
-== Manual Collection Creation with a Policy
+To create a new named policy, use the <<solrcloud-autoscaling-api.adoc#create-and-modify-collection-specific-policy,`set-policy` API>>. Once you have a named policy, you can specify the `policy=<policy_name>` parameter to the CREATE command of the Collection API:
+
+[source,text]
+/admin/collections?action=CREATE&name=coll1&numShards=1&replicationFactor=2&policy=policy1
+
+The above CREATE collection command will associate a policy named `policy1` with the collection named `coll1`. Only a single policy may be associated with a collection.
+
+== Example: Manual Collection Creation with a Policy
The starting state for this example is a Solr cluster with 3 nodes: "nodeA", "nodeB", and "nodeC". An existing 2-shard `FirstCollection` with a `replicationFactor` of 1 has one replica on "nodeB" and one on "nodeC". The default Autoscaling preferences are in effect:
@@ -395,23 +435,11 @@ http://localhost:8983/solr/admin/collections?action=CREATE&name=SecondCollection
For each of the two replicas to be created, each Solr node is tested, in order from least to most loaded: would all policy rules be satisfied if a replica were placed there using an ADDREPLICA sub-command?
* ADDREPLICA for `shard1`: According to the Autoscaling preferences, the least loaded node is the one with the fewest cores: "nodeA", because it hosts no cores, while the other two nodes each host one core. The test to place a replica here succeeds, because doing so causes no policy violations, since the core count after adding the replica would not exceed the configured maximum of 1. Because "nodeA" can host the first shard's replica, Solr skips testing of the other two nodes.
-* ADDREPLICA for `shard2`: After placing the `shard1` replica, all nodes would be equally loaded, since each would have one core. The test to place the `shard2` replica fails on each node, because placement would push the node over its maximum core count. Since there is no node that can host this replica, the CREATE command fails.
-
-Let's try again after increasing the maximum core count on all nodes to 2:
+* ADDREPLICA for `shard2`: After placing the `shard1` replica, all nodes would be equally loaded, since each would have one core. The test to place the `shard2` replica fails on each node, because placement would push the node over its maximum core count. This causes a policy violation.
+
+Since there is no node that can host a replica for `shard2` without causing a violation, the overall CREATE command fails. Let's try again after increasing the maximum core count on all nodes to 2:
[source,json]
[ {"cores": "<3", "node": "#ANY"} ]
After re-issuing the `SecondCollection` CREATE command, the replica for `shard1` will be placed on "nodeA": it's least loaded, so is tested first, and no policy violation will result from placement there. The `shard2` replica could be placed on any of the 3 nodes, since they're all equally loaded, and the chosen node will remain below its maximum core count after placement. The CREATE command succeeds.
-
-== Commands That Use Autoscaling Policy and Preferences
-
-The configured autoscaling policy and preferences are used by these <<collections-api.adoc#collections-api,Collections API>> commands:
-
-* CREATE
-* CREATESHARD
-* ADDREPLICA
-* RESTORE
-* SPLITSHARD
-* UTILIZENODE
-* MOVEREPLICA
[10/29] lucene-solr:jira/http2: LUCENE-8503: Call #getDelegate
instead of direct member access during unwrap
Posted by da...@apache.org.
LUCENE-8503: Call #getDelegate instead of direct member access during unwrap
Filter*Reader instances access the member or the delegate directly instead of
calling getDelegate(). In order to track access of the delegate these methods
should call #getDelegat()
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/44dbfaab
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/44dbfaab
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/44dbfaab
Branch: refs/heads/jira/http2
Commit: 44dbfaab19b5845f27d6174071bd3808756ff552
Parents: 36eae57
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Sep 17 10:39:41 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Sep 17 16:05:26 2018 +0200
----------------------------------------------------------------------
.../core/src/java/org/apache/lucene/index/FilterCodecReader.java | 4 ++--
.../src/java/org/apache/lucene/index/FilterDirectoryReader.java | 2 +-
.../core/src/java/org/apache/lucene/index/FilterLeafReader.java | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44dbfaab/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
index 0f8dcfc..ff3ea18 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
@@ -48,8 +48,8 @@ public abstract class FilterCodecReader extends CodecReader {
}
return reader;
}
- /**
- * The underlying CodecReader instance.
+ /**
+ * The underlying CodecReader instance.
*/
protected final CodecReader in;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44dbfaab/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
index 4a9cd68..70ecd12 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterDirectoryReader.java
@@ -37,7 +37,7 @@ public abstract class FilterDirectoryReader extends DirectoryReader {
* an instance of {@link FilterDirectoryReader}. */
public static DirectoryReader unwrap(DirectoryReader reader) {
while (reader instanceof FilterDirectoryReader) {
- reader = ((FilterDirectoryReader) reader).in;
+ reader = ((FilterDirectoryReader) reader).getDelegate();
}
return reader;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/44dbfaab/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
index 8e1f272..418e7a8 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java
@@ -44,7 +44,7 @@ public abstract class FilterLeafReader extends LeafReader {
* an instance of {@link FilterLeafReader}. */
public static LeafReader unwrap(LeafReader reader) {
while (reader instanceof FilterLeafReader) {
- reader = ((FilterLeafReader) reader).in;
+ reader = ((FilterLeafReader) reader).getDelegate();
}
return reader;
}
[14/29] lucene-solr:jira/http2: SOLR-12759: detect actual problem and
only then "assumeTrue" to bail
Posted by da...@apache.org.
SOLR-12759: detect actual problem and only then "assumeTrue" to bail
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5abee478
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5abee478
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5abee478
Branch: refs/heads/jira/http2
Commit: 5abee478b6c34cdb240674a9dc74830e476c11a5
Parents: 7d0a778
Author: David Smiley <ds...@apache.org>
Authored: Tue Sep 18 13:43:41 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Sep 18 13:43:41 2018 -0400
----------------------------------------------------------------------
.../handler/extraction/ExtractingRequestHandlerTest.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5abee478/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
index 852b7ce..85e1c78 100644
--- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
+++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
@@ -15,8 +15,11 @@
* limitations under the License.
*/
package org.apache.solr.handler.extraction;
+
import java.util.ArrayList;
import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
@@ -41,8 +44,12 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
- assumeFalse("SOLR-12759 JDK 11 (1st release) and Tika 1.x can result in extracting dates in a bad format.",
- System.getProperty("java.version").startsWith("11"));
+ // Is the JDK/env affected by a known bug?
+ if (!TimeZone.getDefault().getDisplayName(false, TimeZone.SHORT, Locale.US).matches("[A-Z]{3}([+-]\\d\\d(:\\d\\d)?)?")) {
+ assert System.getProperty("java.version").startsWith("11") : "Is some other JVM affected? Or bad regex?";
+ assumeTrue("SOLR-12759 JDK 11 (1st release) and Tika 1.x can result in extracting dates in a bad format.", false);
+ }
+
initCore("solrconfig.xml", "schema.xml", getFile("extraction/solr").getAbsolutePath());
}
[02/29] lucene-solr:jira/http2: SOLR-12551: fix Tika version in list
of major components in CHANGES.txt
Posted by da...@apache.org.
SOLR-12551: fix Tika version in list of major components in CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/39733cbf
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/39733cbf
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/39733cbf
Branch: refs/heads/jira/http2
Commit: 39733cbf5ff9d8587d418007bd13087a434a1aa2
Parents: a7a4f68
Author: Cassandra Targett <ct...@apache.org>
Authored: Fri Sep 14 08:55:53 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Fri Sep 14 08:55:53 2018 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/39733cbf/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 64b8e58..3c5eb2d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -22,7 +22,7 @@ Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this r
Versions of Major Components
---------------------
-Apache Tika 1.17
+Apache Tika 1.18
Carrot2 3.16.0
Velocity 1.7 and Velocity Tools 2.0
Apache ZooKeeper 3.4.11
@@ -97,7 +97,7 @@ Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this r
Versions of Major Components
---------------------
-Apache Tika 1.17
+Apache Tika 1.18
Carrot2 3.16.0
Velocity 1.7 and Velocity Tools 2.0
Apache ZooKeeper 3.4.11
@@ -441,7 +441,7 @@ Other Changes
* SOLR-12744: Improve logging messages and verbosity around recoveries (Cao Manh Dat, Varun Thacker)
-* SOLR-8742: In HdfsDirectoryTest replace RAMDirectory usages with ByteBuffersDirectory.
+* SOLR-8742: In HdfsDirectoryTest replace RAMDirectory usages with ByteBuffersDirectory.
(hossman, Mark Miller, Andrzej Bialecki, Steve Rowe)
================== 7.4.0 ==================
@@ -8246,8 +8246,8 @@ Bug Fixes
* SOLR-6693: bin\solr.cmd doesn't support 32-bit JRE/JDK running on Windows due to
parenthesis in JAVA_HOME. (Timothy Potter, Christopher Hewitt, Jan Høydahl)
-
-* SOLR-12662: Reproducing TestPolicy failures: NPE and NoClassDefFoundError.
+
+* SOLR-12662: Reproducing TestPolicy failures: NPE and NoClassDefFoundError.
(Steve Rowe)
Optimizations
[06/29] lucene-solr:jira/http2: LUCENE-8502: Allow access to delegate
in FilterCodecReader
Posted by da...@apache.org.
LUCENE-8502: Allow access to delegate in FilterCodecReader
FilterCodecReader doesn't allow access to it's delegate like other
filter readers. This adds a new getDelegate method to access the
wrapped reader.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6fac57d9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6fac57d9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6fac57d9
Branch: refs/heads/jira/http2
Commit: 6fac57d9ad4e2251118b400215b08901781c9696
Parents: b2b597b
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Sep 17 10:19:44 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Sep 17 11:02:22 2018 +0200
----------------------------------------------------------------------
.../apache/lucene/index/FilterCodecReader.java | 14 ++++++++++++++
.../lucene/index/TestFilterCodecReader.java | 18 ++++++++++++++++++
2 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6fac57d9/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
index 4187051..0f8dcfc 100644
--- a/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/FilterCodecReader.java
@@ -39,6 +39,15 @@ import org.apache.lucene.util.Bits;
* {@link #getCoreCacheHelper()} and {@link #getReaderCacheHelper()}.
*/
public abstract class FilterCodecReader extends CodecReader {
+
+ /** Get the wrapped instance by <code>reader</code> as long as this reader is
+ * an instance of {@link FilterCodecReader}. */
+ public static CodecReader unwrap(CodecReader reader) {
+ while (reader instanceof FilterCodecReader) {
+ reader = ((FilterCodecReader) reader).getDelegate();
+ }
+ return reader;
+ }
/**
* The underlying CodecReader instance.
*/
@@ -127,6 +136,11 @@ public abstract class FilterCodecReader extends CodecReader {
in.checkIntegrity();
}
+ /** Returns the wrapped {@link CodecReader}. */
+ public CodecReader getDelegate() {
+ return in;
+ }
+
/**
* Returns a filtered codec reader with the given live docs and numDocs.
*/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6fac57d9/lucene/core/src/test/org/apache/lucene/index/TestFilterCodecReader.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterCodecReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterCodecReader.java
index feb803f..53ca774 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestFilterCodecReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterCodecReader.java
@@ -16,8 +16,12 @@
*/
package org.apache.lucene.index;
+import java.io.IOException;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
public class TestFilterCodecReader extends LuceneTestCase {
@@ -27,6 +31,20 @@ public class TestFilterCodecReader extends LuceneTestCase {
implTestDeclaredMethodsOverridden(subClass.getSuperclass(), subClass);
}
+ public void testGetDelegate() throws IOException {
+ try (Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir,newIndexWriterConfig())) {
+ w.addDocument(new Document());
+ try (DirectoryReader reader = w.getReader()) {
+ FilterCodecReader r = FilterCodecReader.wrapLiveDocs((CodecReader) reader.getSequentialSubReaders().get(0),
+ null, 1);
+
+ assertSame(FilterCodecReader.unwrap(r), reader.getSequentialSubReaders().get(0));
+ assertSame(r.getDelegate(), reader.getSequentialSubReaders().get(0));
+ }
+ }
+ }
+
private void implTestDeclaredMethodsOverridden(Class<?> superClass, Class<?> subClass) throws Exception {
for (final Method superClassMethod : superClass.getDeclaredMethods()) {
final int modifiers = superClassMethod.getModifiers();
[25/29] lucene-solr:jira/http2: LUCENE-8498: Remove LowerCaseTokenizer
Posted by da...@apache.org.
LUCENE-8498: Remove LowerCaseTokenizer
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c0d29759
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c0d29759
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c0d29759
Branch: refs/heads/jira/http2
Commit: c0d2975970d3de8f5056a20504dec1431d455ab1
Parents: 52bdcf6
Author: Alan Woodward <ro...@apache.org>
Authored: Sat Sep 15 16:56:27 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Thu Sep 20 11:57:05 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
lucene/MIGRATE.txt | 11 ++
.../analysis/core/LowerCaseTokenizer.java | 72 -----------
.../core/LowerCaseTokenizerFactory.java | 75 ------------
.../lucene/analysis/core/SimpleAnalyzer.java | 4 +-
.../lucene/analysis/core/StopAnalyzer.java | 6 +-
.../lucene/analysis/util/CharTokenizer.java | 86 +------------
...apache.lucene.analysis.util.TokenizerFactory | 1 -
.../analysis/br/TestBrazilianAnalyzer.java | 7 +-
.../lucene/analysis/core/TestAnalyzers.java | 8 --
.../analysis/custom/TestCustomAnalyzer.java | 13 +-
.../lucene/analysis/de/TestGermanAnalyzer.java | 8 +-
.../standard/TestStandardFactories.java | 16 ---
.../analysis/util/TestCharTokenizers.java | 122 +++----------------
.../extraction/solr/collection1/conf/schema.xml | 15 ---
.../collection1/conf/schema-copyfield-test.xml | 14 ---
.../solr/collection1/conf/schema-folding.xml | 3 +-
.../solr/collection1/conf/schema-hash.xml | 13 --
.../collection1/conf/schema-required-fields.xml | 14 ---
.../solr/collection1/conf/schema-rest.xml | 9 +-
.../solr/collection1/conf/schema-sql.xml | 13 --
.../collection1/conf/schema-tokenizer-test.xml | 11 +-
.../test-files/solr/collection1/conf/schema.xml | 15 +--
.../solr/collection1/conf/schema12.xml | 9 +-
.../solr/collection1/conf/schema15.xml | 14 ---
.../solr/collection1/conf/schemasurround.xml | 14 ---
.../schema/TestFieldCollectionResource.java | 10 +-
.../solr/rest/schema/TestFieldTypeResource.java | 3 +-
.../solr/util/TestMaxTokenLenTokenizer.java | 20 +--
.../solrj/solr/collection1/conf/schema-sql.xml | 13 --
.../solrj/solr/collection1/conf/schema.xml | 13 --
.../solr/configsets/streaming/conf/schema.xml | 11 --
32 files changed, 78 insertions(+), 568 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index bd8c616..70badd8 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -81,6 +81,9 @@ API Changes
* LUCENE-8352: TokenStreamComponents is now final, and can take a Consumer<Reader>
in its constructor (Mark Harwood, Alan Woodward, Adrien Grand)
+* LUCENE-8498: LowerCaseTokenizer has been removed, and CharTokenizer no longer
+ takes a normalizer function. (Alan Woodward)
+
Changes in Runtime Behavior
* LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index 6008956..1b56b64 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -129,3 +129,14 @@ Most code should just require recompilation, though possibly requiring some adde
Instead of overriding TokenStreamComponents#setReader() to customise analyzer
initialisation, you should now pass a Consumer<Reader> instance to the
TokenStreamComponents constructor.
+
+## LowerCaseTokenizer and LowerCaseTokenizerFactory have been removed ##
+
+LowerCaseTokenizer combined tokenization and filtering in a way that broke token
+normalization, so they have been removed. Instead, use a LetterTokenizer followed by
+a LowerCaseFilter
+
+## CharTokenizer no longer takes a normalizer function ##
+
+CharTokenizer now only performs tokenization. To perform any type of filtering
+use a TokenFilter chain as you would with any other Tokenizer.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java
deleted file mode 100644
index 26b8747..0000000
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.core;
-
-
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.util.AttributeFactory;
-
-/**
- * LowerCaseTokenizer performs the function of LetterTokenizer
- * and LowerCaseFilter together. It divides text at non-letters and converts
- * them to lower case. While it is functionally equivalent to the combination
- * of LetterTokenizer and LowerCaseFilter, there is a performance advantage
- * to doing the two tasks at once, hence this (redundant) implementation.
- * <P>
- * Note: this does a decent job for most European languages, but does a terrible
- * job for some Asian languages, where words are not separated by spaces.
- * </p>
- */
-public final class LowerCaseTokenizer extends LetterTokenizer {
-
- /**
- * Construct a new LowerCaseTokenizer.
- */
- public LowerCaseTokenizer() {
- }
-
- /**
- * Construct a new LowerCaseTokenizer using a given
- * {@link org.apache.lucene.util.AttributeFactory}.
- *
- * @param factory
- * the attribute factory to use for this {@link Tokenizer}
- */
- public LowerCaseTokenizer(AttributeFactory factory) {
- super(factory);
- }
-
- /**
- * Construct a new LowerCaseTokenizer using a given
- * {@link org.apache.lucene.util.AttributeFactory}.
- *
- * @param factory the attribute factory to use for this {@link Tokenizer}
- * @param maxTokenLen maximum token length the tokenizer will emit.
- * Must be greater than 0 and less than MAX_TOKEN_LENGTH_LIMIT (1024*1024)
- * @throws IllegalArgumentException if maxTokenLen is invalid.
- */
- public LowerCaseTokenizer(AttributeFactory factory, int maxTokenLen) {
- super(factory, maxTokenLen);
- }
-
- /** Converts char to lower case
- * {@link Character#toLowerCase(int)}.*/
- @Override
- protected int normalize(int c) {
- return Character.toLowerCase(c);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java
deleted file mode 100644
index 44e2742..0000000
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.analysis.core;
-
-
-import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
-import org.apache.lucene.analysis.util.CharTokenizer;
-import org.apache.lucene.analysis.util.MultiTermAwareComponent;
-import org.apache.lucene.analysis.util.TokenizerFactory;
-import org.apache.lucene.util.AttributeFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.lucene.analysis.standard.StandardTokenizer.MAX_TOKEN_LENGTH_LIMIT;
-
-/**
- * Factory for {@link LowerCaseTokenizer}.
- * <pre class="prettyprint">
- * <fieldType name="text_lwrcase" class="solr.TextField" positionIncrementGap="100">
- * <analyzer>
- * <tokenizer class="solr.LowerCaseTokenizerFactory" maxTokenLen="256"/>
- * </analyzer>
- * </fieldType></pre>
- * <p>
- * Options:
- * <ul>
- * <li>maxTokenLen: max token length, should be greater than 0 and less than MAX_TOKEN_LENGTH_LIMIT (1024*1024).
- * It is rare to need to change this
- * else {@link CharTokenizer}::DEFAULT_MAX_WORD_LEN</li>
- * </ul>
- */
-public class LowerCaseTokenizerFactory extends TokenizerFactory implements MultiTermAwareComponent {
- private final int maxTokenLen;
-
- /**
- * Creates a new LowerCaseTokenizerFactory
- */
- public LowerCaseTokenizerFactory(Map<String, String> args) {
- super(args);
- maxTokenLen = getInt(args, "maxTokenLen", CharTokenizer.DEFAULT_MAX_WORD_LEN);
- if (maxTokenLen > MAX_TOKEN_LENGTH_LIMIT || maxTokenLen <= 0) {
- throw new IllegalArgumentException("maxTokenLen must be greater than 0 and less than " + MAX_TOKEN_LENGTH_LIMIT + " passed: " + maxTokenLen);
- }
- if (!args.isEmpty()) {
- throw new IllegalArgumentException("Unknown parameters: " + args);
- }
- }
-
- @Override
- public LowerCaseTokenizer create(AttributeFactory factory) {
- return new LowerCaseTokenizer(factory, maxTokenLen);
- }
-
- @Override
- public AbstractAnalysisFactory getMultiTermComponent() {
- Map<String,String> map = new HashMap<>(getOriginalArgs());
- map.remove("maxTokenLen"); //removing "maxTokenLen" argument for LowerCaseFilterFactory init
- return new LowerCaseFilterFactory(map);
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/SimpleAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/SimpleAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/SimpleAnalyzer.java
index 6e0f2f0..3fcb92c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/SimpleAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/SimpleAnalyzer.java
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.core;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/** An {@link Analyzer} that filters {@link LetterTokenizer}
* with {@link LowerCaseFilter}
@@ -34,7 +35,8 @@ public final class SimpleAnalyzer extends Analyzer {
@Override
protected TokenStreamComponents createComponents(final String fieldName) {
- return new TokenStreamComponents(new LowerCaseTokenizer());
+ Tokenizer tokenizer = new LetterTokenizer();
+ return new TokenStreamComponents(tokenizer, new LowerCaseFilter(tokenizer));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopAnalyzer.java
index cf7ecdd..dde74c0 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopAnalyzer.java
@@ -60,13 +60,13 @@ public final class StopAnalyzer extends StopwordAnalyzerBase {
* used to tokenize all the text in the provided {@link Reader}.
*
* @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
- * built from a {@link LowerCaseTokenizer} filtered with
+ * built from a {@link LetterTokenizer} filtered with
* {@link StopFilter}
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- final Tokenizer source = new LowerCaseTokenizer();
- return new TokenStreamComponents(source, new StopFilter(source, stopwords));
+ final Tokenizer source = new LetterTokenizer();
+ return new TokenStreamComponents(source, new StopFilter(new LowerCaseFilter(source), stopwords));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
index ff9d6ff..092d25d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
@@ -20,14 +20,11 @@ package org.apache.lucene.analysis.util;
import java.io.IOException;
import java.util.Objects;
import java.util.function.IntPredicate;
-import java.util.function.IntUnaryOperator;
-import org.apache.lucene.analysis.CharacterUtils.CharacterBuffer;
import org.apache.lucene.analysis.CharacterUtils;
-import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.CharacterUtils.CharacterBuffer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LetterTokenizer;
-import org.apache.lucene.analysis.core.LowerCaseTokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@@ -107,48 +104,12 @@ public abstract class CharTokenizer extends Tokenizer {
* </pre>
*/
public static CharTokenizer fromTokenCharPredicate(AttributeFactory factory, final IntPredicate tokenCharPredicate) {
- return fromTokenCharPredicate(factory, tokenCharPredicate, IntUnaryOperator.identity());
- }
-
- /**
- * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
- * The predicate should return {@code true} for all valid token characters.
- * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
- * <p>
- * This factory is intended to be used with lambdas or method references. E.g., an elegant way
- * to create an instance which behaves exactly as {@link LowerCaseTokenizer} is:
- * <pre class="prettyprint lang-java">
- * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(Character::isLetter, Character::toLowerCase);
- * </pre>
- */
- public static CharTokenizer fromTokenCharPredicate(final IntPredicate tokenCharPredicate, final IntUnaryOperator normalizer) {
- return fromTokenCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, tokenCharPredicate, normalizer);
- }
-
- /**
- * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
- * The predicate should return {@code true} for all valid token characters.
- * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
- * <p>
- * This factory is intended to be used with lambdas or method references. E.g., an elegant way
- * to create an instance which behaves exactly as {@link LowerCaseTokenizer} is:
- * <pre class="prettyprint lang-java">
- * Tokenizer tok = CharTokenizer.fromTokenCharPredicate(factory, Character::isLetter, Character::toLowerCase);
- * </pre>
- */
- public static CharTokenizer fromTokenCharPredicate(AttributeFactory factory, final IntPredicate tokenCharPredicate, final IntUnaryOperator normalizer) {
Objects.requireNonNull(tokenCharPredicate, "predicate must not be null.");
- Objects.requireNonNull(normalizer, "normalizer must not be null");
return new CharTokenizer(factory) {
@Override
protected boolean isTokenChar(int c) {
return tokenCharPredicate.test(c);
}
-
- @Override
- protected int normalize(int c) {
- return normalizer.applyAsInt(c);
- }
};
}
@@ -167,7 +128,7 @@ public abstract class CharTokenizer extends Tokenizer {
public static CharTokenizer fromSeparatorCharPredicate(final IntPredicate separatorCharPredicate) {
return fromSeparatorCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, separatorCharPredicate);
}
-
+
/**
* Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate, supplied as method reference or lambda expression.
* The predicate should return {@code true} for all valid token separator characters.
@@ -179,37 +140,7 @@ public abstract class CharTokenizer extends Tokenizer {
* </pre>
*/
public static CharTokenizer fromSeparatorCharPredicate(AttributeFactory factory, final IntPredicate separatorCharPredicate) {
- return fromSeparatorCharPredicate(factory, separatorCharPredicate, IntUnaryOperator.identity());
- }
-
- /**
- * Creates a new instance of CharTokenizer using a custom predicate, supplied as method reference or lambda expression.
- * The predicate should return {@code true} for all valid token separator characters.
- * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
- * <p>
- * This factory is intended to be used with lambdas or method references. E.g., an elegant way
- * to create an instance which behaves exactly as the combination {@link WhitespaceTokenizer} and {@link LowerCaseFilter} is:
- * <pre class="prettyprint lang-java">
- * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(Character::isWhitespace, Character::toLowerCase);
- * </pre>
- */
- public static CharTokenizer fromSeparatorCharPredicate(final IntPredicate separatorCharPredicate, final IntUnaryOperator normalizer) {
- return fromSeparatorCharPredicate(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, separatorCharPredicate, normalizer);
- }
-
- /**
- * Creates a new instance of CharTokenizer with the supplied attribute factory using a custom predicate.
- * The predicate should return {@code true} for all valid token separator characters.
- * This factory also takes a function to normalize chars, e.g., lowercasing them, supplied as method reference or lambda expression.
- * <p>
- * This factory is intended to be used with lambdas or method references. E.g., an elegant way
- * to create an instance which behaves exactly as {@link WhitespaceTokenizer} and {@link LowerCaseFilter} is:
- * <pre class="prettyprint lang-java">
- * Tokenizer tok = CharTokenizer.fromSeparatorCharPredicate(factory, Character::isWhitespace, Character::toLowerCase);
- * </pre>
- */
- public static CharTokenizer fromSeparatorCharPredicate(AttributeFactory factory, final IntPredicate separatorCharPredicate, final IntUnaryOperator normalizer) {
- return fromTokenCharPredicate(factory, separatorCharPredicate.negate(), normalizer);
+ return fromTokenCharPredicate(factory, separatorCharPredicate.negate());
}
private int offset = 0, bufferIndex = 0, dataLen = 0, finalOffset = 0;
@@ -230,15 +161,6 @@ public abstract class CharTokenizer extends Tokenizer {
*/
protected abstract boolean isTokenChar(int c);
- /**
- * Called on each token character to normalize it before it is added to the
- * token. The default implementation does nothing. Subclasses may use this to,
- * e.g., lowercase tokens.
- */
- protected int normalize(int c) {
- return c;
- }
-
@Override
public final boolean incrementToken() throws IOException {
clearAttributes();
@@ -276,7 +198,7 @@ public abstract class CharTokenizer extends Tokenizer {
buffer = termAtt.resizeBuffer(2+length); // make sure a supplementary fits in the buffer
}
end += charCount;
- length += Character.toChars(normalize(c), buffer, length); // buffer it, normalized
+ length += Character.toChars(c, buffer, length); // buffer it, normalized
if (length >= maxTokenLen) { // buffer overflow! make sure to check for >= surrogate pair could break == test
break;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory
index 4b37eb8..e8bceff 100644
--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory
+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.util.TokenizerFactory
@@ -15,7 +15,6 @@
org.apache.lucene.analysis.core.KeywordTokenizerFactory
org.apache.lucene.analysis.core.LetterTokenizerFactory
-org.apache.lucene.analysis.core.LowerCaseTokenizerFactory
org.apache.lucene.analysis.core.WhitespaceTokenizerFactory
org.apache.lucene.analysis.ngram.EdgeNGramTokenizerFactory
org.apache.lucene.analysis.ngram.NGramTokenizerFactory
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java
index 550a62a..5096ee8 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianAnalyzer.java
@@ -25,7 +25,8 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.core.LetterTokenizer;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
/**
@@ -147,9 +148,9 @@ public class TestBrazilianAnalyzer extends BaseTokenStreamTestCase {
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(1, true);
set.add("BrasÃlia");
- Tokenizer tokenizer = new LowerCaseTokenizer();
+ Tokenizer tokenizer = new LetterTokenizer();
tokenizer.setReader(new StringReader("BrasÃlia Brasilia"));
- BrazilianStemFilter filter = new BrazilianStemFilter(new SetKeywordMarkerFilter(tokenizer, set));
+ BrazilianStemFilter filter = new BrazilianStemFilter(new SetKeywordMarkerFilter(new LowerCaseFilter(tokenizer), set));
assertTokenStreamContents(filter, new String[] { "brasÃlia", "brasil" });
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
index b7fc18b..8133b7a 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
@@ -216,14 +216,6 @@ public class TestAnalyzers extends BaseTokenStreamTestCase {
int length = highSurEndingLower.length();
assertEquals('\ud801', termBuffer[length - 1]);
}
-
- public void testLowerCaseTokenizer() throws IOException {
- StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
- LowerCaseTokenizer tokenizer = new LowerCaseTokenizer();
- tokenizer.setReader(reader);
- assertTokenStreamContents(tokenizer, new String[] { "tokenizer",
- "\ud801\udc44test" });
- }
public void testWhitespaceTokenizer() throws IOException {
StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
index 1fa59d1..a4e1ac5 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/custom/TestCustomAnalyzer.java
@@ -31,9 +31,8 @@ import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.charfilter.HTMLStripCharFilterFactory;
import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory;
+import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilterFactory;
-import org.apache.lucene.analysis.core.LowerCaseTokenizer;
-import org.apache.lucene.analysis.core.LowerCaseTokenizerFactory;
import org.apache.lucene.analysis.core.StopFilterFactory;
import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilterFactory;
@@ -419,7 +418,7 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
@Override
public Tokenizer create(AttributeFactory factory) {
- return new LowerCaseTokenizer(factory);
+ return new LetterTokenizer(factory);
}
}
@@ -500,14 +499,6 @@ public class TestCustomAnalyzer extends BaseTokenStreamTestCase {
.build();
assertEquals(new BytesRef("e f c"), analyzer.normalize("dummy", "a b c"));
}
-
- /** test normalize where the TokenizerFactory returns a filter to normalize the text */
- public void testNormalizationWithLowerCaseTokenizer() throws IOException {
- CustomAnalyzer analyzer1 = CustomAnalyzer.builder()
- .withTokenizer(LowerCaseTokenizerFactory.class, Collections.emptyMap())
- .build();
- assertEquals(new BytesRef("abc"), analyzer1.normalize("dummy", "ABC"));
- }
public void testConditions() throws IOException {
CustomAnalyzer analyzer = CustomAnalyzer.builder()
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java
index 4c52c0e..3d8be31 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java
@@ -23,7 +23,9 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CharArraySet;
-import org.apache.lucene.analysis.core.LowerCaseTokenizer;
+import org.apache.lucene.analysis.LowerCaseFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.LetterTokenizer;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
public class TestGermanAnalyzer extends BaseTokenStreamTestCase {
@@ -38,10 +40,10 @@ public class TestGermanAnalyzer extends BaseTokenStreamTestCase {
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet( 1, true);
set.add("fischen");
- final LowerCaseTokenizer in = new LowerCaseTokenizer();
+ final Tokenizer in = new LetterTokenizer();
in.setReader(new StringReader("Fischen Trinken"));
GermanStemFilter filter = new GermanStemFilter(
- new SetKeywordMarkerFilter(in, set));
+ new SetKeywordMarkerFilter(new LowerCaseFilter(in), set));
assertTokenStreamContents(filter, new String[] { "fischen", "trink" });
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
index 00bc7c6..3f3d5c2 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
@@ -126,17 +126,6 @@ public class TestStandardFactories extends BaseTokenStreamFactoryTestCase {
}
/**
- * Test LowerCaseTokenizerFactory
- */
- public void testLowerCaseTokenizer() throws Exception {
- Reader reader = new StringReader("What's this thing do?");
- Tokenizer stream = tokenizerFactory("LowerCase").create(newAttributeFactory());
- stream.setReader(reader);
- assertTokenStreamContents(stream,
- new String[] { "what", "s", "this", "thing", "do" });
- }
-
- /**
* Ensure the ASCIIFoldingFilterFactory works
*/
public void testASCIIFolding() throws Exception {
@@ -169,11 +158,6 @@ public class TestStandardFactories extends BaseTokenStreamFactoryTestCase {
assertTrue(expected.getMessage().contains("Unknown parameters"));
expected = expectThrows(IllegalArgumentException.class, () -> {
- tokenizerFactory("LowerCase", "bogusArg", "bogusValue");
- });
- assertTrue(expected.getMessage().contains("Unknown parameters"));
-
- expected = expectThrows(IllegalArgumentException.class, () -> {
tokenFilterFactory("ASCIIFolding", "bogusArg", "bogusValue");
});
assertTrue(expected.getMessage().contains("Unknown parameters"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
index 4596608..2fcda4f 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
@@ -21,16 +21,12 @@ import java.io.IOException;
import java.io.StringReader;
import java.util.Locale;
-import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.LetterTokenizer;
-import org.apache.lucene.analysis.core.LowerCaseTokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.util.TestUtil;
/**
@@ -54,9 +50,9 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase {
}
// internal buffer size is 1024 make sure we have a surrogate pair right at the border
builder.insert(1023, "\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(newAttributeFactory());
+ Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString()));
- assertTokenStreamContents(tokenizer, builder.toString().toLowerCase(Locale.ROOT).split(" "));
+ assertTokenStreamContents(new LowerCaseFilter(tokenizer), builder.toString().toLowerCase(Locale.ROOT).split(" "));
}
/*
@@ -72,9 +68,9 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase {
builder.append("a");
}
builder.append("\ud801\udc1cabc");
- Tokenizer tokenizer = new LowerCaseTokenizer(newAttributeFactory());
+ Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString()));
- assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT)});
+ assertTokenStreamContents(new LowerCaseFilter(tokenizer), new String[] {builder.toString().toLowerCase(Locale.ROOT)});
}
}
@@ -87,9 +83,9 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase {
for (int i = 0; i < 255; i++) {
builder.append("A");
}
- Tokenizer tokenizer = new LowerCaseTokenizer(newAttributeFactory());
+ Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
- assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
+ assertTokenStreamContents(new LowerCaseFilter(tokenizer), new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
/*
@@ -101,14 +97,14 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase {
for (int i = 0; i < 100; i++) {
builder.append("A");
}
- Tokenizer tokenizer = new LowerCaseTokenizer(newAttributeFactory(), 100);
+ Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory(), 100);
// Tricky, passing two copies of the string to the reader....
tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
- assertTokenStreamContents(tokenizer, new String[]{builder.toString().toLowerCase(Locale.ROOT),
+ assertTokenStreamContents(new LowerCaseFilter(tokenizer), new String[]{builder.toString().toLowerCase(Locale.ROOT),
builder.toString().toLowerCase(Locale.ROOT) });
Exception e = expectThrows(IllegalArgumentException.class, () ->
- new LowerCaseTokenizer(newAttributeFactory(), -1));
+ new LetterTokenizer(newAttributeFactory(), -1));
assertEquals("maxTokenLen must be greater than 0 and less than 1048576 passed: -1", e.getMessage());
tokenizer = new LetterTokenizer(newAttributeFactory(), 100);
@@ -134,16 +130,16 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase {
}
e = expectThrows(IllegalArgumentException.class, () ->
- new LowerCaseTokenizer(newAttributeFactory(), 0));
+ new LetterTokenizer(newAttributeFactory(), 0));
assertEquals("maxTokenLen must be greater than 0 and less than 1048576 passed: 0", e.getMessage());
e = expectThrows(IllegalArgumentException.class, () ->
- new LowerCaseTokenizer(newAttributeFactory(), 10_000_000));
+ new LetterTokenizer(newAttributeFactory(), 10_000_000));
assertEquals("maxTokenLen must be greater than 0 and less than 1048576 passed: 10000000", e.getMessage());
- tokenizer = new LowerCaseTokenizer(newAttributeFactory(), 4800);
+ tokenizer = new LetterTokenizer(newAttributeFactory(), 4800);
tokenizer.setReader(new StringReader(builder.toString()));
- assertTokenStreamContents(tokenizer, new String[]{builder.toString().toLowerCase(Locale.ROOT)});
+ assertTokenStreamContents(new LowerCaseFilter(tokenizer), new String[]{builder.toString().toLowerCase(Locale.ROOT)});
e = expectThrows(IllegalArgumentException.class, () ->
@@ -195,87 +191,9 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase {
builder.append("A");
}
builder.append("\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(newAttributeFactory());
+ Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
- assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
- }
-
- // LUCENE-3642: normalize SMP->BMP and check that offsets are correct
- public void testCrossPlaneNormalization() throws IOException {
- Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory()) {
- @Override
- protected int normalize(int c) {
- if (c > 0xffff) {
- return 'δ';
- } else {
- return c;
- }
- }
- };
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
- int num = 1000 * RANDOM_MULTIPLIER;
- for (int i = 0; i < num; i++) {
- String s = TestUtil.randomUnicodeString(random());
- try (TokenStream ts = analyzer.tokenStream("foo", s)) {
- ts.reset();
- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- while (ts.incrementToken()) {
- String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
- for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
- cp = highlightedText.codePointAt(j);
- assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
- }
- }
- ts.end();
- }
- }
- // just for fun
- checkRandomData(random(), analyzer, num);
- analyzer.close();
- }
-
- // LUCENE-3642: normalize BMP->SMP and check that offsets are correct
- public void testCrossPlaneNormalization2() throws IOException {
- Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory()) {
- @Override
- protected int normalize(int c) {
- if (c <= 0xffff) {
- return 0x1043C;
- } else {
- return c;
- }
- }
- };
- return new TokenStreamComponents(tokenizer, tokenizer);
- }
- };
- int num = 1000 * RANDOM_MULTIPLIER;
- for (int i = 0; i < num; i++) {
- String s = TestUtil.randomUnicodeString(random());
- try (TokenStream ts = analyzer.tokenStream("foo", s)) {
- ts.reset();
- OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class);
- while (ts.incrementToken()) {
- String highlightedText = s.substring(offsetAtt.startOffset(), offsetAtt.endOffset());
- for (int j = 0, cp = 0; j < highlightedText.length(); j += Character.charCount(cp)) {
- cp = highlightedText.codePointAt(j);
- assertTrue("non-letter:" + Integer.toHexString(cp), Character.isLetter(cp));
- }
- }
- ts.end();
- }
- }
- // just for fun
- checkRandomData(random(), analyzer, num);
- analyzer.close();
+ assertTokenStreamContents(new LowerCaseFilter(tokenizer), new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
public void testDefinitionUsingMethodReference1() throws Exception {
@@ -287,16 +205,16 @@ public class TestCharTokenizers extends BaseTokenStreamTestCase {
public void testDefinitionUsingMethodReference2() throws Exception {
final StringReader reader = new StringReader("Tokenizer(Test)");
- final Tokenizer tokenizer = CharTokenizer.fromTokenCharPredicate(Character::isLetter, Character::toUpperCase);
+ final Tokenizer tokenizer = CharTokenizer.fromTokenCharPredicate(Character::isLetter);
tokenizer.setReader(reader);
- assertTokenStreamContents(tokenizer, new String[] { "TOKENIZER", "TEST" });
+ assertTokenStreamContents(tokenizer, new String[] { "Tokenizer", "Test" });
}
public void testDefinitionUsingLambda() throws Exception {
final StringReader reader = new StringReader("Tokenizer\u00A0Test Foo");
- final Tokenizer tokenizer = CharTokenizer.fromSeparatorCharPredicate(c -> c == '\u00A0' || Character.isWhitespace(c), Character::toLowerCase);
+ final Tokenizer tokenizer = CharTokenizer.fromSeparatorCharPredicate(c -> c == '\u00A0' || Character.isWhitespace(c));
tokenizer.setReader(reader);
- assertTokenStreamContents(tokenizer, new String[] { "tokenizer", "test", "foo" });
+ assertTokenStreamContents(tokenizer, new String[] { "Tokenizer", "Test", "Foo" });
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
index 3dbd6aa..475c333 100644
--- a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
+++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
@@ -126,20 +126,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldType>
- <fieldType name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- <filter class="solr.ClassicFilterFactory"/>
- <filter class="solr.StopFilterFactory" words="stopwords.txt"/>
- </analyzer>
- </fieldType>
-
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -386,8 +373,6 @@
<field name="test_hlt_off" type="highlittext" indexed="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml
index f36751e..20dc97a 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-copyfield-test.xml
@@ -90,19 +90,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldType>
- <fieldType name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- <filter class="solr.StopFilterFactory" words="stopwords.txt"/>
- </analyzer>
- </fieldType>
-
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -347,8 +335,6 @@
<field name="test_hlt_off" type="highlittext" indexed="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml b/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml
index 1d20b80..0b13a57 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-folding.xml
@@ -81,7 +81,8 @@
<fieldType name="text_lower_token" class="solr.TextField">
<analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
+ <tokenizer class="solr.LetterTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.ASCIIFoldingFilterFactory"/>
</analyzer>
</fieldType>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema-hash.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-hash.xml b/solr/core/src/test-files/solr/collection1/conf/schema-hash.xml
index 3e8aa15..c2d6b39 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-hash.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-hash.xml
@@ -139,18 +139,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldtype>
- <fieldtype name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldtype>
-
<!-- fieldtypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldtype name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldtype>
<fieldtype name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -484,8 +473,6 @@
termPositions="true" termOffsets="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml
index 4210d5b..0ac0c04 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-required-fields.xml
@@ -73,19 +73,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldType>
- <fieldType name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- <filter class="solr.StopFilterFactory" words="stopwords.txt"/>
- </analyzer>
- </fieldType>
-
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -331,8 +319,6 @@
<field name="test_hlt_off" type="highlittext" indexed="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
index 2a04356..46b735c 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml
@@ -199,17 +199,13 @@
<fieldType name="teststop" class="solr.TextField">
<analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
+ <tokenizer class="solr.LetterTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" words="stopwords.txt"/>
</analyzer>
</fieldType>
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -604,7 +600,6 @@
<!-- fields to test individual tokenizers and tokenfilters -->
<field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
index 40bbe5a..03d9d7e 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-sql.xml
@@ -149,18 +149,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldtype>
- <fieldtype name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldtype>
-
<!-- fieldtypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldtype name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldtype>
<fieldtype name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -501,8 +490,6 @@
termPositions="true" termOffsets="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml b/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml
index 6c33504..5613c66 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema-tokenizer-test.xml
@@ -43,7 +43,8 @@ more concise example.
<fieldType name="lowerCasefieldType" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false">
<analyzer type="index">
- <tokenizer class="solr.LowerCaseTokenizerFactory" maxTokenLen="3" />
+ <tokenizer class="solr.LetterTokenizerFactory" maxTokenLen="3" />
+ <filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.StandardTokenizerFactory"/>
@@ -85,12 +86,6 @@ more concise example.
</analyzer>
</fieldType>
- <fieldType name="lowerCase0fieldType" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory" maxTokenLen="3" />
- </analyzer>
- </fieldType>
-
<fieldType name="whiteSp0fieldType" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false">
<analyzer>
<tokenizer class="solr.WhitespaceTokenizerFactory" maxTokenLen="3" />
@@ -112,13 +107,11 @@ more concise example.
<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
<field name="letter" type="letterfieldType" indexed="true" stored="true"/>
- <field name="lowerCase" type="lowerCasefieldType" indexed="true" stored="true"/>
<field name="whiteSpace" type="whiteSpfieldType" indexed="true" stored="true"/>
<field name="unicodeWhiteSpace" type="uniWhiteSpfieldType" indexed="true" stored="true"/>
<field name="keyword" type="keywordfieldType" indexed="true" stored="true"/>
<field name="letter0" type="letter0fieldType" indexed="true" stored="true"/>
- <field name="lowerCase0" type="lowerCase0fieldType" indexed="true" stored="true"/>
<field name="whiteSpace0" type="whiteSp0fieldType" indexed="true" stored="true"/>
<field name="unicodeWhiteSpace0" type="uniWhiteSp0fieldType" indexed="true" stored="true"/>
<field name="keyword0" type="keyword0fieldType" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema.xml b/solr/core/src/test-files/solr/collection1/conf/schema.xml
index b1a261b..b61bbb1 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema.xml
@@ -142,20 +142,17 @@
<fieldType name="teststop" class="solr.TextField">
<analyzer type="index">
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
+ <tokenizer class="solr.LetterTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" words="stopwords.txt"/>
</analyzer>
<analyzer type="query">
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
+ <tokenizer class="solr.LetterTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -574,7 +571,7 @@
<!-- fields to test individual tokenizers and tokenfilters -->
<field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
+ <field name="lowertok" type="lowerfilt" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
@@ -636,7 +633,7 @@
<field name="store" type="location" indexed="true" stored="true" omitNorms="false"/>
- <field name="lower" type="lowertok" indexed="false" stored="true" multiValued="true"/>
+ <field name="lower" type="lowerfilt" indexed="false" stored="true" multiValued="true"/>
<field name="_route_" type="string" indexed="true" stored="true" multiValued="false"/>
<field name="payloadDelimited" type="payloadDelimited"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema12.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema12.xml b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
index 8947676..e4c3ad2 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema12.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema12.xml
@@ -234,17 +234,13 @@
<fieldType name="teststop" class="solr.TextField">
<analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
+ <tokenizer class="solr.LetterTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
<filter class="solr.StopFilterFactory" words="stopwords.txt"/>
</analyzer>
</fieldType>
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -591,7 +587,6 @@
<!-- fields to test individual tokenizers and tokenfilters -->
<field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schema15.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema15.xml b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
index 80d19e9..361344f 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schema15.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schema15.xml
@@ -163,19 +163,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldType>
- <fieldType name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- <filter class="solr.StopFilterFactory" words="stopwords.txt"/>
- </analyzer>
- </fieldType>
-
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -505,8 +493,6 @@
<field name="test_hlt_off" type="highlittext" indexed="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml
----------------------------------------------------------------------
diff --git a/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml b/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml
index 213acc7..93b11ed 100644
--- a/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml
+++ b/solr/core/src/test-files/solr/collection1/conf/schemasurround.xml
@@ -164,19 +164,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldType>
- <fieldType name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- <filter class="solr.StopFilterFactory" words="stopwords.txt"/>
- </analyzer>
- </fieldType>
-
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -517,8 +505,6 @@
<field name="test_hlt_off" type="highlittext" indexed="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java
index 31fa9f5..bdd3cd2 100644
--- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java
+++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldCollectionResource.java
@@ -77,11 +77,11 @@ public class TestFieldCollectionResource extends SolrRestletTestBase {
"/fields/[0]/name=='HTMLstandardtok'",
"/fields/[1]/name=='HTMLwhitetok'",
"/fields/[2]/name=='_version_'",
- "/fields/[108]/name=='*_d'",
- "/fields/[107]/name=='*_f'",
- "/fields/[106]/name=='*_b'",
- "/fields/[105]/name=='*_t'",
- "/fields/[104]/name=='*_l'"
+ "/fields/[107]/name=='*_d'",
+ "/fields/[106]/name=='*_f'",
+ "/fields/[105]/name=='*_b'",
+ "/fields/[104]/name=='*_t'",
+ "/fields/[103]/name=='*_l'"
);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java
index ea19af0..08a3f1b 100644
--- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java
+++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldTypeResource.java
@@ -81,7 +81,8 @@ public class TestFieldTypeResource extends SolrRestletTestBase {
"count(/response/lst[@name='fieldType']/*) = 3",
"/response/lst[@name='fieldType']/str[@name='name'] = 'teststop'",
"/response/lst[@name='fieldType']/str[@name='class'] = 'solr.TextField'",
- "/response/lst[@name='fieldType']/lst[@name='analyzer']/lst[@name='tokenizer']/str[@name='class'] = 'solr.LowerCaseTokenizerFactory'",
+ "/response/lst[@name='fieldType']/lst[@name='analyzer']/lst[@name='tokenizer']/str[@name='class'] = 'solr.LetterTokenizerFactory'",
+ "/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst/str[@name='class'][.='solr.LowerCaseFilterFactory']",
"/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst/str[@name='class'][.='solr.StopFilterFactory']",
"/response/lst[@name='fieldType']/lst[@name='analyzer']/arr[@name='filters']/lst/str[@name='words'][.='stopwords.txt']"
);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/core/src/test/org/apache/solr/util/TestMaxTokenLenTokenizer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestMaxTokenLenTokenizer.java b/solr/core/src/test/org/apache/solr/util/TestMaxTokenLenTokenizer.java
index c7e0dc3..f66c03e 100644
--- a/solr/core/src/test/org/apache/solr/util/TestMaxTokenLenTokenizer.java
+++ b/solr/core/src/test/org/apache/solr/util/TestMaxTokenLenTokenizer.java
@@ -22,7 +22,6 @@ import org.junit.BeforeClass;
/**
* Tests for:
- * {@link org.apache.lucene.analysis.core.LowerCaseTokenizerFactory}
* {@link org.apache.lucene.analysis.core.LetterTokenizerFactory}
* {@link org.apache.lucene.analysis.core.KeywordTokenizerFactory}
* {@link org.apache.lucene.analysis.core.WhitespaceTokenizerFactory}
@@ -44,25 +43,18 @@ public class TestMaxTokenLenTokenizer extends SolrTestCaseJ4 {
// using fields with definitions, different tokenizer factories respectively at index time and standard tokenizer at query time.
updateJ("{\"add\":{\"doc\": {\"id\":1,\"letter\":\"letter\"}},\"commit\":{}}",null);
- updateJ("{\"add\":{\"doc\": {\"id\":2,\"lowerCase\":\"lowerCase\"}},\"commit\":{}}",null);
updateJ("{\"add\":{\"doc\": {\"id\":3,\"whiteSpace\":\"whiteSpace in\"}},\"commit\":{}}",null);
updateJ("{\"add\":{\"doc\": {\"id\":4,\"unicodeWhiteSpace\":\"unicode in\"}},\"commit\":{}}",null);
updateJ("{\"add\":{\"doc\": {\"id\":5,\"keyword\":\"keyword\"}},\"commit\":{}}",null);
assertU(commit());
- assertQ("Check the total number of docs", req("q","*:*"), "//result[@numFound=5]");
+ assertQ("Check the total number of docs", req("q","*:*"), "//result[@numFound=4]");
//Tokens generated for "letter": "let" "ter" "letter" , maxTokenLen=3
assertQ("Check the total number of docs", req("q","letter:let"), "//result[@numFound=1]");
assertQ("Check the total number of docs", req("q","letter:lett"), "//result[@numFound=0]");
- //Tokens generated for "lowerCase": "low" "erC" "ase" "lowerCase" , maxTokenLen=3
- assertQ("Check the total number of docs", req("q","lowerCase:low"), "//result[@numFound=1]");
- assertQ("Check the total number of docs", req("q","lowerCase:l"), "//result[@numFound=0]");
- assertQ("Check the total number of docs", req("q","lowerCase:lo"), "//result[@numFound=0]");
- assertQ("Check the total number of docs", req("q","lowerCase:lower"), "//result[@numFound=0]");
-
//Tokens generated for "whiteSpace in": "whi" "teS" "pac" "e" "in" "whiteSpace" , maxTokenLen=3
assertQ("Check the total number of docs", req("q","whiteSpace:whi"), "//result[@numFound=1]");
assertQ("Check the total number of docs", req("q","whiteSpace:teS"), "//result[@numFound=1]");
@@ -88,14 +80,13 @@ public class TestMaxTokenLenTokenizer extends SolrTestCaseJ4 {
// using fields with definitions, same tokenizers both at index and query time.
updateJ("{\"add\":{\"doc\": {\"id\":1,\"letter0\":\"letter\"}},\"commit\":{}}",null);
- updateJ("{\"add\":{\"doc\": {\"id\":2,\"lowerCase0\":\"lowerCase\"}},\"commit\":{}}",null);
updateJ("{\"add\":{\"doc\": {\"id\":3,\"whiteSpace0\":\"whiteSpace in\"}},\"commit\":{}}",null);
updateJ("{\"add\":{\"doc\": {\"id\":4,\"unicodeWhiteSpace0\":\"unicode in\"}},\"commit\":{}}",null);
updateJ("{\"add\":{\"doc\": {\"id\":5,\"keyword0\":\"keyword\"}},\"commit\":{}}",null);
assertU(commit());
- assertQ("Check the total number of docs", req("q","*:*"), "//result[@numFound=5]");
+ assertQ("Check the total number of docs", req("q","*:*"), "//result[@numFound=4]");
//Tokens generated for "letter": "let" "ter" "letter" , maxTokenLen=3
// Anything that matches the first three letters should be found when maxLen=3
@@ -104,13 +95,6 @@ public class TestMaxTokenLenTokenizer extends SolrTestCaseJ4 {
assertQ("Check the total number of docs", req("q","letter0:lett"), "//result[@numFound=1]");
assertQ("Check the total number of docs", req("q","letter0:letXYZ"), "//result[@numFound=1]");
- //Tokens generated for "lowerCase": "low" "erC" "ase" "lowerCase" , maxTokenLen=3
- // Anything that matches the first three letters should be found when maxLen=3
- assertQ("Check the total number of docs", req("q","lowerCase0:low"), "//result[@numFound=1]");
- assertQ("Check the total number of docs", req("q","lowerCase0:l"), "//result[@numFound=0]");
- assertQ("Check the total number of docs", req("q","lowerCase0:lo"), "//result[@numFound=0]");
- assertQ("Check the total number of docs", req("q","lowerCase0:lowerXYZ"), "//result[@numFound=1]");
-
//Tokens generated for "whiteSpace in": "whi" "teS" "pac" "e" "in" "whiteSpace" , maxTokenLen=3
// Anything that matches the first three letters should be found when maxLen=3
assertQ("Check the total number of docs", req("q","whiteSpace0:h"), "//result[@numFound=0]");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml
index 3a1f328..974893c 100644
--- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml
+++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml
@@ -141,18 +141,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldtype>
- <fieldtype name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldtype>
-
<!-- fieldtypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldtype name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldtype>
<fieldtype name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -493,8 +482,6 @@
termPositions="true" termOffsets="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml
index 02b5053..079a35f 100644
--- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml
+++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema.xml
@@ -116,18 +116,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldType>
- <fieldType name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
-
<!-- fieldTypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldType name="lowertok" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldType>
<fieldType name="keywordtok" class="solr.TextField">
<analyzer>
<tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/>
@@ -461,8 +450,6 @@
termPositions="true" termOffsets="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c0d29759/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml
index aa96296..6cd4f91 100644
--- a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml
+++ b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml
@@ -137,16 +137,7 @@
<analyzer class="org.apache.lucene.analysis.core.WhitespaceAnalyzer"/>
</fieldtype>
- <fieldtype name="teststop" class="solr.TextField">
- <analyzer>
- <tokenizer class="solr.LowerCaseTokenizerFactory"/>
- </analyzer>
- </fieldtype>
-
<!-- fieldtypes in this section isolate tokenizers and tokenfilters for testing -->
- <fieldtype name="lowertok" class="solr.TextField">
- <analyzer><tokenizer class="solr.LowerCaseTokenizerFactory"/></analyzer>
- </fieldtype>
<fieldtype name="keywordtok" class="solr.TextField">
<analyzer><tokenizer class="solr.MockTokenizerFactory" pattern="keyword"/></analyzer>
</fieldtype>
@@ -479,8 +470,6 @@
termPositions="true" termOffsets="true"/>
<!-- fields to test individual tokenizers and tokenfilters -->
- <field name="teststop" type="teststop" indexed="true" stored="true"/>
- <field name="lowertok" type="lowertok" indexed="true" stored="true"/>
<field name="keywordtok" type="keywordtok" indexed="true" stored="true"/>
<field name="standardtok" type="standardtok" indexed="true" stored="true"/>
<field name="HTMLstandardtok" type="HTMLstandardtok" indexed="true" stored="true"/>
[09/29] lucene-solr:jira/http2: SOLR-12765: Incorrect format of JMX
cache stats.
Posted by da...@apache.org.
SOLR-12765: Incorrect format of JMX cache stats.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/36eae571
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/36eae571
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/36eae571
Branch: refs/heads/jira/http2
Commit: 36eae571636426148ea25028c5b0e0149ce45f88
Parents: ac7969e
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Mon Sep 17 14:40:36 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Mon Sep 17 15:33:16 2018 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../solr/metrics/reporters/jmx/JmxMetricsReporter.java | 10 +++++++---
2 files changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36eae571/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e773904..3f78719 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -355,6 +355,8 @@ Bug Fixes
* SOLR-11836: FacetStream works with bucketSizeLimit of -1 which will fetch all the buckets.
(Alfonso Muñoz-Pomer Fuentes, Amrit Sarkar via Varun Thacker)
+* SOLR-12765: Incorrect format of JMX cache stats. (Bojan Smid, ab)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/36eae571/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java b/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java
index ebf47e6..f7a0127 100644
--- a/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/reporters/jmx/JmxMetricsReporter.java
@@ -50,6 +50,7 @@ import com.codahale.metrics.ObjectNameFactory;
import com.codahale.metrics.Reporter;
import com.codahale.metrics.Timer;
import org.apache.solr.metrics.MetricsMap;
+import org.apache.solr.metrics.SolrMetricManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -558,9 +559,12 @@ public class JmxMetricsReporter implements Reporter, Closeable {
try {
if (filter.matches(name, gauge)) {
final ObjectName objectName = createName("gauges", name);
- if (gauge instanceof MetricsMap) {
- ((MetricsMap)gauge).setAttribute(new Attribute(INSTANCE_TAG, tag));
- registerMBean(gauge, objectName);
+ if (gauge instanceof SolrMetricManager.GaugeWrapper &&
+ ((SolrMetricManager.GaugeWrapper)gauge).getGauge() instanceof MetricsMap) {
+ MetricsMap mm = (MetricsMap)((SolrMetricManager.GaugeWrapper)gauge).getGauge();
+ mm.setAttribute(new Attribute(INSTANCE_TAG, tag));
+ // don't wrap it in a JmxGauge, it already supports all necessary JMX attributes
+ registerMBean(mm, objectName);
} else {
registerMBean(new JmxGauge(gauge, objectName, tag), objectName);
}
[17/29] lucene-solr:jira/http2: LUCENE-8352: Make
TokenStreamComponents final
Posted by da...@apache.org.
LUCENE-8352: Make TokenStreamComponents final
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c696cafc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c696cafc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c696cafc
Branch: refs/heads/jira/http2
Commit: c696cafc0d7fc0e133df20e0b188655fa020fe99
Parents: 10060a6
Author: Alan Woodward <ro...@apache.org>
Authored: Tue Sep 18 11:00:20 2018 +0100
Committer: Alan Woodward <ro...@apache.org>
Committed: Wed Sep 19 10:02:56 2018 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
lucene/MIGRATE.txt | 6 +
.../miscellaneous/LimitTokenCountAnalyzer.java | 2 +-
.../query/QueryAutoStopWordAnalyzer.java | 2 +-
.../shingle/ShingleAnalyzerWrapper.java | 2 +-
.../analysis/standard/ClassicAnalyzer.java | 11 +-
.../standard/UAX29URLEmailAnalyzer.java | 13 +-
.../TestPerFieldAnalyzerWrapper.java | 2 +-
.../org/apache/lucene/analysis/Analyzer.java | 42 ++-
.../analysis/standard/StandardAnalyzer.java | 13 +-
.../lucene/analysis/TestAnalyzerWrapper.java | 58 +++
.../prefix/BytesRefIteratorTokenStream.java | 2 +-
.../spatial/prefix/PrefixTreeStrategy.java | 17 +
.../analyzing/AnalyzingInfixSuggester.java | 2 +-
.../suggest/analyzing/FreeTextSuggester.java | 2 +-
.../suggest/document/CompletionAnalyzer.java | 2 +-
.../analyzing/AnalyzingSuggesterTest.java | 365 +++++++------------
.../suggest/analyzing/FuzzySuggesterTest.java | 159 +++-----
.../analysis/CannedBinaryTokenStream.java | 2 +-
.../lucene/analysis/CannedTokenStream.java | 8 +-
.../AbstractSpatialPrefixTreeFieldType.java | 35 +-
.../apache/solr/schema/PreAnalyzedField.java | 17 +-
.../FieldAnalysisRequestHandlerTest.java | 2 +-
23 files changed, 319 insertions(+), 448 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 98631f3..7a783a5 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -78,6 +78,9 @@ API Changes
* LUCENE-8483: Scorers may no longer take null as a Weight (Alan Woodward)
+* LUCENE-8352: TokenStreamComponents is now final, and can take a Consumer<Reader>
+ in its constructor (Mark Harwood, Alan Woodward, Adrien Grand)
+
Changes in Runtime Behavior
* LUCENE-8333: Switch MoreLikeThis.setMaxDocFreqPct to use maxDoc instead of
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/MIGRATE.txt
----------------------------------------------------------------------
diff --git a/lucene/MIGRATE.txt b/lucene/MIGRATE.txt
index e0e70a3..6008956 100644
--- a/lucene/MIGRATE.txt
+++ b/lucene/MIGRATE.txt
@@ -123,3 +123,9 @@ be replaced with a Scorable instead.
instead of long at suggest time ##
Most code should just require recompilation, though possibly requiring some added casts.
+
+## TokenStreamComponents is now final ##
+
+Instead of overriding TokenStreamComponents#setReader() to customise analyzer
+initialisation, you should now pass a Consumer<Reader> instance to the
+TokenStreamComponents constructor.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
index ca90dfc..8e6cee6 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
@@ -59,7 +59,7 @@ public final class LimitTokenCountAnalyzer extends AnalyzerWrapper {
@Override
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
- return new TokenStreamComponents(components.getTokenizer(),
+ return new TokenStreamComponents(components.getSource(),
new LimitTokenCountFilter(components.getTokenStream(), maxTokenCount, consumeAllTokens));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
index 37f044a..66fdc1b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/query/QueryAutoStopWordAnalyzer.java
@@ -169,7 +169,7 @@ public final class QueryAutoStopWordAnalyzer extends AnalyzerWrapper {
}
StopFilter stopFilter = new StopFilter(components.getTokenStream(),
new CharArraySet(stopWords, false));
- return new TokenStreamComponents(components.getTokenizer(), stopFilter);
+ return new TokenStreamComponents(components.getSource(), stopFilter);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
index fb580cf..4729116 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java
@@ -159,6 +159,6 @@ public final class ShingleAnalyzerWrapper extends AnalyzerWrapper {
filter.setOutputUnigrams(outputUnigrams);
filter.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
filter.setFillerToken(fillerToken);
- return new TokenStreamComponents(components.getTokenizer(), filter);
+ return new TokenStreamComponents(components.getSource(), filter);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
index d65f08b..868834f 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
@@ -92,13 +92,10 @@ public final class ClassicAnalyzer extends StopwordAnalyzerBase {
TokenStream tok = new ClassicFilter(src);
tok = new LowerCaseFilter(tok);
tok = new StopFilter(tok, stopwords);
- return new TokenStreamComponents(src, tok) {
- @Override
- protected void setReader(final Reader reader) {
- src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
- super.setReader(reader);
- }
- };
+ return new TokenStreamComponents(r -> {
+ src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
+ src.setReader(r);
+ }, tok);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
index be52370..9fd15fa 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
@@ -88,15 +88,10 @@ public final class UAX29URLEmailAnalyzer extends StopwordAnalyzerBase {
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new LowerCaseFilter(src);
tok = new StopFilter(tok, stopwords);
- return new TokenStreamComponents(src, tok) {
- @Override
- protected void setReader(final Reader reader) {
- // So that if maxTokenLength was changed, the change takes
- // effect next time tokenStream is called:
- src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength);
- super.setReader(reader);
- }
- };
+ return new TokenStreamComponents(r -> {
+ src.setMaxTokenLength(UAX29URLEmailAnalyzer.this.maxTokenLength);
+ src.setReader(r);
+ }, tok);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
index 54a64ce..11eff95 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
@@ -107,7 +107,7 @@ public class TestPerFieldAnalyzerWrapper extends BaseTokenStreamTestCase {
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
assertNotSame(specialAnalyzer.tokenStream("special", text), components.getTokenStream());
TokenFilter filter = new ASCIIFoldingFilter(components.getTokenStream());
- return new TokenStreamComponents(components.getTokenizer(), filter);
+ return new TokenStreamComponents(components.getSource(), filter);
}
};
ts3 = wrapper2.tokenStream("special", text);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java b/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
index 86766d2..46338dc 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
@@ -23,6 +23,7 @@ import java.io.Reader;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
+import java.util.function.Consumer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@@ -355,16 +356,16 @@ public abstract class Analyzer implements Closeable {
/**
* This class encapsulates the outer components of a token stream. It provides
- * access to the source ({@link Tokenizer}) and the outer end (sink), an
+ * access to the source (a {@link Reader} {@link Consumer} and the outer end (sink), an
* instance of {@link TokenFilter} which also serves as the
* {@link TokenStream} returned by
* {@link Analyzer#tokenStream(String, Reader)}.
*/
- public static class TokenStreamComponents {
+ public static final class TokenStreamComponents {
/**
* Original source of the tokens.
*/
- protected final Tokenizer source;
+ protected final Consumer<Reader> source;
/**
* Sink tokenstream, such as the outer tokenfilter decorating
* the chain. This can be the source if there are no filters.
@@ -378,25 +379,30 @@ public abstract class Analyzer implements Closeable {
* Creates a new {@link TokenStreamComponents} instance.
*
* @param source
- * the analyzer's tokenizer
+ * the source to set the reader on
* @param result
* the analyzer's resulting token stream
*/
- public TokenStreamComponents(final Tokenizer source,
+ public TokenStreamComponents(final Consumer<Reader> source,
final TokenStream result) {
this.source = source;
this.sink = result;
}
-
+
/**
- * Creates a new {@link TokenStreamComponents} instance.
- *
- * @param source
- * the analyzer's tokenizer
+ * Creates a new {@link TokenStreamComponents} instance
+ * @param tokenizer the analyzer's Tokenizer
+ * @param result the analyzer's resulting token stream
*/
- public TokenStreamComponents(final Tokenizer source) {
- this.source = source;
- this.sink = source;
+ public TokenStreamComponents(final Tokenizer tokenizer, final TokenStream result) {
+ this(tokenizer::setReader, result);
+ }
+
+ /**
+ * Creates a new {@link TokenStreamComponents} from a Tokenizer
+ */
+ public TokenStreamComponents(final Tokenizer tokenizer) {
+ this(tokenizer::setReader, tokenizer);
}
/**
@@ -406,8 +412,8 @@ public abstract class Analyzer implements Closeable {
* @param reader
* a reader to reset the source component
*/
- protected void setReader(final Reader reader) {
- source.setReader(reader);
+ private void setReader(final Reader reader) {
+ source.accept(reader);
}
/**
@@ -420,11 +426,9 @@ public abstract class Analyzer implements Closeable {
}
/**
- * Returns the component's {@link Tokenizer}
- *
- * @return Component's {@link Tokenizer}
+ * Returns the component's source
*/
- public Tokenizer getTokenizer() {
+ public Consumer<Reader> getSource() {
return source;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
index cd7262c..76c958b 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
@@ -81,15 +81,10 @@ public final class StandardAnalyzer extends StopwordAnalyzerBase {
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new LowerCaseFilter(src);
tok = new StopFilter(tok, stopwords);
- return new TokenStreamComponents(src, tok) {
- @Override
- protected void setReader(final Reader reader) {
- // So that if maxTokenLength was changed, the change takes
- // effect next time tokenStream is called:
- src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
- super.setReader(reader);
- }
- };
+ return new TokenStreamComponents(r -> {
+ src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
+ src.setReader(r);
+ }, tok);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/core/src/test/org/apache/lucene/analysis/TestAnalyzerWrapper.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/analysis/TestAnalyzerWrapper.java b/lucene/core/src/test/org/apache/lucene/analysis/TestAnalyzerWrapper.java
new file mode 100644
index 0000000..24c8caa
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/analysis/TestAnalyzerWrapper.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestAnalyzerWrapper extends LuceneTestCase {
+
+ public void testSourceDelegation() throws IOException {
+
+ AtomicBoolean sourceCalled = new AtomicBoolean(false);
+
+ Analyzer analyzer = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(r -> {
+ sourceCalled.set(true);
+ }, new CannedTokenStream());
+ }
+ };
+
+ Analyzer wrapped = new AnalyzerWrapper(analyzer.getReuseStrategy()) {
+ @Override
+ protected Analyzer getWrappedAnalyzer(String fieldName) {
+ return analyzer;
+ }
+
+ @Override
+ protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
+ return new TokenStreamComponents(components.getSource(), new LowerCaseFilter(components.getTokenStream()));
+ }
+ };
+
+ try (TokenStream ts = wrapped.tokenStream("", "text")) {
+ assertTrue(sourceCalled.get());
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java
index ca38abf..9be0e71 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/BytesRefIteratorTokenStream.java
@@ -28,7 +28,7 @@ import org.apache.lucene.util.BytesRefIterator;
*
* @lucene.internal
*/
-class BytesRefIteratorTokenStream extends TokenStream {
+public class BytesRefIteratorTokenStream extends TokenStream {
public BytesRefIterator getBytesRefIterator() {
return bytesIter;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
----------------------------------------------------------------------
diff --git a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
index d65e9f5..99a5a4a 100644
--- a/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
+++ b/lucene/spatial-extras/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
@@ -157,6 +157,23 @@ public abstract class PrefixTreeStrategy extends SpatialStrategy {
return new Field[]{field};
}
+ public class ShapeTokenStream extends BytesRefIteratorTokenStream {
+
+ public void setShape(Shape shape) {
+ double distErr = SpatialArgs.calcDistanceFromErrPct(shape, distErrPct, ctx);
+ int detailLevel = grid.getLevelForDistance(distErr);
+ Iterator<Cell> cells = createCellIteratorToIndex(shape, detailLevel, null);
+ CellToBytesRefIterator cellToBytesRefIterator = newCellToBytesRefIterator();
+ cellToBytesRefIterator.reset(cells);
+ setBytesRefIterator(cellToBytesRefIterator);
+ }
+
+ }
+
+ public ShapeTokenStream tokenStream() {
+ return new ShapeTokenStream();
+ }
+
protected CellToBytesRefIterator newCellToBytesRefIterator() {
//subclasses could return one that never emits leaves, or does both, or who knows.
return new CellToBytesRefIterator();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
index a8394e0..6947675 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
@@ -360,7 +360,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable {
if (fieldName.equals(TEXTGRAMS_FIELD_NAME) && minPrefixChars > 0) {
// TODO: should use an EdgeNGramTokenFilterFactory here
TokenFilter filter = new EdgeNGramTokenFilter(components.getTokenStream(), 1, minPrefixChars, false);
- return new TokenStreamComponents(components.getTokenizer(), filter);
+ return new TokenStreamComponents(components.getSource(), filter);
} else {
return components;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
index 3e4845b..1b0a980 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
@@ -236,7 +236,7 @@ public class FreeTextSuggester extends Lookup implements Accountable {
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
ShingleFilter shingles = new ShingleFilter(components.getTokenStream(), 2, grams);
shingles.setTokenSeparator(Character.toString((char) separator));
- return new TokenStreamComponents(components.getTokenizer(), shingles);
+ return new TokenStreamComponents(components.getSource(), shingles);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionAnalyzer.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionAnalyzer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionAnalyzer.java
index 8888382..631a67a 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionAnalyzer.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionAnalyzer.java
@@ -132,6 +132,6 @@ public final class CompletionAnalyzer extends AnalyzerWrapper {
protected TokenStreamComponents wrapComponents(String fieldName, TokenStreamComponents components) {
CompletionTokenStream tokenStream = new CompletionTokenStream(components.getTokenStream(),
preserveSep, preservePositionIncrements, maxGraphExpansions);
- return new TokenStreamComponents(components.getTokenizer(), tokenStream);
+ return new TokenStreamComponents(components.getSource(), tokenStream);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
index 7f3b8b4..cc1e305 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
@@ -44,6 +44,7 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.BytesTermAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
@@ -303,52 +304,82 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
IOUtils.close(a, tempDir);
}
- public void testGraphDups() throws Exception {
+ static final class MultiCannedTokenizer extends Tokenizer {
- final Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
- int tokenStreamCounter = 0;
- final TokenStream[] tokenStreams = new TokenStream[] {
- new CannedTokenStream(new Token[] {
- token("wifi",1,1),
- token("hotspot",0,2),
- token("network",1,1),
- token("is",1,1),
- token("slow",1,1)
- }),
- new CannedTokenStream(new Token[] {
- token("wi",1,1),
- token("hotspot",0,3),
- token("fi",1,1),
- token("network",1,1),
- token("is",1,1),
- token("fast",1,1)
-
- }),
- new CannedTokenStream(new Token[] {
- token("wifi",1,1),
- token("hotspot",0,2),
- token("network",1,1)
- }),
- };
-
- @Override
- public TokenStream getTokenStream() {
- TokenStream result = tokenStreams[tokenStreamCounter];
- tokenStreamCounter++;
- return result;
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
+ int counter = -1;
+ final TokenStream[] tokenStreams;
+
+ MultiCannedTokenizer(TokenStream... tokenStreams) {
+ super(tokenStreams[0].getAttributeFactory());
+ this.tokenStreams = tokenStreams;
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (tokenStreams[counter].incrementToken() == false) {
+ return false;
}
- };
+ this.restoreState(tokenStreams[counter].captureState());
+ return true;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ tokenStreams[counter].reset();
+ }
+ }
+
+ static final class MultiCannedAnalyzer extends Analyzer {
+
+ final MultiCannedTokenizer tokenizer;
+
+ MultiCannedAnalyzer(TokenStream... tokenStreams) {
+ this(false, tokenStreams);
+ }
+
+ MultiCannedAnalyzer(boolean addBytesAtt, TokenStream... tokenStreams) {
+ this.tokenizer = new MultiCannedTokenizer(tokenStreams);
+ if (addBytesAtt) {
+ this.tokenizer.addAttribute(BytesTermAttribute.class);
+ }
+ }
+
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ tokenizer.counter = 0;
+ return new TokenStreamComponents(tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ tokenizer.counter++;
+ if (tokenizer.counter >= tokenizer.tokenStreams.length) {
+ tokenizer.counter = tokenizer.tokenStreams.length - 1;
+ }
+ return super.initReader(fieldName, reader);
+ }
+ }
+
+ public void testGraphDups() throws Exception {
+
+ final Analyzer analyzer = new MultiCannedAnalyzer(
+ new CannedTokenStream(
+ token("wifi",1,1),
+ token("hotspot",0,2),
+ token("network",1,1),
+ token("is",1,1),
+ token("slow",1,1)),
+ new CannedTokenStream(
+ token("wi",1,1),
+ token("hotspot",0,3),
+ token("fi",1,1),
+ token("network",1,1),
+ token("is",1,1),
+ token("fast",1,1)),
+ new CannedTokenStream(
+ token("wifi",1,1),
+ token("hotspot",0,2),
+ token("network",1,1)));
Input keys[] = new Input[] {
new Input("wifi network is slow", 50),
@@ -378,45 +409,20 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
// The Analyzer below mimics the functionality of the SynonymAnalyzer
// using the above map, so that the suggest module does not need a dependency on the
- // synonym module
-
- final Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
- int tokenStreamCounter = 0;
- final TokenStream[] tokenStreams = new TokenStream[] {
- new CannedTokenStream(new Token[] {
- token("ab",1,1),
- token("ba",0,1),
- token("xc",1,1)
- }),
- new CannedTokenStream(new Token[] {
- token("ba",1,1),
- token("xd",1,1)
- }),
- new CannedTokenStream(new Token[] {
- token("ab",1,1),
- token("ba",0,1),
- token("x",1,1)
- })
- };
-
- @Override
- public TokenStream getTokenStream() {
- TokenStream result = tokenStreams[tokenStreamCounter];
- tokenStreamCounter++;
- return result;
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ // synonym module
+
+ final Analyzer analyzer = new MultiCannedAnalyzer(
+ new CannedTokenStream(
+ token("ab", 1, 1),
+ token("ba", 0, 1),
+ token("xc", 1, 1)),
+ new CannedTokenStream(
+ token("ba", 1, 1),
+ token("xd", 1, 1)),
+ new CannedTokenStream(
+ token("ab",1,1),
+ token("ba",0,1),
+ token("x",1,1)));
Input keys[] = new Input[] {
new Input("ab xc", 50),
@@ -426,7 +432,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
suggester.build(new InputArrayIterator(keys));
List<LookupResult> results = suggester.lookup("ab x", false, 1);
- assertTrue(results.size() == 1);
+ assertEquals(1, results.size());
IOUtils.close(analyzer, tempDir);
}
@@ -459,39 +465,15 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
}
*/
- private final Analyzer getUnusualAnalyzer() {
- return new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
-
- int count;
-
- @Override
- public TokenStream getTokenStream() {
- // 4th time we are called, return tokens a b,
- // else just a:
- if (count++ != 3) {
- return new CannedTokenStream(new Token[] {
- token("a", 1, 1),
- });
- } else {
- // After that "a b":
- return new CannedTokenStream(new Token[] {
- token("a", 1, 1),
- token("b", 1, 1),
- });
- }
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ private Analyzer getUnusualAnalyzer() {
+ // First three calls just returns "a", then returns ["a","b"], then "a" again
+ return new MultiCannedAnalyzer(
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1), token("b", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)));
}
public void testExactFirst() throws Exception {
@@ -999,23 +981,10 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
Analyzer a = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
-
- @Override
- public TokenStream getTokenStream() {
- return new CannedTokenStream(new Token[] {
- token("hairy", 1, 1),
- token("smelly", 0, 1),
- token("dog", 1, 1),
- });
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
+ return new TokenStreamComponents(r -> {}, new CannedTokenStream(
+ token("hairy", 1, 1),
+ token("smelly", 0, 1),
+ token("dog", 1, 1)));
}
};
@@ -1056,38 +1025,15 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
}
public void testDupSurfaceFormsMissingResults2() throws Exception {
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
-
- int count;
-
- @Override
- public TokenStream getTokenStream() {
- if (count == 0) {
- count++;
- return new CannedTokenStream(new Token[] {
- token("p", 1, 1),
- token("q", 1, 1),
- token("r", 0, 1),
- token("s", 0, 1),
- });
- } else {
- return new CannedTokenStream(new Token[] {
- token("p", 1, 1),
- });
- }
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ Analyzer a = new MultiCannedAnalyzer(
+ new CannedTokenStream(
+ token("p", 1, 1),
+ token("q", 1, 1),
+ token("r", 0, 1),
+ token("s", 0, 1)),
+ new CannedTokenStream(token("p", 1, 1)),
+ new CannedTokenStream(token("p", 1, 1)),
+ new CannedTokenStream(token("p", 1, 1)));
Directory tempDir = getDirectory();
AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
@@ -1131,24 +1077,7 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
* and checks that they come back in surface-form order.
*/
public void testTieBreakOnSurfaceForm() throws Exception {
- Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
- @Override
- public TokenStream getTokenStream() {
- return new CannedTokenStream(new Token[] {
- token("dog", 1, 1)
- });
- }
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ Analyzer a = new MultiCannedAnalyzer(new CannedTokenStream(token("dog", 1, 1)));
Directory tempDir = getDirectory();
AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
@@ -1187,41 +1116,12 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
}
public void test0ByteKeys() throws Exception {
- final Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
- int tokenStreamCounter = 0;
- final TokenStream[] tokenStreams = new TokenStream[] {
- new CannedBinaryTokenStream(new BinaryToken[] {
- token(new BytesRef(new byte[] {0x0, 0x0, 0x0})),
- }),
- new CannedBinaryTokenStream(new BinaryToken[] {
- token(new BytesRef(new byte[] {0x0, 0x0})),
- }),
- new CannedBinaryTokenStream(new BinaryToken[] {
- token(new BytesRef(new byte[] {0x0, 0x0, 0x0})),
- }),
- new CannedBinaryTokenStream(new BinaryToken[] {
- token(new BytesRef(new byte[] {0x0, 0x0})),
- }),
- };
-
- @Override
- public TokenStream getTokenStream() {
- TokenStream result = tokenStreams[tokenStreamCounter];
- tokenStreamCounter++;
- return result;
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ final Analyzer a = new MultiCannedAnalyzer(true,
+ new CannedBinaryTokenStream(token(new BytesRef(new byte[] {0x0, 0x0, 0x0}))),
+ new CannedBinaryTokenStream(token(new BytesRef(new byte[] {0x0, 0x0}))),
+ new CannedBinaryTokenStream(token(new BytesRef(new byte[] {0x0, 0x0, 0x0}))),
+ new CannedBinaryTokenStream(token(new BytesRef(new byte[] {0x0, 0x0})))
+ );
Directory tempDir = getDirectory();
AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);
@@ -1265,26 +1165,13 @@ public class AnalyzingSuggesterTest extends LuceneTestCase {
public void testTooManyExpansions() throws Exception {
final Analyzer a = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
- @Override
- public TokenStream getTokenStream() {
- Token a = new Token("a", 0, 1);
- a.setPositionIncrement(1);
- Token b = new Token("b", 0, 1);
- b.setPositionIncrement(0);
- return new CannedTokenStream(new Token[] {a, b});
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(r -> {}, new CannedTokenStream(
+ new Token("a", 0, 1),
+ new Token("b", 0, 0, 1)));
+ }
+ };
Directory tempDir = getDirectory();
AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, 1, true);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
----------------------------------------------------------------------
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
index 165c9f2..2368779 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
@@ -17,7 +17,6 @@
package org.apache.lucene.search.suggest.analyzing;
import java.io.IOException;
-import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -36,7 +35,6 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.TokenStreamToAutomaton;
-import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.search.suggest.Input;
@@ -240,50 +238,24 @@ public class FuzzySuggesterTest extends LuceneTestCase {
public void testGraphDups() throws Exception {
- final Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
- int tokenStreamCounter = 0;
- final TokenStream[] tokenStreams = new TokenStream[] {
- new CannedTokenStream(new Token[] {
- token("wifi",1,1),
- token("hotspot",0,2),
- token("network",1,1),
- token("is",1,1),
- token("slow",1,1)
- }),
- new CannedTokenStream(new Token[] {
- token("wi",1,1),
- token("hotspot",0,3),
- token("fi",1,1),
- token("network",1,1),
- token("is",1,1),
- token("fast",1,1)
-
- }),
- new CannedTokenStream(new Token[] {
- token("wifi",1,1),
- token("hotspot",0,2),
- token("network",1,1)
- }),
- };
-
- @Override
- public TokenStream getTokenStream() {
- TokenStream result = tokenStreams[tokenStreamCounter];
- tokenStreamCounter++;
- return result;
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ final Analyzer analyzer = new AnalyzingSuggesterTest.MultiCannedAnalyzer(
+ new CannedTokenStream(
+ token("wifi", 1, 1),
+ token("hotspot", 0, 2),
+ token("network", 1, 1),
+ token("is", 1, 1),
+ token("slow", 1, 1)),
+ new CannedTokenStream(
+ token("wi", 1, 1),
+ token("hotspot", 0, 3),
+ token("fi", 1, 1),
+ token("network", 1, 1),
+ token("is", 1, 1),
+ token("fast", 1, 1)),
+ new CannedTokenStream(
+ token("wifi", 1, 1),
+ token("hotspot",0,2),
+ token("network",1,1)));
Input keys[] = new Input[] {
new Input("wifi network is slow", 50),
@@ -326,43 +298,18 @@ public class FuzzySuggesterTest extends LuceneTestCase {
// using the above map, so that the suggest module does not need a dependency on the
// synonym module
- final Analyzer analyzer = new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
- int tokenStreamCounter = 0;
- final TokenStream[] tokenStreams = new TokenStream[] {
- new CannedTokenStream(new Token[] {
- token("ab",1,1),
- token("ba",0,1),
- token("xc",1,1)
- }),
- new CannedTokenStream(new Token[] {
- token("ba",1,1),
- token("xd",1,1)
- }),
- new CannedTokenStream(new Token[] {
- token("ab",1,1),
- token("ba",0,1),
- token("x",1,1)
- })
- };
-
- @Override
- public TokenStream getTokenStream() {
- TokenStream result = tokenStreams[tokenStreamCounter];
- tokenStreamCounter++;
- return result;
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ final Analyzer analyzer = new AnalyzingSuggesterTest.MultiCannedAnalyzer(
+ new CannedTokenStream(
+ token("ab", 1, 1),
+ token("ba", 0, 1),
+ token("xc", 1, 1)),
+ new CannedTokenStream(
+ token("ba", 1, 1),
+ token("xd", 1, 1)),
+ new CannedTokenStream(
+ token("ab", 1, 1),
+ token("ba", 0, 1),
+ token("x", 1, 1)));
Input keys[] = new Input[] {
new Input("ab xc", 50),
@@ -399,41 +346,17 @@ public class FuzzySuggesterTest extends LuceneTestCase {
ts.end();
ts.close();
}
- */
-
- private final Analyzer getUnusualAnalyzer() {
- return new Analyzer() {
- @Override
- protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
-
- return new TokenStreamComponents(tokenizer) {
-
- int count;
-
- @Override
- public TokenStream getTokenStream() {
- // 4th time we are called, return tokens a b,
- // else just a:
- if (count++ != 3) {
- return new CannedTokenStream(new Token[] {
- token("a", 1, 1),
- });
- } else {
- // After that "a b":
- return new CannedTokenStream(new Token[] {
- token("a", 1, 1),
- token("b", 1, 1),
- });
- }
- }
-
- @Override
- protected void setReader(final Reader reader) {
- }
- };
- }
- };
+ */
+
+ private Analyzer getUnusualAnalyzer() {
+ // First three calls just returns "a", then returns ["a","b"], then "a" again
+ return new AnalyzingSuggesterTest.MultiCannedAnalyzer(
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1), token("b", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)),
+ new CannedTokenStream(token("a", 1, 1)));
}
public void testExactFirst() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java
index b0096bc..7eab0f4 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedBinaryTokenStream.java
@@ -57,7 +57,7 @@ public final class CannedBinaryTokenStream extends TokenStream {
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
public CannedBinaryTokenStream(BinaryToken... tokens) {
- super();
+ super(Token.TOKEN_ATTRIBUTE_FACTORY);
this.tokens = tokens;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java
index 8323882..31c3d5f 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/CannedTokenStream.java
@@ -52,7 +52,13 @@ public final class CannedTokenStream extends TokenStream {
posIncrAtt.setPositionIncrement(finalPosInc);
offsetAtt.setOffset(finalOffset, finalOffset);
}
-
+
+ @Override
+ public void reset() throws IOException {
+ upto = 0;
+ super.reset();
+ }
+
@Override
public boolean incrementToken() {
if (upto < tokens.length) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java
index abf55f1..b6e9106 100644
--- a/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java
+++ b/solr/core/src/java/org/apache/solr/schema/AbstractSpatialPrefixTreeFieldType.java
@@ -17,21 +17,16 @@
package org.apache.solr.schema;
import java.io.IOException;
-import java.io.Reader;
import java.lang.invoke.MethodHandles;
import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.spatial.prefix.PrefixTreeStrategy;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTree;
import org.apache.lucene.spatial.prefix.tree.SpatialPrefixTreeFactory;
import org.apache.lucene.spatial.query.SpatialArgsParser;
import org.apache.solr.util.MapListener;
-
-import org.locationtech.spatial4j.shape.Shape;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,29 +75,19 @@ public abstract class AbstractSpatialPrefixTreeFieldType<T extends PrefixTreeStr
* so that the analysis UI will show reasonable tokens.
*/
@Override
- public Analyzer getIndexAnalyzer()
- {
+ public Analyzer getIndexAnalyzer() {
return new Analyzer() {
-
@Override
- protected TokenStreamComponents createComponents(final String fieldName) {
- return new TokenStreamComponents(new KeywordTokenizer()) {
- private Shape shape = null;
-
- protected void setReader(final Reader reader) {
- source.setReader(reader);
- try {
- shape = parseShape(IOUtils.toString(reader));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- public TokenStream getTokenStream() {
- PrefixTreeStrategy s = newSpatialStrategy(fieldName==null ? getTypeName() : fieldName);
- return s.createIndexableFields(shape)[0].tokenStreamValue();
+ protected TokenStreamComponents createComponents(String fieldName) {
+ PrefixTreeStrategy s = newSpatialStrategy(fieldName == null ? getTypeName() : fieldName);
+ PrefixTreeStrategy.ShapeTokenStream ts = s.tokenStream();
+ return new TokenStreamComponents(r -> {
+ try {
+ ts.setShape(parseShape(IOUtils.toString(r)));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
- };
+ }, ts);
}
};
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
index 368bfde..00da7ed 100644
--- a/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
+++ b/solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
@@ -374,18 +374,13 @@ public class PreAnalyzedField extends TextField implements HasImplicitIndexAnaly
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final PreAnalyzedTokenizer tokenizer = new PreAnalyzedTokenizer(parser);
- return new TokenStreamComponents(tokenizer) {
- @Override
- protected void setReader(final Reader reader) {
- super.setReader(reader);
- try {
- tokenizer.decodeInput(reader);
- } catch (IOException e) {
- // save this exception for reporting when reset() is called
- tokenizer.setReaderConsumptionException(e);
- }
+ return new TokenStreamComponents(r -> {
+ try {
+ tokenizer.decodeInput(r);
+ } catch (IOException e) {
+ tokenizer.setReaderConsumptionException(e);
}
- };
+ }, tokenizer);
}
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c696cafc/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
index ef7de39..b233d11 100644
--- a/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/FieldAnalysisRequestHandlerTest.java
@@ -432,7 +432,7 @@ public class FieldAnalysisRequestHandlerTest extends AnalysisRequestHandlerTestB
NamedList<NamedList> result = handler.handleAnalysisRequest(request, h.getCore().getLatestSchema());
NamedList<List<NamedList>> tokens = (NamedList<List<NamedList>>)
((NamedList)result.get("field_types").get("location_rpt")).get("index");
- List<NamedList> tokenList = tokens.get("org.apache.lucene.spatial.prefix.BytesRefIteratorTokenStream");
+ List<NamedList> tokenList = tokens.get("org.apache.lucene.spatial.prefix.PrefixTreeStrategy$ShapeTokenStream");
List<String> vals = new ArrayList<>(tokenList.size());
[27/29] lucene-solr:jira/http2: Add toString() to help debugging.
Posted by da...@apache.org.
Add toString() to help debugging.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0dd7b704
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0dd7b704
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0dd7b704
Branch: refs/heads/jira/http2
Commit: 0dd7b70447999a7f5502a35fe031c9ffbd60cde3
Parents: 5e6c99c
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Sep 20 15:32:42 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Sep 20 15:32:42 2018 +0200
----------------------------------------------------------------------
.../solr/client/solrj/request/CollectionAdminRequest.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0dd7b704/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index b400688..4cf68d4 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -135,6 +135,11 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
}
+ @Override
+ public String toString() {
+ return jsonStr();
+ }
+
/**
* Base class for asynchronous collection admin requests
*/
[21/29] lucene-solr:jira/http2: LUCENE-8454: Fix incorrect vertex
indexing and other computation errors in shape tessellation that would
sometimes cause an infinite loop.
Posted by da...@apache.org.
LUCENE-8454: Fix incorrect vertex indexing and other computation errors in shape tessellation that would sometimes cause an infinite loop.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d494f6be
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d494f6be
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d494f6be
Branch: refs/heads/jira/http2
Commit: d494f6be13a585c273c6a3dabc76644f4d480c31
Parents: 264110e
Author: Nicholas Knize <Nicholas Knize>
Authored: Wed Sep 19 15:04:48 2018 -0500
Committer: Nicholas Knize <Nicholas Knize>
Committed: Wed Sep 19 15:49:31 2018 -0500
----------------------------------------------------------------------
lucene/CHANGES.txt | 7 ++
.../java/org/apache/lucene/geo/Tessellator.java | 105 ++++++++++---------
.../org/apache/lucene/geo/TestTessellator.java | 22 ++++
3 files changed, 82 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d494f6be/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 7a783a5..bd8c616 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -173,6 +173,13 @@ Build
* LUCENE-8504: Upgrade forbiddenapis to version 2.6. (Uwe Schindler)
+======================= Lucene 7.5.1 =======================
+
+Bug Fixes:
+
+* LUCENE-8454: Fix incorrect vertex indexing and other computation errors in
+ shape tessellation that would sometimes cause an infinite loop. (Nick Knize)
+
======================= Lucene 7.5.0 =======================
API Changes:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d494f6be/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java b/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
index 969facf..a03e57c 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
@@ -150,9 +150,8 @@ final public class Tessellator {
final List<Node> holeList = new ArrayList<>();
// Iterate through each array of hole vertices.
Polygon[] holes = polygon.getHoles();
- int nodeIndex = 0;
+ int nodeIndex = polygon.numPoints();
for(int i = 0; i < polygon.numHoles(); ++i) {
- nodeIndex += holes[i].numPoints();
// create the doubly-linked hole list
Node list = createDoublyLinkedList(holes[i], nodeIndex, WindingOrder.CCW);
if (list == list.next) {
@@ -163,6 +162,7 @@ final public class Tessellator {
// Add the leftmost vertex of the hole.
holeList.add(fetchLeftmost(list));
}
+ nodeIndex += holes[i].numPoints();
}
// Sort the hole vertices by x coordinate
@@ -209,11 +209,15 @@ final public class Tessellator {
// segment's endpoint with lesser x will be potential connection point
{
do {
- if (hy <= p.getY() && hy >= p.next.getY()) {
+ if (hy <= p.getY() && hy >= p.next.getY() && p.next.getY() != p.getY()) {
final double x = p.getX() + (hy - p.getY()) * (p.next.getX() - p.getX()) / (p.next.getY() - p.getY());
if (x <= hx && x > qx) {
qx = x;
- connection = (p.getX() < p.next.getX()) ? p : p.next;
+ if (x == hx) {
+ if (hy == p.getY()) return p;
+ if (hy == p.next.getY()) return p.next;
+ }
+ connection = p.getX() < p.next.getX() ? p : p.next;
}
}
p = p.next;
@@ -222,7 +226,7 @@ final public class Tessellator {
if (connection == null) {
return null;
- } else if (hx == connection.getX()) {
+ } else if (hx == qx) {
return connection.previous;
}
@@ -237,8 +241,9 @@ final public class Tessellator {
p = connection.next;
{
while (p != stop) {
- if (hx > p.getX() && p.getX() >= mx && pointInEar(hy < my ? hx : qx, hy, mx, my, hy < my ? qx : hx, hy, mx, my)) {
- tan = Math.abs(hy - my) / (hx - mx); // tangential
+ if (hx >= p.getX() && p.getX() >= mx && hx != p.getX()
+ && pointInEar(p.getX(), p.getY(), hy < my ? hx : qx, hy, mx, my, hy < my ? qx : hx, hy)) {
+ tan = Math.abs(hy - p.getY()) / (hx - p.getX()); // tangential
if ((tan < tanMin || (tan == tanMin && p.getX() > connection.getX())) && isLocallyInside(p, holeNode)) {
connection = p;
tanMin = tan;
@@ -355,10 +360,6 @@ final public class Tessellator {
double cx = ear.next.x;
double cy = ear.next.y;
- // now make sure we don't have other points inside the potential ear;
- Node node;
- int idx;
-
// triangle bbox (flip the bits so negative encoded values are < positive encoded values)
int minTX = StrictMath.min(StrictMath.min(ear.previous.x, ear.x), ear.next.x) ^ 0x80000000;
int minTY = StrictMath.min(StrictMath.min(ear.previous.y, ear.y), ear.next.y) ^ 0x80000000;
@@ -369,32 +370,42 @@ final public class Tessellator {
long minZ = BitUtil.interleave(minTX, minTY);
long maxZ = BitUtil.interleave(maxTX, maxTY);
- // first look for points inside the triangle in increasing z-order
- node = ear.nextZ;
- while (node != null && Long.compareUnsigned(node.morton, maxZ) <= 0) {
- if (Long.compareUnsigned(node.morton, maxZ) <= 0) {
- idx = node.idx;
- if (idx != ear.previous.idx && idx != ear.next.idx
- && pointInEar(node.x, node.y, ax, ay, bx, by, cx, cy)
- && area(node.previous.x, node.previous.y, node.x, node.y, node.next.x, node.next.y) >= 0) {
+ // now make sure we don't have other points inside the potential ear;
+
+ // look for points inside the triangle in both directions
+ Node p = ear.previousZ;
+ Node n = ear.nextZ;
+ while (p != null && Long.compareUnsigned(p.morton, minZ) >= 0
+ && n != null && Long.compareUnsigned(n.morton, maxZ) <= 0) {
+ if (p.idx != ear.previous.idx && p.idx != ear.next.idx &&
+ pointInEar(p.x, p.y, ax, ay, bx, by, cx, cy) &&
+ area(p.previous.x, p.previous.y, p.x, p.y, p.next.x, p.next.y) >= 0) return false;
+ p = p.previousZ;
+
+ if (n.idx != ear.previous.idx && n.idx != ear.next.idx &&
+ pointInEar(n.x, n.y, ax, ay, bx, by, cx, cy) &&
+ area(n.previous.x, n.previous.y, n.x, n.y, n.next.x, n.next.y) >= 0) return false;
+ n = n.nextZ;
+ }
+
+ // first look for points inside the triangle in decreasing z-order
+ while (p != null && Long.compareUnsigned(p.morton, minZ) >= 0) {
+ if (p.idx != ear.previous.idx && p.idx != ear.next.idx
+ && pointInEar(p.x, p.y, ax, ay, bx, by, cx, cy)
+ && area(p.previous.x, p.previous.y, p.x, p.y, p.next.x, p.next.y) >= 0) {
return false;
}
- }
- node = node.nextZ;
- }
- // then look for points in decreasing z-order
- node = ear.previousZ;
- while (node != null &&
- Long.compareUnsigned(node.morton, minZ) >= 0) {
- if (Long.compareUnsigned(node.morton, maxZ) <= 0) {
- idx = node.idx;
- if (idx != ear.previous.idx && idx != ear.next.idx
- && pointInEar(node.x, node.y, ax, ay, bx, by, cx, cy)
- && area(node.previous.x, node.previous.y, node.x, node.y, node.next.x, node.next.y) >= 0) {
+ p = p.previousZ;
+ }
+ // then look for points in increasing z-order
+ while (n != null &&
+ Long.compareUnsigned(n.morton, maxZ) <= 0) {
+ if (n.idx != ear.previous.idx && n.idx != ear.next.idx
+ && pointInEar(n.x, n.y, ax, ay, bx, by, cx, cy)
+ && area(n.previous.x, n.previous.y, n.x, n.y, n.next.x, n.next.y) >= 0) {
return false;
}
- }
- node = node.previousZ;
+ n = n.nextZ;
}
return true;
}
@@ -464,6 +475,8 @@ final public class Tessellator {
a.nextZ = b;
b.previous = a;
b.previousZ = a;
+ a2.next = an;
+ a2.nextZ = an;
an.previous = a2;
an.previousZ = a2;
b2.next = a2;
@@ -577,24 +590,11 @@ final public class Tessellator {
// now we have two lists; merge
while (pSize > 0 || (qSize > 0 && q != null)) {
- // decide whether next element of merge comes from p or q
- if (pSize == 0) {
- // p is empty; e must come from q
- e = q;
- q = q.nextZ;
- --qSize;
- } else if (qSize == 0 || q == null) {
- // q is empty; e must come from p
- e = p;
- p = p.nextZ;
- --pSize;
- } else if (Long.compareUnsigned(p.morton, q.morton) <= 0) {
- // first element of p is lower (or same); e must come from p
+ if (pSize != 0 && (qSize == 0 || q == null || Long.compareUnsigned(p.morton, q.morton) <= 0)) {
e = p;
p = p.nextZ;
--pSize;
} else {
- // first element of q is lower; e must come from q
e = q;
q = q.nextZ;
--qSize;
@@ -677,7 +677,7 @@ final public class Tessellator {
node.nextZ = node;
} else {
node.next = lastNode.next;
- node.nextZ = lastNode.nextZ;
+ node.nextZ = lastNode.next;
node.previous = lastNode;
node.previousZ = lastNode;
lastNode.next.previous = node;
@@ -716,8 +716,8 @@ final public class Tessellator {
private static boolean pointInEar(final double x, final double y, final double ax, final double ay,
final double bx, final double by, final double cx, final double cy) {
return (cx - x) * (ay - y) - (ax - x) * (cy - y) >= 0 &&
- (ax - x) * (by - y) - (bx - x) * (ay - y) >= 0 &&
- (bx - x) * (cy - y) - (cx - x) * (by - y) >= 0;
+ (ax - x) * (by - y) - (bx - x) * (ay - y) >= 0 &&
+ (bx - x) * (cy - y) - (cx - x) * (by - y) >= 0;
}
/** compute whether the given x, y point is in a triangle; uses the winding order method */
@@ -795,16 +795,17 @@ final public class Tessellator {
this.next = other.next;
this.previousZ = other.previousZ;
this.nextZ = other.nextZ;
+ this.isSteiner = other.isSteiner;
}
/** get the x value */
public final double getX() {
- return polygon.getPolyLon(vrtxIdx);
+ return x;
}
/** get the y value */
public final double getY() {
- return polygon.getPolyLat(vrtxIdx);
+ return y;
}
/** get the longitude value */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d494f6be/lucene/sandbox/src/test/org/apache/lucene/geo/TestTessellator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/geo/TestTessellator.java b/lucene/sandbox/src/test/org/apache/lucene/geo/TestTessellator.java
index 878cf01..055e025 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/geo/TestTessellator.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/geo/TestTessellator.java
@@ -16,6 +16,9 @@
*/
package org.apache.lucene.geo;
+import java.text.ParseException;
+import java.util.List;
+
import org.apache.lucene.util.LuceneTestCase;
import static org.apache.lucene.geo.GeoTestUtil.nextBoxNotCrossingDateline;
@@ -42,4 +45,23 @@ public class TestTessellator extends LuceneTestCase {
poly = new Polygon(poly.getPolyLats(), poly.getPolyLons(), inner, inner2);
assertTrue(Tessellator.tessellate(poly).size() > 0);
}
+
+ public void testLUCENE8454() throws ParseException {
+ String geoJson = "{\"type\": \"Polygon\", \"coordinates\": [[[167.8752929333776, -30.078235509309092], [167.729078, -30.078368], [167.7288750679411, -29.918443128222044], [167.728949, -30.078598], [167.582239, -30.078557], [167.58234527408044, -29.9717026229659], " +
+ "[167.43547018634274, -30.030896196337487], [167.43528, -30.078575], [167.288467, -30.078185], [167.28846777961195, -30.078041819512045], [167.142089, -30.077483], [167.143635, -29.813199], [167.1450859974141, -29.567345798606294], [167.144888, -29.567345], " +
+ "[167.14633281276596, -29.302953194679134], [167.146281, -29.302953], [167.147725, -29.036352], [167.292924, -29.036892], [167.2918703799358, -29.301396273146477], [167.29192460356776, -29.301396365495897], [167.292964, -29.036798], [167.4380298884901, -29.037250444489867], " +
+ "[167.43803, -29.03719], [167.583317, -29.037381], [167.58331697583935, -29.03744011447325], [167.7285250024388, -29.037514998454153], [167.728525, -29.03749], [167.873835, -29.037419], [167.87383543708486, -29.037703808329873], [168.018612, -29.037121], " +
+ "[168.0186121103674, -29.03714161109612], [168.163842, -29.03656], [168.1650939339767, -29.247683610268638], [168.164004, -29.036724], [168.309341, -29.036127], [168.3110870459225, -29.30068025473746], [168.311176, -29.30068], [168.312472, -29.567161], " +
+ "[168.31243194795024, -29.56716111631554], [168.31443, -29.812612], [168.31388505737894, -29.812615143334597], [168.315886, -30.077081], [168.169234, -30.077883], [168.16913368505345, -30.06147402418803], [168.169224, -30.077737], [168.022447, -30.078317], " +
+ "[168.02181920125142, -29.924959173336568], [168.0221, -30.078254], [167.875293, -30.078413], [167.8752929333776, -30.078235509309092]]," + //holes
+ "[[167.43638852926597, -29.811913377451322], [167.43642819713568, -29.81191343893342], [167.43660948310222, -29.684470839430233], [167.43638852926597, -29.811913377451322]], " +
+ "[[167.2900169281376, -29.811700260790584], [167.29007609051774, -29.811700416752192], [167.29022481985885, -29.765019899914726], [167.2900169281376, -29.811700260790584]], " +
+ "[[167.72865676499967, -29.812149953736277], [167.7287401903084, -29.81214997654223], [167.72874, -29.812], [167.72893197342373, -29.81199982820994], [167.72851531939722, -29.568503012044204], [167.72851327553326, -29.568503011862287], [167.72865676499967, -29.812149953736277]], " +
+ "[[167.87424106545097, -29.302014822030415], [167.87432742269175, -29.30201461402921], [167.87418553426855, -29.265830214765142], [167.87424106545097, -29.302014822030415]], " +
+ "[[168.1652103335658, -29.3030088541673], [168.16605788758287, -29.446580625201833], [168.16556735186845, -29.303245228857072], [168.165381, -29.303246], [168.16537977124085, -29.303008170411644], [168.1652103335658, -29.3030088541673]], " +
+ "[[168.02088551865063, -29.647294313012004], [168.02133932508806, -29.811843292379823], [168.02135614030843, -29.811843274349446], [168.021356, -29.811809], [168.02162340579383, -29.811807949652078], [168.02088551865063, -29.647294313012004]]]}";
+ Polygon[] polygons =Polygon.fromGeoJSON(geoJson);
+ List<Tessellator.Triangle> result = Tessellator.tessellate(polygons[0]);
+ assertEquals(result.size(), 84);
+ }
}
\ No newline at end of file
[26/29] lucene-solr:jira/http2: Reset the latch between re-runs of
the same test.
Posted by da...@apache.org.
Reset the latch between re-runs of the same test.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5e6c99cc
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5e6c99cc
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5e6c99cc
Branch: refs/heads/jira/http2
Commit: 5e6c99cc35da4122c21d78e8715d58de4b4026a1
Parents: c0d2975
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Sep 20 14:26:55 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Sep 20 14:27:30 2018 +0200
----------------------------------------------------------------------
.../cloud/autoscaling/ScheduledMaintenanceTriggerTest.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5e6c99cc/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
index 840c0bd..36f4a13 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
@@ -41,6 +41,7 @@ import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.LogLevel;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@@ -78,6 +79,11 @@ public class ScheduledMaintenanceTriggerTest extends SolrCloudTestCase {
timeSource = cloudManager.getTimeSource();
}
+ @Before
+ public void initTest() {
+ triggerFired = new CountDownLatch(1);
+ }
+
@After
public void restoreDefaults() throws Exception {
SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST,
@@ -138,7 +144,7 @@ public class ScheduledMaintenanceTriggerTest extends SolrCloudTestCase {
}
}
- static CountDownLatch triggerFired = new CountDownLatch(1);
+ static CountDownLatch triggerFired;
public static class TestTriggerAction extends TriggerActionBase {
[16/29] lucene-solr:jira/http2: LUCENE-8482: Added feature query in
LatLonPoint to boost results by distance
Posted by da...@apache.org.
LUCENE-8482: Added feature query in LatLonPoint to boost results by distance
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/10060a62
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/10060a62
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/10060a62
Branch: refs/heads/jira/http2
Commit: 10060a6237ccd2785f6cbe248ca7254028f8eb04
Parents: c06a7ab
Author: iverase <iv...@apache.org>
Authored: Wed Sep 19 10:49:56 2018 +0200
Committer: iverase <iv...@apache.org>
Committed: Wed Sep 19 10:49:56 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
.../org/apache/lucene/document/LatLonPoint.java | 24 +
.../LatLonPointDistanceFeatureQuery.java | 480 +++++++++++++++++++
.../TestLatLonPointDistanceFeatureQuery.java | 411 ++++++++++++++++
4 files changed, 919 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10060a62/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 96c1c19..98631f3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -117,6 +117,10 @@ New Features
how close a value of a long field is from an configurable origin. This is
typically useful to boost by recency. (Adrien Grand)
+* LUCENE-8482: LatLonPoint#newDistanceFeatureQuery may be used to boost scores
+ based on the haversine distance of a LatLonPoint field to a provided point. This is
+ typically useful to boost by distance. (Ignacio Vera)
+
Improvements
* LUCENE-7997: Add BaseSimilarityTestCase to sanity check similarities.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10060a62/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java b/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
index 6cf8ab5..e3edead 100644
--- a/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
+++ b/lucene/core/src/java/org/apache/lucene/document/LatLonPoint.java
@@ -21,6 +21,8 @@ import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.PointRangeQuery;
@@ -252,4 +254,26 @@ public class LatLonPoint extends Field {
return new LatLonPointInPolygonQuery(field, polygons);
}
+ /**
+ * Given a field that indexes point values into a {@link LatLonPoint}
+ * and doc values into {@link LatLonDocValuesField}, this returns a query that scores
+ * documents based on their haversine distance in meters to {@code (originLat, originLon)}:
+ * {@code score = weight * pivotDistanceMeters / (pivotDistanceMeters + distance)}, ie.
+ * score is in the {@code [0, weight]} range, is equal to {@code weight} when
+ * the document's value is equal to {@code (originLat, originLon)} and is equal to
+ * {@code weight/2} when the document's value is distant of
+ * {@code pivotDistanceMeters} from {@code (originLat, originLon)}.
+ * In case of multi-valued fields, only the closest point to {@code (originLat, originLon)}
+ * will be considered.
+ * This query is typically useful to boost results based on distance by adding
+ * this query to a {@link Occur#SHOULD} clause of a {@link BooleanQuery}.
+ */
+ public static Query newDistanceFeatureQuery(String field, float weight, double originLat, double originLon, double pivotDistanceMeters) {
+ Query query = new LatLonPointDistanceFeatureQuery(field, originLat, originLon, pivotDistanceMeters);
+ if (weight != 1f) {
+ query = new BoostQuery(query, weight);
+ }
+ return query;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10060a62/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceFeatureQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceFeatureQuery.java b/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceFeatureQuery.java
new file mode 100644
index 0000000..3f30dea
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/document/LatLonPointDistanceFeatureQuery.java
@@ -0,0 +1,480 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.Set;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.geo.GeoUtils;
+import org.apache.lucene.geo.Rectangle;
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.index.PointValues;
+import org.apache.lucene.index.PointValues.IntersectVisitor;
+import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.index.SortedNumericDocValues;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.ScorerSupplier;
+import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.DocIdSetBuilder;
+import org.apache.lucene.util.FutureArrays;
+import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.SloppyMath;
+
+
+final class LatLonPointDistanceFeatureQuery extends Query {
+
+ private final String field;
+ private final double originLat;
+ private final double originLon;
+ private final double pivotDistance;
+
+ LatLonPointDistanceFeatureQuery(String field, double originLat, double originLon, double pivotDistance) {
+ this.field = Objects.requireNonNull(field);
+ GeoUtils.checkLatitude(originLat);
+ GeoUtils.checkLongitude(originLon);
+ this.originLon = originLon;
+ this.originLat = originLat;
+ if (pivotDistance <= 0) {
+ throw new IllegalArgumentException("pivotDistance must be > 0, got " + pivotDistance);
+ }
+ this.pivotDistance = pivotDistance;
+ }
+
+ @Override
+ public final boolean equals(Object o) {
+ return sameClassAs(o) &&
+ equalsTo(getClass().cast(o));
+ }
+
+ private boolean equalsTo(LatLonPointDistanceFeatureQuery other) {
+ return Objects.equals(field, other.field) &&
+ originLon == other.originLon &&
+ originLat == other.originLat &&
+ pivotDistance == other.pivotDistance;
+ }
+
+ @Override
+ public int hashCode() {
+ int h = classHash();
+ h = 31 * h + field.hashCode();
+ h = 31 * h + Double.hashCode(originLat);
+ h = 31 * h + Double.hashCode(originLon);
+ h = 31 * h + Double.hashCode(pivotDistance);
+ return h;
+ }
+
+ @Override
+ public String toString(String field) {
+ return getClass().getSimpleName() + "(field=" + field + ",originLat=" + originLat + ",originLon=" + originLon + ",pivotDistance=" + pivotDistance + ")";
+ }
+
+ @Override
+ public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+ return new Weight(this) {
+
+ @Override
+ public boolean isCacheable(LeafReaderContext ctx) {
+ return false;
+ }
+
+ @Override
+ public void extractTerms(Set<Term> terms) {}
+
+ @Override
+ public Explanation explain(LeafReaderContext context, int doc) throws IOException {
+ SortedNumericDocValues multiDocValues = DocValues.getSortedNumeric(context.reader(), field);
+ if (multiDocValues.advanceExact(doc) == false) {
+ return Explanation.noMatch("Document " + doc + " doesn't have a value for field " + field);
+ }
+ long encoded = selectValue(multiDocValues);
+ int latitudeBits = (int)(encoded >> 32);
+ int longitudeBits = (int)(encoded & 0xFFFFFFFF);
+ double lat = GeoEncodingUtils.decodeLatitude(latitudeBits);
+ double lon = GeoEncodingUtils.decodeLongitude(longitudeBits);
+ double distance = SloppyMath.haversinMeters(originLat, originLon, lat, lon);
+ float score = (float) (boost * (pivotDistance / (pivotDistance + distance)));
+ return Explanation.match(score, "Distance score, computed as weight * pivotDistance / (pivotDistance + abs(distance)) from:",
+ Explanation.match(boost, "weight"),
+ Explanation.match(pivotDistance, "pivotDistance"),
+ Explanation.match(originLat, "originLat"),
+ Explanation.match(originLon, "originLon"),
+ Explanation.match(lat, "current lat"),
+ Explanation.match(lon, "current lon"),
+ Explanation.match(distance, "distance"));
+ }
+
+ private long selectValue(SortedNumericDocValues multiDocValues) throws IOException {
+ int count = multiDocValues.docValueCount();
+ long value = multiDocValues.nextValue();
+ if (count == 1) {
+ return value;
+ }
+ // compute exact sort key: avoid any asin() computations
+ double distance = getDistanceKeyFromEncoded(value);
+ for (int i = 1; i < count; ++i) {
+ long nextValue = multiDocValues.nextValue();
+ double nextDistance = getDistanceKeyFromEncoded(nextValue);
+ if (nextDistance < distance) {
+ distance = nextDistance;
+ value = nextValue;
+ }
+ }
+ return value;
+ }
+
+ private NumericDocValues selectValues(SortedNumericDocValues multiDocValues) {
+ final NumericDocValues singleton = DocValues.unwrapSingleton(multiDocValues);
+ if (singleton != null) {
+ return singleton;
+ }
+ return new NumericDocValues() {
+
+ long value;
+
+ @Override
+ public long longValue() throws IOException {
+ return value;
+ }
+
+ @Override
+ public boolean advanceExact(int target) throws IOException {
+ if (multiDocValues.advanceExact(target)) {
+ value = selectValue(multiDocValues);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int docID() {
+ return multiDocValues.docID();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ return multiDocValues.nextDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return multiDocValues.advance(target);
+ }
+
+ @Override
+ public long cost() {
+ return multiDocValues.cost();
+ }
+
+ };
+ }
+
+ @Override
+ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
+ PointValues pointValues = context.reader().getPointValues(field);
+ if (pointValues == null) {
+ // No data on this segment
+ return null;
+ }
+ final SortedNumericDocValues multiDocValues = DocValues.getSortedNumeric(context.reader(), field);
+ final NumericDocValues docValues = selectValues(multiDocValues);
+
+ final Weight weight = this;
+ return new ScorerSupplier() {
+
+ @Override
+ public Scorer get(long leadCost) throws IOException {
+ return new DistanceScorer(weight, context.reader().maxDoc(), leadCost, boost, pointValues, docValues);
+ }
+
+ @Override
+ public long cost() {
+ return docValues.cost();
+ }
+ };
+ }
+
+ @Override
+ public Scorer scorer(LeafReaderContext context) throws IOException {
+ ScorerSupplier scorerSupplier = scorerSupplier(context);
+ if (scorerSupplier == null) {
+ return null;
+ }
+ return scorerSupplier.get(Long.MAX_VALUE);
+ }
+
+ };
+ }
+
+ private double getDistanceFromEncoded(long encoded) {
+ return SloppyMath.haversinMeters(getDistanceKeyFromEncoded(encoded));
+ }
+
+ private double getDistanceKeyFromEncoded(long encoded) {
+ int latitudeBits = (int)(encoded >> 32);
+ int longitudeBits = (int)(encoded & 0xFFFFFFFF);
+ double lat = GeoEncodingUtils.decodeLatitude(latitudeBits);
+ double lon = GeoEncodingUtils.decodeLongitude(longitudeBits);
+ return SloppyMath.haversinSortKey(originLat, originLon, lat, lon);
+ }
+
+ private class DistanceScorer extends Scorer {
+
+ private final int maxDoc;
+ private DocIdSetIterator it;
+ private int doc = -1;
+ private final long leadCost;
+ private final float boost;
+ private final PointValues pointValues;
+ private final NumericDocValues docValues;
+ private double maxDistance = GeoUtils.EARTH_MEAN_RADIUS_METERS * Math.PI;
+
+ protected DistanceScorer(Weight weight, int maxDoc, long leadCost, float boost,
+ PointValues pointValues, NumericDocValues docValues) {
+ super(weight);
+ this.maxDoc = maxDoc;
+ this.leadCost = leadCost;
+ this.boost = boost;
+ this.pointValues = pointValues;
+ this.docValues = docValues;
+ // initially use doc values in order to iterate all documents that have
+ // a value for this field
+ this.it = docValues;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private float score(double distance) {
+ return (float) (boost * (pivotDistance / (pivotDistance + distance)));
+ }
+
+ /**
+ * Inverting the score computation is very hard due to all potential
+ * rounding errors, so we binary search the maximum distance. The limit
+ * is set to 1 meter.
+ */
+ private double computeMaxDistance(float minScore, double previousMaxDistance) {
+ assert score(0) >= minScore;
+ if (score(previousMaxDistance) >= minScore) {
+ // minScore did not decrease enough to require an update to the max distance
+ return previousMaxDistance;
+ }
+ assert score(previousMaxDistance) < minScore;
+ double min = 0, max = previousMaxDistance;
+ // invariant: score(min) >= minScore && score(max) < minScore
+ while (max - min > 1) {
+ double mid = (min + max) / 2;
+ float score = score(mid);
+ if (score >= minScore) {
+ min = mid;
+ } else {
+ max = mid;
+ }
+ }
+ assert score(min) >= minScore;
+ assert min == Double.MAX_VALUE || score(min + 1) < minScore;
+ return min;
+ }
+
+ @Override
+ public float score() throws IOException {
+ if (docValues.advanceExact(docID()) == false) {
+ return 0;
+ }
+ return score(getDistanceFromEncoded(docValues.longValue()));
+ }
+
+ @Override
+ public DocIdSetIterator iterator() {
+ // add indirection so that if 'it' is updated then it will
+ // be taken into account
+ return new DocIdSetIterator() {
+
+ @Override
+ public int nextDoc() throws IOException {
+ return doc = it.nextDoc();
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public long cost() {
+ return it.cost();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ return doc = it.advance(target);
+ }
+ };
+ }
+
+ @Override
+ public float getMaxScore(int upTo) {
+ return boost;
+ }
+
+ private int setMinCompetitiveScoreCounter = 0;
+
+
+ @Override
+ public void setMinCompetitiveScore(float minScore) throws IOException {
+ if (minScore > boost) {
+ it = DocIdSetIterator.empty();
+ return;
+ }
+
+ setMinCompetitiveScoreCounter++;
+ // We sample the calls to this method as it is expensive to recalculate the iterator.
+ if (setMinCompetitiveScoreCounter > 256 && (setMinCompetitiveScoreCounter & 0x1f) != 0x1f) {
+ return;
+ }
+
+ double previousMaxDistance = maxDistance;
+ maxDistance = computeMaxDistance(minScore, maxDistance);
+ if (maxDistance == previousMaxDistance) {
+ // nothing to update
+ return;
+ }
+
+ //Ideally we would be doing a distance query but that is too expensive so we approximate
+ //with a box query which performs better.
+ Rectangle box = Rectangle.fromPointDistance(originLat, originLon, maxDistance);
+ final byte minLat[] = new byte[LatLonPoint.BYTES];
+ final byte maxLat[] = new byte[LatLonPoint.BYTES];
+ final byte minLon[] = new byte[LatLonPoint.BYTES];
+ final byte maxLon[] = new byte[LatLonPoint.BYTES];
+ final boolean crossDateLine = box.crossesDateline();
+
+
+ NumericUtils.intToSortableBytes(GeoEncodingUtils.encodeLatitude(box.minLat), minLat, 0);
+ NumericUtils.intToSortableBytes(GeoEncodingUtils.encodeLatitude(box.maxLat), maxLat, 0);
+ NumericUtils.intToSortableBytes(GeoEncodingUtils.encodeLongitude(box.minLon), minLon, 0);
+ NumericUtils.intToSortableBytes(GeoEncodingUtils.encodeLongitude(box.maxLon), maxLon, 0);
+
+ DocIdSetBuilder result = new DocIdSetBuilder(maxDoc);
+ final int doc = docID();
+ IntersectVisitor visitor = new IntersectVisitor() {
+
+ DocIdSetBuilder.BulkAdder adder;
+
+ @Override
+ public void grow(int count) {
+ adder = result.grow(count);
+ }
+
+ @Override
+ public void visit(int docID) {
+ if (docID <= doc) {
+ // Already visited or skipped
+ return;
+ }
+ adder.add(docID);
+ }
+
+ @Override
+ public void visit(int docID, byte[] packedValue) {
+ if (docID <= doc) {
+ // Already visited or skipped
+ return;
+ }
+ if (FutureArrays.compareUnsigned(packedValue, 0, LatLonPoint.BYTES, maxLat, 0, LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(packedValue, 0, LatLonPoint.BYTES, minLat, 0, LatLonPoint.BYTES) < 0) {
+ //Latitude out of range
+ return;
+ }
+ if (crossDateLine) {
+ if (FutureArrays.compareUnsigned(packedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, minLon, 0, LatLonPoint.BYTES) < 0 &&
+ FutureArrays.compareUnsigned(packedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, maxLon, 0, LatLonPoint.BYTES) > 0) {
+ //Longitude out of range
+ return;
+ }
+
+ } else {
+ if (FutureArrays.compareUnsigned(packedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, maxLon, 0, LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(packedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, minLon, 0, LatLonPoint.BYTES) < 0) {
+ //Longitude out of range
+ return;
+ }
+ }
+ adder.add(docID);
+ }
+
+ @Override
+ public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+
+ if (FutureArrays.compareUnsigned(minPackedValue, 0, LatLonPoint.BYTES, maxLat, 0, LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(maxPackedValue, 0, LatLonPoint.BYTES, minLat, 0, LatLonPoint.BYTES) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+ boolean crosses = FutureArrays.compareUnsigned(minPackedValue, 0, LatLonPoint.BYTES, minLat, 0, LatLonPoint.BYTES) < 0 ||
+ FutureArrays.compareUnsigned(maxPackedValue, 0, LatLonPoint.BYTES, maxLat, 0, LatLonPoint.BYTES) > 0;
+
+ if (crossDateLine) {
+ if (FutureArrays.compareUnsigned(minPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, maxLon, 0, LatLonPoint.BYTES) > 0 &&
+ FutureArrays.compareUnsigned(maxPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, minLon, 0, LatLonPoint.BYTES) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+ crosses |= FutureArrays.compareUnsigned(minPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, maxLon, 0, LatLonPoint.BYTES) < 0 ||
+ FutureArrays.compareUnsigned(maxPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, minLon, 0, LatLonPoint.BYTES) > 0;
+
+ } else {
+ if (FutureArrays.compareUnsigned(minPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, maxLon, 0, LatLonPoint.BYTES) > 0 ||
+ FutureArrays.compareUnsigned(maxPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, minLon, 0, LatLonPoint.BYTES) < 0) {
+ return Relation.CELL_OUTSIDE_QUERY;
+ }
+ crosses |= FutureArrays.compareUnsigned(minPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, minLon, 0, LatLonPoint.BYTES) < 0 ||
+ FutureArrays.compareUnsigned(maxPackedValue, LatLonPoint.BYTES, 2 * LatLonPoint.BYTES, maxLon, 0, LatLonPoint.BYTES) > 0;
+ }
+ if (crosses) {
+ return Relation.CELL_CROSSES_QUERY;
+ } else {
+ return Relation.CELL_INSIDE_QUERY;
+ }
+ }
+ };
+
+ final long currentQueryCost = Math.min(leadCost, it.cost());
+ final long threshold = currentQueryCost >>> 3;
+ long estimatedNumberOfMatches = pointValues.estimatePointCount(visitor); // runs in O(log(numPoints))
+ // TODO: what is the right factor compared to the current disi? Is 8 optimal?
+ if (estimatedNumberOfMatches >= threshold) {
+ // the new range is not selective enough to be worth materializing
+ return;
+ }
+ pointValues.intersect(visitor);
+ it = result.build().iterator();
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/10060a62/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java b/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java
new file mode 100644
index 0000000..2010bc5
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/document/TestLatLonPointDistanceFeatureQuery.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.document;
+
+import java.io.IOException;
+
+import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.geo.GeoUtils;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MultiReader;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.search.CheckHits;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryUtils;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.TopScoreDocCollector;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.SloppyMath;
+
+public class TestLatLonPointDistanceFeatureQuery extends LuceneTestCase {
+
+ public void testEqualsAndHashcode() {
+ Query q1 = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, 5);
+ Query q2 = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, 5);
+ QueryUtils.checkEqual(q1, q2);
+
+ Query q3 = LatLonPoint.newDistanceFeatureQuery("bar", 3, 10, 10,5);
+ QueryUtils.checkUnequal(q1, q3);
+
+ Query q4 = LatLonPoint.newDistanceFeatureQuery("foo", 4, 10, 10, 5);
+ QueryUtils.checkUnequal(q1, q4);
+
+ Query q5 = LatLonPoint.newDistanceFeatureQuery("foo", 3, 9, 10, 5);
+ QueryUtils.checkUnequal(q1, q5);
+
+ Query q6 = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 9, 5);
+ QueryUtils.checkUnequal(q1, q6);
+
+ Query q7 = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, 6);
+ QueryUtils.checkUnequal(q1, q7);
+ }
+
+ public void testBasics() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig()
+ .setMergePolicy(newLogMergePolicy(random().nextBoolean())));
+ Document doc = new Document();
+ LatLonPoint point = new LatLonPoint("foo", 0.0, 0.0);
+ doc.add(point);
+ LatLonDocValuesField docValue = new LatLonDocValuesField("foo",0.0, 0.0);
+ doc.add(docValue);
+
+ double pivotDistance = 5000;//5k
+
+ point.setLocationValue(-7, -7);
+ docValue.setLocationValue(-7, -7);
+ w.addDocument(doc);
+
+ point.setLocationValue(9, 9);
+ docValue.setLocationValue(9, 9);
+ w.addDocument(doc);
+
+
+ point.setLocationValue(8, 8);
+ docValue.setLocationValue(8, 8);
+ w.addDocument(doc);
+
+ point.setLocationValue(4, 4);
+ docValue.setLocationValue(4, 4);
+ w.addDocument(doc);
+
+ point.setLocationValue(-1, -1);
+ docValue.setLocationValue(-1, -1);
+ w.addDocument(doc);
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, pivotDistance);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1);
+ searcher.search(q, collector);
+ TopDocs topHits = collector.topDocs();
+ assertEquals(2, topHits.scoreDocs.length);
+
+ double distance1 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(9)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(9)), 10,10);
+ double distance2 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(8)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(8)), 10,10);
+
+ CheckHits.checkEqual(q,
+ new ScoreDoc[] {
+ new ScoreDoc(1, (float) (3f * (pivotDistance / (pivotDistance + distance1)))),
+ new ScoreDoc(2, (float) (3f * (pivotDistance / (pivotDistance + distance2))))
+ },
+ topHits.scoreDocs);
+
+ distance1 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(9)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(9)), 9,9);
+ distance2 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(8)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(8)), 9,9);
+
+ q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 9, 9, pivotDistance);
+ collector = TopScoreDocCollector.create(2, null, 1);
+ searcher.search(q, collector);
+ topHits = collector.topDocs();
+ assertEquals(2, topHits.scoreDocs.length);
+ CheckHits.checkExplanations(q, "", searcher);
+
+ CheckHits.checkEqual(q,
+ new ScoreDoc[] {
+ new ScoreDoc(1, (float) (3f * (pivotDistance / (pivotDistance + distance1)))),
+ new ScoreDoc(2, (float) (3f * (pivotDistance / (pivotDistance + distance2))))
+ },
+ topHits.scoreDocs);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testCrossesDateLine() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig()
+ .setMergePolicy(newLogMergePolicy(random().nextBoolean())));
+ Document doc = new Document();
+ LatLonPoint point = new LatLonPoint("foo", 0.0, 0.0);
+ doc.add(point);
+ LatLonDocValuesField docValue = new LatLonDocValuesField("foo",0.0, 0.0);
+ doc.add(docValue);
+
+ double pivotDistance = 5000;//5k
+
+ point.setLocationValue(0, -179);
+ docValue.setLocationValue(0, -179);
+ w.addDocument(doc);
+
+ point.setLocationValue(0, 176);
+ docValue.setLocationValue(0, 176);
+ w.addDocument(doc);
+
+ point.setLocationValue(0, -150);
+ docValue.setLocationValue(0, -150);
+ w.addDocument(doc);
+
+ point.setLocationValue(0, -140);
+ docValue.setLocationValue(0, -140);
+ w.addDocument(doc);
+
+ point.setLocationValue(0, 140);
+ docValue.setLocationValue(01, 140);
+ w.addDocument(doc);
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 0, 179, pivotDistance);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1);
+ searcher.search(q, collector);
+ TopDocs topHits = collector.topDocs();
+ assertEquals(2, topHits.scoreDocs.length);
+
+ double distance1 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(0)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(-179)), 0,179);
+ double distance2 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(0)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(176)), 0,179);
+
+ CheckHits.checkEqual(q,
+ new ScoreDoc[] {
+ new ScoreDoc(0, (float) (3f * (pivotDistance / (pivotDistance + distance1)))),
+ new ScoreDoc(1, (float) (3f * (pivotDistance / (pivotDistance + distance2))))
+ },
+ topHits.scoreDocs);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMissingField() throws IOException {
+ IndexReader reader = new MultiReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, 5000);
+ TopDocs topHits = searcher.search(q, 2);
+ assertEquals(0, topHits.totalHits.value);
+ }
+
+ public void testMissingValue() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig()
+ .setMergePolicy(newLogMergePolicy(random().nextBoolean())));
+ Document doc = new Document();
+ LatLonPoint point = new LatLonPoint("foo", 0, 0);
+ doc.add(point);
+ LatLonDocValuesField docValue = new LatLonDocValuesField("foo", 0, 0);
+ doc.add(docValue);
+
+ point.setLocationValue(3, 3);
+ docValue.setLocationValue(3, 3);
+ w.addDocument(doc);
+
+ w.addDocument(new Document());
+
+ point.setLocationValue(7, 7);
+ docValue.setLocationValue(7, 7);
+ w.addDocument(doc);
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 10, 10, 5);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(3, null, 1);
+ searcher.search(q, collector);
+ TopDocs topHits = collector.topDocs();
+ assertEquals(2, topHits.scoreDocs.length);
+
+ double distance1 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(7)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(7)), 10,10);
+ double distance2 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(3)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(3)), 10,10);
+
+ CheckHits.checkEqual(q,
+ new ScoreDoc[] {
+ new ScoreDoc(2, (float) (3f * (5. / (5. + distance1)))),
+ new ScoreDoc(0, (float) (3f * (5. / (5. + distance2))))
+ },
+ topHits.scoreDocs);
+
+ CheckHits.checkExplanations(q, "", searcher);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testMultiValued() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig()
+ .setMergePolicy(newLogMergePolicy(random().nextBoolean())));
+
+ Document doc = new Document();
+ for (double[] point : new double[][] {{0, 0}, {30, 30}, {60, 60}}) {
+ doc.add(new LatLonPoint("foo", point[0], point[1]));
+ doc.add(new LatLonDocValuesField("foo", point[0], point[1]));
+ }
+ w.addDocument(doc);
+
+ doc = new Document();
+ for (double[] point : new double[][] {{45, 0}, {-45, 0}, {-90, 0}, {90, 0}}) {
+ doc.add(new LatLonPoint("foo", point[0], point[1]));
+ doc.add(new LatLonDocValuesField("foo", point[0], point[1]));
+ }
+ w.addDocument(doc);
+
+ doc = new Document();
+ for (double[] point : new double[][] {{0, 90}, {0, -90}, {0, 180}, {0, -180}}) {
+ doc.add(new LatLonPoint("foo", point[0], point[1]));
+ doc.add(new LatLonDocValuesField("foo", point[0], point[1]));
+ }
+ w.addDocument(doc);
+
+ doc = new Document();
+ for (double[] point : new double[][] {{3, 2}}) {
+ doc.add(new LatLonPoint("foo", point[0], point[1]));
+ doc.add(new LatLonDocValuesField("foo", point[0], point[1]));
+ }
+ w.addDocument(doc);
+
+ doc = new Document();
+ for (double[] point : new double[][] {{45, 45}, {-45, -45}}) {
+ doc.add(new LatLonPoint("foo", point[0], point[1]));
+ doc.add(new LatLonDocValuesField("foo", point[0], point[1]));
+ }
+ w.addDocument(doc);
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ Query q = LatLonPoint.newDistanceFeatureQuery("foo", 3, 0, 0, 200);
+ TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 1);
+ searcher.search(q, collector);
+ TopDocs topHits = collector.topDocs();
+ assertEquals(2, topHits.scoreDocs.length);
+
+ double distance1 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(0)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(0)), 0,0);
+ double distance2 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(3)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(2)), 0,0);
+
+ CheckHits.checkEqual(q,
+ new ScoreDoc[] {
+ new ScoreDoc(0, (float) (3f * (200 / (200 + distance1)))),
+ new ScoreDoc(3, (float) (3f * (200 / (200 + distance2))))
+ },
+ topHits.scoreDocs);
+
+ q = LatLonPoint.newDistanceFeatureQuery("foo", 3, -90, 0, 10000.);
+ collector = TopScoreDocCollector.create(2, null, 1);
+ searcher.search(q, collector);
+ topHits = collector.topDocs();
+ assertEquals(2, topHits.scoreDocs.length);
+ CheckHits.checkExplanations(q, "", searcher);
+
+ distance1 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-90)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(0)), -90,0);
+ distance2 = SloppyMath.haversinMeters(GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-45)) , GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(-45)), -90,0);
+
+ CheckHits.checkEqual(q,
+ new ScoreDoc[] {
+ new ScoreDoc(1, (float) (3f * (10000. / (10000. + distance1)))),
+ new ScoreDoc(4, (float) (3f * (10000. / (10000. + distance2))))
+ },
+ topHits.scoreDocs);
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testRandom() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig()
+ .setMergePolicy(newLogMergePolicy(random().nextBoolean())));
+ Document doc = new Document();
+ LatLonPoint point = new LatLonPoint("foo", 0., 0.);
+ doc.add(point);
+ LatLonDocValuesField docValue = new LatLonDocValuesField("foo", 0., 0.);
+ doc.add(docValue);
+
+ int numDocs = atLeast(10000);
+ for (int i = 0; i < numDocs; ++i) {
+ double lat = random().nextDouble() * 180 - 90;
+ double lon = random().nextDouble() * 360 - 180;
+ point.setLocationValue(lat, lon);
+ docValue.setLocationValue(lat, lon);
+ w.addDocument(doc);
+ }
+
+ IndexReader reader = DirectoryReader.open(w);
+ IndexSearcher searcher = newSearcher(reader);
+
+ for (int iter = 0; iter < 10; ++iter) {
+ double lat = random().nextDouble() * 180 - 90;
+ double lon = random().nextDouble() * 360 - 180;
+ double pivotDistance = random().nextDouble() * random().nextDouble() * Math.PI * GeoUtils.EARTH_MEAN_RADIUS_METERS;
+ float boost = (1 + random().nextInt(10)) / 3f;
+ Query q = LatLonPoint.newDistanceFeatureQuery("foo", boost, lat, lon, pivotDistance);
+
+ CheckHits.checkTopScores(random(), q, searcher);
+ }
+
+ reader.close();
+ w.close();
+ dir.close();
+ }
+
+ public void testCompareSorting() throws IOException {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig()
+ .setMergePolicy(newLogMergePolicy(random().nextBoolean())));
+
+ Document doc = new Document();
+ LatLonPoint point = new LatLonPoint("foo", 0., 0.);
+ doc.add(point);
+ LatLonDocValuesField docValue = new LatLonDocValuesField("foo", 0., 0.);
+ doc.add(docValue);
+
+ int numDocs = atLeast(10000);
+ for (int i = 0; i < numDocs; ++i) {
+ double lat = random().nextDouble() * 180 - 90;
+ double lon = random().nextDouble() * 360 - 180;
+ point.setLocationValue(lat, lon);
+ docValue.setLocationValue(lat, lon);
+ w.addDocument(doc);
+ }
+
+ DirectoryReader reader = w.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+
+ double lat = random().nextDouble() * 180 - 90;
+ double lon = random().nextDouble() * 360 - 180;
+ double pivotDistance = random().nextDouble() * random().nextDouble() * GeoUtils.EARTH_MEAN_RADIUS_METERS * Math.PI;
+ float boost = (1 + random().nextInt(10)) / 3f;
+
+ Query query1 = LatLonPoint.newDistanceFeatureQuery("foo", boost, lat, lon, pivotDistance);
+ Sort sort1 = new Sort(SortField.FIELD_SCORE, LatLonDocValuesField.newDistanceSort("foo", lat, lon));
+
+ Query query2 = new MatchAllDocsQuery();
+ Sort sort2 = new Sort(LatLonDocValuesField.newDistanceSort("foo", lat, lon));
+
+ TopDocs topDocs1 = searcher.search(query1, 10, sort1);
+ TopDocs topDocs2 = searcher.search(query2, 10, sort2);
+ for (int i =0; i< 10; i++) {
+ assertTrue(topDocs1.scoreDocs[i].doc == topDocs2.scoreDocs[i].doc);
+ }
+ reader.close();
+ w.close();
+ dir.close();
+ }
+}
[07/29] lucene-solr:jira/http2: LUCENE-8504: Update forbiddenapis to
2.6
Posted by da...@apache.org.
LUCENE-8504: Update forbiddenapis to 2.6
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4940b366
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4940b366
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4940b366
Branch: refs/heads/jira/http2
Commit: 4940b3666ef4eb6a32f7720fcd7c34994682901f
Parents: 6fac57d
Author: Uwe Schindler <us...@apache.org>
Authored: Mon Sep 17 13:02:11 2018 +0200
Committer: Uwe Schindler <us...@apache.org>
Committed: Mon Sep 17 13:02:11 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 ++++-
lucene/common-build.xml | 2 +-
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4940b366/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6b8cb67..f62fef2b 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -161,7 +161,10 @@ Optimizations
(Jim Ferenczi, Adrien Grand)
======================= Lucene 7.6.0 =======================
-(No Changes)
+
+Build
+
+* LUCENE-8504: Upgrade forbiddenapis to version 2.6. (Uwe Schindler)
======================= Lucene 7.5.0 =======================
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4940b366/lucene/common-build.xml
----------------------------------------------------------------------
diff --git a/lucene/common-build.xml b/lucene/common-build.xml
index 9fbddc3..0d38822 100644
--- a/lucene/common-build.xml
+++ b/lucene/common-build.xml
@@ -2404,7 +2404,7 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
<property name="forbidden-sysout-excludes" value=""/>
<target name="-install-forbidden-apis" unless="forbidden-apis.loaded" depends="ivy-availability-check,ivy-configure">
- <ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="2.5"
+ <ivy:cachepath organisation="de.thetaphi" module="forbiddenapis" revision="2.6"
inline="true" conf="default" transitive="true" pathid="forbidden-apis.classpath"/>
<taskdef name="forbidden-apis" classname="de.thetaphi.forbiddenapis.ant.AntTask" classpathref="forbidden-apis.classpath"/>
<property name="forbidden-apis.loaded" value="true"/>
[11/29] lucene-solr:jira/http2: Add missing changes entries for
LUCENE-8502 and LUCENE-8503
Posted by da...@apache.org.
Add missing changes entries for LUCENE-8502 and LUCENE-8503
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/895bff46
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/895bff46
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/895bff46
Branch: refs/heads/jira/http2
Commit: 895bff46b2c3d3d027e693e11a2feac9f518191d
Parents: 44dbfaa
Author: Simon Willnauer <si...@apache.org>
Authored: Mon Sep 17 16:07:35 2018 +0200
Committer: Simon Willnauer <si...@apache.org>
Committed: Mon Sep 17 16:08:01 2018 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/895bff46/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f62fef2b..96c1c19 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -270,6 +270,10 @@ Bug Fixes:
* LUCENE-8466: IndexWriter.deleteDocs(Query... query) incorrectly applies deletes on flush
if the index is sorted. (Adrien Grand, Jim Ferenczi, Vish Ramachandran)
+* LUCENE-8502: Allow access to delegate in FilterCodecReader. FilterCodecReader didn't
+ allow access to it's delegate like other filter readers. This adds a new #getDelegate method
+ to access the wrapped reader. (Simon Willnauer)
+
Changes in Runtime Behavior:
* LUCENE-7976: TieredMergePolicy now respects maxSegmentSizeMB by default when executing
@@ -279,6 +283,11 @@ Changes in Runtime Behavior:
aggressively by default ensuring that no more than ~1/3 of the index size is
used by deleted documents. (Adrien Grand)
+* LUCENE-8503: Call #getDelegate instead of direct member access during unwrap.
+ Filter*Reader instances access the member or the delegate directly instead of
+ calling getDelegate(). In order to track access of the delegate these methods
+ should call #getDelegate() (Simon Willnauer)
+
Improvements
* LUCENE-8468: A ByteBuffer based Directory implementation. (Dawid Weiss)
[03/29] lucene-solr:jira/http2: LUCENE-8500: Fix release smoke test
to always use the downloaded KEYS file
Posted by da...@apache.org.
LUCENE-8500: Fix release smoke test to always use the downloaded KEYS file
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a02959c8
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a02959c8
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a02959c8
Branch: refs/heads/jira/http2
Commit: a02959c87e132ae341a689b55afdd566f90f7e5c
Parents: 39733cb
Author: Jim Ferenczi <ji...@apache.org>
Authored: Sat Sep 15 21:28:12 2018 +0200
Committer: Jim Ferenczi <ji...@apache.org>
Committed: Sat Sep 15 21:28:12 2018 +0200
----------------------------------------------------------------------
dev-tools/scripts/smokeTestRelease.py | 39 ++++++++++++++----------------
1 file changed, 18 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a02959c8/dev-tools/scripts/smokeTestRelease.py
----------------------------------------------------------------------
diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py
index af26ae1..70bea24 100644
--- a/dev-tools/scripts/smokeTestRelease.py
+++ b/dev-tools/scripts/smokeTestRelease.py
@@ -285,7 +285,7 @@ def checkAllJARs(topDir, project, gitRevision, version, tmpDir, baseURL):
% (fullPath, luceneDistFilenames[jarFilename]))
-def checkSigs(project, urlString, version, tmpDir, isSigned, local_keys):
+def checkSigs(project, urlString, version, tmpDir, isSigned, keysFile):
print(' test basics...')
ents = getDirEntries(urlString)
@@ -344,16 +344,6 @@ def checkSigs(project, urlString, version, tmpDir, isSigned, local_keys):
actual = [x[0] for x in artifacts]
if expected != actual:
raise RuntimeError('%s: wrong artifacts: expected %s but got %s' % (project, expected, actual))
-
- print(' get KEYS')
- if local_keys is not None:
- print(" Using local KEYS file %s" % local_keys)
- keysFile = local_keys
- else:
- keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
- print(" Downloading online KEYS file %s" % keysFileURL)
- download('KEYS', keysFileURL, tmpDir)
- keysFile = '%s/KEYS' % (tmpDir)
# Set up clean gpg world; import keys file:
gpgHomeDir = '%s/%s.gpg' % (tmpDir, project)
@@ -978,7 +968,7 @@ def testDemo(run_java, isSrc, version, jdk):
def removeTrailingZeros(version):
return re.sub(r'(\.0)*$', '', version)
-def checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned):
+def checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile):
POMtemplates = defaultdict()
getPOMtemplates(solrSrcUnpackPath, POMtemplates, tmpDir)
print(' download artifacts')
@@ -996,7 +986,7 @@ def checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigne
checkJavadocAndSourceArtifacts(artifacts, version)
verifyDeployedPOMsCoordinates(artifacts, version)
if isSigned:
- verifyMavenSigs(baseURL, tmpDir, artifacts)
+ verifyMavenSigs(baseURL, tmpDir, artifacts, keysFile)
distFiles = getBinaryDistFilesForMavenChecks(tmpDir, version, baseURL)
checkIdenticalMavenArtifacts(distFiles, artifacts, version)
@@ -1118,13 +1108,9 @@ def getPOMcoordinate(treeRoot):
packaging = 'jar' if packaging is None else packaging.text.strip()
return groupId, artifactId, packaging, version
-def verifyMavenSigs(baseURL, tmpDir, artifacts):
+def verifyMavenSigs(baseURL, tmpDir, artifacts, keysFile):
print(' verify maven artifact sigs', end=' ')
for project in ('lucene', 'solr'):
- keysFile = '%s/%s.KEYS' % (tmpDir, project)
- if not os.path.exists(keysFile):
- keysURL = '%s/%s/KEYS' % (baseURL, project)
- download('%s.KEYS' % project, keysURL, tmpDir, quiet=True)
# Set up clean gpg world; import keys file:
gpgHomeDir = '%s/%s.gpg' % (tmpDir, project)
@@ -1505,15 +1491,26 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
raise RuntimeError('could not find solr subdir')
print()
+ print('Get KEYS...')
+ if local_keys is not None:
+ print(" Using local KEYS file %s" % local_keys)
+ keysFile = local_keys
+ else:
+ keysFileURL = "https://archive.apache.org/dist/lucene/KEYS"
+ print(" Downloading online KEYS file %s" % keysFileURL)
+ download('KEYS', keysFileURL, tmpDir)
+ keysFile = '%s/KEYS' % (tmpDir)
+
+ print()
print('Test Lucene...')
- checkSigs('lucene', lucenePath, version, tmpDir, isSigned, local_keys)
+ checkSigs('lucene', lucenePath, version, tmpDir, isSigned, keysFile)
for artifact in ('lucene-%s.tgz' % version, 'lucene-%s.zip' % version):
unpackAndVerify(java, 'lucene', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
unpackAndVerify(java, 'lucene', tmpDir, 'lucene-%s-src.tgz' % version, gitRevision, version, testArgs, baseURL)
print()
print('Test Solr...')
- checkSigs('solr', solrPath, version, tmpDir, isSigned, local_keys)
+ checkSigs('solr', solrPath, version, tmpDir, isSigned, keysFile)
for artifact in ('solr-%s.tgz' % version, 'solr-%s.zip' % version):
unpackAndVerify(java, 'solr', tmpDir, artifact, gitRevision, version, testArgs, baseURL)
solrSrcUnpackPath = unpackAndVerify(java, 'solr', tmpDir, 'solr-%s-src.tgz' % version,
@@ -1521,7 +1518,7 @@ def smokeTest(java, baseURL, gitRevision, version, tmpDir, isSigned, local_keys,
print()
print('Test Maven artifacts for Lucene and Solr...')
- checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned)
+ checkMaven(solrSrcUnpackPath, baseURL, tmpDir, gitRevision, version, isSigned, keysFile)
print('\nSUCCESS! [%s]\n' % (datetime.datetime.now() - startTime))
[15/29] lucene-solr:jira/http2: SOLR-12759: fix regexp
Posted by da...@apache.org.
SOLR-12759: fix regexp
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c06a7ab0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c06a7ab0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c06a7ab0
Branch: refs/heads/jira/http2
Commit: c06a7ab0c79306c221f6441641bde973b2ff1e56
Parents: 5abee47
Author: David Smiley <ds...@apache.org>
Authored: Tue Sep 18 20:21:44 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Tue Sep 18 20:21:44 2018 -0400
----------------------------------------------------------------------
.../solr/handler/extraction/ExtractingRequestHandlerTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c06a7ab0/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
index 85e1c78..4401745 100644
--- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
+++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/ExtractingRequestHandlerTest.java
@@ -45,7 +45,7 @@ public class ExtractingRequestHandlerTest extends SolrTestCaseJ4 {
@BeforeClass
public static void beforeClass() throws Exception {
// Is the JDK/env affected by a known bug?
- if (!TimeZone.getDefault().getDisplayName(false, TimeZone.SHORT, Locale.US).matches("[A-Z]{3}([+-]\\d\\d(:\\d\\d)?)?")) {
+ if (!TimeZone.getDefault().getDisplayName(false, TimeZone.SHORT, Locale.US).matches("[A-Z]{3,4}([+-]\\d\\d(:\\d\\d)?)?")) {
assert System.getProperty("java.version").startsWith("11") : "Is some other JVM affected? Or bad regex?";
assumeTrue("SOLR-12759 JDK 11 (1st release) and Tika 1.x can result in extracting dates in a bad format.", false);
}
[18/29] lucene-solr:jira/http2: Fix occasional test failure due to a
missing trigger config.
Posted by da...@apache.org.
Fix occasional test failure due to a missing trigger config.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/40823263
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/40823263
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/40823263
Branch: refs/heads/jira/http2
Commit: 408232635a43dff3946b2ed82e9d0c7e0577e308
Parents: c696caf
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Sep 19 11:57:39 2018 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Sep 19 11:58:30 2018 +0200
----------------------------------------------------------------------
.../solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/40823263/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
index dded18c..8acab5f 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/sim/TestSimTriggerIntegration.java
@@ -127,7 +127,10 @@ public class TestSimTriggerIntegration extends SimSolrCloudTestCase {
SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
SolrClient solrClient = cluster.simGetSolrClient();
NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
+ String result = response.get("result").toString();
+ if (!"success".equals(result) && !result.contains("No trigger exists")) {
+ fail("Unexpected response: " + result);
+ }
waitForSeconds = 1 + random().nextInt(3);
actionConstructorCalled = new CountDownLatch(1);
[28/29] lucene-solr:jira/http2: LUCENE-8454: Fix tessellator to use
original polygon vertices.
Posted by da...@apache.org.
LUCENE-8454: Fix tessellator to use original polygon vertices.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d7e97fb7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d7e97fb7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d7e97fb7
Branch: refs/heads/jira/http2
Commit: d7e97fb7f84d8613683a080610f177b7cae2b31a
Parents: 0dd7b70
Author: Nicholas Knize <nk...@gmail.com>
Authored: Thu Sep 20 10:39:24 2018 -0500
Committer: Nicholas Knize <nk...@gmail.com>
Committed: Thu Sep 20 10:39:24 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/lucene/geo/Tessellator.java | 4 ++--
.../document/TestLatLonPolygonShapeQueries.java | 6 -----
.../apache/lucene/document/TestLatLonShape.java | 23 ++++++++++++++++++++
3 files changed, 25 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7e97fb7/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java b/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
index a03e57c..9a63ad2 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/geo/Tessellator.java
@@ -800,12 +800,12 @@ final public class Tessellator {
/** get the x value */
public final double getX() {
- return x;
+ return polygon.getPolyLon(vrtxIdx);
}
/** get the y value */
public final double getY() {
- return y;
+ return polygon.getPolyLat(vrtxIdx);
}
/** get the longitude value */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7e97fb7/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
index 9825e6a..ce76a82 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonPolygonShapeQueries.java
@@ -99,10 +99,4 @@ public class TestLatLonPolygonShapeQueries extends BaseLatLonShapeTestCase {
return queryRelation == QueryRelation.INTERSECTS ? false : true;
}
}
-
- @Override
- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8454")
- public void testRandomMedium() throws Exception {
- super.testRandomMedium();
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d7e97fb7/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
----------------------------------------------------------------------
diff --git a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
index 2f6e5e1..9a125ba 100644
--- a/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
+++ b/lucene/sandbox/src/test/org/apache/lucene/document/TestLatLonShape.java
@@ -200,4 +200,27 @@ public class TestLatLonShape extends LuceneTestCase {
IOUtils.close(reader, dir);
}
+
+ public void testLUCENE8454() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
+
+ Polygon poly = new Polygon(new double[] {-1.490648725633769E-132d, 90d, 90d, -1.490648725633769E-132d},
+ new double[] {0d, 0d, 180d, 0d});
+
+ Document document = new Document();
+ addPolygonsToDoc(FIELDNAME, document, poly);
+ writer.addDocument(document);
+
+ ///// search //////
+ IndexReader reader = writer.getReader();
+ writer.close();
+ IndexSearcher searcher = newSearcher(reader);
+
+ // search a bbox in the hole
+ Query q = LatLonShape.newBoxQuery(FIELDNAME, QueryRelation.DISJOINT,-29.46555603761226d, 0.0d, 8.381903171539307E-8d, 0.9999999403953552d);
+ assertEquals(0, searcher.count(q));
+
+ IOUtils.close(reader, dir);
+ }
}
[13/29] lucene-solr:jira/http2: SOLR-12771: add CHANGES entry
Posted by da...@apache.org.
SOLR-12771: add CHANGES entry
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7d0a7782
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7d0a7782
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7d0a7782
Branch: refs/heads/jira/http2
Commit: 7d0a7782fa7e99250bccfb4d3e995485c3f0ca19
Parents: ca86de1
Author: Steve Rowe <sa...@apache.org>
Authored: Mon Sep 17 15:02:10 2018 -0400
Committer: Steve Rowe <sa...@apache.org>
Committed: Mon Sep 17 15:02:10 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7d0a7782/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3f78719..249f681 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -451,6 +451,8 @@ Other Changes
* SOLR-8742: In HdfsDirectoryTest replace RAMDirectory usages with ByteBuffersDirectory.
(hossman, Mark Miller, Andrzej Bialecki, Steve Rowe)
+
+* SOLR-12771: Improve Autoscaling Policy and Preferences documentation. (hossman, Steve Rowe)
================== 7.4.0 ==================
[04/29] lucene-solr:jira/http2: SOLR-12028: BadApple and AwaitsFix
annotations usage
Posted by da...@apache.org.
SOLR-12028: BadApple and AwaitsFix annotations usage
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/910a1c03
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/910a1c03
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/910a1c03
Branch: refs/heads/jira/http2
Commit: 910a1c03124d31f940259682340fef402a42c9ae
Parents: a02959c
Author: Erick Erickson <Er...@gmail.com>
Authored: Sat Sep 15 21:26:19 2018 -0700
Committer: Erick Erickson <Er...@gmail.com>
Committed: Sat Sep 15 21:26:19 2018 -0700
----------------------------------------------------------------------
.../core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java | 2 +-
.../test/org/apache/solr/cloud/DocValuesNotIndexedTest.java | 3 +--
.../org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java | 3 +--
.../src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java | 6 +++---
.../src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java | 1 +
.../src/test/org/apache/solr/cloud/TestWithCollection.java | 1 +
solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java | 1 +
.../org/apache/solr/cloud/api/collections/ShardSplitTest.java | 2 ++
.../solr/cloud/api/collections/TestHdfsCloudBackupRestore.java | 2 +-
.../solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java | 3 +--
.../cloud/autoscaling/ScheduledTriggerIntegrationTest.java | 3 +--
.../cloud/autoscaling/SearchRateTriggerIntegrationTest.java | 1 +
solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java | 1 +
.../apache/solr/handler/admin/MetricsHistoryHandlerTest.java | 1 +
.../solr/metrics/reporters/solr/SolrCloudReportersTest.java | 3 +--
.../org/apache/solr/client/solrj/impl/CloudSolrClientTest.java | 3 +--
.../test/org/apache/solr/client/solrj/io/graph/GraphTest.java | 2 +-
17 files changed, 20 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
index a82dd7c..5fa3d56 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteReplicaTest.java
@@ -219,7 +219,7 @@ public class DeleteReplicaTest extends SolrCloudTestCase {
@Test
@Slow
//28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018
+ // commented 15-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018
public void raceConditionOnDeleteAndRegisterReplica() throws Exception {
raceConditionOnDeleteAndRegisterReplica("true");
raceConditionOnDeleteAndRegisterReplica("false");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
index bbd1389..51c2cd0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/DocValuesNotIndexedTest.java
@@ -30,7 +30,6 @@ import java.util.Locale;
import java.util.Map;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -314,7 +313,7 @@ public class DocValuesNotIndexedTest extends SolrCloudTestCase {
// make sure all the values for each field are unique. We need to have docs that have values that are _not_
// unique.
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testGroupingDVOnly() throws IOException, SolrServerException {
List<SolrInputDocument> docs = new ArrayList<>(50);
for (int idx = 0; idx < 49; ++idx) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
index b31e8f7f..c7cc9e4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
@@ -21,7 +21,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrClient;
@@ -69,7 +68,7 @@ public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase
@Test
@ShardsFixed(num = 6)
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void test() throws Exception {
handle.clear();
handle.put("timestamp", SKIPVAL);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java b/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
index 336cef3..88be4db 100644
--- a/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/LIRRollingUpdatesTest.java
@@ -97,7 +97,7 @@ public class LIRRollingUpdatesTest extends SolrCloudTestCase {
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
+ // commented 15-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void testNewReplicaOldLeader() throws Exception {
String collection = "testNewReplicaOldLeader";
@@ -165,7 +165,7 @@ public class LIRRollingUpdatesTest extends SolrCloudTestCase {
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
+ // commented 15-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void testNewLeaderOldReplica() throws Exception {
// in case of new leader & old replica, new leader can still put old replica into LIR
@@ -335,7 +335,7 @@ public class LIRRollingUpdatesTest extends SolrCloudTestCase {
@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 04-May-2018
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
+ // commented 15-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void testNewLeaderAndMixedReplicas() throws Exception {
testLeaderAndMixedReplicas(false);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
index 000d092..3422633 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SaslZkACLProviderTest.java
@@ -115,6 +115,7 @@ public class SaslZkACLProviderTest extends SolrTestCaseJ4 {
@Test
//commented 9-Aug-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 05-Jul-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testSaslZkACLProvider() throws Exception {
// Test with Sasl enabled
SolrZkClient zkClient = new SolrZkClientWithACLs(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java b/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
index b822a24..d21b32b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestWithCollection.java
@@ -411,6 +411,7 @@ public class TestWithCollection extends SolrCloudTestCase {
* Tests that when a new node is added to the cluster and autoscaling framework
* moves replicas to the new node, we maintain all co-locating guarantees
*/
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testNodeAdded() throws Exception {
String prefix = "testNodeAdded";
String xyz = prefix + "_xyz";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java
index 22029c1..201de26 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkShardTermsTest.java
@@ -52,6 +52,7 @@ public class ZkShardTermsTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testParticipationOfReplicas() throws IOException, SolrServerException, InterruptedException {
String collection = "collection1";
try (ZkShardTerms zkShardTerms = new ZkShardTerms(collection, "shard2", cluster.getZkClient())) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
index 6619ee5..39ae618 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java
@@ -267,6 +267,7 @@ public class ShardSplitTest extends AbstractFullDistribZkTestBase {
*/
@Test
//05-Jul-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testSplitAfterFailedSplit() throws Exception {
waitForThingsToLevelOut(15);
@@ -327,6 +328,7 @@ public class ShardSplitTest extends AbstractFullDistribZkTestBase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testSplitMixedReplicaTypes() throws Exception {
doSplitMixedReplicaTypes(SolrIndexSplitter.SplitMethod.REWRITE);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index 2700419..e81bc4b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -208,7 +208,7 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
}
@Override
@Test
- @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
+ // commented 15-Sep-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 09-Aug-2018
public void test() throws Exception {
super.test();
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
index e80252f..b1ac6ee 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -61,7 +60,7 @@ public class AutoAddReplicasIntegrationTest extends SolrCloudTestCase {
@Test
// This apparently fails in both subclasses.
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testSimple() throws Exception {
JettySolrRunner jetty1 = cluster.getJettySolrRunner(0);
JettySolrRunner jetty2 = cluster.getJettySolrRunner(1);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
index 6197dc9..8d1fcda 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
@@ -27,7 +27,6 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -73,7 +72,7 @@ public class ScheduledTriggerIntegrationTest extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testScheduledTrigger() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
index 437b8ce..5d7ce99 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
@@ -274,6 +274,7 @@ public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
@Test
//17-Aug-2018 commented @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testBelowSearchRate() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
String COLL1 = "belowRate_collection";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
index b860d6e..220d9ec 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
@@ -79,6 +79,7 @@ public class TestSQLHandler extends AbstractFullDistribZkTestBase {
@Test
//28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 21-May-2018
//@LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void doTest() throws Exception {
waitForRecoveriesToFinish(false);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
index 735b427..79bded9 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHistoryHandlerTest.java
@@ -110,6 +110,7 @@ public class MetricsHistoryHandlerTest extends SolrCloudTestCase {
}
@Test
+ @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 15-Sep-2018
public void testBasic() throws Exception {
timeSource.sleep(10000);
List<Pair<String, Long>> list = handler.getFactory().list(100);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
index e395fac..359cdf5 100644
--- a/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
+++ b/solr/core/src/test/org/apache/solr/metrics/reporters/solr/SolrCloudReportersTest.java
@@ -21,7 +21,6 @@ import java.util.Map;
import com.codahale.metrics.Metric;
import org.apache.commons.io.IOUtils;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.core.CoreContainer;
@@ -155,7 +154,7 @@ public class SolrCloudReportersTest extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testDefaultPlugins() throws Exception {
String solrXml = IOUtils.toString(SolrCloudReportersTest.class.getResourceAsStream("/solr/solr.xml"), "UTF-8");
configureCluster(2)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
index fefe868..8570e8e 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientTest.java
@@ -36,7 +36,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.client.solrj.SolrClient;
@@ -855,7 +854,7 @@ public class CloudSolrClientTest extends SolrCloudTestCase {
* limits the distributed query to locally hosted shards only
*/
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void preferReplicaTypesTest() throws Exception {
String collectionName = "replicaTypesTestColl";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/910a1c03/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphTest.java
index fc24a63..9e99224 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/graph/GraphTest.java
@@ -72,7 +72,7 @@ public class GraphTest extends SolrCloudTestCase {
}
@Test
- @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
+ // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
public void testShortestPathStream() throws Exception {
new UpdateRequest()
[20/29] lucene-solr:jira/http2: SOLR-12784: Fix broken link to
stemdict.txt by including it in the Guide directly
Posted by da...@apache.org.
SOLR-12784: Fix broken link to stemdict.txt by including it in the Guide directly
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/264110e7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/264110e7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/264110e7
Branch: refs/heads/jira/http2
Commit: 264110e7b92fdca471db593390901c632cb83bcb
Parents: 5981895
Author: Cassandra Targett <ct...@apache.org>
Authored: Wed Sep 19 14:15:01 2018 -0500
Committer: Cassandra Targett <ct...@apache.org>
Committed: Wed Sep 19 14:15:01 2018 -0500
----------------------------------------------------------------------
solr/solr-ref-guide/src/language-analysis.adoc | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/264110e7/solr/solr-ref-guide/src/language-analysis.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/language-analysis.adoc b/solr/solr-ref-guide/src/language-analysis.adoc
index 2749ce1..d31d295 100644
--- a/solr/solr-ref-guide/src/language-analysis.adoc
+++ b/solr/solr-ref-guide/src/language-analysis.adoc
@@ -1,4 +1,5 @@
= Language Analysis
+:example-source-dir: {solr-root-path}core/src/test-files/solr/collection1/conf/
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -69,9 +70,8 @@ IMPORTANT: When adding the same token twice, it will also score twice (double),
Overrides stemming algorithms by applying a custom mapping, then protecting these terms from being modified by stemmers.
-A customized mapping of words to stems, in a tab-separated file, can be specified to the "dictionary" attribute in the schema. Words in this mapping will be stemmed to the stems from the file, and will not be further changed by any stemmer.
+A customized mapping of words to stems, in a tab-separated file, can be specified to the `dictionary` attribute in the schema. Words in this mapping will be stemmed to the stems from the file, and will not be further changed by any stemmer.
-A sample http://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/test-files/solr/collection1/conf/stemdict.txt[stemdict.txt] with comments can be found in the Source Repository.
[source,xml]
----
@@ -84,6 +84,15 @@ A sample http://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/test-fil
</fieldtype>
----
+A sample `stemdict.txt` file is shown below:
+
+[source,text]
+----
+include::{example-source-dir}stemdict.txt[lines=18..22]
+----
+
+If you have a checkout of Solr's source code locally, you can also find this example in Solr's test resources at `solr/core/src/test-files/solr/collection1/conf/stemdict.txt`.
+
== Dictionary Compound Word Token Filter
This filter splits, or _decompounds_, compound words into individual words using a dictionary of the component words. Each input token is passed through unchanged. If it can also be decompounded into subwords, each subword is also added to the stream at the same logical position.
[08/29] lucene-solr:jira/http2: SOLR-6280: CollapseQParser now throws
an error when pointing to a multi-valued field.
Posted by da...@apache.org.
SOLR-6280: CollapseQParser now throws an error when pointing to a multi-valued field.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ac7969e3
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ac7969e3
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ac7969e3
Branch: refs/heads/jira/http2
Commit: ac7969e3c05cf9db28dbe52d0911d64a864d8c97
Parents: 4940b36
Author: David Smiley <ds...@apache.org>
Authored: Mon Sep 17 09:08:54 2018 -0400
Committer: David Smiley <ds...@apache.org>
Committed: Mon Sep 17 09:08:54 2018 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 6 ++++++
.../apache/solr/search/CollapsingQParserPlugin.java | 15 +++++++++++++--
.../org/apache/solr/search/QueryEqualityTest.java | 2 +-
.../solr/search/TestCollapseQParserPlugin.java | 16 ++++++++++++++++
4 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ac7969e3/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8873834..e773904 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -83,6 +83,12 @@ Velocity 1.7 and Velocity Tools 2.0
Apache ZooKeeper 3.4.11
Jetty 9.4.11.v20180605
+New Features
+----------------------
+
+* SOLR-6280: {!collapse}: if you attempt to use CollapseQParser on a field that is multi-valued, you will now get an
+ error. Previously, the collapsing behavior was unreliable and undefined despite no explicit error.
+ (Munendra S N, David Smiley)
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ac7969e3/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
index 08867db..13417ab 100644
--- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
+++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java
@@ -73,6 +73,7 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.NumberType;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
import org.apache.solr.uninverting.UninvertingReader;
@@ -270,12 +271,19 @@ public class CollapsingQParserPlugin extends QParserPlugin {
return s;
}
- public CollapsingPostFilter(SolrParams localParams, SolrParams params, SolrQueryRequest request) throws IOException {
+ public CollapsingPostFilter(SolrParams localParams, SolrParams params, SolrQueryRequest request) {
this.collapseField = localParams.get("field");
if (this.collapseField == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Required 'field' param is missing.");
}
+ // if unknown field, this would fail fast
+ SchemaField collapseFieldSf = request.getSchema().getField(this.collapseField);
+ // collapseFieldSf won't be null
+ if (collapseFieldSf.multiValued()) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collapsing not supported on multivalued fields");
+ }
+
this.groupHeadSelector = GroupHeadSelector.build(localParams);
if (groupHeadSelector.type.equals(GroupHeadSelectorType.SORT) &&
@@ -334,7 +342,7 @@ public class CollapsingQParserPlugin extends QParserPlugin {
} else if(nPolicy.equals((NULL_EXPAND))) {
this.nullPolicy = NULL_POLICY_EXPAND;
} else {
- throw new IOException("Invalid nullPolicy:"+nPolicy);
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid nullPolicy:"+nPolicy);
}
}
@@ -369,6 +377,9 @@ public class CollapsingQParserPlugin extends QParserPlugin {
boostDocsMap,
searcher);
+ } catch (SolrException e) {
+ // handle SolrException separately
+ throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ac7969e3/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
index d39a42f..be7fe91 100644
--- a/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
+++ b/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
@@ -275,7 +275,7 @@ public class QueryEqualityTest extends SolrTestCaseJ4 {
}
public void testQueryCollapse() throws Exception {
- SolrQueryRequest req = req("myField","foo_s",
+ SolrQueryRequest req = req("myField","foo_s1",
"g_sort","foo_s1 asc, foo_i desc");
try {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ac7969e3/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
index d0521d3..9f65ba5 100644
--- a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
+++ b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java
@@ -920,6 +920,22 @@ public class TestCollapseQParserPlugin extends SolrTestCaseJ4 {
}
@Test
+ public void testForNotSupportedCases() {
+ String[] doc = {"id","3", "term_s", "YYYY", "test_ii", "5000", "test_l", "100", "test_f", "200"};
+ assertU(adoc(doc));
+ assertU(commit());
+
+ // collapsing on multivalued field
+ assertQEx("Should Fail with Bad Request", "Collapsing not supported on multivalued fields",
+ req("q","*:*", "fq","{!collapse field=test_ii}"), SolrException.ErrorCode.BAD_REQUEST);
+
+ // collapsing on unknown field
+ assertQEx("Should Fail with Bad Request", "org.apache.solr.search.SyntaxError: undefined field: \"bleh\"",
+ req("q","*:*", "fq","{!collapse field=bleh}"), SolrException.ErrorCode.BAD_REQUEST);
+
+ }
+
+ @Test
public void test64BitCollapseFieldException() {
ModifiableSolrParams doubleParams = new ModifiableSolrParams();
doubleParams.add("q", "*:*");
[19/29] lucene-solr:jira/http2: LUCENE-8484: Account dropped
sub-readers in test
Posted by da...@apache.org.
LUCENE-8484: Account dropped sub-readers in test
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5981895c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5981895c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5981895c
Branch: refs/heads/jira/http2
Commit: 5981895cb48416f1133702a35bf9811f963f3ed9
Parents: 4082326
Author: Nhat Nguyen <nh...@elastic.co>
Authored: Tue Sep 18 21:00:41 2018 -0400
Committer: Nhat Nguyen <nh...@elastic.co>
Committed: Wed Sep 19 10:04:26 2018 -0400
----------------------------------------------------------------------
lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5981895c/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
index 6b9312b..1d0a319 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
@@ -1448,6 +1448,8 @@ public class TestAddIndexes extends LuceneTestCase {
Directory dir3 = newDirectory();
iwc1 = newIndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("soft_delete");
writer = new IndexWriter(dir3, iwc1);
+ // Resize as some fully deleted sub-readers might be dropped in the wrappedReader
+ readers = new CodecReader[(wrappedReader.leaves().size())];
for (int i = 0; i < readers.length; i++) {
readers[i] = (CodecReader)wrappedReader.leaves().get(i).reader();
}