You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/09/14 03:31:00 UTC
[43/43] lucene-solr:jira/http2: Merge with master
Merge with master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/9c65fe4f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/9c65fe4f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/9c65fe4f
Branch: refs/heads/jira/http2
Commit: 9c65fe4fe338721b1dd342210cd4b12eee6efb31
Parents: 802f367 d6641ff
Author: Cao Manh Dat <da...@apache.org>
Authored: Fri Sep 14 10:30:03 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Fri Sep 14 10:30:03 2018 +0700
----------------------------------------------------------------------
dev-tools/doap/solr.rdf | 2 +-
dev-tools/scripts/buildAndPushRelease.py | 35 +--
dev-tools/scripts/smokeTestRelease.py | 36 +--
lucene/CHANGES.txt | 23 ++
lucene/MIGRATE.txt | 10 +
lucene/build.xml | 15 +-
lucene/common-build.xml | 26 --
.../java/org/apache/lucene/index/Sorter.java | 27 --
.../lucene/search/Boolean2ScorerSupplier.java | 4 +-
.../org/apache/lucene/search/BooleanScorer.java | 14 +-
.../org/apache/lucene/search/BooleanWeight.java | 2 +-
.../apache/lucene/search/CachingCollector.java | 16 +-
.../org/apache/lucene/search/FakeScorer.java | 62 -----
.../apache/lucene/search/MatchAllDocsQuery.java | 2 +-
.../org/apache/lucene/search/ScoreAndDoc.java | 35 +++
.../java/org/apache/lucene/search/Scorer.java | 7 +-
.../org/apache/lucene/search/SortRescorer.java | 10 +-
.../java/org/apache/lucene/util/Version.java | 7 +
.../org/apache/lucene/util/bkd/BKDReader.java | 58 ++++-
.../org/apache/lucene/util/bkd/BKDWriter.java | 56 ++++-
.../lucene/search/MultiCollectorTest.java | 20 +-
.../search/TestBoolean2ScorerSupplier.java | 80 ++++--
.../org/apache/lucene/search/TestBooleanOr.java | 2 +-
.../apache/lucene/search/TestBooleanScorer.java | 2 +-
.../lucene/search/TestCachingCollector.java | 23 +-
.../lucene/search/TestConjunctionDISI.java | 50 +++-
.../lucene/search/TestConstantScoreQuery.java | 4 +-
.../search/TestMaxScoreSumPropagator.java | 34 ++-
.../lucene/search/TestMultiCollector.java | 16 +-
.../apache/lucene/search/TestQueryRescorer.java | 2 +-
.../lucene/search/TestTopDocsCollector.java | 22 +-
.../lucene/search/TestTopFieldCollector.java | 2 +-
.../apache/lucene/expressions/FakeScorer.java | 53 ----
.../lucene/facet/DrillSidewaysScorer.java | 24 +-
.../facet/taxonomy/SearcherTaxonomyManager.java | 14 ++
.../taxonomy/TestSearcherTaxonomyManager.java | 14 ++
.../search/grouping/BlockGroupingCollector.java | 19 +-
.../lucene/search/grouping/FakeScorer.java | 52 ----
.../apache/lucene/search/join/FakeScorer.java | 52 ----
.../queries/function/FunctionRangeQuery.java | 4 +-
.../lucene/queries/function/FunctionValues.java | 15 +-
.../lucene/queries/function/ValueSource.java | 26 +-
.../queries/function/ValueSourceScorer.java | 8 +-
.../docvalues/DocTermsIndexDocValues.java | 5 +-
.../function/docvalues/DoubleDocValues.java | 11 +-
.../function/docvalues/IntDocValues.java | 5 +-
.../function/docvalues/LongDocValues.java | 5 +-
.../function/valuesource/EnumFieldSource.java | 5 +-
.../document/FloatPointNearestNeighbor.java | 2 +-
.../apache/lucene/search/NearestNeighbor.java | 2 +-
.../Geo3dShapeWGS84ModelRectRelationTest.java | 2 +-
.../analyzing/BlendedInfixSuggester.java | 7 +-
.../analyzing/BlendedInfixSuggesterTest.java | 73 ++++--
solr/CHANGES.txt | 80 +++---
solr/build.xml | 12 +-
.../ExtractingRequestHandlerTest.java | 5 +-
solr/core/ivy.xml | 2 +-
.../cloud/autoscaling/AutoScalingHandler.java | 13 +-
.../solr/handler/component/QueryComponent.java | 31 +--
.../solr/search/CollapsingQParserPlugin.java | 29 +--
.../src/java/org/apache/solr/search/Filter.java | 2 +-
.../apache/solr/search/FunctionRangeQuery.java | 12 +-
.../search/function/ValueSourceRangeFilter.java | 9 +-
.../apache/solr/update/SolrCmdDistributor.java | 5 +-
solr/core/src/test-files/log4j2.xml | 14 +-
.../apache/solr/handler/RequestLoggingTest.java | 27 +-
.../org/apache/solr/logging/TestLogWatcher.java | 91 +++----
.../apache/solr/search/TestRankQueryPlugin.java | 32 +--
.../solr/store/hdfs/HdfsDirectoryTest.java | 7 +-
solr/server/resources/log4j2-console.xml | 8 +-
solr/server/resources/log4j2.xml | 30 +--
solr/solr-ref-guide/build.xml | 18 +-
.../src/common-query-parameters.adoc | 4 +-
solr/solr-ref-guide/src/css/ref-guide.css | 5 +
solr/solr-ref-guide/src/curve-fitting.adoc | 26 +-
solr/solr-ref-guide/src/dsp.adoc | 66 ++---
.../src/indexconfig-in-solrconfig.adoc | 14 +-
.../src/initparams-in-solrconfig.adoc | 2 +-
solr/solr-ref-guide/src/machine-learning.adoc | 238 ++++++++----------
solr/solr-ref-guide/src/math-expressions.adoc | 2 +-
solr/solr-ref-guide/src/matrix-math.adoc | 20 +-
solr/solr-ref-guide/src/numerical-analysis.adoc | 114 ++++-----
solr/solr-ref-guide/src/other-parsers.adoc | 6 +-
.../src/other-schema-elements.adoc | 2 +-
.../src/pagination-of-results.adoc | 2 +-
.../src/probability-distributions.adoc | 100 ++++----
solr/solr-ref-guide/src/regression.adoc | 77 +++---
.../src/resource-and-plugin-loading.adoc | 24 +-
.../src/rule-based-replica-placement.adoc | 2 +-
solr/solr-ref-guide/src/scalar-math.adoc | 4 +-
solr/solr-ref-guide/src/simulations.adoc | 125 +++++----
solr/solr-ref-guide/src/solr-upgrade-notes.adoc | 73 +++++-
solr/solr-ref-guide/src/spell-checking.adoc | 2 +-
solr/solr-ref-guide/src/statistics.adoc | 22 +-
.../src/stream-decorator-reference.adoc | 2 +-
.../src/stream-evaluator-reference.adoc | 4 +-
.../src/stream-source-reference.adoc | 6 +-
solr/solr-ref-guide/src/suggester.adoc | 8 +-
solr/solr-ref-guide/src/term-vectors.adoc | 91 ++++---
solr/solr-ref-guide/src/time-series.adoc | 14 +-
.../transforming-and-indexing-custom-json.adoc | 2 +-
.../src/uploading-data-with-index-handlers.adoc | 83 +++---
solr/solr-ref-guide/src/variables.adoc | 62 ++---
solr/solr-ref-guide/src/vector-math.adoc | 50 ++--
solr/solr-ref-guide/src/vectorization.adoc | 108 ++++++--
.../client/solrj/cloud/autoscaling/Clause.java | 252 ++++++++++++-------
.../solrj/cloud/autoscaling/Condition.java | 18 +-
.../solrj/cloud/autoscaling/CoresVariable.java | 5 +-
.../cloud/autoscaling/FreeDiskVariable.java | 40 ++-
.../client/solrj/cloud/autoscaling/Operand.java | 1 +
.../client/solrj/cloud/autoscaling/Policy.java | 4 +-
.../solrj/cloud/autoscaling/ReplicaCount.java | 48 +++-
.../client/solrj/cloud/autoscaling/Row.java | 35 ++-
.../solrj/cloud/autoscaling/Suggestion.java | 43 +++-
.../solrj/cloud/autoscaling/Variable.java | 18 +-
.../solrj/cloud/autoscaling/VariableBase.java | 16 +-
.../solrj/cloud/autoscaling/Violation.java | 31 +--
.../autoscaling/WithCollectionVariable.java | 3 +-
.../org/apache/solr/client/solrj/io/Lang.java | 2 +-
.../solrj/io/eval/LatLonVectorsEvaluator.java | 115 +++++++++
.../solrj/io/eval/LocationVectorsEvaluator.java | 105 --------
.../client/solrj/io/stream/FacetStream.java | 7 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 25 +-
.../solrj/cloud/autoscaling/TestPolicy2.java | 193 +++++++++++++-
.../apache/solr/client/solrj/io/TestLang.java | 2 +-
.../solrj/io/stream/MathExpressionTest.java | 4 +-
.../client/solrj/io/stream/StreamingTest.java | 9 +
.../apache/solr/cloud/SolrCloudTestCase.java | 40 ++-
128 files changed, 2143 insertions(+), 1762 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9c65fe4f/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
index 710f03f,d7388f0..983617f
--- a/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCmdDistributor.java
@@@ -46,27 -56,140 +46,28 @@@ import org.slf4j.LoggerFactory
/**
* Used for distributing commands from a shard leader to its replicas.
*/
-public class SolrCmdDistributor implements Closeable {
+public class SolrCmdDistributor {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private StreamingSolrClients clients;
- private boolean finished = false; // see finish()
- private int retryPause = 500;
-
private final List<Error> allErrors = new ArrayList<>();
- private final List<Error> errors = Collections.synchronizedList(new ArrayList<Error>());
-
- private final CompletionService<Object> completionService;
- private final Set<Future<Object>> pending = new HashSet<>();
-
- public static interface AbortCheck {
- public boolean abortCheck();
- }
-
+ private Http2SolrClient client;
+ private Phaser pendingTasksPhaser = new Phaser(1);
+ private int retryPause = 500;
+
public SolrCmdDistributor(UpdateShardHandler updateShardHandler) {
- this.clients = new StreamingSolrClients(updateShardHandler);
- this.completionService = new ExecutorCompletionService<>(updateShardHandler.getUpdateExecutor());
+ this.client = updateShardHandler.getUpdateOnlyHttpClient();
}
-
- public SolrCmdDistributor(Http2SolrClient client, int retryPause) {
+
+ /* For tests only */
- SolrCmdDistributor(StreamingSolrClients clients, int retryPause) {
- this.clients = clients;
++ SolrCmdDistributor(Http2SolrClient client, int retryPause) {
+ this.client = client;
this.retryPause = retryPause;
- completionService = new ExecutorCompletionService<>(clients.getUpdateExecutor());
}
- public void finish() {
- try {
- assert ! finished : "lifecycle sanity check";
- finished = true;
-
- blockAndDoRetries();
- } finally {
- clients.shutdown();
- }
- }
-
- public void close() {
- clients.shutdown();
+ public void finish() {
+ blockUntilFinished();
}
- private void doRetriesIfNeeded() {
- // NOTE: retries will be forwards to a single url
-
- List<Error> errors = new ArrayList<>(this.errors);
- errors.addAll(clients.getErrors());
- List<Error> resubmitList = new ArrayList<>();
-
- if (log.isInfoEnabled() && errors.size() > 0) {
- log.info("SolrCmdDistributor found {} errors", errors.size());
- }
-
- if (log.isDebugEnabled() && errors.size() > 0) {
- StringBuilder builder = new StringBuilder("SolrCmdDistributor found:");
- int maxErrorsToShow = 10;
- for (Error e:errors) {
- if (maxErrorsToShow-- <= 0) break;
- builder.append("\n" + e);
- }
- if (errors.size() > 10) {
- builder.append("\n... and ");
- builder.append(errors.size() - 10);
- builder.append(" more");
- }
- log.debug(builder.toString());
- }
-
- for (Error err : errors) {
- try {
- String oldNodeUrl = err.req.node.getUrl();
-
- /*
- * if this is a retryable request we may want to retry, depending on the error we received and
- * the number of times we have already retried
- */
- boolean isRetry = err.req.shouldRetry(err);
-
- if (testing_errorHook != null) Diagnostics.call(testing_errorHook,
- err.e);
-
- // this can happen in certain situations such as close
- if (isRetry) {
- err.req.retries++;
-
- if (err.req.node instanceof ForwardNode) {
- SolrException.log(SolrCmdDistributor.log, "forwarding update to "
- + oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
- + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
- } else {
- SolrException.log(SolrCmdDistributor.log, "FROMLEADER request to "
- + oldNodeUrl + " failed - retrying ... retries: "
- + err.req.retries + "/" + err.req.node.getMaxRetries() + ". "
- + err.req.cmd.toString() + " params:"
- + err.req.uReq.getParams() + " rsp:" + err.statusCode, err.e);
- }
- resubmitList.add(err);
- } else {
- allErrors.add(err);
- }
- } catch (Exception e) {
- // continue on
- log.error("Unexpected Error while doing request retries", e);
- }
- }
-
- if (resubmitList.size() > 0) {
- // Only backoff once for the full batch
- try {
- int backoffTime = Math.min(retryPause * resubmitList.get(0).req.retries, 2000);
- log.debug("Sleeping {}ms before re-submitting {} requests", backoffTime, resubmitList.size());
- Thread.sleep(backoffTime);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- log.warn(null, e);
- }
- }
-
- clients.clearErrors();
- this.errors.clear();
- for (Error err : resubmitList) {
- submit(err.req, false);
- }
-
- if (resubmitList.size() > 0) {
- blockAndDoRetries();
- }
- }
-
public void distribDelete(DeleteUpdateCommand cmd, List<Node> nodes, ModifiableSolrParams params) throws IOException {
distribDelete(cmd, nodes, params, false, null, null);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/9c65fe4f/solr/core/src/test-files/log4j2.xml
----------------------------------------------------------------------
diff --cc solr/core/src/test-files/log4j2.xml
index 0240fcb,7d0ebf7..612cfe8
--- a/solr/core/src/test-files/log4j2.xml
+++ b/solr/core/src/test-files/log4j2.xml
@@@ -27,14 -27,13 +27,14 @@@
</Console>
</Appenders>
<Loggers>
- <AsyncLogger name="org.apache.zookeeper" level="WARN"/>
- <AsyncLogger name="org.apache.hadoop" level="WARN"/>
- <AsyncLogger name="org.apache.directory" level="WARN"/>
- <AsyncLogger name="org.apache.solr.hadoop" level="INFO"/>
- <AsyncLogger name="org.eclipse.jetty" level="INFO"/>
+ <Logger name="org.apache.zookeeper" level="WARN"/>
+ <Logger name="org.apache.hadoop" level="WARN"/>
+ <Logger name="org.apache.directory" level="WARN"/>
+ <Logger name="org.apache.solr.hadoop" level="INFO"/>
++ <Logger name="org.eclipse.jetty" level="INFO"/>
- <AsyncRoot level="INFO">
+ <Root level="INFO">
<AppenderRef ref="STDERR"/>
- </AsyncRoot>
+ </Root>
</Loggers>
</Configuration>