You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/11/05 11:01:44 UTC
[17/19] lucene-solr:jira/http2: SOLR-12023: Autoscaling policy engine
shuffles replicas needlessly
SOLR-12023: Autoscaling policy engine shuffles replicas needlessly
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/eb359ca0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/eb359ca0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/eb359ca0
Branch: refs/heads/jira/http2
Commit: eb359ca0790af505debf33a57c3bfb18eecbab4e
Parents: f669a1f
Author: Noble Paul <no...@apache.org>
Authored: Tue Oct 30 21:35:38 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Sun Nov 4 19:23:56 2018 +1100
----------------------------------------------------------------------
solr/CHANGES.txt | 123 ++++---------------
.../solrj/cloud/autoscaling/Suggester.java | 8 +-
.../org/apache/solr/common/NavigableObject.java | 2 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 25 ++--
4 files changed, 43 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb359ca0/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 90cce5b..be31fe7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -16,88 +16,6 @@ In this release, there is an example Solr server including a bundled
servlet container in the directory named "example".
See the Solr tutorial at https://lucene.apache.org/solr/guide/solr-tutorial.html
-================== 8.0.0 ==================
-
-Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
-
-Versions of Major Components
----------------------
-Apache Tika 1.19.1
-Carrot2 3.16.0
-Velocity 1.7 and Velocity Tools 2.0
-Apache ZooKeeper 3.4.11
-Jetty 9.4.11.v20180605
-
-Upgrade Notes
-----------------------
-
-* LUCENE-7996: The 'func' query parser now returns scores that are equal to 0
- when a negative value is produced. This change is due to the fact that
- Lucene now requires scores to be positive. (Adrien Grand)
-
-* SOLR-11882: SolrMetric registries retained references to SolrCores when closed. A
- change of SolrMetricMAnager.registerGauge and SolrMetricProducer.initializeMetrics
- method signatures was required to fix it. Third party components that use this API
- need to be updated. (Eros Taborelli, Erick Erickson, ab)
-
-* LUCENE-8267: Memory codecs have been removed from the codebase (MemoryPostings,
- MemoryDocValues). If you used postingsFormat="Memory" or docValuesFormat="Memory"
- then either remove it to use the default or experiment with one of the others. (Dawid Weiss)
-
-* SOLR-12586: The date format patterns used by ParseDateFieldUpdateProcessorFactory (present in "schemaless mode")
- are now interpreted by Java 8's java.time.DateTimeFormatter instead of Joda Time. The pattern language is very
- similar but not the same. Typically, simply update the pattern by changing an uppercase 'Z' to lowercase 'z' and
- that's it. For the current recommended set of patterns in schemaless mode, see "Schemaless Mode" in the ref guide,
- or simply examine the default configSet. Also note that the set of patterns (formats) here have
- expanded from before to subsume those patterns previously handled by the "extract" contrib (Solr Cell / Tika).
- (David Smiley, Bar Rotstein)
-
-* SOLR-12593: The "extraction" contrib (Solr Cell) no longer does any date parsing, and thus no longer has the
- "date.formats" configuration. To ensure date strings are properly parsed, use ParseDateFieldUpdateProcessorFactory
- (an URP) commonly registered with the name "parse-date" in "schemaless mode". (David Smiley, Bar Rotstein)
-
-* SOLR-12754: The UnifiedHighlighter hl.weightMatches now defaults to true. If there are unforseen highlight problems,
- this may be the culprit.
-
-New Features
-----------------------
-
-* SOLR-12591: Expand the set of recognized date format patterns of schemaless mode to subsume those handled by the
- "extract" contrib (Solr Cell / Tika). This is primarily a change in configuration of the default configSet for more
- patterns, but also included enabling "lenient" parsing in ParseDateFieldUpdateProcessorFactory. The default
- locale was changed from ROOT to en_US since well-known patterns assume this locale.
- (David Smiley, Bar Rotstein)
-
-* SOLR-12879: MinHash query parser that builds queries providing a measure of Jaccard similarity (Andy Hind via Tommaso Teofili)
-
-* SOLR-12593: The default configSet now includes an "ignored_*" dynamic field. (David Smiley)
-
-Optimizations
-----------------------
-
-* SOLR-12725: ParseDateFieldUpdateProcessorFactory should reuse ParsePosition. (ab)
-
-Other Changes
-----------------------
-
-* SOLR-12614: Make "Nodes" view the default in AdminUI "Cloud" tab (janhoy)
-
-* SOLR-12586: Upgrade ParseDateFieldUpdateProcessorFactory (present in "schemaless mode") to use Java 8's
- java.time.DateTimeFormatter instead of Joda time (see upgrade notes). "Lenient" is enabled. Removed Joda Time dependency.
- (David Smiley, Bar Rotstein)
-
-* SOLR-5163: edismax now throws an exception when qf refers to a nonexistent field (Charles Sanders, David Smiley)
-
-* SOLR-12805: Store previous term (generation) of replica when start recovery process (Cao Manh Dat)
-
-* SOLR-12652: Remove SolrMetricManager.overridableRegistryName method (Peter Somogyi via David Smiley)
-
-* LUCENE-8513: SlowCompositeReaderWrapper now uses MultiTerms directly instead of MultiFields (David Smiley)
-
-* SOLR-11812: Remove backward compatibility of old LIR implementation in 8.0 (Cao Manh Dat)
-
-* SOLR-12620: Remove the Admin UI Cloud -> Graph (Radial) view (janhoy)
-
================== 7.6.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
@@ -172,8 +90,6 @@ New Features
* SOLR-12862: Add log10 Stream Evaluator and allow the pow Stream Evaluator to accept a vector of exponents (Joel Bernstein)
-* SOLR-12942: Add an option in IndexSizeTrigger to select the split shard method. (ab)
-
Other Changes
----------------------
@@ -233,7 +149,7 @@ Bug Fixes
* SOLR-12875: fix ArrayIndexOutOfBoundsException when unique(field) or uniqueBlock(_root_) is
used with DVHASH method in json.facet. (Tim Underwood via Mikhail Khludnev)
-* SOLR-12954: fix facet.pivot refinement bugs when using facet.sort=index and facet.mincount>1 (hossman)
+* SOLR-12023: Autoscaling policy engine shuffles replicas needlessly (noble)
Improvements
----------------------
@@ -254,8 +170,6 @@ Improvements
* SOLR-12892: MapWriter to use CharSequence instead of String (noble)
-* SOLR-12882: Eliminate excessive lambda allocation in json facets FacetFieldProcessorByHashDV (Tim Underwood)
-
================== 7.5.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
@@ -525,12 +439,13 @@ Optimizations
* SOLR-11654: Time Routed Alias will now route documents to the ideal shard of a collection, thus avoiding a hop.
Usually documents were already routed well but not always. (Gus Heck, David Smiley)
+* SOLR-12305: When a replica is applying updates, some kind of updates can skip buffering for faster recovery.
+ (Cao Manh Dat)
+
* SOLR-11598: The export handler does not limit users to 4 sort fields and is now unlimited. However the speed at
which we can export is directly proportional to the number of sort fields specified. This change also allows streaming
expressions to group by on more than 4 fields. (Aroop Ganguly, Amrit Sarkar, Varun Thacker)
-* SOLR-12305: When a replica is applying updates, some kind of updates can skip buffering for faster recovery.
- (Cao Manh Dat)
* SOLR-12509: Improve SplitShardCmd performance and reliability. A new method of splitting has been
introduced (splitMethod=link) which uses hard-linking of index files when possible, resulting in
@@ -580,8 +495,6 @@ Other Changes
* SOLR-12617: Remove Commons BeanUtils as a dependency (Varun Thacker)
-* SOLR-11008: Use a lighter config for MetricsHandlerTest and ensure the core is up before the test starts (Varun Thacker)
-
* SOLR-11766: Move Streaming Expressions section in Ref Guide to be a top-level section. (Cassandra Targett)
* SOLR-12656: ShardSplitTest should extend AbstractFullDistribZkTestBase instead of BasicDistributedZkTest. (shalin)
@@ -691,6 +604,11 @@ New Features
* SOLR-9480: A new 'relatedness()' aggregate function for JSON Faceting to enable building Semantic
Knowledge Graphs. (Trey Grainger, hossman)
+* SOLR-11453: Configuring slowQueryThresholdMillis logs slow requests to a separate file - solr_slow_requests.log.
+ (Shawn Heisey, Remko Popma, Varun Thacker)
+
+* SOLR-12401: Add getValue() and setValue() Stream Evaluators (Joel Bernstein, janhoy)
+
* SOLR-12378: Support missing versionField on indexed docs in DocBasedVersionConstraintsURP.
(Oliver Bates, Michael Braun via Mark Miller)
@@ -707,11 +625,6 @@ New Features
* SOLR-12328: JSON Facet API: Domain change with graph query.
(Daniel Meehl, Kevin Watters, yonik)
-* SOLR-11453: Configuring slowQueryThresholdMillis logs slow requests to a separate file - solr_slow_requests.log.
- (Shawn Heisey, Remko Popma, Varun Thacker)
-
-* SOLR-12401: Add getValue() and setValue() Stream Evaluators (Joel Bernstein, janhoy)
-
* SOLR-11779, SOLR-12438: Basic long-term collection of aggregated metrics. Historical data is
maintained as multi-resolution time series using round-robin databases in the '.system'
collection. New /admin/metrics/history API allows retrieval of this data in numeric
@@ -772,6 +685,11 @@ Bug Fixes
* SOLR-11929: UpdateLog metrics are not initialized on core reload. (ab, Steve Rowe)
+* SOLR-11882: SolrMetric registries retained references to SolrCores when closed. A
+ change of SolrMetricMAnager.registerGauge and SolrMetricProducer.initializeMetrics
+ method signatures was required to fix it. Third party components may continue to use the old API
+ but should be updated to avoid this bug (Eros Taborelli, Erick Erickson, ab)
+
* SOLR-12199: TestReplicationHandler.doTestRepeater(): TEST_PORT interpolation failure:
Server refused connection at: http://127.0.0.1:TEST_PORT/solr (Mikhail Khludnev, Dawid Weiss, Steve Rowe)
@@ -856,14 +774,17 @@ Bug Fixes
* SOLR-3567: Spellcheck custom parameters not being passed through due to wrong prefix creation.
(Josh Lucas via shalin)
-* SOLR-12358: Autoscaling suggestions fail randomly with sorting (noble)
-
* SOLR-12294: update processors loaded from runtime jars fail to load if they are specified
- in an update processor chain (noble)
+ in an update processor chain (noble)
+
+* SOLR-12358: Autoscaling suggestions fail randomly with sorting (noble)
* SOLR-12314: Use http timeout's defined in solr.xml for creating ConcurrentUpdateSolrClient during
indexing requests between leader and replica ( Mark Miller, Varun Thacker)
+* SOLR-12290: Do not close any servlet streams and improve our servlet stream closing prevention code for users
+ and devs. (Mark Miller)
+
* SOLR-12374: SnapShooter.getIndexCommit can forget to decref the searcher; though it's not clear in practice when.
(David Smiley)
@@ -911,7 +832,7 @@ Optimizations
* SOLR-11880: Avoid creating new exceptions for every request made to MDCAwareThreadPoolExecutor by distributed
search and update operations. (Varun Thacker, shalin)
-* SOLR-12375: Optimize Lucene ScoreMode use:
+* SOLR-12375: Optimize Lucene needsScore / ScoreMode use:
A non-cached filter query could be told incorrectly that scores were needed.
The /export (ExportQParserPlugin) would declare incorrectly that scores are needed.
Expanded docs (expand component) could be told incorrectly that scores are needed. (David Smiley)
@@ -1004,8 +925,6 @@ Other Changes
* SOLR-12435: Fix bin/solr help and ref guide text to describe ZK_HOST in solr.in.sh/solr.in.cmd
as an alternative to -z cmdline param. (Steve Rowe)
-* SOLR-12428: Solr LTR jar now included in _default configset's solrconfig.xml (Ishan Chattopadhyaya)
-
================== 7.3.1 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb359ca0/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index 39ad8bf..f052ae9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -299,7 +299,11 @@ public abstract class Suggester implements MapWriter {
for (Map.Entry<String, List<ReplicaInfo>> shard : e.getValue().entrySet()) {
if (!isAllowed(new Pair<>(e.getKey(), shard.getKey()), Hint.COLL_SHARD)) continue;//todo fix
if (shard.getValue() == null || shard.getValue().isEmpty()) continue;
- replicaList.add(new Pair<>(shard.getValue().get(0), r));
+ for (ReplicaInfo replicaInfo : shard.getValue()) {
+ if (replicaInfo.getName().startsWith("SYNTHETIC.")) continue;
+ replicaList.add(new Pair<>(shard.getValue().get(0), r));
+ break;
+ }
}
}
}
@@ -311,10 +315,8 @@ public abstract class Suggester implements MapWriter {
List<Violation> errors = new ArrayList<>();
for (Clause clause : session.expandedClauses) {
Clause originalClause = clause.derivedFrom == null ? clause : clause.derivedFrom;
-// if (!executeInStrictMode && !clause.strict) {
if (this.deviations == null) this.deviations = new LinkedHashMap<>();
this.deviations.put(originalClause, new double[1]);
-// }
List<Violation> errs = clause.test(session, this.deviations == null ? null : this.deviations.get(originalClause));
if (!errs.isEmpty() &&
(executeInStrictMode || clause.strict)) errors.addAll(errs);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb359ca0/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java b/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java
index be50a17..ccef7e2 100644
--- a/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java
+++ b/solr/solrj/src/java/org/apache/solr/common/NavigableObject.java
@@ -24,7 +24,7 @@ import org.apache.solr.common.util.Utils;
/**This class contains helper methods for navigating deeply nested Objects. Keep in mind that
* it may be expensive depending on the underlying implementation. each level needs an extra lookup
- * and the lookup may be as expensive as O(log(n)) to O(o) depending on the underlying impl
+ * and the lookup may be as expensive as O(log(n)) to O(n) depending on the underlying impl
*
*/
public interface NavigableObject {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/eb359ca0/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 7b32567..8dc0c24 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -1972,16 +1972,25 @@ public class TestPolicy extends SolrTestCaseJ4 {
public void testMoveReplicaSuggester() {
String autoScalingjson = " '{cluster-policy':[" +
- " { 'cores':'<10', 'node':'#ANY'}," +
- " { 'replica':'<2', 'shard':'#EACH', 'node':'#ANY'}," +
- " { 'nodeRole':'overseer','replica':0}]," +
- " 'cluster-preferences':[{'minimize':'cores'}]}";
+ "{'cores':'<10', 'node':'#ANY'}," +
+ "{'replica':'<2', 'shard':'#EACH','node':'#ANY'}]," +
+ "'cluster-preferences':[{'minimize':'cores'}]}";
Policy policy = new Policy((Map<String, Object>) Utils.fromJSONString(autoScalingjson));
Policy.Session session = policy.createSession(cloudManagerWithData((Map) loadFromResource("testMoveReplicaSuggester.json")));
- Suggester suggester = session.getSuggester(MOVEREPLICA).hint(Hint.TARGET_NODE, "10.0.0.6:7574_solr");
+ Suggester suggester = session.getSuggester(MOVEREPLICA)
+ .hint(Hint.TARGET_NODE, "10.0.0.6:7574_solr");
SolrRequest op = suggester.getSuggestion();
assertNotNull(op);
- suggester = suggester.getSession().getSuggester(MOVEREPLICA).hint(Hint.TARGET_NODE, "10.0.0.6:7574_solr");
+ suggester = suggester.getSession()
+ .getSuggester(MOVEREPLICA)
+ .hint(Hint.TARGET_NODE, "10.0.0.6:7574_solr");
+ op = suggester.getSuggestion();
+ assertNull(op);
+
+ suggester = suggester.getSession()
+ .getSuggester(MOVEREPLICA)
+ .forceOperation(true)
+ .hint(Hint.TARGET_NODE, "10.0.0.6:8983_solr");
op = suggester.getSuggestion();
assertNull(op);
}
@@ -2183,7 +2192,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals(0, violations.get(0).getViolatingReplicas().size());
l = PolicyHelper.getSuggestions(cfg, cloudManagerWithData((Map) loadFromResource("testFreeDiskSuggestions.json")));
- assertEquals(4, l.size());
+ assertEquals(3, l.size());
assertEquals("r4", l.get(0)._get("operation/command/move-replica/replica", null));
assertEquals("node1", l.get(0)._get("operation/command/move-replica/targetNode", null));
@@ -2193,8 +2202,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("r2", l.get(2)._get("operation/command/move-replica/replica", null));
assertEquals("node1", l.get(2)._get("operation/command/move-replica/targetNode", null));
- assertEquals("improvement", l.get(3)._get("type", null));
-
}