You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/09/28 22:09:17 UTC
[01/24] lucene-solr:jira/solr-11285: Revert "SOLR-11072: Fix
searchRate trigger hookup to ComputePlanAction. Make the trigger more robust
by supporting host collections with warm shards. Add unit test."
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11285 18b993689 -> 185bcb774
Revert "SOLR-11072: Fix searchRate trigger hookup to ComputePlanAction. Make the trigger more robust by supporting host collections with warm shards. Add unit test."
This reverts commit 6fb2803
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bc25e846
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bc25e846
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bc25e846
Branch: refs/heads/jira/solr-11285
Commit: bc25e846bca3900717bf6e1a0952cc8c410348bd
Parents: de3107c
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Sep 18 09:25:27 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Sep 18 09:25:27 2017 -0700
----------------------------------------------------------------------
.../solr/cloud/autoscaling/AutoScaling.java | 2 -
.../cloud/autoscaling/ComputePlanAction.java | 30 +++--------
.../cloud/autoscaling/SearchRateTrigger.java | 21 ++------
.../autoscaling/ComputePlanActionTest.java | 53 --------------------
.../solr/common/params/AutoScalingParams.java | 1 -
5 files changed, 11 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc25e846/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index 420d4f3..ed24bf7 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@ -130,8 +130,6 @@ public class AutoScaling {
return new NodeAddedTrigger(name, props, coreContainer);
case NODELOST:
return new NodeLostTrigger(name, props, coreContainer);
- case SEARCHRATE:
- return new SearchRateTrigger(name, props, coreContainer);
default:
throw new IllegalArgumentException("Unknown event type: " + type + " in trigger: " + name);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc25e846/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 6451c67..44b3583 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -20,7 +20,6 @@ package org.apache.solr.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -36,7 +35,6 @@ import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.util.Pair;
import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -128,27 +126,13 @@ public class ComputePlanAction extends TriggerActionBase {
} else {
// collection || shard || replica -> ADDREPLICA
suggester = session.getSuggester(CollectionParams.CollectionAction.ADDREPLICA);
- Map<String, Set<String>> collShards = new HashMap<>();
- // AddReplicaSuggester needs a list of Pair(coll, shard)
- hotReplicas.forEach(r -> collShards.computeIfAbsent(r.getCollection(), c -> new HashSet<>()).add(r.getShard()));
- hotShards.forEach((coll, shards) -> collShards.computeIfAbsent(coll, c -> new HashSet<>()).addAll(shards.keySet()));
- // if we only have hotCollections then use warmShards to pick ones to replicate
- Map<String, String> warmShards = (Map<String, String>)event.getProperty(AutoScalingParams.WARM_SHARD);
- hotCollections.forEach((coll, rate) -> {
- Set<String> shards = collShards.get(coll);
- if (shards == null || shards.isEmpty()) {
- String warmShard = warmShards.get(coll);
- if (warmShard == null) {
- log.warn("Got hot collection '" + coll + "' but no warm shard! Ignoring...");
- return;
- }
- collShards.computeIfAbsent(coll, s -> new HashSet<>()).add(warmShard);
- }
- });
- for (Map.Entry<String, Set<String>> e : collShards.entrySet()) {
- for (String shard : e.getValue()) {
- suggester = suggester.hint(Policy.Suggester.Hint.COLL_SHARD, new Pair<>(e.getKey(), shard));
- }
+ Set<String> collections = new HashSet<>();
+ // XXX improve this when AddReplicaSuggester supports coll_shard hint
+ hotReplicas.forEach(r -> collections.add(r.getCollection()));
+ hotShards.forEach((coll, shards) -> collections.add(coll));
+ hotCollections.forEach((coll, rate) -> collections.add(coll));
+ for (String coll : collections) {
+ suggester = suggester.hint(Policy.Suggester.Hint.COLL, coll);
}
}
break;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc25e846/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
index f8a4eeb..6692fda 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
@@ -204,10 +204,8 @@ public class SearchRateTrigger extends TriggerBase {
.collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().get()));
Map<String, Map<String, Double>> hotShards = new HashMap<>();
- Map<String, String> warmShards = new HashMap<>();
List<ReplicaInfo> hotReplicas = new ArrayList<>();
collectionRates.forEach((coll, shardRates) -> {
- final Object[] warmShard = new Object[2];
shardRates.forEach((sh, replicaRates) -> {
double shardRate = replicaRates.stream()
.map(r -> {
@@ -218,14 +216,6 @@ public class SearchRateTrigger extends TriggerBase {
return r;
})
.mapToDouble(r -> (Double)r.getVariable(AutoScalingParams.RATE)).sum();
- if (warmShard[0] == null) {
- warmShard[0] = sh;
- warmShard[1] = shardRate;
- }
- if (shardRate > (double)warmShard[1]) {
- warmShard[0] = sh;
- warmShard[1] = shardRate;
- }
if (waitForElapsed(coll + "." + sh, now, lastShardEvent) &&
(shardRate > rate) &&
(collection.equals(Policy.ANY) || collection.equals(coll)) &&
@@ -233,12 +223,11 @@ public class SearchRateTrigger extends TriggerBase {
hotShards.computeIfAbsent(coll, s -> new HashMap<>()).put(sh, shardRate);
}
});
- warmShards.put(coll, (String)warmShard[0]);
});
Map<String, Double> hotCollections = new HashMap<>();
- collectionRates.forEach((coll, shRates) -> {
- double total = shRates.entrySet().stream()
+ collectionRates.forEach((coll, shardRates) -> {
+ double total = shardRates.entrySet().stream()
.mapToDouble(e -> e.getValue().stream()
.mapToDouble(r -> (Double)r.getVariable(AutoScalingParams.RATE)).sum()).sum();
if (waitForElapsed(coll, now, lastCollectionEvent) &&
@@ -254,7 +243,7 @@ public class SearchRateTrigger extends TriggerBase {
// generate event
- if (processor.process(new SearchRateEvent(getName(), now, hotNodes, hotCollections, hotShards, hotReplicas, warmShards))) {
+ if (processor.process(new SearchRateEvent(getName(), now, hotNodes, hotCollections, hotShards, hotReplicas))) {
// update lastEvent times
hotNodes.keySet().forEach(node -> lastNodeEvent.put(node, now));
hotCollections.keySet().forEach(coll -> lastCollectionEvent.put(coll, now));
@@ -277,14 +266,12 @@ public class SearchRateTrigger extends TriggerBase {
public static class SearchRateEvent extends TriggerEvent {
public SearchRateEvent(String source, long eventTime, Map<String, Double> hotNodes,
Map<String, Double> hotCollections,
- Map<String, Map<String, Double>> hotShards, List<ReplicaInfo> hotReplicas,
- Map<String, String> warmShards) {
+ Map<String, Map<String, Double>> hotShards, List<ReplicaInfo> hotReplicas) {
super(TriggerEventType.SEARCHRATE, source, eventTime, null);
properties.put(AutoScalingParams.COLLECTION, hotCollections);
properties.put(AutoScalingParams.SHARD, hotShards);
properties.put(AutoScalingParams.REPLICA, hotReplicas);
properties.put(AutoScalingParams.NODE, hotNodes);
- properties.put(AutoScalingParams.WARM_SHARD, warmShards);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc25e846/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 93a13b0..b0bf4e6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -38,7 +38,6 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.LogLevel;
@@ -49,7 +48,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
/**
@@ -117,12 +115,6 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
// expected if testNodeWithMultipleReplicasLost hasn't run already
}
- try {
- CollectionAdminRequest.deleteCollection("testSearchRate").process(solrClient);
- } catch (Exception e) {
- // expected if testSearchRate hasn't run already
- }
-
String setClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
" {'cores':'<10', 'node':'#ANY'}," +
@@ -351,51 +343,6 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
assertEquals("Unexpected node in computed operation", runner.getNodeName(), nodeAdded);
}
- @Test
- public void testSearchRate() throws Exception {
- // create an empty node
- cluster.startJettySolrRunner();
- CloudSolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'search_rate_trigger'," +
- "'event' : 'searchRate'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'rate' : 1.0," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testSearchRate",
- "conf",1, 1);
- create.process(solrClient);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "testSearchRate", (liveNodes, collectionState) -> collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes)));
-
- // generate some dummy traffic
- SolrParams query = params(CommonParams.Q, "*:*");
- for (int i = 0; i < 500; i++) {
- solrClient.query("testSearchRate", query);
- }
-
- assertTrue("Trigger was not fired", triggerFiredLatch.await(10, TimeUnit.SECONDS));
- assertTrue(fired.get());
- Map context = actionContextPropsRef.get();
- assertNotNull(context);
- List<SolrRequest> operations = (List<SolrRequest>) context.get("operations");
- assertNotNull("The operations computed by ComputePlanAction should not be null", operations);
- assertEquals("ComputePlanAction should have computed exactly 1 operation", 1, operations.size());
- SolrRequest request = operations.get(0);
- SolrParams params = request.getParams();
- assertEquals("Expected ADDREPLICA action after exceeding searchRate", ADDREPLICA, CollectionParams.CollectionAction.get(params.get("action")));
-
- }
-
public static class AssertingTriggerAction implements TriggerAction {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc25e846/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java b/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
index b78e23e..f0c3d77 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
@@ -41,7 +41,6 @@ public interface AutoScalingParams {
String SHARD = "shard";
String REPLICA = "replica";
String NODE = "node";
- String WARM_SHARD = "warmShard";
String HANDLER = "handler";
String RATE = "rate";
String REMOVE_LISTENERS = "removeListeners";
[23/24] lucene-solr:jira/solr-11285: Merge branch
'feature/autoscaling' into jira/solr-11285
Posted by ab...@apache.org.
Merge branch 'feature/autoscaling' into jira/solr-11285
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1f25f3ec
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1f25f3ec
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1f25f3ec
Branch: refs/heads/jira/solr-11285
Commit: 1f25f3ec293592f34b4c24acfd92dd58d97c6c31
Parents: 18b9936 fba51f3
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Thu Sep 28 22:40:13 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Thu Sep 28 22:40:13 2017 +0200
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../lucene/analysis/synonym/SynonymMap.java | 3 +-
.../apache/lucene/search/ConjunctionScorer.java | 4 +-
.../apache/lucene/search/TestConjunctions.java | 56 +++-
solr/CHANGES.txt | 66 ++---
.../java/org/apache/solr/cloud/Overseer.java | 75 +----
.../solr/cloud/autoscaling/AutoScaling.java | 11 +-
.../cloud/autoscaling/AutoScalingHandler.java | 6 +-
.../cloud/autoscaling/ComputePlanAction.java | 50 ----
.../cloud/autoscaling/ExecutePlanAction.java | 110 +++++++-
.../cloud/autoscaling/NodeAddedTrigger.java | 10 +-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 5 +
.../autoscaling/OverseerTriggerThread.java | 52 +++-
.../cloud/autoscaling/ScheduledTriggers.java | 84 +++++-
.../cloud/autoscaling/SearchRateTrigger.java | 275 -------------------
.../solr/handler/CdcrReplicatorManager.java | 8 +-
.../solr/handler/admin/MetricsHandler.java | 11 +-
.../org/apache/solr/response/DocsStreamer.java | 87 ++----
.../solr/response/RetrieveFieldsOptimizer.java | 130 +++++++++
.../apache/solr/search/SolrDocumentFetcher.java | 25 +-
.../facet/FacetFieldProcessorByHashDV.java | 11 +-
.../org/apache/solr/search/facet/HLLAgg.java | 1 +
.../org/apache/solr/search/facet/UniqueAgg.java | 1 +
.../solr/search/join/GraphPointsCollector.java | 1 +
.../apache/solr/cloud/CdcrBootstrapTest.java | 1 -
.../cloud/SharedFSAutoReplicaFailoverTest.java | 25 +-
.../cloud/TestCloudJSONFacetJoinDomain.java | 24 +-
.../AutoAddReplicasIntegrationTest.java | 2 +
.../autoscaling/ComputePlanActionTest.java | 57 +---
.../autoscaling/ExecutePlanActionTest.java | 42 ++-
.../cloud/autoscaling/NodeLostTriggerTest.java | 1 +
.../autoscaling/SearchRateTriggerTest.java | 168 -----------
.../admin/AutoscalingHistoryHandlerTest.java | 21 +-
.../solr/handler/admin/MetricsHandlerTest.java | 9 -
.../response/TestRetrieveFieldsOptimizer.java | 56 ++++
.../solr/search/facet/TestJsonFacets.java | 26 ++
.../conf/solrconfig.xml | 1 -
.../cloud/autoscaling/ClusterDataProvider.java | 1 -
.../solrj/cloud/autoscaling/NoneSuggester.java | 4 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 1 -
.../solrj/cloud/autoscaling/Preference.java | 8 +
.../solrj/cloud/autoscaling/ReplicaInfo.java | 24 --
.../autoscaling/SolrCloudDataProvider.java | 10 +-
.../solrj/impl/SolrClientCloudDataProvider.java | 12 +
.../impl/SolrClientClusterDataProvider.java | 2 +-
.../apache/solr/common/cloud/ZkStateReader.java | 3 +-
.../solr/common/params/AutoScalingParams.java | 7 -
.../solrj/cloud/autoscaling/TestPolicy.java | 38 ---
48 files changed, 762 insertions(+), 866 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 1ad8eef,f665857..baa5d25
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@@ -19,19 -19,17 +19,14 @@@ package org.apache.solr.cloud
import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
import java.util.Collections;
--import java.util.LinkedList;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
--import java.util.concurrent.atomic.AtomicInteger;
import com.codahale.metrics.Timer;
--import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.cloud.DistributedQueue;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
- import org.apache.solr.cloud.autoscaling.AutoScaling;
- import org.apache.solr.cloud.autoscaling.AutoScalingHandler;
import org.apache.solr.cloud.autoscaling.OverseerTriggerThread;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
import org.apache.solr.cloud.overseer.CollectionMutator;
@@@ -53,11 -48,8 +45,9 @@@ import org.apache.solr.common.util.IOUt
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.component.ShardHandler;
- import org.apache.solr.request.LocalSolrQueryRequest;
- import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@@ -533,14 -523,8 +521,9 @@@ public class Overseer implements Closea
ccThread = new OverseerThread(ccTg, overseerCollectionConfigSetProcessor, "OverseerCollectionConfigSetProcessor-" + id);
ccThread.setDaemon(true);
- //TODO nocommit, autoscaling framework should start autoAddReplicas trigger automatically (implicitly)
- autoscalingTriggerCreator = new Thread(createAutoscalingTriggerIfNotExist(), "AutoscalingTriggerCreator");
- autoscalingTriggerCreator.setDaemon(true);
- autoscalingTriggerCreator.start();
-
ThreadGroup triggerThreadGroup = new ThreadGroup("Overseer autoscaling triggers");
- OverseerTriggerThread trigger = new OverseerTriggerThread(zkController, config);
+ OverseerTriggerThread trigger = new OverseerTriggerThread(zkController.getCoreContainer().getResourceLoader(),
- zkController.getSolrCloudDataProvider());
++ zkController.getSolrCloudDataProvider(), config);
triggerThread = new OverseerThread(triggerThreadGroup, trigger, "OverseerAutoScalingTriggerThread-" + id);
updaterThread.start();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index c314cd6,bd74c84..3842ac3
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@@ -24,9 -24,10 +24,10 @@@ import java.util.Map
import com.google.common.base.Preconditions;
import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+ import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.cloud.ZkController;
+import org.apache.solr.core.SolrResourceLoader;
public class AutoScaling {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 42cdf18,5d5e3eb..47468c4
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@@ -20,22 -20,19 +20,15 @@@ package org.apache.solr.cloud.autoscali
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
--import java.util.HashMap;
- import java.util.HashSet;
import java.util.List;
import java.util.Map;
- import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
-import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
import org.apache.solr.common.params.CollectionParams;
- import org.apache.solr.client.solrj.cloud.autoscaling.NoneSuggester;
- import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
- import org.apache.solr.common.params.AutoScalingParams;
- import org.apache.solr.common.util.Pair;
-import org.apache.solr.core.CoreContainer;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
index 0a536ed,8221ff8..a8bfd15
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
@@@ -17,15 -17,30 +17,28 @@@
package org.apache.solr.cloud.autoscaling;
+ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+ import java.util.Collections;
import java.util.List;
+ import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.TimeoutException;
+ import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
++import org.apache.solr.client.solrj.cloud.autoscaling.AlreadyExistsException;
++import org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
+ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+ import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.SolrException;
+ import org.apache.solr.common.cloud.SolrZkClient;
+ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
+ import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreContainer;
+ import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -49,21 -72,106 +65,113 @@@ public class ExecutePlanAction extends
for (SolrRequest operation : operations) {
log.info("Executing operation: {}", operation.getParams());
try {
- SolrResponse response = null;
- if (operation instanceof CollectionAdminRequest.AsyncCollectionAdminRequest) {
- CollectionAdminRequest.AsyncCollectionAdminRequest req = (CollectionAdminRequest.AsyncCollectionAdminRequest) operation;
- String asyncId = event.getSource() + '/' + event.getId() + '/' + counter;
- String znode = saveAsyncId(event, context, asyncId);
- log.debug("Saved requestId: {} in znode: {}", asyncId, znode);
- asyncId = req.processAsync(asyncId, cloudSolrClient);
- CollectionAdminRequest.RequestStatusResponse statusResponse = waitForTaskToFinish(cloudSolrClient, asyncId,
- DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
- if (statusResponse != null) {
- RequestStatusState state = statusResponse.getRequestStatus();
- if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
- try {
- zkClient.delete(znode, -1, true);
- } catch (KeeperException e) {
- log.warn("Unexpected exception while trying to delete znode: " + znode, e);
- }
- }
- response = statusResponse;
- }
- } else {
- response = operation.process(cloudSolrClient);
- }
+ SolrResponse response = dataProvider.request(operation);
++ // nocommit
++//=======
++// try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder()
++// .withZkHost(container.getZkController().getZkServerAddress())
++// .withHttpClient(container.getUpdateShardHandler().getHttpClient())
++// .build()) {
++// int counter = 0;
++// for (SolrRequest operation : operations) {
++// log.info("Executing operation: {}", operation.getParams());
++// try {
++// SolrResponse response = null;
++// if (operation instanceof CollectionAdminRequest.AsyncCollectionAdminRequest) {
++// CollectionAdminRequest.AsyncCollectionAdminRequest req = (CollectionAdminRequest.AsyncCollectionAdminRequest) operation;
++// String asyncId = event.getSource() + '/' + event.getId() + '/' + counter;
++// String znode = saveAsyncId(event, context, asyncId);
++// log.debug("Saved requestId: {} in znode: {}", asyncId, znode);
++// asyncId = req.processAsync(asyncId, cloudSolrClient);
++// CollectionAdminRequest.RequestStatusResponse statusResponse = waitForTaskToFinish(cloudSolrClient, asyncId,
++// DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
++// if (statusResponse != null) {
++// RequestStatusState state = statusResponse.getRequestStatus();
++// if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
++// try {
++// zkClient.delete(znode, -1, true);
++// } catch (KeeperException e) {
++// log.warn("Unexpected exception while trying to delete znode: " + znode, e);
++// }
++// }
++// response = statusResponse;
++// }
++// } else {
++// response = operation.process(cloudSolrClient);
++// }
++//>>>>>>> feature/autoscaling
+ NamedList<Object> result = response.getResponse();
context.getProperties().compute("responses", (s, o) -> {
List<NamedList<Object>> responses = (List<NamedList<Object>>) o;
if (responses == null) responses = new ArrayList<>(operations.size());
- responses.add(response.getResponse());
+ responses.add(result);
return responses;
});
- } catch (SolrServerException | HttpSolrClient.RemoteSolrException e) {
++ } catch (IOException e) {
++ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to talk to ZooKeeper", e);
++// } catch (InterruptedException e) {
++// Thread.currentThread().interrupt();
++// throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ExecutePlanAction was interrupted", e);
+ } catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unexpected exception executing operation: " + operation.getParams(), e);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ExecutePlanAction was interrupted", e);
- } catch (KeeperException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to talk to ZooKeeper", e);
}
+
- counter++;
++// counter++;
}
- } catch (IOException e) {
+ } catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unexpected IOException while processing event: " + event, e);
}
}
+
- static CollectionAdminRequest.RequestStatusResponse waitForTaskToFinish(CloudSolrClient cloudSolrClient, String requestId, long duration, TimeUnit timeUnit) throws SolrServerException, IOException, InterruptedException {
++
++ static CollectionAdminRequest.RequestStatusResponse waitForTaskToFinish(SolrCloudDataProvider dataProvider, String requestId, long duration, TimeUnit timeUnit) throws IOException, InterruptedException {
+ long timeoutSeconds = timeUnit.toSeconds(duration);
+ RequestStatusState state = RequestStatusState.NOT_FOUND;
+ CollectionAdminRequest.RequestStatusResponse statusResponse = null;
+ for (int i = 0; i < timeoutSeconds; i++) {
+ try {
- statusResponse = CollectionAdminRequest.requestStatus(requestId).process(cloudSolrClient);
++ statusResponse = (CollectionAdminRequest.RequestStatusResponse)dataProvider.request(CollectionAdminRequest.requestStatus(requestId));
+ state = statusResponse.getRequestStatus();
+ if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED) {
+ log.info("Task with requestId={} finished with state={} in {}s", requestId, state, i * 5);
- CollectionAdminRequest.deleteAsyncId(requestId).process(cloudSolrClient);
++ dataProvider.request(CollectionAdminRequest.deleteAsyncId(requestId));
+ return statusResponse;
+ } else if (state == RequestStatusState.NOT_FOUND) {
+ // the request for this id was never actually submitted! no harm done, just bail out
+ log.warn("Task with requestId={} was not found on overseer", requestId);
- CollectionAdminRequest.deleteAsyncId(requestId).process(cloudSolrClient);
++ dataProvider.request(CollectionAdminRequest.deleteAsyncId(requestId));
+ return statusResponse;
+ }
+ } catch (Exception e) {
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause instanceof IllegalStateException && rootCause.getMessage().contains("Connection pool shut down")) {
+ throw e;
+ }
+ if (rootCause instanceof TimeoutException && rootCause.getMessage().contains("Could not connect to ZooKeeper")) {
+ throw e;
+ }
+ log.error("Unexpected Exception while querying status of requestId=" + requestId, e);
+ }
+ if (i > 0 && i % 5 == 0) {
+ log.debug("Task with requestId={} still not complete after {}s. Last state={}", requestId, i * 5, state);
+ }
+ TimeUnit.SECONDS.sleep(5);
+ }
+ log.debug("Task with requestId={} did not complete within 5 minutes. Last state={}", requestId, state);
+ return statusResponse;
+ }
+
+ /**
+ * Saves the given asyncId in ZK as a persistent sequential node.
+ *
+ * @return the path of the newly created node in ZooKeeper
+ */
- private String saveAsyncId(TriggerEvent event, ActionContext context, String asyncId) throws InterruptedException, KeeperException {
++ private String saveAsyncId(DistribStateManager stateManager, TriggerEvent event, ActionContext context, String asyncId) throws InterruptedException, AlreadyExistsException, IOException {
+ String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + context.getSource().getName() + "/" + getName();
- CoreContainer container = context.getCoreContainer();
- SolrZkClient zkClient = container.getZkController().getZkClient();
- try {
- zkClient.makePath(parentPath, new byte[0], CreateMode.PERSISTENT, true);
- } catch (KeeperException.NodeExistsException e) {
- // ignore
- }
- return zkClient.create(parentPath + "/" + PREFIX, Utils.toJSON(Collections.singletonMap("requestid", asyncId)), CreateMode.PERSISTENT_SEQUENTIAL, true);
++ stateManager.makePath(parentPath);
++ return stateManager.createData(parentPath + "/" + PREFIX, Utils.toJSON(Collections.singletonMap("requestid", asyncId)), CreateMode.PERSISTENT_SEQUENTIAL);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index 438e383,bbfd098..120414d
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@@ -63,10 -96,16 +63,16 @@@ public class NodeAddedTrigger extends T
@Override
public void init() {
- super.init();
+ List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
+ if (o != null && !o.isEmpty()) {
+ for (int i = 0; i < o.size(); i++) {
+ Map<String, String> map = o.get(i);
+ actions.get(i).init(map);
+ }
+ }
// pick up added nodes for which marker paths were created
try {
- List<String> added = zkClient.getChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, null, true);
+ List<String> added = stateManager.listData(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
added.forEach(n -> {
// don't add nodes that have since gone away
if (lastLiveNodes.contains(n)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 2174d41,cbcbb3e..c51ecdd
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@@ -24,17 -26,21 +26,20 @@@ import java.util.HashMap
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+ import java.util.Locale;
import java.util.Map;
-import java.util.Objects;
+import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicReference;
-import org.apache.lucene.util.IOUtils;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+ import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.TimeSource;
-import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
index dbe2ea4,367c450..8cf3707
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
@@@ -32,12 -30,16 +32,15 @@@ import java.util.concurrent.locks.Reent
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
++import org.apache.solr.client.solrj.cloud.autoscaling.BadVersionException;
+import org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.cloud.ZkController;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.IOUtils;
+ import org.apache.solr.common.util.Utils;
+ import org.apache.solr.core.CloudConfig;
-import org.apache.zookeeper.KeeperException;
+import org.apache.solr.core.SolrResourceLoader;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
import org.slf4j.Logger;
@@@ -51,8 -55,14 +56,10 @@@ public class OverseerTriggerThread impl
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private final ZkController zkController;
+ private final SolrCloudDataProvider dataProvider;
+ private final CloudConfig cloudConfig;
+
- private final ZkStateReader zkStateReader;
-
- private final SolrZkClient zkClient;
-
private final ScheduledTriggers scheduledTriggers;
private final AutoScaling.TriggerFactory triggerFactory;
@@@ -72,10 -82,13 +79,11 @@@
private AutoScalingConfig autoScalingConfig;
- public OverseerTriggerThread(SolrResourceLoader loader, SolrCloudDataProvider dataProvider) {
- public OverseerTriggerThread(ZkController zkController, CloudConfig cloudConfig) {
- this.zkController = zkController;
++ public OverseerTriggerThread(SolrResourceLoader loader, SolrCloudDataProvider dataProvider, CloudConfig cloudConfig) {
+ this.dataProvider = dataProvider;
+ this.cloudConfig = cloudConfig;
- zkStateReader = zkController.getZkStateReader();
- zkClient = zkController.getZkClient();
- scheduledTriggers = new ScheduledTriggers(zkController);
- triggerFactory = new AutoScaling.TriggerFactory(zkController.getCoreContainer(), zkController);
+ scheduledTriggers = new ScheduledTriggers(loader, dataProvider);
+ triggerFactory = new AutoScaling.TriggerFactoryImpl(loader, dataProvider);
}
@Override
@@@ -97,10 -110,36 +105,33 @@@
public void run() {
int lastZnodeVersion = znodeVersion;
+ // we automatically add a trigger for auto add replicas if it does not exists already
+ while (!isClosed) {
+ try {
- AutoScalingConfig autoScalingConfig = zkStateReader.getAutoScalingConfig();
++ AutoScalingConfig autoScalingConfig = dataProvider.getDistribStateManager().getAutoScalingConfig();
+ AutoScalingConfig withAutoAddReplicasTrigger = withAutoAddReplicasTrigger(autoScalingConfig);
+ if (withAutoAddReplicasTrigger.equals(autoScalingConfig)) break;
+ log.debug("Adding .autoAddReplicas trigger");
- zkClient.setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(withAutoAddReplicasTrigger), withAutoAddReplicasTrigger.getZkVersion(), true);
++ dataProvider.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(withAutoAddReplicasTrigger), withAutoAddReplicasTrigger.getZkVersion());
+ break;
- } catch (KeeperException.BadVersionException bve) {
++ } catch (BadVersionException bve) {
+ // somebody else has changed the configuration so we must retry
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("Interrupted", e);
+ break;
- } catch (KeeperException e) {
++ } catch (IOException e) {
+ log.error("A ZK error has occurred", e);
+ }
+ }
+
+ if (isClosed || Thread.currentThread().isInterrupted()) return;
+
try {
refreshAutoScalingConf(new AutoScalingWatcher());
- } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
+ } catch (ConnectException e) {
log.warn("ZooKeeper watch triggered for autoscaling conf, but Solr cannot talk to ZK: [{}]", e.getMessage());
- } catch (KeeperException e) {
- log.error("A ZK error has occurred", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
diff --cc solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
index c1cea01,7a96552..d98ea45
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
@@@ -39,20 -40,27 +40,27 @@@ import java.util.concurrent.atomic.Atom
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
+ import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
+import org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
++import org.apache.solr.client.solrj.cloud.autoscaling.VersionedData;
+ import org.apache.solr.client.solrj.impl.CloudSolrClient;
+ import org.apache.solr.client.solrj.request.CollectionAdminRequest.RequestStatusResponse;
+ import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.cloud.ActionThrottle;
-import org.apache.solr.cloud.Overseer;
-import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.Stats;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
+ import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.util.DefaultSolrThreadFactory;
-import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Op;
+ import org.apache.zookeeper.OpResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -157,6 -159,12 +167,12 @@@ public class ScheduledTriggers implemen
scheduledTriggers.replace(newTrigger.getName(), scheduledTrigger);
}
newTrigger.setProcessor(event -> {
- if (coreContainer.isShutDown()) {
++ if (dataProvider.isClosed()) {
+ String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s because Solr has been shutdown.", event.toString());
+ log.warn(msg);
+ listeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.ABORTED, msg);
+ return false;
+ }
ScheduledTrigger scheduledSource = scheduledTriggers.get(event.getSource());
if (scheduledSource == null) {
String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s because the source trigger: %s doesn't exist.", event.toString(), event.getSource());
@@@ -190,7 -199,12 +207,12 @@@
// let the action executor thread wait instead of the trigger thread so we use the throttle here
actionThrottle.minimumWaitBetweenActions();
actionThrottle.markAttemptingAction();
+
+ // in future, we could wait for pending tasks in a different thread and re-enqueue
+ // this event so that we continue processing other events and not block this action executor
+ waitForPendingTasks(newTrigger, actions);
+
- ActionContext actionContext = new ActionContext(coreContainer, newTrigger, new HashMap<>());
+ ActionContext actionContext = new ActionContext(dataProvider, newTrigger, new HashMap<>());
for (TriggerAction action : actions) {
List<String> beforeActions = (List<String>)actionContext.getProperties().computeIfAbsent(TriggerEventProcessorStage.BEFORE_ACTION.toString(), k -> new ArrayList<String>());
beforeActions.add(action.getName());
@@@ -236,6 -250,69 +258,65 @@@
scheduledTrigger.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay(scheduledTrigger, 0, DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS, TimeUnit.SECONDS);
}
+ private void waitForPendingTasks(AutoScaling.Trigger newTrigger, List<TriggerAction> actions) throws AlreadyClosedException {
- try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder()
- .withZkHost(coreContainer.getZkController().getZkServerAddress())
- .withHttpClient(coreContainer.getUpdateShardHandler().getHttpClient())
- .build()) {
-
- SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
++ DistribStateManager stateManager = dataProvider.getDistribStateManager();
++ try {
+
+ for (TriggerAction action : actions) {
+ if (action instanceof ExecutePlanAction) {
+ String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + newTrigger.getName() + "/" + action.getName();
- if (!zkClient.exists(parentPath, true)) {
++ if (!stateManager.hasData(parentPath)) {
+ break;
+ }
- List<String> children = zkClient.getChildren(parentPath, null, true);
++ List<String> children = stateManager.listData(parentPath);
+ if (children != null) {
+ for (String child : children) {
+ String path = parentPath + '/' + child;
- byte[] data = zkClient.getData(path, null, null, true);
++ VersionedData data = stateManager.getData(path, null);
+ if (data != null) {
- Map map = (Map) Utils.fromJSON(data);
++ Map map = (Map) Utils.fromJSON(data.getData());
+ String requestid = (String) map.get("requestid");
+ try {
+ log.debug("Found pending task with requestid={}", requestid);
- RequestStatusResponse statusResponse = waitForTaskToFinish(cloudSolrClient, requestid,
++ RequestStatusResponse statusResponse = waitForTaskToFinish(dataProvider, requestid,
+ ExecutePlanAction.DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ if (statusResponse != null) {
+ RequestStatusState state = statusResponse.getRequestStatus();
+ if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
- zkClient.delete(path, -1, true);
++ stateManager.removeData(path, -1);
+ }
+ }
+ } catch (Exception e) {
- if (coreContainer.isShutDown()) {
++ if (dataProvider.isClosed()) {
+ throw e; // propagate the abort to the caller
+ }
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause instanceof IllegalStateException && rootCause.getMessage().contains("Connection pool shut down")) {
+ throw e;
+ }
+ if (rootCause instanceof TimeoutException && rootCause.getMessage().contains("Could not connect to ZooKeeper")) {
+ throw e;
+ }
+ log.error("Unexpected exception while waiting for pending task with requestid: " + requestid + " to finish", e);
+ }
+ }
+ }
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Thread interrupted", e);
+ } catch (Exception e) {
- if (coreContainer.isShutDown()) {
++ if (dataProvider.isClosed()) {
+ throw new AlreadyClosedException("The Solr instance has been shutdown");
+ }
+ // we catch but don't rethrow because a failure to wait for pending tasks
+ // should not keep the actions from executing
+ log.error("Unexpected exception while waiting for pending tasks to finish", e);
+ }
+ }
+
/**
* Removes and stops the trigger with the given name. Also cleans up any leftover
* state / events in ZK.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 93a13b0,d9b88b5..94b4094
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@@ -63,7 -61,7 +61,7 @@@ public class ComputePlanActionTest exte
private static final int NODE_COUNT = 1;
private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
private static final AtomicReference<Map> actionContextPropsRef = new AtomicReference<>();
-- private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
++ private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();o
@BeforeClass
public static void setupCluster() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
index 1ffb00b,58972af..2753c3d
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
@@@ -23,15 -23,8 +23,14 @@@ import java.util.Collection
import java.util.List;
import java.util.Map;
+import org.apache.solr.common.cloud.ClusterState;
- import org.apache.zookeeper.Watcher;
+
+/**
+ * This interface abstracts the access to cluster data.
+ */
public interface ClusterDataProvider extends Closeable {
- /**Get the value of each tag for a given node
+ /**
+ * Get the value of each tag for a given node
*
* @param node node name
* @param tags tag names
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 5dd8a33,bc2a5d7..3ed12a6
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@@ -18,9 -18,8 +18,8 @@@
package org.apache.solr.client.solrj.cloud.autoscaling;
+import java.io.IOException;
import java.util.ArrayList;
--import java.util.Collection;
import java.util.EnumMap;
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SolrCloudDataProvider.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SolrCloudDataProvider.java
index 995de9f,0000000..2c30ecc
mode 100644,000000..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SolrCloudDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SolrCloudDataProvider.java
@@@ -1,63 -1,0 +1,61 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.cloud.autoscaling;
+
+import java.io.Closeable;
+import java.io.IOException;
- import java.util.List;
+import java.util.Map;
- import java.util.NoSuchElementException;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.cloud.DistributedQueue;
- import org.apache.zookeeper.CreateMode;
- import org.apache.zookeeper.Op;
- import org.apache.zookeeper.OpResult;
- import org.apache.zookeeper.Watcher;
+
+/**
+ * This interface abstracts the access to a SolrCloud cluster, including interactions with Zookeeper, Solr
+ * and generic HTTP calls.
+ */
+public interface SolrCloudDataProvider extends Closeable {
+
+ ClusterDataProvider getClusterDataProvider();
+
+ DistribStateManager getDistribStateManager();
+
+ interface DistributedQueueFactory {
+ DistributedQueue makeQueue(String path) throws IOException;
+ void removeQueue(String path) throws IOException;
+ }
+
+ DistributedQueueFactory getDistributedQueueFactory();
+
+ // Solr-like methods
+
+ SolrResponse request(SolrRequest req) throws IOException;
+
+ byte[] httpRequest(String url, SolrRequest.METHOD method, Map<String, String> headers, String payload, int timeout, boolean followRedirects) throws IOException;
+
+ // distributed queue implementation
+
+ @Override
+ default void close() {
+
+ }
++
++ default boolean isClosed() {
++ return false;
++ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientCloudDataProvider.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientCloudDataProvider.java
index 246ef88,0000000..59d4d97
mode 100644,000000..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientCloudDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientCloudDataProvider.java
@@@ -1,156 -1,0 +1,168 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpDelete;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
+import org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager;
+import org.apache.solr.client.solrj.cloud.autoscaling.SolrCloudDataProvider;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Op;
+import org.apache.zookeeper.OpResult;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class that implements {@link SolrCloudDataProvider} using a SolrClient
+ */
+public class SolrClientCloudDataProvider implements SolrCloudDataProvider {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CloudSolrClient solrClient;
+ private final ZkDistribStateManager stateManager;
+ private final DistributedQueueFactory queueFactory;
+ private final ZkStateReader zkStateReader;
+ private final SolrZkClient zkClient;
++ private volatile boolean isClosed;
+
+ public SolrClientCloudDataProvider(DistributedQueueFactory queueFactory, CloudSolrClient solrClient) {
+ this.queueFactory = queueFactory;
+ this.solrClient = solrClient;
+ this.zkStateReader = solrClient.getZkStateReader();
+ this.zkClient = zkStateReader.getZkClient();
+ this.stateManager = new ZkDistribStateManager(zkClient);
++ this.isClosed = false;
++ }
++
++ @Override
++ public void close() {
++ isClosed = true;
++ }
++
++ @Override
++ public boolean isClosed() {
++ return isClosed;
+ }
+
+ @Override
+ public ClusterDataProvider getClusterDataProvider() {
+ return new SolrClientClusterDataProvider(solrClient);
+ }
+
+ @Override
+ public DistribStateManager getDistribStateManager() {
+ return stateManager;
+ }
+
+ @Override
+ public SolrResponse request(SolrRequest req) throws IOException {
+ try {
+ return req.process(solrClient);
+ } catch (SolrServerException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private static final byte[] EMPTY = new byte[0];
+
+ @Override
+ public byte[] httpRequest(String url, SolrRequest.METHOD method, Map<String, String> headers, String payload, int timeout, boolean followRedirects) throws IOException {
+ HttpClient client = solrClient.getHttpClient();
+ final HttpRequestBase req;
+ HttpEntity entity = null;
+ if (payload != null) {
+ entity = new StringEntity(payload, "UTF-8");
+ }
+ switch (method) {
+ case GET:
+ req = new HttpGet(url);
+ break;
+ case POST:
+ req = new HttpPost(url);
+ if (entity != null) {
+ ((HttpPost)req).setEntity(entity);
+ }
+ break;
+ case PUT:
+ req = new HttpPut(url);
+ if (entity != null) {
+ ((HttpPut)req).setEntity(entity);
+ }
+ break;
+ case DELETE:
+ req = new HttpDelete(url);
+ break;
+ default:
+ throw new IOException("Unsupported method " + method);
+ }
+ if (headers != null) {
+ headers.forEach((k, v) -> req.addHeader(k, v));
+ }
+ RequestConfig.Builder requestConfigBuilder = HttpClientUtil.createDefaultRequestConfigBuilder();
+ if (timeout > 0) {
+ requestConfigBuilder.setSocketTimeout(timeout);
+ requestConfigBuilder.setConnectTimeout(timeout);
+ }
+ requestConfigBuilder.setRedirectsEnabled(followRedirects);
+ req.setConfig(requestConfigBuilder.build());
+ HttpClientContext httpClientRequestContext = HttpClientUtil.createNewHttpClientRequestContext();
+ HttpResponse rsp = client.execute(req, httpClientRequestContext);
+ int statusCode = rsp.getStatusLine().getStatusCode();
+ if (statusCode != 200) {
+ throw new IOException("Error sending request to " + url + ", HTTP response: " + rsp.toString());
+ }
+ HttpEntity responseEntity = rsp.getEntity();
+ if (responseEntity != null && responseEntity.getContent() != null) {
+ return EntityUtils.toByteArray(responseEntity);
+ } else {
+ return EMPTY;
+ }
+ }
+
+ @Override
+ public DistributedQueueFactory getDistributedQueueFactory() {
+ return queueFactory;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
index fd0722a,0000000..2cd24ae
mode 100644,000000..100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
@@@ -1,249 -1,0 +1,249 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.impl;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.cloud.autoscaling.ClusterDataProvider;
+import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.response.SimpleSolrResponse;
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.cloud.rule.ImplicitSnitch;
+import org.apache.solr.common.cloud.rule.RemoteCallback;
+import org.apache.solr.common.cloud.rule.SnitchContext;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.client.solrj.cloud.autoscaling.Clause.METRICS_PREFIX;
+
+/**
+ *
+ */
+public class SolrClientClusterDataProvider implements ClusterDataProvider, MapWriter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CloudSolrClient solrClient;
+ private final ZkStateReader zkStateReader;
+ private final Map<String, Map<String, Map<String, List<ReplicaInfo>>>> data = new HashMap<>();
+ private Map<String, Object> snitchSession = new HashMap<>();
+ private Map<String, Map> nodeVsTags = new HashMap<>();
+
+ public SolrClientClusterDataProvider(CloudSolrClient solrClient) {
+ this.solrClient = solrClient;
+ this.zkStateReader = solrClient.getZkStateReader();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Map<String, ClusterState.CollectionRef> all = clusterState.getCollectionStates();
+ all.forEach((collName, ref) -> {
+ DocCollection coll = ref.get();
+ if (coll == null) return;
+ coll.forEachReplica((shard, replica) -> {
+ Map<String, Map<String, List<ReplicaInfo>>> nodeData = data.computeIfAbsent(replica.getNodeName(), k -> new HashMap<>());
+ Map<String, List<ReplicaInfo>> collData = nodeData.computeIfAbsent(collName, k -> new HashMap<>());
+ List<ReplicaInfo> replicas = collData.computeIfAbsent(shard, k -> new ArrayList<>());
- replicas.add(new ReplicaInfo(replica.getName(), collName, shard, replica.getType(), replica.getProperties()));
++ replicas.add(new ReplicaInfo(replica.getName(), collName, shard, replica.getType(), new HashMap<>()));
+ });
+ });
+ }
+
+ public Collection<String> getLiveNodes() {
+ return zkStateReader.getClusterState().getLiveNodes();
+ }
+
+ @Override
+ public String getPolicyNameByCollection(String coll) {
+ ClusterState.CollectionRef state = solrClient.getClusterStateProvider().getState(coll);
+ return state == null || state.get() == null ? null : (String) state.get().getProperties().get("policy");
+ }
+
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
+ AutoScalingSnitch snitch = new AutoScalingSnitch();
+ ClientSnitchCtx ctx = new ClientSnitchCtx(null, node, snitchSession, solrClient);
+ snitch.getTags(node, new HashSet<>(tags), ctx);
+ nodeVsTags.put(node, ctx.getTags());
+ return ctx.getTags();
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return data.computeIfAbsent(node, s -> Collections.emptyMap());//todo fill other details
+ }
+
+ @Override
+ public Map<String, Object> getClusterProperties() {
+ return zkStateReader.getClusterProperties();
+ }
+
+ @Override
+ public ClusterState getClusterState() {
+ return zkStateReader.getClusterState();
+ }
+
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put("clusterState", zkStateReader.getClusterState());
+ ew.put("replicaInfo", Utils.getDeepCopy(data, 5));
+ ew.put("nodeValues", nodeVsTags);
+ }
+
+ static class ClientSnitchCtx
+ extends SnitchContext {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ ZkClientClusterStateProvider zkClientClusterStateProvider;
+ CloudSolrClient solrClient;
+
+ public ClientSnitchCtx(SnitchInfo perSnitch,
+ String node, Map<String, Object> session,
+ CloudSolrClient solrClient) {
+ super(perSnitch, node, session);
+ this.solrClient = solrClient;
+ this.zkClientClusterStateProvider = (ZkClientClusterStateProvider) solrClient.getClusterStateProvider();
+ }
+
+
+ @Override
+ public Map getZkJson(String path) throws KeeperException, InterruptedException {
+ return Utils.getJson(zkClientClusterStateProvider.getZkStateReader().getZkClient(), path, true);
+ }
+
+ public void invokeRemote(String node, ModifiableSolrParams params, String klas, RemoteCallback callback) {
+
+ }
+
+ public SimpleSolrResponse invoke(String solrNode, String path, SolrParams params)
+ throws IOException, SolrServerException {
+ String url = zkClientClusterStateProvider.getZkStateReader().getBaseUrlForNodeName(solrNode);
+
+ GenericSolrRequest request = new GenericSolrRequest(SolrRequest.METHOD.GET, path, params);
+ try (HttpSolrClient client = new HttpSolrClient.Builder()
+ .withHttpClient(solrClient.getHttpClient())
+ .withBaseSolrUrl(url)
+ .withResponseParser(new BinaryResponseParser())
+ .build()) {
+ NamedList<Object> rsp = client.request(request);
+ request.response.nl = rsp;
+ return request.response;
+ }
+ }
+
+ }
+
+ //uses metrics API to get node information
+ static class AutoScalingSnitch extends ImplicitSnitch {
+ @Override
+ protected void getRemoteInfo(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
+ ClientSnitchCtx snitchContext = (ClientSnitchCtx) ctx;
+ Map<String, String> metricsKeyVsTag = new HashMap<>();
+ for (String tag : requestedTags) {
+ if (tag.startsWith(SYSPROP)) {
+ metricsKeyVsTag.put("solr.jvm:system.properties:" + tag.substring(SYSPROP.length()), tag);
+ } else if (tag.startsWith(METRICS_PREFIX)) {
+ metricsKeyVsTag.put(tag.substring(METRICS_PREFIX.length()), tag);
+ }
+ }
+ if (!metricsKeyVsTag.isEmpty()) {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("key", metricsKeyVsTag.keySet().toArray(new String[metricsKeyVsTag.size()]));
+ try {
+ SimpleSolrResponse rsp = snitchContext.invoke(solrNode, CommonParams.METRICS_PATH, params);
+ metricsKeyVsTag.forEach((key, tag) -> {
+ Object v = Utils.getObjectByPath(rsp.nl, true, Arrays.asList("metrics", key));
+ if (v != null) snitchContext.getTags().put(tag, v);
+ });
+ } catch (Exception e) {
+ log.warn("could not get tags from node " + solrNode, e);
+ }
+ }
+
+ Set<String> groups = new HashSet<>();
+ List<String> prefixes = new ArrayList<>();
+ if (requestedTags.contains(DISK)) {
+ groups.add("solr.node");
+ prefixes.add("CONTAINER.fs.usableSpace");
+ }
+ if (requestedTags.contains(CORES)) {
+ groups.add("solr.core");
+ prefixes.add("CORE.coreName");
+ }
+ if (requestedTags.contains(SYSLOADAVG)) {
+ groups.add("solr.jvm");
+ prefixes.add("os.systemLoadAverage");
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ groups.add("solr.jvm");
+ prefixes.add("memory.heap.usage");
+ }
+ if (groups.isEmpty() || prefixes.isEmpty()) return;
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add("group", StrUtils.join(groups, ','));
+ params.add("prefix", StrUtils.join(prefixes, ','));
+
+ try {
+ SimpleSolrResponse rsp = snitchContext.invoke(solrNode, CommonParams.METRICS_PATH, params);
+ Map m = rsp.nl.asMap(4);
+ if (requestedTags.contains(DISK)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.node/CONTAINER.fs.usableSpace");
+ if (n != null) ctx.getTags().put(DISK, n.doubleValue() / 1024.0d / 1024.0d / 1024.0d);
+ }
+ if (requestedTags.contains(CORES)) {
+ int count = 0;
+ Map cores = (Map) m.get("metrics");
+ for (Object o : cores.keySet()) {
+ if (o.toString().startsWith("solr.core.")) count++;
+ }
+ ctx.getTags().put(CORES, count);
+ }
+ if (requestedTags.contains(SYSLOADAVG)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.jvm/os.systemLoadAverage");
+ if (n != null) ctx.getTags().put(SYSLOADAVG, n.doubleValue() * 100.0d);
+ }
+ if (requestedTags.contains(HEAPUSAGE)) {
+ Number n = (Number) Utils.getObjectByPath(m, true, "metrics/solr.jvm/memory.heap.usage");
+ if (n != null) ctx.getTags().put(HEAPUSAGE, n.doubleValue() * 100.0d);
+ }
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1f25f3ec/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --cc solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 5516902,96e2257..22afc51
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@@ -1226,42 -1172,10 +1200,36 @@@ public class TestPolicy extends SolrTes
}
@Override
- public Collection<String> getNodes() {
+ public Collection<String> getLiveNodes() {
return Arrays.asList( "127.0.0.1:50097_solr", "127.0.0.1:50096_solr");
}
-
- @Override
- public ClusterState getClusterState() {
- throw new UnsupportedOperationException("getClusterState");
- }
-
};
+ SolrCloudDataProvider dataProvider = new SolrCloudDataProvider() {
+ @Override
+ public ClusterDataProvider getClusterDataProvider() {
+ return provider;
+ }
+
+ @Override
+ public DistribStateManager getDistribStateManager() {
+ return null;
+ }
+
+ @Override
+ public DistributedQueueFactory getDistributedQueueFactory() {
+ return null;
+ }
+
+ @Override
+ public SolrResponse request(SolrRequest req) throws IOException {
+ return null;
+ }
+
+ @Override
+ public byte[] httpRequest(String url, SolrRequest.METHOD method, Map<String, String> headers, String payload, int timeout, boolean followRedirects) throws IOException {
+ return new byte[0];
+ }
+ };
List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
"newColl", new AutoScalingConfig((Map<String, Object>)Utils.fromJSONString(autoScaleJson)),
dataProvider, Collections.singletonMap("newColl", "c1"), Arrays.asList("shard1", "shard2"), 1, 0, 0, null);
@@@ -1315,42 -1229,10 +1283,36 @@@
}
@Override
- public Collection<String> getNodes() {
+ public Collection<String> getLiveNodes() {
return Arrays.asList("node1", "node2", "node3", "node4");
}
-
- @Override
- public ClusterState getClusterState() {
- throw new UnsupportedOperationException("getClusterState");
- }
-
};
+ SolrCloudDataProvider dataProvider = new SolrCloudDataProvider() {
+ @Override
+ public ClusterDataProvider getClusterDataProvider() {
+ return provider;
+ }
+
+ @Override
+ public DistribStateManager getDistribStateManager() {
+ return null;
+ }
+
+ @Override
+ public DistributedQueueFactory getDistributedQueueFactory() {
+ return null;
+ }
+
+ @Override
+ public SolrResponse request(SolrRequest req) throws IOException {
+ return null;
+ }
+
+ @Override
+ public byte[] httpRequest(String url, SolrRequest.METHOD method, Map<String, String> headers, String payload, int timeout, boolean followRedirects) throws IOException {
+ return new byte[0];
+ }
+ };
List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
"newColl", new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScaleJson)),
dataProvider, Collections.singletonMap("newColl", "policy1"), Arrays.asList("shard1", "shard2"), 3,0,0, null);
[18/24] lucene-solr:jira/solr-11285: SOLR-11085: Improve resiliency
of autoscaling actions against overseer restarts and operation failures
Posted by ab...@apache.org.
SOLR-11085: Improve resiliency of autoscaling actions against overseer restarts and operation failures
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/e13c4c34
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/e13c4c34
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/e13c4c34
Branch: refs/heads/jira/solr-11285
Commit: e13c4c34f0edbf7b74c3d0a8552556d57a3b1416
Parents: d58e5fe
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Sep 20 16:40:27 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Sep 20 16:40:27 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 1 +
.../solr/cloud/autoscaling/AutoScaling.java | 10 +-
.../cloud/autoscaling/ExecutePlanAction.java | 105 +++++++++++++++-
.../cloud/autoscaling/NodeAddedTrigger.java | 19 +--
.../solr/cloud/autoscaling/NodeLostTrigger.java | 17 +--
.../autoscaling/OverseerTriggerThread.java | 2 +-
.../cloud/autoscaling/ScheduledTriggers.java | 122 +++++++++++++++++--
.../cloud/SharedFSAutoReplicaFailoverTest.java | 25 ++--
.../autoscaling/ExecutePlanActionTest.java | 44 ++++++-
.../cloud/autoscaling/NodeAddedTriggerTest.java | 14 +--
.../cloud/autoscaling/NodeLostTriggerTest.java | 14 +--
.../solrj/cloud/autoscaling/Preference.java | 8 ++
.../apache/solr/common/cloud/ZkStateReader.java | 3 +-
13 files changed, 324 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d2a2213..0383f8b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -177,6 +177,7 @@ Bug Fixes
* SOLR-11363: JSON Facet API: repeated values in a numeric points field with docValues enabled
were double counted. (Hossman, yonik)
+* SOLR-11085: Improve resiliency of autoscaling actions against overseer restarts and operation failures. (shalin)
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index 760a425..bd74c84 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@ -27,6 +27,7 @@ import org.apache.lucene.store.AlreadyClosedException;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.cloud.ZkController;
public class AutoScaling {
@@ -114,12 +115,15 @@ public class AutoScaling {
public static class TriggerFactory implements Closeable {
private final CoreContainer coreContainer;
+ private final ZkController zkController;
private boolean isClosed = false;
- public TriggerFactory(CoreContainer coreContainer) {
+ public TriggerFactory(CoreContainer coreContainer, ZkController zkController) {
Preconditions.checkNotNull(coreContainer);
+ Preconditions.checkNotNull(zkController);
this.coreContainer = coreContainer;
+ this.zkController = zkController;
}
public synchronized Trigger create(TriggerEventType type, String name, Map<String, Object> props) {
@@ -128,9 +132,9 @@ public class AutoScaling {
}
switch (type) {
case NODEADDED:
- return new NodeAddedTrigger(name, props, coreContainer);
+ return new NodeAddedTrigger(name, props, coreContainer, zkController);
case NODELOST:
- return new NodeLostTrigger(name, props, coreContainer);
+ return new NodeLostTrigger(name, props, coreContainer, zkController);
default:
throw new IllegalArgumentException("Unknown event type: " + type + " in trigger: " + name);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
index 2cd9824..8221ff8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
@@ -20,16 +20,27 @@ package org.apache.solr.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,11 +50,15 @@ import org.slf4j.LoggerFactory;
*/
public class ExecutePlanAction extends TriggerActionBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private static final String PREFIX = "op-";
+
+ static final int DEFAULT_TASK_TIMEOUT_SECONDS = 120;
@Override
public void process(TriggerEvent event, ActionContext context) {
log.debug("-- processing event: {} with context properties: {}", event, context.getProperties());
CoreContainer container = context.getCoreContainer();
+ SolrZkClient zkClient = container.getZkController().getZkClient();
List<SolrRequest> operations = (List<SolrRequest>) context.getProperty("operations");
if (operations == null || operations.isEmpty()) {
log.info("No operations to execute for event: {}", event);
@@ -53,24 +68,110 @@ public class ExecutePlanAction extends TriggerActionBase {
.withZkHost(container.getZkController().getZkServerAddress())
.withHttpClient(container.getUpdateShardHandler().getHttpClient())
.build()) {
+ int counter = 0;
for (SolrRequest operation : operations) {
log.info("Executing operation: {}", operation.getParams());
try {
- SolrResponse response = operation.process(cloudSolrClient);
+ SolrResponse response = null;
+ if (operation instanceof CollectionAdminRequest.AsyncCollectionAdminRequest) {
+ CollectionAdminRequest.AsyncCollectionAdminRequest req = (CollectionAdminRequest.AsyncCollectionAdminRequest) operation;
+ String asyncId = event.getSource() + '/' + event.getId() + '/' + counter;
+ String znode = saveAsyncId(event, context, asyncId);
+ log.debug("Saved requestId: {} in znode: {}", asyncId, znode);
+ asyncId = req.processAsync(asyncId, cloudSolrClient);
+ CollectionAdminRequest.RequestStatusResponse statusResponse = waitForTaskToFinish(cloudSolrClient, asyncId,
+ DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ if (statusResponse != null) {
+ RequestStatusState state = statusResponse.getRequestStatus();
+ if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
+ try {
+ zkClient.delete(znode, -1, true);
+ } catch (KeeperException e) {
+ log.warn("Unexpected exception while trying to delete znode: " + znode, e);
+ }
+ }
+ response = statusResponse;
+ }
+ } else {
+ response = operation.process(cloudSolrClient);
+ }
+ NamedList<Object> result = response.getResponse();
context.getProperties().compute("responses", (s, o) -> {
List<NamedList<Object>> responses = (List<NamedList<Object>>) o;
if (responses == null) responses = new ArrayList<>(operations.size());
- responses.add(response.getResponse());
+ responses.add(result);
return responses;
});
} catch (SolrServerException | HttpSolrClient.RemoteSolrException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unexpected exception executing operation: " + operation.getParams(), e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ExecutePlanAction was interrupted", e);
+ } catch (KeeperException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unable to talk to ZooKeeper", e);
}
+
+ counter++;
}
} catch (IOException e) {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unexpected IOException while processing event: " + event, e);
}
}
+
+ static CollectionAdminRequest.RequestStatusResponse waitForTaskToFinish(CloudSolrClient cloudSolrClient, String requestId, long duration, TimeUnit timeUnit) throws SolrServerException, IOException, InterruptedException {
+ long timeoutSeconds = timeUnit.toSeconds(duration);
+ RequestStatusState state = RequestStatusState.NOT_FOUND;
+ CollectionAdminRequest.RequestStatusResponse statusResponse = null;
+ for (int i = 0; i < timeoutSeconds; i++) {
+ try {
+ statusResponse = CollectionAdminRequest.requestStatus(requestId).process(cloudSolrClient);
+ state = statusResponse.getRequestStatus();
+ if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED) {
+ log.info("Task with requestId={} finished with state={} in {}s", requestId, state, i * 5);
+ CollectionAdminRequest.deleteAsyncId(requestId).process(cloudSolrClient);
+ return statusResponse;
+ } else if (state == RequestStatusState.NOT_FOUND) {
+ // the request for this id was never actually submitted! no harm done, just bail out
+ log.warn("Task with requestId={} was not found on overseer", requestId);
+ CollectionAdminRequest.deleteAsyncId(requestId).process(cloudSolrClient);
+ return statusResponse;
+ }
+ } catch (Exception e) {
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause instanceof IllegalStateException && rootCause.getMessage().contains("Connection pool shut down")) {
+ throw e;
+ }
+ if (rootCause instanceof TimeoutException && rootCause.getMessage().contains("Could not connect to ZooKeeper")) {
+ throw e;
+ }
+ log.error("Unexpected Exception while querying status of requestId=" + requestId, e);
+ }
+ if (i > 0 && i % 5 == 0) {
+ log.debug("Task with requestId={} still not complete after {}s. Last state={}", requestId, i * 5, state);
+ }
+ TimeUnit.SECONDS.sleep(5);
+ }
+ log.debug("Task with requestId={} did not complete within 5 minutes. Last state={}", requestId, state);
+ return statusResponse;
+ }
+
+ /**
+ * Saves the given asyncId in ZK as a persistent sequential node.
+ *
+ * @return the path of the newly created node in ZooKeeper
+ */
+ private String saveAsyncId(TriggerEvent event, ActionContext context, String asyncId) throws InterruptedException, KeeperException {
+ String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + context.getSource().getName() + "/" + getName();
+ CoreContainer container = context.getCoreContainer();
+ SolrZkClient zkClient = container.getZkController().getZkClient();
+ try {
+ zkClient.makePath(parentPath, new byte[0], CreateMode.PERSISTENT, true);
+ } catch (KeeperException.NodeExistsException e) {
+ // ignore
+ }
+ return zkClient.create(parentPath + "/" + PREFIX, Utils.toJSON(Collections.singletonMap("requestid", asyncId)), CreateMode.PERSISTENT_SEQUENTIAL, true);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index 0c0278b..bbfd098 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.CoreContainer;
@@ -52,6 +53,7 @@ public class NodeAddedTrigger extends TriggerBase {
private final String name;
private final Map<String, Object> properties;
private final CoreContainer container;
+ private final ZkController zkController;
private final List<TriggerAction> actions;
private final AtomicReference<AutoScaling.TriggerEventProcessor> processorRef;
private final boolean enabled;
@@ -66,11 +68,12 @@ public class NodeAddedTrigger extends TriggerBase {
private Map<String, Long> nodeNameVsTimeAdded = new HashMap<>();
public NodeAddedTrigger(String name, Map<String, Object> properties,
- CoreContainer container) {
- super(container.getZkController().getZkClient());
+ CoreContainer container, ZkController zkController) {
+ super(zkController.getZkClient());
this.name = name;
this.properties = properties;
this.container = container;
+ this.zkController = zkController;
this.timeSource = TimeSource.CURRENT_TIME;
this.processorRef = new AtomicReference<>();
List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
@@ -83,7 +86,7 @@ public class NodeAddedTrigger extends TriggerBase {
} else {
actions = Collections.emptyList();
}
- lastLiveNodes = new HashSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
+ lastLiveNodes = new HashSet<>(zkController.getZkStateReader().getClusterState().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
this.enabled = Boolean.parseBoolean(String.valueOf(properties.getOrDefault("enabled", "true")));
this.waitForSecond = ((Long) properties.getOrDefault("waitFor", -1L)).intValue();
@@ -102,7 +105,7 @@ public class NodeAddedTrigger extends TriggerBase {
}
// pick up added nodes for which marker paths were created
try {
- List<String> added = container.getZkController().getZkClient().getChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, null, true);
+ List<String> added = zkClient.getChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, null, true);
added.forEach(n -> {
// don't add nodes that have since gone away
if (lastLiveNodes.contains(n)) {
@@ -229,7 +232,7 @@ public class NodeAddedTrigger extends TriggerBase {
}
log.debug("Running NodeAddedTrigger {}", name);
- ZkStateReader reader = container.getZkController().getZkStateReader();
+ ZkStateReader reader = zkController.getZkStateReader();
Set<String> newLiveNodes = reader.getClusterState().getLiveNodes();
log.debug("Found livenodes: {}", newLiveNodes);
@@ -278,7 +281,7 @@ public class NodeAddedTrigger extends TriggerBase {
});
}
}
- lastLiveNodes = new HashSet(newLiveNodes);
+ lastLiveNodes = new HashSet<>(newLiveNodes);
} catch (RuntimeException e) {
log.error("Unexpected exception in NodeAddedTrigger", e);
}
@@ -287,8 +290,8 @@ public class NodeAddedTrigger extends TriggerBase {
private void removeMarker(String nodeName) {
String path = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + nodeName;
try {
- if (container.getZkController().getZkClient().exists(path, true)) {
- container.getZkController().getZkClient().delete(path, -1, true);
+ if (zkClient.exists(path, true)) {
+ zkClient.delete(path, -1, true);
}
} catch (KeeperException.NoNodeException e) {
// ignore
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 18dafcb..cbcbb3e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicReference;
import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.core.CoreContainer;
@@ -52,6 +53,7 @@ public class NodeLostTrigger extends TriggerBase {
private final String name;
private final Map<String, Object> properties;
private final CoreContainer container;
+ private final ZkController zkController;
private final List<TriggerAction> actions;
private final AtomicReference<AutoScaling.TriggerEventProcessor> processorRef;
private final boolean enabled;
@@ -66,11 +68,12 @@ public class NodeLostTrigger extends TriggerBase {
private Map<String, Long> nodeNameVsTimeRemoved = new HashMap<>();
public NodeLostTrigger(String name, Map<String, Object> properties,
- CoreContainer container) {
- super(container.getZkController().getZkClient());
+ CoreContainer container, ZkController zkController) {
+ super(zkController.getZkClient());
this.name = name;
this.properties = properties;
this.container = container;
+ this.zkController = zkController;
this.timeSource = TimeSource.CURRENT_TIME;
this.processorRef = new AtomicReference<>();
List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
@@ -83,7 +86,7 @@ public class NodeLostTrigger extends TriggerBase {
} else {
actions = Collections.emptyList();
}
- lastLiveNodes = new HashSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
+ lastLiveNodes = new HashSet<>(zkController.getZkStateReader().getClusterState().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
this.enabled = Boolean.parseBoolean(String.valueOf(properties.getOrDefault("enabled", "true")));
this.waitForSecond = ((Long) properties.getOrDefault("waitFor", -1L)).intValue();
@@ -101,7 +104,7 @@ public class NodeLostTrigger extends TriggerBase {
}
// pick up lost nodes for which marker paths were created
try {
- List<String> lost = container.getZkController().getZkClient().getChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH, null, true);
+ List<String> lost = zkClient.getChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH, null, true);
lost.forEach(n -> {
// don't add nodes that have since came back
if (!lastLiveNodes.contains(n)) {
@@ -226,7 +229,7 @@ public class NodeLostTrigger extends TriggerBase {
}
}
- ZkStateReader reader = container.getZkController().getZkStateReader();
+ ZkStateReader reader = zkController.getZkStateReader();
Set<String> newLiveNodes = reader.getClusterState().getLiveNodes();
log.debug("Running NodeLostTrigger: {} with currently live nodes: {}", name, newLiveNodes);
@@ -286,8 +289,8 @@ public class NodeLostTrigger extends TriggerBase {
private void removeMarker(String nodeName) {
String path = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + nodeName;
try {
- if (container.getZkController().getZkClient().exists(path, true)) {
- container.getZkController().getZkClient().delete(path, -1, true);
+ if (zkClient.exists(path, true)) {
+ zkClient.delete(path, -1, true);
}
} catch (KeeperException.NoNodeException e) {
// ignore
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
index 34a6eaa..367c450 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
@@ -88,7 +88,7 @@ public class OverseerTriggerThread implements Runnable, Closeable {
zkStateReader = zkController.getZkStateReader();
zkClient = zkController.getZkClient();
scheduledTriggers = new ScheduledTriggers(zkController);
- triggerFactory = new AutoScaling.TriggerFactory(zkController.getCoreContainer());
+ triggerFactory = new AutoScaling.TriggerFactory(zkController.getCoreContainer(), zkController);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
index ec9ffc1..7a96552 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
@@ -35,27 +35,37 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest.RequestStatusResponse;
+import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.cloud.ActionThrottle;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.ZkController;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Op;
+import org.apache.zookeeper.OpResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish;
+
/**
* Responsible for scheduling active triggers, starting and stopping them and
* performing actions when they fire
@@ -149,6 +159,12 @@ public class ScheduledTriggers implements Closeable {
scheduledTriggers.replace(newTrigger.getName(), scheduledTrigger);
}
newTrigger.setProcessor(event -> {
+ if (coreContainer.isShutDown()) {
+ String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s because Solr has been shutdown.", event.toString());
+ log.warn(msg);
+ listeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.ABORTED, msg);
+ return false;
+ }
ScheduledTrigger scheduledSource = scheduledTriggers.get(event.getSource());
if (scheduledSource == null) {
String msg = String.format(Locale.ROOT, "Ignoring autoscaling event %s because the source trigger: %s doesn't exist.", event.toString(), event.getSource());
@@ -166,13 +182,14 @@ public class ScheduledTriggers implements Closeable {
return false;
}
if (hasPendingActions.compareAndSet(false, true)) {
- listeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.STARTED);
final boolean enqueued;
if (replaying) {
enqueued = false;
} else {
enqueued = scheduledTrigger.enqueue(event);
}
+ // fire STARTED event listeners after enqueuing the event is successful
+ listeners.fireListeners(event.getSource(), event, TriggerEventProcessorStage.STARTED);
List<TriggerAction> actions = source.getActions();
if (actions != null) {
actionExecutor.submit(() -> {
@@ -182,6 +199,11 @@ public class ScheduledTriggers implements Closeable {
// let the action executor thread wait instead of the trigger thread so we use the throttle here
actionThrottle.minimumWaitBetweenActions();
actionThrottle.markAttemptingAction();
+
+ // in future, we could wait for pending tasks in a different thread and re-enqueue
+ // this event so that we continue processing other events and not block this action executor
+ waitForPendingTasks(newTrigger, actions);
+
ActionContext actionContext = new ActionContext(coreContainer, newTrigger, new HashMap<>());
for (TriggerAction action : actions) {
List<String> beforeActions = (List<String>)actionContext.getProperties().computeIfAbsent(TriggerEventProcessorStage.BEFORE_ACTION.toString(), k -> new ArrayList<String>());
@@ -228,6 +250,69 @@ public class ScheduledTriggers implements Closeable {
scheduledTrigger.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay(scheduledTrigger, 0, DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS, TimeUnit.SECONDS);
}
+ private void waitForPendingTasks(AutoScaling.Trigger newTrigger, List<TriggerAction> actions) throws AlreadyClosedException {
+ try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder()
+ .withZkHost(coreContainer.getZkController().getZkServerAddress())
+ .withHttpClient(coreContainer.getUpdateShardHandler().getHttpClient())
+ .build()) {
+
+ SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
+
+ for (TriggerAction action : actions) {
+ if (action instanceof ExecutePlanAction) {
+ String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + newTrigger.getName() + "/" + action.getName();
+ if (!zkClient.exists(parentPath, true)) {
+ break;
+ }
+ List<String> children = zkClient.getChildren(parentPath, null, true);
+ if (children != null) {
+ for (String child : children) {
+ String path = parentPath + '/' + child;
+ byte[] data = zkClient.getData(path, null, null, true);
+ if (data != null) {
+ Map map = (Map) Utils.fromJSON(data);
+ String requestid = (String) map.get("requestid");
+ try {
+ log.debug("Found pending task with requestid={}", requestid);
+ RequestStatusResponse statusResponse = waitForTaskToFinish(cloudSolrClient, requestid,
+ ExecutePlanAction.DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ if (statusResponse != null) {
+ RequestStatusState state = statusResponse.getRequestStatus();
+ if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
+ zkClient.delete(path, -1, true);
+ }
+ }
+ } catch (Exception e) {
+ if (coreContainer.isShutDown()) {
+ throw e; // propagate the abort to the caller
+ }
+ Throwable rootCause = ExceptionUtils.getRootCause(e);
+ if (rootCause instanceof IllegalStateException && rootCause.getMessage().contains("Connection pool shut down")) {
+ throw e;
+ }
+ if (rootCause instanceof TimeoutException && rootCause.getMessage().contains("Could not connect to ZooKeeper")) {
+ throw e;
+ }
+ log.error("Unexpected exception while waiting for pending task with requestid: " + requestid + " to finish", e);
+ }
+ }
+ }
+ }
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Thread interrupted", e);
+ } catch (Exception e) {
+ if (coreContainer.isShutDown()) {
+ throw new AlreadyClosedException("The Solr instance has been shutdown");
+ }
+ // we catch but don't rethrow because a failure to wait for pending tasks
+ // should not keep the actions from executing
+ log.error("Unexpected exception while waiting for pending tasks to finish", e);
+ }
+ }
+
/**
* Removes and stops the trigger with the given name. Also cleans up any leftover
* state / events in ZK.
@@ -244,27 +329,38 @@ public class ScheduledTriggers implements Closeable {
String statePath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + triggerName;
String eventsPath = ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH + "/" + triggerName;
try {
- if (zkClient.exists(statePath, true)) {
- zkClient.delete(statePath, -1, true);
- }
+ zkDelTree(zkClient, statePath);
} catch (KeeperException | InterruptedException e) {
log.warn("Failed to remove state for removed trigger " + statePath, e);
}
try {
- if (zkClient.exists(eventsPath, true)) {
- List<String> events = zkClient.getChildren(eventsPath, null, true);
- List<Op> ops = new ArrayList<>(events.size() + 1);
- events.forEach(ev -> {
- ops.add(Op.delete(eventsPath + "/" + ev, -1));
- });
- ops.add(Op.delete(eventsPath, -1));
- zkClient.multi(ops, true);
- }
+ zkDelTree(zkClient, eventsPath);
} catch (KeeperException | InterruptedException e) {
log.warn("Failed to remove events for removed trigger " + eventsPath, e);
}
}
+ static List<OpResult> zkDelTree(SolrZkClient zkClient, String znode) throws KeeperException, InterruptedException {
+ if (zkClient.exists(znode, true)) {
+ ArrayList<Op> ops = new ArrayList<>();
+ zkDelTree(zkClient, znode, ops);
+ return zkClient.multi(ops, true);
+ }
+ return Collections.emptyList();
+ }
+
+ private static void zkDelTree(SolrZkClient zkClient, String znode, ArrayList<Op> ops) throws KeeperException, InterruptedException {
+ if (zkClient.exists(znode, true)) {
+ List<String> children = zkClient.getChildren(znode, null, true);
+ if (children != null) {
+ for (String child : children) {
+ String path = znode + "/" + child;
+ zkDelTree(zkClient, path, ops);
+ }
+ }
+ ops.add(Op.delete(znode, -1));
+ }
+ }
/**
* @return an unmodifiable set of names of all triggers being managed by this class
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
index 294ada7..41c7c40 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
@@ -17,6 +17,7 @@
package org.apache.solr.cloud;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -57,6 +58,7 @@ import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.BadHdfsThreadsFilter;
+import org.apache.solr.util.LogLevel;
import org.apache.solr.util.TimeOut;
import org.junit.AfterClass;
import org.junit.Before;
@@ -64,6 +66,8 @@ import org.junit.BeforeClass;
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Nightly
@Slow
@@ -71,7 +75,9 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.*=DEBUG")
public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final boolean DEBUG = true;
private static MiniDFSCluster dfsCluster;
@@ -234,8 +240,9 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
assertUlogDir(collections);
- //TODO SOLR-11085, we must test overseer failover
- List<JettySolrRunner> stoppedJetties = notOverseerJetties();
+ boolean allowOverseerRestart = random().nextBoolean();
+ List<JettySolrRunner> stoppedJetties = allowOverseerRestart
+ ? jettys.stream().filter(jettySolrRunner -> random().nextBoolean()).collect(Collectors.toList()) : notOverseerJetties();
ChaosMonkey.stop(stoppedJetties);
ChaosMonkey.stop(controlJetty);
@@ -276,10 +283,14 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
}
SolrParams queryAll = new SolrQuery("*:*");
cloudClient.setDefaultCollection(collection);
- QueryResponse queryResponse = cloudClient.query(queryAll);
- actualResultSize = queryResponse.getResults().getNumFound();
- if(expectedResultSize == actualResultSize) {
- return;
+ try {
+ QueryResponse queryResponse = cloudClient.query(queryAll);
+ actualResultSize = queryResponse.getResults().getNumFound();
+ if(expectedResultSize == actualResultSize) {
+ return;
+ }
+ } catch (SolrServerException | IOException e) {
+ log.warn("Querying solr threw an exception. This can be expected to happen during restarts.", e);
}
Thread.sleep(1000);
@@ -399,7 +410,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
}
Thread.sleep(200);
}
- fail("Expected numSlices=" + numSlices + " numReplicas=" + numReplicas + " but found " + cloudClient.getZkStateReader().getClusterState().getCollection(collection));
+ fail("Expected numSlices=" + numSlices + " numReplicas=" + numReplicas + " but found " + cloudClient.getZkStateReader().getClusterState().getCollection(collection) + " with /live_nodes: " + cloudClient.getZkStateReader().getClusterState().getLiveNodes());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
index 1cb692e..e92d8a0 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
@@ -17,13 +17,19 @@
package org.apache.solr.cloud.autoscaling;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
+import com.google.common.collect.Lists;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -33,6 +39,8 @@ import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
import org.apache.solr.util.LogLevel;
@@ -105,18 +113,46 @@ public class ExecutePlanActionTest extends SolrCloudTestCase {
try (ExecutePlanAction action = new ExecutePlanAction()) {
action.init(Collections.singletonMap("name", "execute_plan"));
- CollectionAdminRequest.MoveReplica moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
- List<SolrRequest> operations = Collections.singletonList(moveReplica);
+
+ // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
+ AtomicBoolean znodeCreated = new AtomicBoolean(false);
+
+ CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
+ CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.LIST) {
+ @Override
+ public String processAsync(String asyncId, SolrClient client) throws IOException, SolrServerException {
+ String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
+ try {
+ if (zkClient().exists(parentPath, true)) {
+ java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
+ if (!children.isEmpty()) {
+ String child = children.get(0);
+ byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true);
+ Map m = (Map) Utils.fromJSON(data);
+ if (m.containsKey("requestid")) {
+ znodeCreated.set(m.get("requestid").equals(asyncId));
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ super.processAsync(asyncId, client);
+ return asyncId;
+ }
+ };
+ List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST,
"mock_trigger_name", Collections.singletonList(TimeSource.CURRENT_TIME.getTime()),
Collections.singletonList(sourceNodeName));
- ActionContext actionContext = new ActionContext(survivor.getCoreContainer(), null,
+ ActionContext actionContext = new ActionContext(survivor.getCoreContainer(), new NodeLostTrigger("xyz", Collections.singletonMap("event", TriggerEventType.NODELOST.name()), survivor.getCoreContainer(), survivor.getCoreContainer().getZkController()),
new HashMap<>(Collections.singletonMap("operations", operations)));
action.process(nodeLostEvent, actionContext);
+ assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
assertNotNull(responses);
- assertEquals(1, responses.size());
+ assertEquals(2, responses.size());
NamedList<Object> response = responses.get(0);
assertNull(response.get("failure"));
assertNotNull(response.get("success"));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java
index 687aec7..67f1d08 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java
@@ -72,7 +72,7 @@ public class NodeAddedTriggerTest extends SolrCloudTestCase {
long waitForSeconds = 1 + random().nextInt(5);
Map<String, Object> props = createTriggerProps(waitForSeconds);
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container)) {
+ try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container, container.getZkController())) {
trigger.setProcessor(noFirstRunProcessor);
trigger.run();
@@ -112,7 +112,7 @@ public class NodeAddedTriggerTest extends SolrCloudTestCase {
// add a new node but remove it before the waitFor period expires
// and assert that the trigger doesn't fire at all
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container)) {
+ try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container, container.getZkController())) {
final long waitTime = 2;
props.put("waitFor", waitTime);
trigger.setProcessor(noFirstRunProcessor);
@@ -157,7 +157,7 @@ public class NodeAddedTriggerTest extends SolrCloudTestCase {
action.put("name", "testActionInit");
action.put("class", NodeAddedTriggerTest.AssertInitTriggerAction.class.getName());
actions.add(action);
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container)) {
+ try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container, container.getZkController())) {
assertEquals(true, actionConstructorCalled.get());
assertEquals(false, actionInitCalled.get());
assertEquals(false, actionCloseCalled.get());
@@ -198,7 +198,7 @@ public class NodeAddedTriggerTest extends SolrCloudTestCase {
public void testListenerAcceptance() throws Exception {
CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
Map<String, Object> props = createTriggerProps(0);
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container)) {
+ try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container, container.getZkController())) {
trigger.setProcessor(noFirstRunProcessor);
trigger.run(); // starts tracking live nodes
@@ -234,7 +234,7 @@ public class NodeAddedTriggerTest extends SolrCloudTestCase {
// add a new node but update the trigger before the waitFor period expires
// and assert that the new trigger still fires
- NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container);
+ NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger", props, container, container.getZkController());
trigger.setProcessor(noFirstRunProcessor);
trigger.run();
@@ -242,7 +242,7 @@ public class NodeAddedTriggerTest extends SolrCloudTestCase {
trigger.run(); // this run should detect the new node
trigger.close(); // close the old trigger
- try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("some_different_name", props, container)) {
+ try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("some_different_name", props, container, container.getZkController())) {
try {
newTrigger.restoreState(trigger);
fail("Trigger should only be able to restore state from an old trigger of the same name");
@@ -251,7 +251,7 @@ public class NodeAddedTriggerTest extends SolrCloudTestCase {
}
}
- try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("node_added_trigger", props, container)) {
+ try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("node_added_trigger", props, container, container.getZkController())) {
AtomicBoolean fired = new AtomicBoolean(false);
AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
newTrigger.setProcessor(event -> {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
index 9833c5d..eb73792 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
@@ -73,7 +73,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
long waitForSeconds = 1 + random().nextInt(5);
Map<String, Object> props = createTriggerProps(waitForSeconds);
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger", props, container)) {
+ try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger", props, container, container.getZkController())) {
trigger.setProcessor(noFirstRunProcessor);
trigger.run();
String lostNodeName1 = cluster.getJettySolrRunner(1).getNodeName();
@@ -117,7 +117,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
// remove a node but add it back before the waitFor period expires
// and assert that the trigger doesn't fire at all
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger", props, container)) {
+ try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger", props, container, container.getZkController())) {
final long waitTime = 2;
props.put("waitFor", waitTime);
trigger.setProcessor(noFirstRunProcessor);
@@ -173,7 +173,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
action.put("name", "testActionInit");
action.put("class", AssertInitTriggerAction.class.getName());
actions.add(action);
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger", props, container)) {
+ try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger", props, container, container.getZkController())) {
assertEquals(true, actionConstructorCalled.get());
assertEquals(false, actionInitCalled.get());
assertEquals(false, actionCloseCalled.get());
@@ -214,7 +214,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
public void testListenerAcceptance() throws Exception {
CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
Map<String, Object> props = createTriggerProps(0);
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger", props, container)) {
+ try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger", props, container, container.getZkController())) {
trigger.setProcessor(noFirstRunProcessor);
JettySolrRunner newNode = cluster.startJettySolrRunner();
@@ -267,7 +267,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
// remove a node but update the trigger before the waitFor period expires
// and assert that the new trigger still fires
- NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger", props, container);
+ NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger", props, container, container.getZkController());
trigger.setProcessor(noFirstRunProcessor);
trigger.run();
@@ -284,7 +284,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
trigger.run(); // this run should detect the lost node
trigger.close(); // close the old trigger
- try (NodeLostTrigger newTrigger = new NodeLostTrigger("some_different_name", props, container)) {
+ try (NodeLostTrigger newTrigger = new NodeLostTrigger("some_different_name", props, container, container.getZkController())) {
try {
newTrigger.restoreState(trigger);
fail("Trigger should only be able to restore state from an old trigger of the same name");
@@ -293,7 +293,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
}
}
- try (NodeLostTrigger newTrigger = new NodeLostTrigger("node_lost_trigger", props, container)) {
+ try (NodeLostTrigger newTrigger = new NodeLostTrigger("node_lost_trigger", props, container, container.getZkController())) {
AtomicBoolean fired = new AtomicBoolean(false);
AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
newTrigger.setProcessor(event -> {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
index a7fa963..0142107 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Preference.java
@@ -18,14 +18,19 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Map;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class Preference implements MapWriter {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
final Policy.SortParam name;
Integer precision;
final Policy.Sort sort;
@@ -76,6 +81,9 @@ public class Preference implements MapWriter {
void setApproxVal(List<Row> tmpMatrix) {
Object prevVal = null;
for (Row row : tmpMatrix) {
+ if (!row.isLive) {
+ continue;
+ }
prevVal = row.cells[idx].approxVal =
(prevVal == null || Double.compare(Math.abs(((Number) prevVal).doubleValue() - ((Number) row.cells[idx].val).doubleValue()), precision) > 0) ?
row.cells[idx].val :
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e13c4c34/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 2462a45..e45565f 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -810,7 +810,8 @@ public class ZkStateReader implements Closeable {
Thread.sleep(GET_LEADER_RETRY_INTERVAL_MS);
}
throw new SolrException(ErrorCode.SERVICE_UNAVAILABLE, "No registered leader was found after waiting for "
- + timeout + "ms " + ", collection: " + collection + " slice: " + shard);
+ + timeout + "ms " + ", collection: " + collection + " slice: " + shard + " saw state=" + clusterState.getCollectionOrNull(collection)
+ + " with live_nodes=" + clusterState.getLiveNodes());
}
/**
[10/24] lucene-solr:jira/solr-11285: SOLR-10397: Pass CloudConfig
from Overseer to the OverseerTriggerThread. Avoid throwing exceptions on
session expiry if thread has been closed already. Abort earlier if thread has
been closed.
Posted by ab...@apache.org.
SOLR-10397: Pass CloudConfig from Overseer to the OverseerTriggerThread. Avoid throwing exceptions on session expiry if thread has been closed already. Abort earlier if thread has been closed.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/61fe5248
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/61fe5248
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/61fe5248
Branch: refs/heads/jira/solr-11285
Commit: 61fe5248b0d9ab840781e7fd90aa23ef71685d67
Parents: bd08ca0
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Sep 19 11:09:00 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Sep 19 11:09:00 2017 -0700
----------------------------------------------------------------------
.../src/java/org/apache/solr/cloud/Overseer.java | 16 +---------------
.../cloud/autoscaling/OverseerTriggerThread.java | 17 ++++++++++++-----
2 files changed, 13 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/61fe5248/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index 7b81e90..f665857 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -473,8 +473,6 @@ public class Overseer implements Closeable {
private OverseerThread triggerThread;
- private Thread autoscalingTriggerCreator;
-
private final ZkStateReader reader;
private final ShardHandler shardHandler;
@@ -526,7 +524,7 @@ public class Overseer implements Closeable {
ccThread.setDaemon(true);
ThreadGroup triggerThreadGroup = new ThreadGroup("Overseer autoscaling triggers");
- OverseerTriggerThread trigger = new OverseerTriggerThread(zkController);
+ OverseerTriggerThread trigger = new OverseerTriggerThread(zkController, config);
triggerThread = new OverseerThread(triggerThreadGroup, trigger, "OverseerAutoScalingTriggerThread-" + id);
updaterThread.start();
@@ -576,10 +574,6 @@ public class Overseer implements Closeable {
IOUtils.closeQuietly(triggerThread);
triggerThread.interrupt();
}
- if (autoscalingTriggerCreator != null) {
- autoscalingTriggerCreator.interrupt();
- }
-
if (updaterThread != null) {
try {
updaterThread.join();
@@ -595,17 +589,9 @@ public class Overseer implements Closeable {
triggerThread.join();
} catch (InterruptedException e) {}
}
- if (autoscalingTriggerCreator != null) {
- try {
- log.info("Waiting for autoscaling trigger creator join");
- autoscalingTriggerCreator.join();
- } catch (InterruptedException e) {}
- }
-
updaterThread = null;
ccThread = null;
triggerThread = null;
- autoscalingTriggerCreator = null;
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/61fe5248/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
index fcc5b68..1a1d341 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
@@ -57,6 +57,8 @@ public class OverseerTriggerThread implements Runnable, Closeable {
private final ZkController zkController;
+ private final CloudConfig cloudConfig;
+
private final ZkStateReader zkStateReader;
private final SolrZkClient zkClient;
@@ -80,8 +82,9 @@ public class OverseerTriggerThread implements Runnable, Closeable {
private AutoScalingConfig autoScalingConfig;
- public OverseerTriggerThread(ZkController zkController) {
+ public OverseerTriggerThread(ZkController zkController, CloudConfig cloudConfig) {
this.zkController = zkController;
+ this.cloudConfig = cloudConfig;
zkStateReader = zkController.getZkStateReader();
zkClient = zkController.getZkClient();
scheduledTriggers = new ScheduledTriggers(zkController);
@@ -108,7 +111,7 @@ public class OverseerTriggerThread implements Runnable, Closeable {
int lastZnodeVersion = znodeVersion;
// we automatically add a trigger for auto add replicas if it does not exists already
- while (true) {
+ while (!isClosed) {
try {
AutoScalingConfig autoScalingConfig = zkStateReader.getAutoScalingConfig();
AutoScalingConfig withAutoAddReplicasTrigger = withAutoAddReplicasTrigger(autoScalingConfig);
@@ -124,17 +127,22 @@ public class OverseerTriggerThread implements Runnable, Closeable {
log.warn("Interrupted", e);
} catch (KeeperException e) {
log.error("A ZK error has occurred", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
}
}
+ if (!isClosed || Thread.currentThread().isInterrupted()) return;
+
try {
refreshAutoScalingConf(new AutoScalingWatcher());
} catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
log.warn("ZooKeeper watch triggered for autoscaling conf, but Solr cannot talk to ZK: [{}]", e.getMessage());
} catch (KeeperException e) {
log.error("A ZK error has occurred", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
+ if (!isClosed) {
+ // throw exception only if we haven't been closed already
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
+ }
+ return; // silently!
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
@@ -327,7 +335,6 @@ public class OverseerTriggerThread implements Runnable, Closeable {
}
private AutoScalingConfig withAutoAddReplicasTrigger(AutoScalingConfig autoScalingConfig) {
- CloudConfig cloudConfig = zkController.getCoreContainer().getConfig().getCloudConfig();
Map<String, Object> triggerProps = AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_PROPS;
String triggerName = (String) triggerProps.get("name");
Map<String, AutoScalingConfig.TriggerConfig> configs = autoScalingConfig.getTriggerConfigs();
[12/24] lucene-solr:jira/solr-11285: SOLR-10397: Break on
interruption and always throw KeeperException on read otherwise trigger
thread cannot function
Posted by ab...@apache.org.
SOLR-10397: Break on interruption and always throw KeeperException on read otherwise trigger thread cannot function
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bb408c35
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bb408c35
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bb408c35
Branch: refs/heads/jira/solr-11285
Commit: bb408c35a13c9598e494e974486c601eb20b2d05
Parents: 61fe524
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Sep 19 11:34:53 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Sep 19 11:34:53 2017 -0700
----------------------------------------------------------------------
.../solr/cloud/autoscaling/OverseerTriggerThread.java | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bb408c35/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
index 1a1d341..34a6eaa 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
@@ -125,12 +125,13 @@ public class OverseerTriggerThread implements Runnable, Closeable {
// Restore the interrupted status
Thread.currentThread().interrupt();
log.warn("Interrupted", e);
+ break;
} catch (KeeperException e) {
log.error("A ZK error has occurred", e);
}
}
- if (!isClosed || Thread.currentThread().isInterrupted()) return;
+ if (isClosed || Thread.currentThread().isInterrupted()) return;
try {
refreshAutoScalingConf(new AutoScalingWatcher());
@@ -138,11 +139,7 @@ public class OverseerTriggerThread implements Runnable, Closeable {
log.warn("ZooKeeper watch triggered for autoscaling conf, but Solr cannot talk to ZK: [{}]", e.getMessage());
} catch (KeeperException e) {
log.error("A ZK error has occurred", e);
- if (!isClosed) {
- // throw exception only if we haven't been closed already
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
- }
- return; // silently!
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
} catch (InterruptedException e) {
// Restore the interrupted status
Thread.currentThread().interrupt();
[13/24] lucene-solr:jira/solr-11285: SOLR-11371: Removing nocommits
related to instantiating plugins from the blob store
Posted by ab...@apache.org.
SOLR-11371: Removing nocommits related to instantiating plugins from the blob store
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2f94136a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2f94136a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2f94136a
Branch: refs/heads/jira/solr-11285
Commit: 2f94136a69bf78855f877fd5d6795a0db80625e0
Parents: bb408c3
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Sep 19 11:52:17 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Sep 19 11:52:17 2017 -0700
----------------------------------------------------------------------
.../org/apache/solr/cloud/autoscaling/AutoScalingHandler.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2f94136a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index 38a2e1e..6ad84b8 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -459,7 +459,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
if (op.hasError()) return currentConfig;
// validate that we can load the listener class
- // todo nocommit -- what about MemClassLoader?
+ // todo allow creation from blobstore
try {
container.getResourceLoader().findClass(listenerClass, TriggerListener.class);
} catch (Exception e) {
@@ -518,7 +518,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
}
// validate that we can load all the actions
- // todo nocommit -- what about MemClassLoader?
+ // todo allow creation from blobstore
for (Map<String, String> action : actions) {
if (!action.containsKey(NAME) || !action.containsKey(CLASS)) {
op.addError("No 'name' or 'class' specified for action: " + action);
[05/24] lucene-solr:jira/solr-11285: SOLR-10397: Create auto add
replicas trigger directly in ZK instead of using API
Posted by ab...@apache.org.
SOLR-10397: Create auto add replicas trigger directly in ZK instead of using API
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bd08ca09
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bd08ca09
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bd08ca09
Branch: refs/heads/jira/solr-11285
Commit: bd08ca0918204c1b9295f7c8c5dc5201d2d59f85
Parents: cda1d2e
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Sep 18 18:43:42 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Sep 18 18:43:42 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/solr/cloud/Overseer.java | 56 --------------------
.../solr/cloud/autoscaling/AutoScaling.java | 11 ++--
.../cloud/autoscaling/AutoScalingHandler.java | 2 +-
.../autoscaling/OverseerTriggerThread.java | 46 +++++++++++++++-
.../AutoAddReplicasIntegrationTest.java | 2 +
5 files changed, 54 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd08ca09/solr/core/src/java/org/apache/solr/cloud/Overseer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/Overseer.java b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
index e2493b7..7b81e90 100644
--- a/solr/core/src/java/org/apache/solr/cloud/Overseer.java
+++ b/solr/core/src/java/org/apache/solr/cloud/Overseer.java
@@ -30,8 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger;
import com.codahale.metrics.Timer;
import org.apache.solr.client.solrj.SolrResponse;
-import org.apache.solr.cloud.autoscaling.AutoScaling;
-import org.apache.solr.cloud.autoscaling.AutoScalingHandler;
import org.apache.solr.cloud.autoscaling.OverseerTriggerThread;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
import org.apache.solr.cloud.overseer.CollectionMutator;
@@ -41,22 +39,17 @@ import org.apache.solr.cloud.overseer.ReplicaMutator;
import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.cloud.overseer.ZkStateWriter;
import org.apache.solr.cloud.overseer.ZkWriteCommand;
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectReleaseTracker;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.component.ShardHandler;
-import org.apache.solr.request.LocalSolrQueryRequest;
-import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -532,11 +525,6 @@ public class Overseer implements Closeable {
ccThread = new OverseerThread(ccTg, overseerCollectionConfigSetProcessor, "OverseerCollectionConfigSetProcessor-" + id);
ccThread.setDaemon(true);
- //TODO nocommit, autoscaling framework should start autoAddReplicas trigger automatically (implicitly)
- autoscalingTriggerCreator = new Thread(createAutoscalingTriggerIfNotExist(), "AutoscalingTriggerCreator");
- autoscalingTriggerCreator.setDaemon(true);
- autoscalingTriggerCreator.start();
-
ThreadGroup triggerThreadGroup = new ThreadGroup("Overseer autoscaling triggers");
OverseerTriggerThread trigger = new OverseerTriggerThread(zkController);
triggerThread = new OverseerThread(triggerThreadGroup, trigger, "OverseerAutoScalingTriggerThread-" + id);
@@ -574,50 +562,6 @@ public class Overseer implements Closeable {
assert ObjectReleaseTracker.release(this);
}
- private Runnable createAutoscalingTriggerIfNotExist() {
- return new Runnable() {
- @Override
- public void run() {
- try {
- boolean triggerExist = getZkStateReader().getAutoScalingConfig()
- .getTriggerConfigs().get(".auto_add_replicas") != null;
- if (triggerExist) return;
- } catch (InterruptedException | KeeperException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
- "Failed when creating .auto_add_replicas trigger");
- }
- try {
- while (getZkController().getCoreContainer().getRequestHandler(AutoScalingHandler.HANDLER_PATH) == null) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- break;
- }
- }
- if (getZkController().getCoreContainer().isShutDown()) {
- return;
- }
- } catch (Exception e) {}
-
- String dsl = AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_DSL.replace("{{waitFor}}",
- String.valueOf(config.getAutoReplicaFailoverWaitAfterExpiration()/1000));
- LocalSolrQueryRequest request = new LocalSolrQueryRequest(null, new ModifiableSolrParams());
- request.getContext().put("httpMethod", "POST");
- request.setContentStreams(Collections.singleton(new ContentStreamBase.StringStream(dsl)));
- SolrQueryResponse response = new SolrQueryResponse();
- try {
- getZkController().getCoreContainer()
- .getRequestHandler(AutoScalingHandler.HANDLER_PATH).handleRequest(request, response);
- if (!"success".equals(response.getValues().get("result"))) {
- log.error("Failed when creating .auto_add_replicas trigger, return {}",response);
- }
- } catch (Exception e) {
- log.error("Failed when creating .auto_add_replicas trigger ", e);
- }
- }
- };
- }
-
private void doClose() {
if (updaterThread != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd08ca09/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index ed24bf7..760a425 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@ -25,6 +25,7 @@ import java.util.Map;
import com.google.common.base.Preconditions;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CoreContainer;
public class AutoScaling {
@@ -144,11 +145,10 @@ public class AutoScaling {
}
public static final String AUTO_ADD_REPLICAS_TRIGGER_DSL =
- "{" +
- " 'set-trigger' : {" +
+ " {" +
" 'name' : '.auto_add_replicas'," +
" 'event' : 'nodeLost'," +
- " 'waitFor' : '{{waitFor}}s'," +
+ " 'waitFor' : -1," +
" 'enabled' : true," +
" 'actions' : [" +
" {" +
@@ -160,6 +160,7 @@ public class AutoScaling {
" 'class':'solr.ExecutePlanAction'" +
" }" +
" ]" +
- " }" +
- "}";
+ " }";
+
+ public static final Map<String, Object> AUTO_ADD_REPLICAS_TRIGGER_PROPS = (Map) Utils.fromJSONString(AUTO_ADD_REPLICAS_TRIGGER_DSL);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd08ca09/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index a5d1bc7..38a2e1e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -542,7 +542,7 @@ public class AutoScalingHandler extends RequestHandlerBase implements Permission
private static String fullName = SystemLogListener.class.getName();
private static String solrName = "solr." + SystemLogListener.class.getSimpleName();
- private static AutoScalingConfig withSystemLogListener(AutoScalingConfig autoScalingConfig, String triggerName) {
+ static AutoScalingConfig withSystemLogListener(AutoScalingConfig autoScalingConfig, String triggerName) {
Map<String, AutoScalingConfig.TriggerListenerConfig> configs = autoScalingConfig.getTriggerListenerConfigs();
for (AutoScalingConfig.TriggerListenerConfig cfg : configs.values()) {
if (triggerName.equals(cfg.trigger)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd08ca09/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
index 6b7aa91..fcc5b68 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/OverseerTriggerThread.java
@@ -37,13 +37,16 @@ import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
import org.apache.solr.common.util.IOUtils;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CloudConfig;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
-import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
+
/**
* Overseer thread responsible for reading triggers from zookeeper and
* adding/removing them from {@link ScheduledTriggers}
@@ -104,6 +107,27 @@ public class OverseerTriggerThread implements Runnable, Closeable {
public void run() {
int lastZnodeVersion = znodeVersion;
+ // we automatically add a trigger for auto add replicas if it does not exists already
+ while (true) {
+ try {
+ AutoScalingConfig autoScalingConfig = zkStateReader.getAutoScalingConfig();
+ AutoScalingConfig withAutoAddReplicasTrigger = withAutoAddReplicasTrigger(autoScalingConfig);
+ if (withAutoAddReplicasTrigger.equals(autoScalingConfig)) break;
+ log.debug("Adding .autoAddReplicas trigger");
+ zkClient.setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(withAutoAddReplicasTrigger), withAutoAddReplicasTrigger.getZkVersion(), true);
+ break;
+ } catch (KeeperException.BadVersionException bve) {
+ // somebody else has changed the configuration so we must retry
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ log.warn("Interrupted", e);
+ } catch (KeeperException e) {
+ log.error("A ZK error has occurred", e);
+ throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
+ }
+ }
+
try {
refreshAutoScalingConf(new AutoScalingWatcher());
} catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
@@ -302,6 +326,26 @@ public class OverseerTriggerThread implements Runnable, Closeable {
}
}
+ private AutoScalingConfig withAutoAddReplicasTrigger(AutoScalingConfig autoScalingConfig) {
+ CloudConfig cloudConfig = zkController.getCoreContainer().getConfig().getCloudConfig();
+ Map<String, Object> triggerProps = AutoScaling.AUTO_ADD_REPLICAS_TRIGGER_PROPS;
+ String triggerName = (String) triggerProps.get("name");
+ Map<String, AutoScalingConfig.TriggerConfig> configs = autoScalingConfig.getTriggerConfigs();
+ for (AutoScalingConfig.TriggerConfig cfg : configs.values()) {
+ if (triggerName.equals(cfg.name)) {
+ // already has this trigger
+ return autoScalingConfig;
+ }
+ }
+ // need to add
+ triggerProps.computeIfPresent("waitFor", (k, v) -> (long) (cloudConfig.getAutoReplicaFailoverWaitAfterExpiration() / 1000));
+ AutoScalingConfig.TriggerConfig config = new AutoScalingConfig.TriggerConfig(triggerName, triggerProps);
+ autoScalingConfig = autoScalingConfig.withTriggerConfig(config);
+ // need to add SystemLogListener explicitly here
+ autoScalingConfig = AutoScalingHandler.withSystemLogListener(autoScalingConfig, triggerName);
+ return autoScalingConfig;
+ }
+
private static Map<String, AutoScaling.Trigger> loadTriggers(AutoScaling.TriggerFactory triggerFactory, AutoScalingConfig autoScalingConfig) {
Map<String, AutoScalingConfig.TriggerConfig> triggers = autoScalingConfig.getTriggerConfigs();
if (triggers == null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bd08ca09/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
index 4886c59..7b1a0c3 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
@@ -36,12 +36,14 @@ import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.util.LogLevel;
import org.apache.solr.util.TimeOut;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.apache.solr.common.util.Utils.makeMap;
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
public class AutoAddReplicasIntegrationTest extends SolrCloudTestCase {
private static final String COLLECTION1 = "testSimple1";
private static final String COLLECTION2 = "testSimple2";
[08/24] lucene-solr:jira/solr-11285: SOLR-11278: Fix race-condition
in CDCR submitting a bootstrap call and checking it's status
Posted by ab...@apache.org.
SOLR-11278: Fix race-condition in CDCR submitting a bootstrap call and checking it's status
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8e12f201
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8e12f201
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8e12f201
Branch: refs/heads/jira/solr-11285
Commit: 8e12f201131d44034efe0cd97dee23df4b16b015
Parents: cec5b41
Author: Varun Thacker <va...@apache.org>
Authored: Tue Sep 19 09:30:07 2017 -0700
Committer: Varun Thacker <va...@apache.org>
Committed: Tue Sep 19 09:30:07 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/solr/handler/CdcrReplicatorManager.java | 8 ++++++--
.../src/test/org/apache/solr/cloud/CdcrBootstrapTest.java | 1 -
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e12f201/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java
index 677f5d3..6ef5ebd 100644
--- a/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java
+++ b/solr/core/src/java/org/apache/solr/handler/CdcrReplicatorManager.java
@@ -312,7 +312,9 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver {
timeOut = new TimeOut(BOOTSTRAP_TIMEOUT_SECONDS, TimeUnit.SECONDS); // reset the timer
retries++;
}
- } else if (status == BootstrapStatus.NOTFOUND) {
+ } else if (status == BootstrapStatus.NOTFOUND || status == BootstrapStatus.CANCELLED) {
+ log.info("CDCR bootstrap " + (status == BootstrapStatus.NOTFOUND ? "not found" : "cancelled") + "in {} seconds",
+ BOOTSTRAP_TIMEOUT_SECONDS - timeOut.timeLeft(TimeUnit.SECONDS));
// the leader of the target shard may have changed and therefore there is no record of the
// bootstrap process so we must retry the operation
while (!closed && sendBootstrapCommand() != BootstrapStatus.SUBMITTED) {
@@ -320,7 +322,9 @@ class CdcrReplicatorManager implements CdcrStateManager.CdcrStateObserver {
}
retries = 1;
timeOut = new TimeOut(6L * 3600L * 3600L, TimeUnit.SECONDS); // reset the timer
- } else if (status == BootstrapStatus.UNKNOWN) {
+ } else if (status == BootstrapStatus.UNKNOWN || status == BootstrapStatus.SUBMITTED) {
+ log.info("CDCR bootstrap is " + (status == BootstrapStatus.UNKNOWN ? "unknown" : "submitted"),
+ BOOTSTRAP_TIMEOUT_SECONDS - timeOut.timeLeft(TimeUnit.SECONDS));
// we were not able to query the status on the remote end
// so just sleep for a bit and try again
Thread.sleep(BOOTSTRAP_RETRY_DELAY_MS);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e12f201/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java b/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java
index a5b37d8..6959bd8 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CdcrBootstrapTest.java
@@ -239,7 +239,6 @@ public class CdcrBootstrapTest extends SolrTestCaseJ4 {
}
}
- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-11278")
public void testBootstrapWithContinousIndexingOnSourceCluster() throws Exception {
// start the target first so that we know its zkhost
MiniSolrCloudCluster target = new MiniSolrCloudCluster(1, createTempDir("cdcr-target"), buildJettyConfig("/solr"));
[07/24] lucene-solr:jira/solr-11285: LUCENE-7957:
ConjunctionScorer.getChildren was failing to return all child scorers
Posted by ab...@apache.org.
LUCENE-7957: ConjunctionScorer.getChildren was failing to return all child scorers
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cec5b418
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cec5b418
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cec5b418
Branch: refs/heads/jira/solr-11285
Commit: cec5b418ea0b68098cf136f26c07505cb2d4acf9
Parents: bc95209
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Sep 19 10:46:51 2017 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Sep 19 10:46:51 2017 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../apache/lucene/search/ConjunctionScorer.java | 4 +-
.../apache/lucene/search/TestConjunctions.java | 56 +++++++++++++++++++-
3 files changed, 60 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cec5b418/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 5a95d6c..84880ab 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -65,6 +65,9 @@ Bug Fixes
it did not properly break ties on the surface forms when both the weights and
the analyzed forms were equal. (Robert Muir)
+* LUCENE-7957: ConjunctionScorer.getChildren was failing to return all
+ child scorers (Adrien Grand, Mike McCandless)
+
Build
* SOLR-11181: Switch order of maven artifact publishing procedure: deploy first
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cec5b418/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java b/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
index 9cddab8..6ad4f40 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
@@ -26,6 +26,7 @@ class ConjunctionScorer extends Scorer {
final DocIdSetIterator disi;
final Scorer[] scorers;
+ final Collection<Scorer> required;
/** Create a new {@link ConjunctionScorer}, note that {@code scorers} must be a subset of {@code required}. */
ConjunctionScorer(Weight weight, Collection<Scorer> required, Collection<Scorer> scorers) {
@@ -33,6 +34,7 @@ class ConjunctionScorer extends Scorer {
assert required.containsAll(scorers);
this.disi = ConjunctionDISI.intersectScorers(required);
this.scorers = scorers.toArray(new Scorer[scorers.size()]);
+ this.required = required;
}
@Override
@@ -67,7 +69,7 @@ class ConjunctionScorer extends Scorer {
@Override
public Collection<ChildScorer> getChildren() {
ArrayList<ChildScorer> children = new ArrayList<>();
- for (Scorer scorer : scorers) {
+ for (Scorer scorer : required) {
children.add(new ChildScorer(scorer, "MUST"));
}
return children;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cec5b418/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
index 8b26121..de18dda 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestConjunctions.java
@@ -18,22 +18,29 @@ package org.apache.lucene.search;
import java.io.IOException;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
public class TestConjunctions extends LuceneTestCase {
@@ -119,4 +126,49 @@ public class TestConjunctions extends LuceneTestCase {
};
}
}
+
+ public void testScorerGetChildren() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+ Document doc = new Document();
+ doc.add(newTextField("field", "a b", Field.Store.NO));
+ w.addDocument(doc);
+ IndexReader r = DirectoryReader.open(w);
+ BooleanQuery.Builder b = new BooleanQuery.Builder();
+ b.add(new TermQuery(new Term("field", "a")), BooleanClause.Occur.MUST);
+ b.add(new TermQuery(new Term("field", "b")), BooleanClause.Occur.FILTER);
+ Query q = b.build();
+ IndexSearcher s = new IndexSearcher(r);
+ final boolean[] setScorerCalled = new boolean[1];
+ s.search(q, new SimpleCollector() {
+ @Override
+ public void setScorer(Scorer s) throws IOException {
+ Collection<Scorer.ChildScorer> childScorers = s.getChildren();
+ setScorerCalled[0] = true;
+ assertEquals(2, childScorers.size());
+ Set<String> terms = new HashSet<>();
+ for (Scorer.ChildScorer childScorer : childScorers) {
+ Query query = childScorer.child.getWeight().getQuery();
+ assertTrue(query instanceof TermQuery);
+ Term term = ((TermQuery) query).getTerm();
+ assertEquals("field", term.field());
+ terms.add(term.text());
+ }
+ assertEquals(2, terms.size());
+ assertTrue(terms.contains("a"));
+ assertTrue(terms.contains("b"));
+ }
+
+ @Override
+ public void collect(int doc) {
+ }
+
+ @Override
+ public boolean needsScores() {
+ return true;
+ }
+ });
+ assertTrue(setScorerCalled[0]);
+ IOUtils.close(r, w, dir);
+ }
}
[15/24] lucene-solr:jira/solr-11285: Merge branch 'master' into
feature/autoscaling
Posted by ab...@apache.org.
Merge branch 'master' into feature/autoscaling
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/1b3cd346
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/1b3cd346
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/1b3cd346
Branch: refs/heads/jira/solr-11285
Commit: 1b3cd346603070f9ee59e9ada98d74a084467481
Parents: ab994e7 4e55977
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Sep 19 11:57:46 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Sep 19 11:57:46 2017 -0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +
.../lucene/analysis/synonym/SynonymMap.java | 3 +-
.../apache/lucene/search/ConjunctionScorer.java | 4 +-
.../apache/lucene/search/TestConjunctions.java | 56 +++++++-
solr/CHANGES.txt | 8 ++
.../solr/handler/CdcrReplicatorManager.java | 8 +-
.../org/apache/solr/response/DocsStreamer.java | 87 +++----------
.../solr/response/RetrieveFieldsOptimizer.java | 130 +++++++++++++++++++
.../apache/solr/search/SolrDocumentFetcher.java | 25 +++-
.../facet/FacetFieldProcessorByHashDV.java | 11 +-
.../org/apache/solr/search/facet/HLLAgg.java | 1 +
.../org/apache/solr/search/facet/UniqueAgg.java | 1 +
.../solr/search/join/GraphPointsCollector.java | 1 +
.../apache/solr/cloud/CdcrBootstrapTest.java | 1 -
.../cloud/TestCloudJSONFacetJoinDomain.java | 24 ++--
.../response/TestRetrieveFieldsOptimizer.java | 56 ++++++++
.../solr/search/facet/TestJsonFacets.java | 26 ++++
.../conf/solrconfig.xml | 1 -
18 files changed, 357 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/1b3cd346/solr/CHANGES.txt
----------------------------------------------------------------------
[03/24] lucene-solr:jira/solr-11285: Revert "SOLR-11072: Implement
trigger for searchRate event type."
Posted by ab...@apache.org.
Revert "SOLR-11072: Implement trigger for searchRate event type."
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cda1d2ee
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cda1d2ee
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cda1d2ee
Branch: refs/heads/jira/solr-11285
Commit: cda1d2ee1a326240aa4f9ae9636cf3f927a82f1a
Parents: 341a730
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Sep 18 12:21:44 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Sep 18 12:21:44 2017 -0700
----------------------------------------------------------------------
.../cloud/autoscaling/SearchRateTrigger.java | 277 -------------------
.../autoscaling/SearchRateTriggerTest.java | 167 -----------
2 files changed, 444 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cda1d2ee/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
deleted file mode 100644
index 6692fda..0000000
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/SearchRateTrigger.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import com.google.common.util.concurrent.AtomicDouble;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
-import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.metrics.SolrCoreMetricManager;
-import org.apache.solr.util.TimeSource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Trigger for the {@link org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType#SEARCHRATE} event.
- */
-public class SearchRateTrigger extends TriggerBase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private final TimeSource timeSource;
- private final CoreContainer container;
- private final String handler;
- private final String collection;
- private final String shard;
- private final String node;
- private final double rate;
- private final Map<String, Long> lastCollectionEvent = new ConcurrentHashMap<>();
- private final Map<String, Long> lastNodeEvent = new ConcurrentHashMap<>();
- private final Map<String, Long> lastShardEvent = new ConcurrentHashMap<>();
- private final Map<String, Long> lastReplicaEvent = new ConcurrentHashMap<>();
- private final Map<String, Object> state = new HashMap<>();
-
- public SearchRateTrigger(String name, Map<String, Object> properties, CoreContainer container) {
- super(TriggerEventType.SEARCHRATE, name, properties, container.getResourceLoader(), container.getZkController().getZkClient());
- this.timeSource = TimeSource.CURRENT_TIME;
- this.container = container;
- this.state.put("lastCollectionEvent", lastCollectionEvent);
- this.state.put("lastNodeEvent", lastNodeEvent);
- this.state.put("lastShardEvent", lastShardEvent);
- this.state.put("lastReplicaEvent", lastReplicaEvent);
-
- // parse config options
- collection = (String)properties.getOrDefault(AutoScalingParams.COLLECTION, Policy.ANY);
- shard = (String)properties.getOrDefault(AutoScalingParams.SHARD, Policy.ANY);
- if (collection.equals(Policy.ANY) && !shard.equals(Policy.ANY)) {
- throw new IllegalArgumentException("When 'shard' is other than #ANY collection name must be also other than #ANY");
- }
- node = (String)properties.getOrDefault(AutoScalingParams.NODE, Policy.ANY);
- handler = (String)properties.getOrDefault(AutoScalingParams.HANDLER, "/select");
-
- if (properties.get("rate") == null) {
- throw new IllegalArgumentException("No 'rate' specified in configuration");
- }
- String rateString = String.valueOf(properties.get("rate"));
- try {
- rate = Double.parseDouble(rateString);
- } catch (Exception e) {
- throw new IllegalArgumentException("Invalid 'rate' configuration value: '" + rateString + "'", e);
- }
- }
-
- @Override
- protected Map<String, Object> getState() {
- return state;
- }
-
- @Override
- protected void setState(Map<String, Object> state) {
- lastCollectionEvent.clear();
- lastNodeEvent.clear();
- lastShardEvent.clear();
- lastReplicaEvent.clear();
- Map<String, Long> collTimes = (Map<String, Long>)state.get("lastCollectionEvent");
- if (collTimes != null) {
- lastCollectionEvent.putAll(collTimes);
- }
- Map<String, Long> nodeTimes = (Map<String, Long>)state.get("lastNodeEvent");
- if (nodeTimes != null) {
- lastNodeEvent.putAll(nodeTimes);
- }
- Map<String, Long> shardTimes = (Map<String, Long>)state.get("lastShardEvent");
- if (shardTimes != null) {
- lastShardEvent.putAll(shardTimes);
- }
- Map<String, Long> replicaTimes = (Map<String, Long>)state.get("lastReplicaEvent");
- if (replicaTimes != null) {
- lastReplicaEvent.putAll(replicaTimes);
- }
- }
-
- @Override
- public void restoreState(AutoScaling.Trigger old) {
- assert old.isClosed();
- if (old instanceof SearchRateTrigger) {
- SearchRateTrigger that = (SearchRateTrigger)old;
- assert this.name.equals(that.name);
- this.lastCollectionEvent.clear();
- this.lastNodeEvent.clear();
- this.lastShardEvent.clear();
- this.lastReplicaEvent.clear();
- this.lastCollectionEvent.putAll(that.lastCollectionEvent);
- this.lastNodeEvent.putAll(that.lastNodeEvent);
- this.lastShardEvent.putAll(that.lastShardEvent);
- this.lastReplicaEvent.putAll(that.lastReplicaEvent);
- } else {
- throw new SolrException(SolrException.ErrorCode.INVALID_STATE,
- "Unable to restore state from an unknown type of trigger");
- }
-
- }
-
- @Override
- public void run() {
- AutoScaling.TriggerEventProcessor processor = processorRef.get();
- if (processor == null) {
- return;
- }
-
- Map<String, Map<String, List<ReplicaInfo>>> collectionRates = new HashMap<>();
- Map<String, AtomicDouble> nodeRates = new HashMap<>();
-
- try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder()
- .withClusterStateProvider(new ZkClientClusterStateProvider(container.getZkController().getZkStateReader()))
- .build()) {
-
- SolrClientDataProvider dataProvider = new SolrClientDataProvider(cloudSolrClient);
-
- for (String node : dataProvider.getNodes()) {
- Map<String, ReplicaInfo> metricTags = new HashMap<>();
- // coll, shard, replica
- Map<String, Map<String, List<ReplicaInfo>>> infos = dataProvider.getReplicaInfo(node, Collections.emptyList());
- infos.forEach((coll, shards) -> {
- shards.forEach((sh, replicas) -> {
- replicas.forEach(replica -> {
- // we have to translate to the metrics registry name, which uses "_replica_nN" as suffix
- String replicaName = SolrCoreMetricManager.parseReplicaName(coll, replica.getCore());
- if (replicaName == null) { // should never happen???
- replicaName = replica.getName(); // which is actually coreNode name...
- }
- String registry = SolrCoreMetricManager.createRegistryName(true, coll, sh, replicaName, null);
- String tag = "metrics:" + registry
- + ":QUERY." + handler + ".requestTimes:1minRate";
- metricTags.put(tag, replica);
- });
- });
- });
- Map<String, Object> rates = dataProvider.getNodeValues(node, metricTags.keySet());
- rates.forEach((tag, rate) -> {
- ReplicaInfo info = metricTags.get(tag);
- if (info == null) {
- log.warn("Missing replica info for response tag " + tag);
- } else {
- Map<String, List<ReplicaInfo>> perCollection = collectionRates.computeIfAbsent(info.getCollection(), s -> new HashMap<>());
- List<ReplicaInfo> perShard = perCollection.computeIfAbsent(info.getShard(), s -> new ArrayList<>());
- info.getVariables().put(AutoScalingParams.RATE, rate);
- perShard.add(info);
- AtomicDouble perNode = nodeRates.computeIfAbsent(node, s -> new AtomicDouble());
- perNode.addAndGet((Double)rate);
- }
- });
- }
- } catch (IOException e) {
- log.warn("Exception getting node values", e);
- return;
- }
-
- long now = timeSource.getTime();
- // check for exceeded rates and filter out those with less than waitFor from previous events
- Map<String, Double> hotNodes = nodeRates.entrySet().stream()
- .filter(entry -> node.equals(Policy.ANY) || node.equals(entry.getKey()))
- .filter(entry -> waitForElapsed(entry.getKey(), now, lastNodeEvent))
- .filter(entry -> entry.getValue().get() > rate)
- .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().get()));
-
- Map<String, Map<String, Double>> hotShards = new HashMap<>();
- List<ReplicaInfo> hotReplicas = new ArrayList<>();
- collectionRates.forEach((coll, shardRates) -> {
- shardRates.forEach((sh, replicaRates) -> {
- double shardRate = replicaRates.stream()
- .map(r -> {
- if (waitForElapsed(r.getCollection() + "." + r.getCore(), now, lastReplicaEvent) &&
- ((Double)r.getVariable(AutoScalingParams.RATE) > rate)) {
- hotReplicas.add(r);
- }
- return r;
- })
- .mapToDouble(r -> (Double)r.getVariable(AutoScalingParams.RATE)).sum();
- if (waitForElapsed(coll + "." + sh, now, lastShardEvent) &&
- (shardRate > rate) &&
- (collection.equals(Policy.ANY) || collection.equals(coll)) &&
- (shard.equals(Policy.ANY) || shard.equals(sh))) {
- hotShards.computeIfAbsent(coll, s -> new HashMap<>()).put(sh, shardRate);
- }
- });
- });
-
- Map<String, Double> hotCollections = new HashMap<>();
- collectionRates.forEach((coll, shardRates) -> {
- double total = shardRates.entrySet().stream()
- .mapToDouble(e -> e.getValue().stream()
- .mapToDouble(r -> (Double)r.getVariable(AutoScalingParams.RATE)).sum()).sum();
- if (waitForElapsed(coll, now, lastCollectionEvent) &&
- (total > rate) &&
- (collection.equals(Policy.ANY) || collection.equals(coll))) {
- hotCollections.put(coll, total);
- }
- });
-
- if (hotCollections.isEmpty() && hotShards.isEmpty() && hotReplicas.isEmpty() && hotNodes.isEmpty()) {
- return;
- }
-
- // generate event
-
- if (processor.process(new SearchRateEvent(getName(), now, hotNodes, hotCollections, hotShards, hotReplicas))) {
- // update lastEvent times
- hotNodes.keySet().forEach(node -> lastNodeEvent.put(node, now));
- hotCollections.keySet().forEach(coll -> lastCollectionEvent.put(coll, now));
- hotShards.entrySet().forEach(e -> e.getValue()
- .forEach((sh, rate) -> lastShardEvent.put(e.getKey() + "." + sh, now)));
- hotReplicas.forEach(r -> lastReplicaEvent.put(r.getCollection() + "." + r.getCore(), now));
- }
- }
-
- private boolean waitForElapsed(String name, long now, Map<String, Long> lastEventMap) {
- Long lastTime = lastEventMap.computeIfAbsent(name, s -> now);
- long elapsed = TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS);
- log.info("name=" + name + ", lastTime=" + lastTime + ", elapsed=" + elapsed);
- if (TimeUnit.SECONDS.convert(now - lastTime, TimeUnit.NANOSECONDS) < getWaitForSecond()) {
- return false;
- }
- return true;
- }
-
- public static class SearchRateEvent extends TriggerEvent {
- public SearchRateEvent(String source, long eventTime, Map<String, Double> hotNodes,
- Map<String, Double> hotCollections,
- Map<String, Map<String, Double>> hotShards, List<ReplicaInfo> hotReplicas) {
- super(TriggerEventType.SEARCHRATE, source, eventTime, null);
- properties.put(AutoScalingParams.COLLECTION, hotCollections);
- properties.put(AutoScalingParams.SHARD, hotShards);
- properties.put(AutoScalingParams.REPLICA, hotReplicas);
- properties.put(AutoScalingParams.NODE, hotNodes);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cda1d2ee/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
deleted file mode 100644
index 0719c22..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-package org.apache.solr.cloud.autoscaling;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.core.CoreContainer;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- *
- */
-public class SearchRateTriggerTest extends SolrCloudTestCase {
-
- private static final String PREFIX = SearchRateTriggerTest.class.getSimpleName() + "-";
- private static final String COLL1 = PREFIX + "collection1";
- private static final String COLL2 = PREFIX + "collection2";
-
- private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> {
- fail("Did not expect the listener to fire on first run!");
- return true;
- };
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(4)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
- "conf", 2, 2);
- CloudSolrClient solrClient = cluster.getSolrClient();
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
- create = CollectionAdminRequest.createCollection(COLL2,
- "conf", 2, 2);
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
- }
-
- @Test
- public void testTrigger() throws Exception {
- double rate = 1.0;
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- URL baseUrl = cluster.getJettySolrRunners().get(1).getBaseUrl();
- long waitForSeconds = 5 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds, rate);
- final List<TriggerEvent> events = new ArrayList<>();
- CloudSolrClient solrClient = cluster.getSolrClient();
-
- try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger", props, container)) {
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
- trigger.setProcessor(event -> events.add(event));
-
- // generate replica traffic
- String coreName = container.getLoadedCoreNames().iterator().next();
- String url = baseUrl.toString() + "/" + coreName;
- try (HttpSolrClient simpleClient = new HttpSolrClient.Builder(url).build()) {
- SolrParams query = params(CommonParams.Q, "*:*", CommonParams.DISTRIB, "false");
- for (int i = 0; i < 200; i++) {
- simpleClient.query(query);
- }
- trigger.run();
- // waitFor delay
- assertEquals(0, events.size());
- Thread.sleep(waitForSeconds * 1000 + 2000);
- // should generate replica event
- trigger.run();
- assertEquals(1, events.size());
- TriggerEvent event = events.get(0);
- assertEquals(TriggerEventType.SEARCHRATE, event.eventType);
- List<ReplicaInfo> infos = (List<ReplicaInfo>)event.getProperty(AutoScalingParams.REPLICA);
- assertEquals(1, infos.size());
- ReplicaInfo info = infos.get(0);
- assertEquals(coreName, info.getCore());
- assertTrue((Double)info.getVariable(AutoScalingParams.RATE) > rate);
- }
- // close that jetty to remove the violation - alternatively wait for 1 min...
- cluster.stopJettySolrRunner(1);
- events.clear();
- SolrParams query = params(CommonParams.Q, "*:*");
- for (int i = 0; i < 500; i++) {
- solrClient.query(COLL1, query);
- }
- Thread.sleep(waitForSeconds * 1000 + 2000);
- trigger.run();
- // should generate collection event
- assertEquals(1, events.size());
- TriggerEvent event = events.get(0);
- Map<String, Double> hotCollections = (Map<String, Double>)event.getProperty(AutoScalingParams.COLLECTION);
- assertEquals(1, hotCollections.size());
- Double Rate = hotCollections.get(COLL1);
- assertNotNull(Rate);
- assertTrue(Rate > rate);
- events.clear();
-
- for (int i = 0; i < 1000; i++) {
- solrClient.query(COLL2, query);
- }
- Thread.sleep(waitForSeconds * 1000 + 2000);
- trigger.run();
- // should generate node and collection event but not for COLL2 because of waitFor
- assertEquals(1, events.size());
- event = events.get(0);
- Map<String, Double> hotNodes = (Map<String, Double>)event.getProperty(AutoScalingParams.NODE);
- assertEquals(3, hotNodes.size());
- hotNodes.forEach((n, r) -> assertTrue(n, r > rate));
- hotCollections = (Map<String, Double>)event.getProperty(AutoScalingParams.COLLECTION);
- assertEquals(2, hotCollections.size());
- Rate = hotCollections.get(COLL1);
- assertNotNull(Rate);
- Rate = hotCollections.get(COLL2);
- assertNotNull(Rate);
-
- events.clear();
- // assert that waitFor prevents new events from being generated
- trigger.run();
- // should not generate any events
- assertEquals(0, events.size());
-
- Thread.sleep(waitForSeconds * 1000 + 2000);
- trigger.run();
- // should generate node and collection event
- assertEquals(1, events.size());
- hotCollections = (Map<String, Double>)event.getProperty(AutoScalingParams.COLLECTION);
- assertEquals(2, hotCollections.size());
- Rate = hotCollections.get(COLL1);
- assertNotNull(Rate);
- Rate = hotCollections.get(COLL2);
- assertNotNull(Rate);
- hotNodes = (Map<String, Double>)event.getProperty(AutoScalingParams.NODE);
- assertEquals(3, hotNodes.size());
- hotNodes.forEach((n, r) -> assertTrue(n, r > rate));
- }
- }
-
- private Map<String, Object> createTriggerProps(long waitForSeconds, double rate) {
- Map<String, Object> props = new HashMap<>();
- props.put("rate", rate);
- props.put("event", "searchRate");
- props.put("waitFor", waitForSeconds);
- props.put("enabled", true);
- List<Map<String, String>> actions = new ArrayList<>(3);
- Map<String, String> map = new HashMap<>(2);
- map.put("name", "compute_plan");
- map.put("class", "solr.ComputePlanAction");
- actions.add(map);
- map = new HashMap<>(2);
- map.put("name", "execute_plan");
- map.put("class", "solr.ExecutePlanAction");
- actions.add(map);
- props.put("actions", actions);
- return props;
- }
-}
[02/24] lucene-solr:jira/solr-11285: Revert "SOLR-11072: Implement
trigger for searchRate event type."
Posted by ab...@apache.org.
Revert "SOLR-11072: Implement trigger for searchRate event type."
This reverts commit 8ef83bf
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/341a730a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/341a730a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/341a730a
Branch: refs/heads/jira/solr-11285
Commit: 341a730ad5b988aa0a75d911bd5e0a364d2e9f99
Parents: bc25e84
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Mon Sep 18 09:26:49 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Mon Sep 18 09:26:49 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +-
.../cloud/autoscaling/ComputePlanAction.java | 34 -----
.../cloud/autoscaling/NodeAddedTrigger.java | 111 +++++++++++++++-
.../solr/cloud/autoscaling/NodeLostTrigger.java | 101 ++++++++++++++-
.../solr/cloud/autoscaling/TriggerBase.java | 127 +------------------
.../solr/handler/admin/MetricsHandler.java | 11 +-
.../solr/handler/admin/MetricsHandlerTest.java | 9 --
.../cloud/autoscaling/ClusterDataProvider.java | 4 -
.../solrj/cloud/autoscaling/NoneSuggester.java | 4 +-
.../client/solrj/cloud/autoscaling/Policy.java | 2 +-
.../solrj/cloud/autoscaling/PolicyHelper.java | 6 -
.../solrj/cloud/autoscaling/ReplicaInfo.java | 24 ----
.../solrj/impl/SolrClientDataProvider.java | 24 ++--
.../solr/common/params/AutoScalingParams.java | 6 -
.../solrj/cloud/autoscaling/TestPolicy.java | 38 ------
15 files changed, 228 insertions(+), 277 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a1a23e6..16e0e02 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -69,7 +69,7 @@ Upgrade Notes
* SOLR-11254: the abstract DocTransformer class now has an abstract score-less transform method variant.
* SOLR-11283: all Stream Evaluators in solrj.io.eval have been refactored to have a simplier and more
- robust structure. This simplifies and condenses the code required to implement a new Evaluator and
+ robust structure. This simplifies and condenses the code required to implement a new Evaluator and
makes it much easier for evaluators to handle differing data types (primitives, objects, arrays,
lists, and so forth). (Dennis Gove)
@@ -98,8 +98,6 @@ New Features
* SOLR-11244: Query DSL for Solr (Cao Manh Dat)
-* SOLR-11072: Implement trigger for searchRate event type. (ab)
-
* SOLR-11317: JSON Facet API: min/max aggregations on numeric fields are now typed better so int/long
fields return an appropriate integral type rather than a double. (yonik)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 44b3583..93441fe 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -20,20 +20,15 @@ package org.apache.solr.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.NoneSuggester;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.SolrClientDataProvider;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.AutoScalingParams;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.KeeperException;
@@ -107,35 +102,6 @@ public class ComputePlanAction extends TriggerActionBase {
.hint(Policy.Suggester.Hint.SRC_NODE, event.getProperty(TriggerEvent.NODE_NAMES));
log.debug("Created suggester with srcNode: {}", event.getProperty(TriggerEvent.NODE_NAMES));
break;
- case SEARCHRATE:
- Map<String, Map<String, Double>> hotShards = (Map<String, Map<String, Double>>)event.getProperty(AutoScalingParams.SHARD);
- Map<String, Double> hotCollections = (Map<String, Double>)event.getProperty(AutoScalingParams.COLLECTION);
- List<ReplicaInfo> hotReplicas = (List<ReplicaInfo>)event.getProperty(AutoScalingParams.REPLICA);
- Map<String, Double> hotNodes = (Map<String, Double>)event.getProperty(AutoScalingParams.NODE);
-
- if (hotShards.isEmpty() && hotCollections.isEmpty() && hotReplicas.isEmpty()) {
- // node -> MOVEREPLICA
- if (hotNodes.isEmpty()) {
- log.warn("Neither hot replicas / collection nor nodes are reported in event: " + event);
- return NoneSuggester.INSTANCE;
- }
- suggester = session.getSuggester(CollectionParams.CollectionAction.MOVEREPLICA);
- for (String node : hotNodes.keySet()) {
- suggester = suggester.hint(Policy.Suggester.Hint.SRC_NODE, node);
- }
- } else {
- // collection || shard || replica -> ADDREPLICA
- suggester = session.getSuggester(CollectionParams.CollectionAction.ADDREPLICA);
- Set<String> collections = new HashSet<>();
- // XXX improve this when AddReplicaSuggester supports coll_shard hint
- hotReplicas.forEach(r -> collections.add(r.getCollection()));
- hotShards.forEach((coll, shards) -> collections.add(coll));
- hotCollections.forEach((coll, rate) -> collections.add(coll));
- for (String coll : collections) {
- suggester = suggester.hint(Policy.Suggester.Hint.COLL, coll);
- }
- }
- break;
default:
throw new UnsupportedOperationException("No support for events other than nodeAdded and nodeLost, received: " + event.getEventType());
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index 5ae28c2..0c0278b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@ -17,17 +17,23 @@
package org.apache.solr.cloud.autoscaling;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -43,26 +49,57 @@ import org.slf4j.LoggerFactory;
public class NodeAddedTrigger extends TriggerBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final String name;
+ private final Map<String, Object> properties;
private final CoreContainer container;
+ private final List<TriggerAction> actions;
+ private final AtomicReference<AutoScaling.TriggerEventProcessor> processorRef;
+ private final boolean enabled;
+ private final int waitForSecond;
+ private final TriggerEventType eventType;
private final TimeSource timeSource;
+ private boolean isClosed = false;
+
private Set<String> lastLiveNodes;
private Map<String, Long> nodeNameVsTimeAdded = new HashMap<>();
public NodeAddedTrigger(String name, Map<String, Object> properties,
CoreContainer container) {
- super(TriggerEventType.NODEADDED, name, properties, container.getResourceLoader(), container.getZkController().getZkClient());
+ super(container.getZkController().getZkClient());
+ this.name = name;
+ this.properties = properties;
this.container = container;
this.timeSource = TimeSource.CURRENT_TIME;
+ this.processorRef = new AtomicReference<>();
+ List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
+ if (o != null && !o.isEmpty()) {
+ actions = new ArrayList<>(3);
+ for (Map<String, String> map : o) {
+ TriggerAction action = container.getResourceLoader().newInstance(map.get("class"), TriggerAction.class);
+ actions.add(action);
+ }
+ } else {
+ actions = Collections.emptyList();
+ }
lastLiveNodes = new HashSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
+ this.enabled = Boolean.parseBoolean(String.valueOf(properties.getOrDefault("enabled", "true")));
+ this.waitForSecond = ((Long) properties.getOrDefault("waitFor", -1L)).intValue();
+ this.eventType = TriggerEventType.valueOf(properties.get("event").toString().toUpperCase(Locale.ROOT));
log.debug("NodeAddedTrigger {} instantiated with properties: {}", name, properties);
}
@Override
public void init() {
- super.init();
+ List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
+ if (o != null && !o.isEmpty()) {
+ for (int i = 0; i < o.size(); i++) {
+ Map<String, String> map = o.get(i);
+ actions.get(i).init(map);
+ }
+ }
// pick up added nodes for which marker paths were created
try {
List<String> added = container.getZkController().getZkClient().getChildren(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, null, true);
@@ -83,6 +120,69 @@ public class NodeAddedTrigger extends TriggerBase {
}
@Override
+ public void setProcessor(AutoScaling.TriggerEventProcessor processor) {
+ processorRef.set(processor);
+ }
+
+ @Override
+ public AutoScaling.TriggerEventProcessor getProcessor() {
+ return processorRef.get();
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public TriggerEventType getEventType() {
+ return eventType;
+ }
+
+ @Override
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ @Override
+ public int getWaitForSecond() {
+ return waitForSecond;
+ }
+
+ @Override
+ public Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ @Override
+ public List<TriggerAction> getActions() {
+ return actions;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof NodeAddedTrigger) {
+ NodeAddedTrigger that = (NodeAddedTrigger) obj;
+ return this.name.equals(that.name)
+ && this.properties.equals(that.properties);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, properties);
+ }
+
+ @Override
+ public void close() throws IOException {
+ synchronized (this) {
+ isClosed = true;
+ IOUtils.closeWhileHandlingException(actions);
+ }
+ }
+
+ @Override
public void restoreState(AutoScaling.Trigger old) {
assert old.isClosed();
if (old instanceof NodeAddedTrigger) {
@@ -198,6 +298,13 @@ public class NodeAddedTrigger extends TriggerBase {
}
+ @Override
+ public boolean isClosed() {
+ synchronized (this) {
+ return isClosed;
+ }
+ }
+
public static class NodeAddedEvent extends TriggerEvent {
public NodeAddedEvent(TriggerEventType eventType, String source, List<Long> times, List<String> nodeNames) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 063f9d2..18dafcb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -17,17 +17,23 @@
package org.apache.solr.cloud.autoscaling;
+import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.lucene.util.IOUtils;
import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -43,7 +49,14 @@ import org.slf4j.LoggerFactory;
public class NodeLostTrigger extends TriggerBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ private final String name;
+ private final Map<String, Object> properties;
private final CoreContainer container;
+ private final List<TriggerAction> actions;
+ private final AtomicReference<AutoScaling.TriggerEventProcessor> processorRef;
+ private final boolean enabled;
+ private final int waitForSecond;
+ private final TriggerEventType eventType;
private final TimeSource timeSource;
private boolean isClosed = false;
@@ -54,11 +67,27 @@ public class NodeLostTrigger extends TriggerBase {
public NodeLostTrigger(String name, Map<String, Object> properties,
CoreContainer container) {
- super(TriggerEventType.NODELOST, name, properties, container.getResourceLoader(), container.getZkController().getZkClient());
+ super(container.getZkController().getZkClient());
+ this.name = name;
+ this.properties = properties;
this.container = container;
this.timeSource = TimeSource.CURRENT_TIME;
+ this.processorRef = new AtomicReference<>();
+ List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
+ if (o != null && !o.isEmpty()) {
+ actions = new ArrayList<>(3);
+ for (Map<String, String> map : o) {
+ TriggerAction action = container.getResourceLoader().newInstance(map.get("class"), TriggerAction.class);
+ actions.add(action);
+ }
+ } else {
+ actions = Collections.emptyList();
+ }
lastLiveNodes = new HashSet<>(container.getZkController().getZkStateReader().getClusterState().getLiveNodes());
log.debug("Initial livenodes: {}", lastLiveNodes);
+ this.enabled = Boolean.parseBoolean(String.valueOf(properties.getOrDefault("enabled", "true")));
+ this.waitForSecond = ((Long) properties.getOrDefault("waitFor", -1L)).intValue();
+ this.eventType = TriggerEventType.valueOf(properties.get("event").toString().toUpperCase(Locale.ROOT));
}
@Override
@@ -89,6 +118,69 @@ public class NodeLostTrigger extends TriggerBase {
}
@Override
+ public void setProcessor(AutoScaling.TriggerEventProcessor processor) {
+ processorRef.set(processor);
+ }
+
+ @Override
+ public AutoScaling.TriggerEventProcessor getProcessor() {
+ return processorRef.get();
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public TriggerEventType getEventType() {
+ return eventType;
+ }
+
+ @Override
+ public boolean isEnabled() {
+ return enabled;
+ }
+
+ @Override
+ public int getWaitForSecond() {
+ return waitForSecond;
+ }
+
+ @Override
+ public Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ @Override
+ public List<TriggerAction> getActions() {
+ return actions;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof NodeLostTrigger) {
+ NodeLostTrigger that = (NodeLostTrigger) obj;
+ return this.name.equals(that.name)
+ && this.properties.equals(that.properties);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, properties);
+ }
+
+ @Override
+ public void close() throws IOException {
+ synchronized (this) {
+ isClosed = true;
+ IOUtils.closeWhileHandlingException(actions);
+ }
+ }
+
+ @Override
public void restoreState(AutoScaling.Trigger old) {
assert old.isClosed();
if (old instanceof NodeLostTrigger) {
@@ -204,6 +296,13 @@ public class NodeLostTrigger extends TriggerBase {
}
}
+ @Override
+ public boolean isClosed() {
+ synchronized (this) {
+ return isClosed;
+ }
+ }
+
public static class NodeLostEvent extends TriggerEvent {
public NodeLostEvent(TriggerEventType eventType, String source, List<Long> times, List<String> nodeNames) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
index fec6b05..7aff846 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
@@ -16,25 +16,14 @@
*/
package org.apache.solr.cloud.autoscaling;
-import java.io.IOException;
import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
import java.util.Map;
-import java.util.Objects;
import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.lucene.util.IOUtils;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
@@ -48,129 +37,19 @@ import org.slf4j.LoggerFactory;
public abstract class TriggerBase implements AutoScaling.Trigger {
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- protected final String name;
- protected final Map<String, Object> properties = new HashMap<>();
- protected final TriggerEventType eventType;
- protected final int waitForSecond;
protected SolrZkClient zkClient;
protected Map<String,Object> lastState;
- protected final List<TriggerAction> actions;
- protected final AtomicReference<AutoScaling.TriggerEventProcessor> processorRef = new AtomicReference<>();
- protected final boolean enabled;
- protected boolean isClosed;
- protected TriggerBase(TriggerEventType eventType, String name, Map<String, Object> properties, SolrResourceLoader loader, SolrZkClient zkClient) {
- this.eventType = eventType;
- this.name = name;
+
+ protected TriggerBase(SolrZkClient zkClient) {
this.zkClient = zkClient;
- if (properties != null) {
- this.properties.putAll(properties);
- }
- this.enabled = Boolean.parseBoolean(String.valueOf(this.properties.getOrDefault("enabled", "true")));
- this.waitForSecond = ((Number) this.properties.getOrDefault("waitFor", -1L)).intValue();
- List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
- if (o != null && !o.isEmpty()) {
- actions = new ArrayList<>(3);
- for (Map<String, String> map : o) {
- TriggerAction action = loader.newInstance(map.get("class"), TriggerAction.class);
- actions.add(action);
- }
- } else {
- actions = Collections.emptyList();
- }
try {
- if (!zkClient.exists(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH, true)) {
- zkClient.makePath(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH, false, true);
- }
+ zkClient.makePath(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH, false, true);
} catch (KeeperException | InterruptedException e) {
LOG.warn("Exception checking ZK path " + ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH, e);
}
}
- @Override
- public void init() {
- List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
- if (o != null && !o.isEmpty()) {
- for (int i = 0; i < o.size(); i++) {
- Map<String, String> map = o.get(i);
- actions.get(i).init(map);
- }
- }
- }
-
- @Override
- public void setProcessor(AutoScaling.TriggerEventProcessor processor) {
- processorRef.set(processor);
- }
-
- @Override
- public AutoScaling.TriggerEventProcessor getProcessor() {
- return processorRef.get();
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public TriggerEventType getEventType() {
- return eventType;
- }
-
- @Override
- public boolean isEnabled() {
- return enabled;
- }
-
- @Override
- public int getWaitForSecond() {
- return waitForSecond;
- }
-
- @Override
- public Map<String, Object> getProperties() {
- return properties;
- }
-
- @Override
- public List<TriggerAction> getActions() {
- return actions;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (obj.getClass().equals(this.getClass())) {
- TriggerBase that = (TriggerBase) obj;
- return this.name.equals(that.name)
- && this.properties.equals(that.properties);
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(name, properties);
- }
-
- @Override
- public void close() throws IOException {
- synchronized (this) {
- isClosed = true;
- IOUtils.closeWhileHandlingException(actions);
- }
- }
-
- @Override
- public boolean isClosed() {
- synchronized (this) {
- return isClosed;
- }
- }
-
/**
* Prepare and return internal state of this trigger in a format suitable for persisting in ZK.
* @return map of internal state properties. Note: values must be supported by {@link Utils#toJSON(Object)}.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
index ed1e474..a9a2d1f 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java
@@ -22,7 +22,6 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -140,18 +139,14 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
continue;
}
MetricUtils.PropertyFilter propertyFilter = MetricUtils.PropertyFilter.ALL;
+ boolean simple = false;
if (propertyName != null) {
propertyFilter = (name) -> name.equals(propertyName);
+ simple = true;
// use escaped versions
key = parts[0] + ":" + parts[1];
}
- MetricUtils.convertMetric(key, m, propertyFilter, false, true, true, false, ":", (k, v) -> {
- if ((v instanceof Map) && propertyName != null) {
- ((Map)v).forEach((k1, v1) -> result.add(k + ":" + k1, v1));
- } else {
- result.add(k, v);
- }
- });
+ MetricUtils.convertMetric(key, m, propertyFilter, false, true, true, simple, ":", (k, v) -> result.add(k, v));
}
rsp.getValues().add("metrics", result);
if (errors.size() > 0) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
index 0fe5ad7..7eb8f12 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java
@@ -277,15 +277,6 @@ public class MetricsHandlerTest extends SolrTestCaseJ4 {
val = values.findRecursive("metrics", key3);
assertNotNull(val);
- String key4 = "solr.core.collection1:QUERY./select.requestTimes:1minRate";
- resp = new SolrQueryResponse();
- handler.handleRequestBody(req(CommonParams.QT, "/admin/metrics", CommonParams.WT, "json",
- MetricsHandler.KEY_PARAM, key4), resp);
- values = resp.getValues();
- val = values.findRecursive("metrics", key4);
- assertNotNull(val);
- assertTrue(val instanceof Number);
-
// test errors
// invalid keys
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
index c974219..58972af 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ClusterDataProvider.java
@@ -23,8 +23,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
-import org.apache.solr.common.cloud.ClusterState;
-
public interface ClusterDataProvider extends Closeable {
/**Get the value of each tag for a given node
*
@@ -44,8 +42,6 @@ public interface ClusterDataProvider extends Closeable {
Collection<String> getNodes();
- ClusterState getClusterState();
-
/**Get the collection-specific policy
*/
String getPolicyNameByCollection(String coll);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java
index 619f5e3..f686014 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/NoneSuggester.java
@@ -19,9 +19,7 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import org.apache.solr.client.solrj.SolrRequest;
-public class NoneSuggester extends Policy.Suggester {
- public static final NoneSuggester INSTANCE = new NoneSuggester();
-
+public class NoneSuggester extends Policy.Suggester{
@Override
SolrRequest init() {
return null;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
index 866f541..761f1b8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Policy.java
@@ -203,7 +203,7 @@ public class Policy implements MapWriter {
}
Session(ClusterDataProvider dataProvider) {
- this.nodes = new ArrayList<>(dataProvider.getClusterState().getLiveNodes());
+ this.nodes = new ArrayList<>(dataProvider.getNodes());
this.dataProvider = dataProvider;
for (String node : nodes) {
collections.addAll(dataProvider.getReplicaInfo(node, Collections.emptyList()).keySet());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index d7fd99b..bc2a5d7 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -30,7 +30,6 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.util.Pair;
@@ -68,11 +67,6 @@ public class PolicyHelper {
}
@Override
- public ClusterState getClusterState() {
- return delegate.getClusterState();
- }
-
- @Override
public String getPolicyNameByCollection(String coll) {
return policyMapping.get() != null && policyMapping.get().containsKey(coll) ?
optionalPolicyMapping.get(coll) :
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
index 42c1727..f6b9db1 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/ReplicaInfo.java
@@ -36,7 +36,6 @@ public class ReplicaInfo implements MapWriter {
this.collection = coll;
this.shard = shard;
this.type = type;
- this.core = (String)vals.get("core");
}
@Override
@@ -55,10 +54,6 @@ public class ReplicaInfo implements MapWriter {
return core;
}
- public String getName() {
- return name;
- }
-
public String getCollection() {
return collection;
}
@@ -66,23 +61,4 @@ public class ReplicaInfo implements MapWriter {
public String getShard() {
return shard;
}
-
- public Map<String, Object> getVariables() {
- return variables;
- }
-
- public Object getVariable(String name) {
- return variables != null ? variables.get(name) : null;
- }
-
- @Override
- public String toString() {
- return "ReplicaInfo{" +
- "name='" + name + '\'' +
- ", collection='" + collection + '\'' +
- ", shard='" + shard + '\'' +
- ", type=" + type +
- ", variables=" + variables +
- '}';
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
index 2b45ccf..d63f5f0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
@@ -64,14 +64,15 @@ public class SolrClientDataProvider implements ClusterDataProvider, MapWriter {
private final CloudSolrClient solrClient;
private final Map<String, Map<String, Map<String, List<ReplicaInfo>>>> data = new HashMap<>();
- private ClusterState clusterState;
+ private Set<String> liveNodes;
private Map<String, Object> snitchSession = new HashMap<>();
private Map<String, Map> nodeVsTags = new HashMap<>();
public SolrClientDataProvider(CloudSolrClient solrClient) {
this.solrClient = solrClient;
ZkStateReader zkStateReader = solrClient.getZkStateReader();
- clusterState = zkStateReader.getClusterState();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ this.liveNodes = clusterState.getLiveNodes();
Map<String, ClusterState.CollectionRef> all = clusterState.getCollectionStates();
all.forEach((collName, ref) -> {
DocCollection coll = ref.get();
@@ -80,22 +81,12 @@ public class SolrClientDataProvider implements ClusterDataProvider, MapWriter {
Map<String, Map<String, List<ReplicaInfo>>> nodeData = data.computeIfAbsent(replica.getNodeName(), k -> new HashMap<>());
Map<String, List<ReplicaInfo>> collData = nodeData.computeIfAbsent(collName, k -> new HashMap<>());
List<ReplicaInfo> replicas = collData.computeIfAbsent(shard, k -> new ArrayList<>());
- replicas.add(new ReplicaInfo(replica.getName(), collName, shard, replica.getType(), replica.getProperties()));
+ replicas.add(new ReplicaInfo(replica.getName(), collName, shard, replica.getType(), new HashMap<>()));
});
});
}
@Override
- public Collection<String> getNodes() {
- return clusterState.getLiveNodes();
- }
-
- @Override
- public ClusterState getClusterState() {
- return clusterState;
- }
-
- @Override
public String getPolicyNameByCollection(String coll) {
ClusterState.CollectionRef state = solrClient.getClusterStateProvider().getState(coll);
return state == null || state.get() == null ? null : (String) state.get().getProperties().get("policy");
@@ -116,8 +107,13 @@ public class SolrClientDataProvider implements ClusterDataProvider, MapWriter {
}
@Override
+ public Collection<String> getNodes() {
+ return liveNodes;
+ }
+
+ @Override
public void writeMap(EntryWriter ew) throws IOException {
- ew.put("clusterState", clusterState);
+ ew.put("liveNodes", liveNodes);
ew.put("replicaInfo", Utils.getDeepCopy(data, 5));
ew.put("nodeValues", nodeVsTags);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java b/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
index f0c3d77..74850ea 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/AutoScalingParams.java
@@ -37,12 +37,6 @@ public interface AutoScalingParams {
String BEFORE_ACTION = "beforeAction";
String AFTER_ACTION = "afterAction";
String TIMEOUT = "timeout";
- String COLLECTION = "collection";
- String SHARD = "shard";
- String REPLICA = "replica";
- String NODE = "node";
- String HANDLER = "handler";
- String RATE = "rate";
String REMOVE_LISTENERS = "removeListeners";
String ZK_VERSION = "zkVersion";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/341a730a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 8273d2c..96e2257 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -34,7 +34,6 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.cloud.autoscaling.Clause.Violation;
import org.apache.solr.client.solrj.cloud.autoscaling.Policy.Suggester.Hint;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -440,11 +439,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public ClusterState getClusterState() {
- throw new UnsupportedOperationException("getClusterState");
- }
-
- @Override
public String getPolicyNameByCollection(String coll) {
return null;
}
@@ -975,11 +969,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public ClusterState getClusterState() {
- return clusterDataProvider.getClusterState();
- }
-
- @Override
public String getPolicyNameByCollection(String coll) {
return null;
}
@@ -1055,11 +1044,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public ClusterState getClusterState() {
- throw new UnsupportedOperationException("getClusterState");
- }
-
- @Override
public String getPolicyNameByCollection(String coll) {
return null;
}
@@ -1118,11 +1102,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public ClusterState getClusterState() {
- return clusterDataProvider.getClusterState();
- }
-
- @Override
public String getPolicyNameByCollection(String coll) {
return "p1";
}
@@ -1151,11 +1130,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public ClusterState getClusterState() {
- throw new UnsupportedOperationException("getClusterState");
- }
-
- @Override
public String getPolicyNameByCollection(String coll) {
return null;
}
@@ -1201,12 +1175,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
public Collection<String> getNodes() {
return Arrays.asList( "127.0.0.1:50097_solr", "127.0.0.1:50096_solr");
}
-
- @Override
- public ClusterState getClusterState() {
- throw new UnsupportedOperationException("getClusterState");
- }
-
};
List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
"newColl", new AutoScalingConfig((Map<String, Object>)Utils.fromJSONString(autoScaleJson)),
@@ -1264,12 +1232,6 @@ public class TestPolicy extends SolrTestCaseJ4 {
public Collection<String> getNodes() {
return Arrays.asList("node1", "node2", "node3", "node4");
}
-
- @Override
- public ClusterState getClusterState() {
- throw new UnsupportedOperationException("getClusterState");
- }
-
};
List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
"newColl", new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScaleJson)),
[14/24] lucene-solr:jira/solr-11285: SOLR-11085: Removing nocommit
Posted by ab...@apache.org.
SOLR-11085: Removing nocommit
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ab994e7b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ab994e7b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ab994e7b
Branch: refs/heads/jira/solr-11285
Commit: ab994e7b32d45f1a5f37e76bfd7a89ab01c23a51
Parents: 2f94136
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Sep 19 11:53:15 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Sep 19 11:53:15 2017 -0700
----------------------------------------------------------------------
.../org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ab994e7b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
index 042fac3..294ada7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
@@ -234,7 +234,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
assertUlogDir(collections);
- //TODO nocommit, we must test overseer failover
+ //TODO SOLR-11085, we must test overseer failover
List<JettySolrRunner> stoppedJetties = notOverseerJetties();
ChaosMonkey.stop(stoppedJetties);
ChaosMonkey.stop(controlJetty);
[16/24] lucene-solr:jira/solr-11285: SOLR-11076: Set log level to
DEBUG for the autoscaling package to make debugging easier
Posted by ab...@apache.org.
SOLR-11076: Set log level to DEBUG for the autoscaling package to make debugging easier
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b4680e58
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b4680e58
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b4680e58
Branch: refs/heads/jira/solr-11285
Commit: b4680e5869f997b9a31878a8822512dc685858e4
Parents: 1b3cd34
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Sep 20 10:36:43 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Sep 20 10:36:43 2017 -0700
----------------------------------------------------------------------
.../apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b4680e58/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
index 478966f..ff6a916 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
@@ -25,6 +25,7 @@ import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.LogLevel;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -36,6 +37,7 @@ import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAut
/**
*
*/
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
[22/24] lucene-solr:jira/solr-11285: SOLR-11407: Add some debugging
information when the test fails.
Posted by ab...@apache.org.
SOLR-11407: Add some debugging information when the test fails.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fba51f34
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fba51f34
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fba51f34
Branch: refs/heads/jira/solr-11285
Commit: fba51f34d7506c8c9d23d139900600949e34f732
Parents: d47af82
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Sep 27 19:25:15 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Sep 27 19:25:15 2017 +0200
----------------------------------------------------------------------
.../handler/admin/AutoscalingHistoryHandlerTest.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fba51f34/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
index 39dd5a7..56c920e 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
@@ -175,7 +175,9 @@ public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase {
boolean await = actionFiredLatch.await(60, TimeUnit.SECONDS);
assertTrue("action did not execute", await);
// commit on the history collection
+ Thread.sleep(2000);
solrClient.commit(CollectionAdminParams.SYSTEM_COLL);
+ Thread.sleep(2000);
// verify that new docs exist
ModifiableSolrParams query = params(CommonParams.Q, "type:" + SystemLogListener.DOC_TYPE,
CommonParams.FQ, "event.source_s:" + PREFIX + "_node_added_trigger");
@@ -185,11 +187,13 @@ public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase {
query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH,
AutoscalingHistoryHandler.TRIGGER_PARAM, PREFIX + "_node_added_trigger");
- docs = solrClient.query(query).getResults();
+ QueryResponse rsp = solrClient.query(query);
+ docs = rsp.getResults();
if (docs.size() != 8) {
- zkClient().printLayoutToStdOut();
- query = params(CommonParams.QT, CommonParams.AUTOSCALING_DIAGNOSTICS_PATH);
- log.info("Diagnostic output: ", solrClient.query(query).getResponse());
+ log.info("Cluster state: " + solrClient.getZkStateReader().getClusterState());
+ query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH);
+ log.info("Wrong response: ", rsp);
+ log.info("Full response: " + solrClient.query(query));
}
assertEquals(8, docs.size());
[04/24] lucene-solr:jira/solr-11285: SOLR-8344: Decide default when
requested fields are both column and row stored.
Posted by ab...@apache.org.
SOLR-8344: Decide default when requested fields are both column and row stored.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/40f78dd2
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/40f78dd2
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/40f78dd2
Branch: refs/heads/jira/solr-11285
Commit: 40f78dd2740122e5fa18f2515effc9272fd2d902
Parents: 2e5f9a4
Author: Cao Manh Dat <da...@apache.org>
Authored: Tue Sep 19 06:02:14 2017 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Tue Sep 19 06:02:14 2017 +0700
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/response/DocsStreamer.java | 87 +++----------
.../solr/response/RetrieveFieldsOptimizer.java | 130 +++++++++++++++++++
.../apache/solr/search/SolrDocumentFetcher.java | 25 +++-
.../response/TestRetrieveFieldsOptimizer.java | 56 ++++++++
5 files changed, 229 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/40f78dd2/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5d035c3..28fb506 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -145,6 +145,8 @@ Optimizations
* SOLR-11314: FastCharStream: re-use the READ_PAST_EOF exception. (Michael Braun via David Smiley)
+* SOLR-8344: Decide default when requested fields are both column and row stored. (Cao Manh Dat, David Smiley)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/40f78dd2/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
index 7149645..f4104a4 100644
--- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
+++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java
@@ -49,9 +49,7 @@ import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
-import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrDocumentFetcher;
-import org.apache.solr.search.SolrReturnFields;
/**
* This streams SolrDocuments from a DocList and applies transformer
@@ -66,9 +64,7 @@ public class DocsStreamer implements Iterator<SolrDocument> {
private final DocTransformer transformer;
private final DocIterator docIterator;
- private final Set<String> fnames; // returnFields.getLuceneFieldNames(). Maybe null. Not empty.
- private final boolean onlyPseudoFields;
- private final Set<String> dvFieldsToReturn; // maybe null. Not empty.
+ private final RetrieveFieldsOptimizer retrieveFieldsOptimizer;
private int idx = -1;
@@ -77,63 +73,13 @@ public class DocsStreamer implements Iterator<SolrDocument> {
this.docs = rctx.getDocList();
transformer = rctx.getReturnFields().getTransformer();
docIterator = this.docs.iterator();
- fnames = rctx.getReturnFields().getLuceneFieldNames();
- //TODO move onlyPseudoFields calc to ReturnFields
- onlyPseudoFields = (fnames == null && !rctx.getReturnFields().wantsAllFields() && !rctx.getReturnFields().hasPatternMatching())
- || (fnames != null && fnames.size() == 1 && SolrReturnFields.SCORE.equals(fnames.iterator().next()));
-
- // add non-stored DV fields that may have been requested
docFetcher = rctx.getSearcher().getDocFetcher();
- dvFieldsToReturn = calcDocValueFieldsForReturn(docFetcher, rctx.getReturnFields());
+ retrieveFieldsOptimizer = RetrieveFieldsOptimizer.create(docFetcher, rctx.getReturnFields());
+ retrieveFieldsOptimizer.optimize(docFetcher);
if (transformer != null) transformer.setContext(rctx);
}
- // TODO move to ReturnFields ? Or SolrDocumentFetcher ?
- public static Set<String> calcDocValueFieldsForReturn(SolrDocumentFetcher docFetcher, ReturnFields returnFields) {
- Set<String> result = null;
- if (returnFields.wantsAllFields()) {
- // check whether there are no additional fields
- Set<String> fieldNames = returnFields.getLuceneFieldNames(true);
- if (fieldNames == null) {
- result = docFetcher.getNonStoredDVs(true);
- } else {
- result = new HashSet<>(docFetcher.getNonStoredDVs(true)); // copy
- // add all requested fields that may be useDocValuesAsStored=false
- for (String fl : fieldNames) {
- if (docFetcher.getNonStoredDVs(false).contains(fl)) {
- result.add(fl);
- }
- }
- }
- } else {
- if (returnFields.hasPatternMatching()) {
- for (String s : docFetcher.getNonStoredDVs(true)) {
- if (returnFields.wantsField(s)) {
- if (null == result) {
- result = new HashSet<>();
- }
- result.add(s);
- }
- }
- } else {
- Set<String> fnames = returnFields.getLuceneFieldNames();
- if (fnames == null) {
- return null;
- }
- result = new HashSet<>(fnames); // copy
- // here we get all non-stored dv fields because even if a user has set
- // useDocValuesAsStored=false in schema, he may have requested a field
- // explicitly using the fl parameter
- result.retainAll(docFetcher.getNonStoredDVs(false));
- }
- }
- if (result != null && result.isEmpty()) {
- return null;
- }
- return result;
- }
-
public int currentIndex() {
return idx;
}
@@ -147,21 +93,22 @@ public class DocsStreamer implements Iterator<SolrDocument> {
idx++;
SolrDocument sdoc = null;
- if (onlyPseudoFields) {
- // no need to get stored fields of the document, see SOLR-5968
- sdoc = new SolrDocument();
- } else {
- try {
- Document doc = docFetcher.doc(id, fnames);
- sdoc = convertLuceneDocToSolrDoc(doc, rctx.getSearcher().getSchema()); // make sure to use the schema from the searcher and not the request (cross-core)
+ try {
+ if (retrieveFieldsOptimizer.returnStoredFields()) {
+ Document doc = docFetcher.doc(id, retrieveFieldsOptimizer.getStoredFields());
+ // make sure to use the schema from the searcher and not the request (cross-core)
+ sdoc = convertLuceneDocToSolrDoc(doc, rctx.getSearcher().getSchema());
+ } else {
+ // no need to get stored fields of the document, see SOLR-5968
+ sdoc = new SolrDocument();
+ }
- // decorate the document with non-stored docValues fields
- if (dvFieldsToReturn != null) {
- docFetcher.decorateDocValueFields(sdoc, id, dvFieldsToReturn);
- }
- } catch (IOException e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading document with docId " + id, e);
+ // decorate the document with non-stored docValues fields
+ if (retrieveFieldsOptimizer.returnDVFields()) {
+ docFetcher.decorateDocValueFields(sdoc, id, retrieveFieldsOptimizer.getDvFields());
}
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error reading document with docId " + id, e);
}
if (transformer != null) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/40f78dd2/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java b/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java
new file mode 100644
index 0000000..c818650
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/response/RetrieveFieldsOptimizer.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.response;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.solr.search.ReturnFields;
+import org.apache.solr.search.SolrDocumentFetcher;
+import org.apache.solr.search.SolrReturnFields;
+
+public class RetrieveFieldsOptimizer {
+ // null means get all available stored fields
+ private final Set<String> storedFields;
+ // always non null
+ private final Set<String> dvFields;
+
+ RetrieveFieldsOptimizer(Set<String> storedFields, Set<String> dvFields) {
+ this.storedFields = storedFields;
+ this.dvFields = dvFields;
+ }
+
+ /**
+ * Sometimes we could fetch a field value from either the stored document or docValues.
+ * Such fields have both and are single-valued.
+ * If choosing docValues allows us to avoid accessing the stored document altogether
+ * for all fields to be returned then we do it,
+ * otherwise we prefer the stored value when we have a choice.
+ */
+ void optimize(SolrDocumentFetcher docFetcher) {
+ optimize(docFetcher.getAllSingleDV());
+ }
+
+ void optimize(Set<String> singleDVs) {
+ if (storedFields == null) return;
+ if (!singleDVs.containsAll(storedFields)) return;
+ dvFields.addAll(storedFields);
+ storedFields.clear();
+ }
+
+ boolean returnStoredFields() {
+ return !(storedFields != null && storedFields.isEmpty());
+ }
+
+ boolean returnDVFields() {
+ return !dvFields.isEmpty();
+ }
+
+ Set<String> getStoredFields() {
+ return storedFields;
+ }
+
+ Set<String> getDvFields() {
+ return dvFields;
+ }
+
+ public static RetrieveFieldsOptimizer create(SolrDocumentFetcher docFetcher, ReturnFields returnFields) {
+ Set<String> storedFields = calcStoredFieldsForReturn(docFetcher, returnFields);
+ Set<String> dvFields = calcDocValueFieldsForReturn(docFetcher, returnFields);
+
+ return new RetrieveFieldsOptimizer(storedFields, dvFields);
+ }
+
+ private static Set<String> calcStoredFieldsForReturn(SolrDocumentFetcher docFetcher, ReturnFields returnFields) {
+ final Set<String> storedFields = new HashSet<>();
+ Set<String> fnames = returnFields.getLuceneFieldNames();
+ if (returnFields.wantsAllFields()) {
+ return null;
+ } else if (returnFields.hasPatternMatching()) {
+ for (String s : docFetcher.getAllStored()) {
+ if (returnFields.wantsField(s)) {
+ storedFields.add(s);
+ }
+ }
+ } else if (fnames != null) {
+ storedFields.addAll(fnames);
+ }
+ storedFields.remove(SolrReturnFields.SCORE);
+ return storedFields;
+ }
+
+ private static Set<String> calcDocValueFieldsForReturn(SolrDocumentFetcher docFetcher, ReturnFields returnFields) {
+ // always return not null
+ final Set<String> result = new HashSet<>();
+ if (returnFields.wantsAllFields()) {
+ result.addAll(docFetcher.getNonStoredDVs(true));
+ // check whether there are no additional fields
+ Set<String> fieldNames = returnFields.getLuceneFieldNames(true);
+ if (fieldNames != null) {
+ // add all requested fields that may be useDocValuesAsStored=false
+ for (String fl : fieldNames) {
+ if (docFetcher.getNonStoredDVs(false).contains(fl)) {
+ result.add(fl);
+ }
+ }
+ }
+ } else if (returnFields.hasPatternMatching()) {
+ for (String s : docFetcher.getNonStoredDVs(true)) {
+ if (returnFields.wantsField(s)) {
+ result.add(s);
+ }
+ }
+ } else {
+ Set<String> fnames = returnFields.getLuceneFieldNames();
+ if (fnames != null) {
+ result.addAll(fnames);
+ // here we get all non-stored dv fields because even if a user has set
+ // useDocValuesAsStored=false in schema, he may have requested a field
+ // explicitly using the fl parameter
+ result.retainAll(docFetcher.getNonStoredDVs(false));
+ }
+ }
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/40f78dd2/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
index 8db8f6d..f5a59a4 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java
@@ -80,6 +80,10 @@ public class SolrDocumentFetcher {
private final SolrCache<Integer,Document> documentCache;
+ private final Set<String> allStored;
+
+ private final Set<String> allSingleDV;
+
/** Contains the names/patterns of all docValues=true,stored=false fields in the schema. */
private final Set<String> allNonStoredDVs;
@@ -108,12 +112,20 @@ public class SolrDocumentFetcher {
final Set<String> allNonStoredDVs = new HashSet<>();
final Set<String> nonStoredDVsWithoutCopyTargets = new HashSet<>();
final Set<String> storedLargeFields = new HashSet<>();
+ final Set<String> allSingleDVs = new HashSet<>();
+ final Set<String> allStoreds = new HashSet<>();
for (FieldInfo fieldInfo : searcher.getFieldInfos()) { // can find materialized dynamic fields, unlike using the Solr IndexSchema.
final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldInfo.name);
if (schemaField == null) {
continue;
}
+ if (schemaField.hasDocValues() && !schemaField.multiValued()) {
+ allSingleDVs.add(fieldInfo.name);
+ }
+ if (schemaField.stored()) {
+ allStoreds.add(fieldInfo.name);
+ }
if (!schemaField.stored() && schemaField.hasDocValues()) {
if (schemaField.useDocValuesAsStored()) {
nonStoredDVsUsedAsStored.add(fieldInfo.name);
@@ -132,6 +144,8 @@ public class SolrDocumentFetcher {
this.allNonStoredDVs = Collections.unmodifiableSet(allNonStoredDVs);
this.nonStoredDVsWithoutCopyTargets = Collections.unmodifiableSet(nonStoredDVsWithoutCopyTargets);
this.largeFields = Collections.unmodifiableSet(storedLargeFields);
+ this.allSingleDV = Collections.unmodifiableSet(allSingleDVs);
+ this.allStored = Collections.unmodifiableSet(allStoreds);
}
public boolean isLazyFieldLoadingEnabled() {
@@ -412,7 +426,7 @@ public class SolrDocumentFetcher {
final LeafReader leafReader = leafContexts.get(subIndex).reader();
for (String fieldName : fields) {
final SchemaField schemaField = searcher.getSchema().getFieldOrNull(fieldName);
- if (schemaField == null || !schemaField.hasDocValues() || doc.containsKey(fieldName)) {
+ if (schemaField == null || !schemaField.hasDocValues()) {
log.warn("Couldn't decorate docValues for field: [{}], schemaField: [{}]", fieldName, schemaField);
continue;
}
@@ -420,6 +434,7 @@ public class SolrDocumentFetcher {
if (fi == null) {
continue; // Searcher doesn't have info about this field, hence ignore it.
}
+ doc.remove(fieldName);
final DocValuesType dvType = fi.getDocValuesType();
switch (dvType) {
case NUMERIC:
@@ -555,6 +570,14 @@ public class SolrDocumentFetcher {
}
}
+ public Set<String> getAllSingleDV() {
+ return allSingleDV;
+ }
+
+ public Set<String> getAllStored() {
+ return allStored;
+ }
+
/**
* Returns an unmodifiable set of non-stored docValues field names.
*
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/40f78dd2/solr/core/src/test/org/apache/solr/response/TestRetrieveFieldsOptimizer.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestRetrieveFieldsOptimizer.java b/solr/core/src/test/org/apache/solr/response/TestRetrieveFieldsOptimizer.java
new file mode 100644
index 0000000..6ea0f42
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/response/TestRetrieveFieldsOptimizer.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.response;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+public class TestRetrieveFieldsOptimizer extends SolrTestCaseJ4{
+
+ @Test
+ public void testOptimizer() {
+ RetrieveFieldsOptimizer optimizer = new RetrieveFieldsOptimizer(
+ new HashSet<>(Arrays.asList("id", "title")),
+ new HashSet<>()
+ );
+ optimizer.optimize(new HashSet<>(Arrays.asList("id", "title")));
+ assertTrue(optimizer.returnDVFields());
+ assertFalse(optimizer.returnStoredFields());
+
+ optimizer = new RetrieveFieldsOptimizer(
+ new HashSet<>(Arrays.asList("id", "title")),
+ new HashSet<>()
+ );
+ optimizer.optimize(new HashSet<>(Collections.singletonList("title")));
+ assertFalse(optimizer.returnDVFields());
+ assertTrue(optimizer.returnStoredFields());
+
+ optimizer = new RetrieveFieldsOptimizer(
+ null,
+ new HashSet<>(Collections.singletonList("id"))
+ );
+ optimizer.optimize(new HashSet<>(Collections.singletonList("id")));
+ assertNull(optimizer.getStoredFields());
+ assertTrue(optimizer.getDvFields().contains("id"));
+
+ }
+}
[09/24] lucene-solr:jira/solr-11285: SOLR-11363: handle repeated
values in points docvalues fields
Posted by ab...@apache.org.
SOLR-11363: handle repeated values in points docvalues fields
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7cc9ee65
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7cc9ee65
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7cc9ee65
Branch: refs/heads/jira/solr-11285
Commit: 7cc9ee65633f653cb5ebe5c031db4d339b809466
Parents: 8e12f20
Author: yonik <yo...@apache.org>
Authored: Tue Sep 19 13:39:45 2017 -0400
Committer: yonik <yo...@apache.org>
Committed: Tue Sep 19 13:39:45 2017 -0400
----------------------------------------------------------------------
solr/CHANGES.txt | 4 +++
.../facet/FacetFieldProcessorByHashDV.java | 11 +++++++--
.../org/apache/solr/search/facet/HLLAgg.java | 1 +
.../org/apache/solr/search/facet/UniqueAgg.java | 1 +
.../solr/search/join/GraphPointsCollector.java | 1 +
.../cloud/TestCloudJSONFacetJoinDomain.java | 24 ++++++++++++------
.../solr/search/facet/TestJsonFacets.java | 26 ++++++++++++++++++++
7 files changed, 58 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7cc9ee65/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 69710a6..885f1f6 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -133,6 +133,10 @@ Bug Fixes
* SOLR-11348: Fix the DIH database example (James Dyer)
+* SOLR-11363: JSON Facet API: repeated values in a numeric points field with docValues enabled
+ were double counted. (Hossman, yonik)
+
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7cc9ee65/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
index 4237c31..2383a14 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java
@@ -390,9 +390,16 @@ class FacetFieldProcessorByHashDV extends FacetFieldProcessor {
values.advance(segDoc);
}
if (segDoc == values.docID()) {
- for (int i = 0; i < values.docValueCount(); i++) {
- collectValFirstPhase(segDoc, values.nextValue());
+ long l = values.nextValue(); // This document must have at least one value
+ collectValFirstPhase(segDoc, l);
+ for (int i = 1; i < values.docValueCount(); i++) {
+ long lnew = values.nextValue();
+ if (lnew != l) { // Skip the value if it's equal to the last one, we don't want to double-count it
+ collectValFirstPhase(segDoc, lnew);
+ }
+ l = lnew;
}
+
}
}
});
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7cc9ee65/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
index 3c55329..85964f7 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java
@@ -235,6 +235,7 @@ public class HLLAgg extends StrAggValueSource {
@Override
protected void collectValues(int doc, HLL hll) throws IOException {
for (int i = 0; i < values.docValueCount(); i++) {
+ // duplicates may be produced for a single doc, but won't matter here.
long val = values.nextValue();
long hash = Hash.fmix64(val);
hll.addRaw(hash);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7cc9ee65/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
index 4541975..d858b5b 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueAgg.java
@@ -254,6 +254,7 @@ public class UniqueAgg extends StrAggValueSource {
@Override
protected void collectValues(int doc, LongSet set) throws IOException {
for (int i = 0; i < values.docValueCount(); i++) {
+ // duplicates may be produced for a single doc, but won't matter here.
set.add(values.nextValue());
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7cc9ee65/solr/core/src/java/org/apache/solr/search/join/GraphPointsCollector.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/join/GraphPointsCollector.java b/solr/core/src/java/org/apache/solr/search/join/GraphPointsCollector.java
index 290136a..e868465 100644
--- a/solr/core/src/java/org/apache/solr/search/join/GraphPointsCollector.java
+++ b/solr/core/src/java/org/apache/solr/search/join/GraphPointsCollector.java
@@ -60,6 +60,7 @@ public class GraphPointsCollector extends GraphEdgeCollector {
if (valuesDoc == doc) {
int count = values.docValueCount();
for (int i = 0; i < count; i++) {
+ // duplicates may be produced for a single doc, but won't matter here.
long v = values.nextValue();
set.add(v);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7cc9ee65/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
index e18312c..48de49a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudJSONFacetJoinDomain.java
@@ -17,7 +17,6 @@
package org.apache.solr.cloud;
import java.io.IOException;
-
import java.lang.invoke.MethodHandles;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -29,10 +28,9 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
-
+
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.util.TestUtil;
-import org.apache.solr.SolrTestCaseJ4.SuppressPointFields;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
@@ -46,7 +44,6 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
-
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
@@ -62,7 +59,6 @@ import org.slf4j.LoggerFactory;
*
* @see TestCloudPivotFacet
*/
-@SuppressPointFields(bugUrl="https://issues.apache.org/jira/browse/SOLR-10939")
public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -600,9 +596,21 @@ public class TestCloudJSONFacetJoinDomain extends SolrCloudTestCase {
final String[] suffixes = random().nextBoolean() ? STR_FIELD_SUFFIXES : INT_FIELD_SUFFIXES;
final boolean noJoin = random().nextBoolean();
- final String from = noJoin ? null : field(suffixes, random().nextInt(MAX_FIELD_NUM));
- final String to = noJoin ? null : field(suffixes, random().nextInt(MAX_FIELD_NUM));
-
+
+ String from = null;
+ String to = null;
+ for (;;) {
+ if (noJoin) break;
+ from = field(suffixes, random().nextInt(MAX_FIELD_NUM));
+ to = field(suffixes, random().nextInt(MAX_FIELD_NUM));
+ // HACK: joined numeric point fields need docValues.. for now just skip _is fields if we are dealing with points.
+ if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP) && (from.endsWith("_is") || to.endsWith("_is")))
+ {
+ continue;
+ }
+ break;
+ }
+
// keep it simple, only filter on string fields - not point of test
final String filterField = strfield(random().nextInt(MAX_FIELD_NUM));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7cc9ee65/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
index 33d7fa8..a1c8cf7 100644
--- a/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
+++ b/solr/core/src/test/org/apache/solr/search/facet/TestJsonFacets.java
@@ -42,6 +42,10 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+// Related tests:
+// TestCloudJSONFacetJoinDomain for random field faceting tests with domain modifications
+// TestJsonFacetRefinement for refinement tests
+
@LuceneTestCase.SuppressCodecs({"Lucene3x","Lucene40","Lucene41","Lucene42","Lucene45","Appending"})
public class TestJsonFacets extends SolrTestCaseHS {
@@ -208,6 +212,28 @@ public class TestJsonFacets extends SolrTestCaseHS {
client.commit();
}
+ @Test
+ public void testRepeatedNumerics() throws Exception {
+ Client client = Client.localClient();
+ String field = "num_is"; // docValues of multi-valued points field can contain duplicate values... make sure they don't mess up our counts.
+ client.add(sdoc("id", "1", "cat_s", "A", "where_s", "NY", "num_d", "4", "num_i", "2", "val_b", "true", "sparse_s", "one", field,"0", field,"0"), null);
+ client.commit();
+
+ client.testJQ(params("q", "id:1", "field", field
+ , "json.facet", "{" +
+ "f1:{terms:${field}}" +
+ ",f2:'hll(${field})'" +
+ ",f3:{type:range, field:${field}, start:0, end:1, gap:1}" +
+ "}"
+ )
+ , "facets=={count:1, " +
+ "f1:{buckets:[{val:0, count:1}]}" +
+ ",f2:1" +
+ ",f3:{buckets:[{val:0, count:1}]}" +
+ "}"
+ );
+ }
+
public void testDomainJoinSelf() throws Exception {
Client client = Client.localClient();
indexSimple(client);
[19/24] lucene-solr:jira/solr-11285: SOLR-10643: Ensure that the
stopped node is removed from live nodes before proceeding
Posted by ab...@apache.org.
SOLR-10643: Ensure that the stopped node is removed from live nodes before proceeding
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ac9ee36f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ac9ee36f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ac9ee36f
Branch: refs/heads/jira/solr-11285
Commit: ac9ee36f059f65611530d764f0c00afcf5639df5
Parents: e13c4c3
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Sep 22 11:08:49 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Sep 22 11:08:49 2017 -0700
----------------------------------------------------------------------
.../test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ac9ee36f/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
index eb73792..0419c40 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
@@ -231,6 +231,7 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
break;
}
}
+ cluster.waitForAllNodes(5);
AtomicInteger callCount = new AtomicInteger(0);
AtomicBoolean fired = new AtomicBoolean(false);
[20/24] lucene-solr:jira/solr-11285: SOLR-11076: Added more debug
logging. Ensure collections are active before we exercise autoscaling. Added
workaround for SOLR-9440.
Posted by ab...@apache.org.
SOLR-11076: Added more debug logging. Ensure collections are active before we exercise autoscaling. Added workaround for SOLR-9440.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/5982d873
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/5982d873
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/5982d873
Branch: refs/heads/jira/solr-11285
Commit: 5982d8734adf44e12cae1985574ca682f07839ca
Parents: ac9ee36
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Sep 22 13:27:24 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Sep 22 13:27:24 2017 -0700
----------------------------------------------------------------------
.../solr/cloud/autoscaling/ComputePlanAction.java | 7 ++++++-
.../handler/admin/AutoscalingHistoryHandlerTest.java | 15 ++++++++++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5982d873/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
index 93441fe..5d5e3eb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ComputePlanAction.java
@@ -20,6 +20,7 @@ package org.apache.solr.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -61,7 +62,11 @@ public class ComputePlanAction extends TriggerActionBase {
return;
}
Policy policy = autoScalingConf.getPolicy();
- Policy.Session session = policy.createSession(new SolrClientDataProvider(cloudSolrClient));
+ SolrClientDataProvider dataProvider = new SolrClientDataProvider(cloudSolrClient);
+ if (log.isDebugEnabled()) {
+ log.debug("Cluster data provider: {}", dataProvider.toMap(new HashMap<>()));
+ }
+ Policy.Session session = policy.createSession(dataProvider);
Policy.Suggester suggester = getSuggester(session, event, zkStateReader);
while (true) {
SolrRequest operation = suggester.getOperation();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/5982d873/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
index ff6a916..39dd5a7 100644
--- a/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/handler/admin/AutoscalingHistoryHandlerTest.java
@@ -37,7 +37,7 @@ import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAut
/**
*
*/
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;")
public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -162,6 +162,14 @@ public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase {
@Test
public void testHistory() throws Exception {
+ waitForState("Timed out wait for collection be active", PREFIX + "_collection",
+ clusterShape(1, 3));
+ waitForState("Timed out wait for collection be active", CollectionAdminParams.SYSTEM_COLL,
+ clusterShape(1, 3));
+ // todo remove this workaround after SOLR-9440
+ cluster.getSolrClient().getZkStateReader().registerCore(".system");
+ cluster.getSolrClient().getZkStateReader().registerCore(PREFIX + "_collection");
+
JettySolrRunner jetty = cluster.startJettySolrRunner();
String nodeAddedName = jetty.getNodeName();
boolean await = actionFiredLatch.await(60, TimeUnit.SECONDS);
@@ -178,6 +186,11 @@ public class AutoscalingHistoryHandlerTest extends SolrCloudTestCase {
query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH,
AutoscalingHistoryHandler.TRIGGER_PARAM, PREFIX + "_node_added_trigger");
docs = solrClient.query(query).getResults();
+ if (docs.size() != 8) {
+ zkClient().printLayoutToStdOut();
+ query = params(CommonParams.QT, CommonParams.AUTOSCALING_DIAGNOSTICS_PATH);
+ log.info("Diagnostic output: ", solrClient.query(query).getResponse());
+ }
assertEquals(8, docs.size());
query = params(CommonParams.QT, CommonParams.AUTOSCALING_HISTORY_PATH,
[24/24] lucene-solr:jira/solr-11285: SOLR-11285: Fix some fallout
from the merge.
Posted by ab...@apache.org.
SOLR-11285: Fix some fallout from the merge.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/185bcb77
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/185bcb77
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/185bcb77
Branch: refs/heads/jira/solr-11285
Commit: 185bcb77478ebc48344e942d4a0c5bbff630c3d5
Parents: 1f25f3e
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Fri Sep 29 00:08:48 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Fri Sep 29 00:08:48 2017 +0200
----------------------------------------------------------------------
.../src/java/org/apache/solr/api/ApiBag.java | 4 ++
.../solr/cloud/autoscaling/AutoScaling.java | 2 -
.../cloud/autoscaling/ExecutePlanAction.java | 69 +++++++++-----------
.../autoscaling/ComputePlanActionTest.java | 2 +-
.../autoscaling/ExecutePlanActionTest.java | 49 +++++++-------
.../impl/SolrClientClusterDataProvider.java | 10 ++-
.../solrj/request/CollectionAdminRequest.java | 4 ++
7 files changed, 76 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/185bcb77/solr/core/src/java/org/apache/solr/api/ApiBag.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/api/ApiBag.java b/solr/core/src/java/org/apache/solr/api/ApiBag.java
index 55730b8..2caaeb9 100644
--- a/solr/core/src/java/org/apache/solr/api/ApiBag.java
+++ b/solr/core/src/java/org/apache/solr/api/ApiBag.java
@@ -327,6 +327,10 @@ public class ApiBag {
public List<Map> getErrs() {
return errs;
}
+
+ public String toString() {
+ return super.toString() + ", errors: " + getErrs() + ", ";
+ }
}
public static class LazyLoadedApi extends Api {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/185bcb77/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index 3842ac3..715ccfe 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@ -153,8 +153,6 @@ public class AutoScaling {
return new NodeAddedTrigger(name, props, loader, dataProvider);
case NODELOST:
return new NodeLostTrigger(name, props, loader, dataProvider);
- case SEARCHRATE:
- return new SearchRateTrigger(name, props, loader, dataProvider);
default:
throw new IllegalArgumentException("Unknown event type: " + type + " in trigger: " + name);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/185bcb77/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
index a8bfd15..8728ecd 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ExecutePlanAction.java
@@ -65,41 +65,36 @@ public class ExecutePlanAction extends TriggerActionBase {
for (SolrRequest operation : operations) {
log.info("Executing operation: {}", operation.getParams());
try {
- SolrResponse response = dataProvider.request(operation);
- // nocommit
-//=======
-// try (CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder()
-// .withZkHost(container.getZkController().getZkServerAddress())
-// .withHttpClient(container.getUpdateShardHandler().getHttpClient())
-// .build()) {
-// int counter = 0;
-// for (SolrRequest operation : operations) {
-// log.info("Executing operation: {}", operation.getParams());
-// try {
-// SolrResponse response = null;
-// if (operation instanceof CollectionAdminRequest.AsyncCollectionAdminRequest) {
-// CollectionAdminRequest.AsyncCollectionAdminRequest req = (CollectionAdminRequest.AsyncCollectionAdminRequest) operation;
-// String asyncId = event.getSource() + '/' + event.getId() + '/' + counter;
-// String znode = saveAsyncId(event, context, asyncId);
-// log.debug("Saved requestId: {} in znode: {}", asyncId, znode);
-// asyncId = req.processAsync(asyncId, cloudSolrClient);
-// CollectionAdminRequest.RequestStatusResponse statusResponse = waitForTaskToFinish(cloudSolrClient, asyncId,
-// DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
-// if (statusResponse != null) {
-// RequestStatusState state = statusResponse.getRequestStatus();
-// if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
-// try {
-// zkClient.delete(znode, -1, true);
-// } catch (KeeperException e) {
-// log.warn("Unexpected exception while trying to delete znode: " + znode, e);
-// }
-// }
-// response = statusResponse;
-// }
-// } else {
-// response = operation.process(cloudSolrClient);
-// }
-//>>>>>>> feature/autoscaling
+ SolrResponse response = null;
+ int counter = 0;
+ if (operation instanceof CollectionAdminRequest.AsyncCollectionAdminRequest) {
+ CollectionAdminRequest.AsyncCollectionAdminRequest req = (CollectionAdminRequest.AsyncCollectionAdminRequest) operation;
+ String asyncId = event.getSource() + '/' + event.getId() + '/' + counter;
+ String znode = saveAsyncId(dataProvider.getDistribStateManager(), event, asyncId);
+ log.debug("Saved requestId: {} in znode: {}", asyncId, znode);
+ // nocommit - make sure the async logic below is correct when using dataProvider API !!!
+ req.setAsyncId(asyncId);
+ SolrResponse asyncResponse = dataProvider.request(req);
+ if (asyncResponse.getResponse().get("error") != null) {
+ throw new IOException("" + asyncResponse.getResponse().get("error"));
+ }
+ asyncId = (String)asyncResponse.getResponse().get("requestid");
+ CollectionAdminRequest.RequestStatusResponse statusResponse = waitForTaskToFinish(dataProvider, asyncId,
+ DEFAULT_TASK_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+ if (statusResponse != null) {
+ RequestStatusState state = statusResponse.getRequestStatus();
+ if (state == RequestStatusState.COMPLETED || state == RequestStatusState.FAILED || state == RequestStatusState.NOT_FOUND) {
+ try {
+ dataProvider.getDistribStateManager().removeData(znode, -1);
+ } catch (Exception e) {
+ log.warn("Unexpected exception while trying to delete znode: " + znode, e);
+ }
+ }
+ response = statusResponse;
+ }
+ } else {
+ response = dataProvider.request(operation);
+ }
NamedList<Object> result = response.getResponse();
context.getProperties().compute("responses", (s, o) -> {
List<NamedList<Object>> responses = (List<NamedList<Object>>) o;
@@ -168,8 +163,8 @@ public class ExecutePlanAction extends TriggerActionBase {
*
* @return the path of the newly created node in ZooKeeper
*/
- private String saveAsyncId(DistribStateManager stateManager, TriggerEvent event, ActionContext context, String asyncId) throws InterruptedException, AlreadyExistsException, IOException {
- String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + context.getSource().getName() + "/" + getName();
+ private String saveAsyncId(DistribStateManager stateManager, TriggerEvent event, String asyncId) throws InterruptedException, AlreadyExistsException, IOException {
+ String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + event.getSource() + "/" + getName();
stateManager.makePath(parentPath);
return stateManager.createData(parentPath + "/" + PREFIX, Utils.toJSON(Collections.singletonMap("requestid", asyncId)), CreateMode.PERSISTENT_SEQUENTIAL);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/185bcb77/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index 94b4094..d9b88b5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -61,7 +61,7 @@ public class ComputePlanActionTest extends SolrCloudTestCase {
private static final int NODE_COUNT = 1;
private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
private static final AtomicReference<Map> actionContextPropsRef = new AtomicReference<>();
- private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();o
+ private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
@BeforeClass
public static void setupCluster() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/185bcb77/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
index 0061f78..b947556 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
@@ -34,6 +34,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
@@ -118,28 +119,30 @@ public class ExecutePlanActionTest extends SolrCloudTestCase {
AtomicBoolean znodeCreated = new AtomicBoolean(false);
CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
- CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.LIST) {
- @Override
- public String processAsync(String asyncId, SolrClient client) throws IOException, SolrServerException {
- String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
- try {
- if (zkClient().exists(parentPath, true)) {
- java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
- if (!children.isEmpty()) {
- String child = children.get(0);
- byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true);
- Map m = (Map) Utils.fromJSON(data);
- if (m.containsKey("requestid")) {
- znodeCreated.set(m.get("requestid").equals(asyncId));
- }
- }
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- super.processAsync(asyncId, client);
- return asyncId;
- }
+ CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
+ // nocommit: this doesn't work because async requests are now processed differently in ExecutePlanAction
+//
+// @Override
+// public String processAsync(String asyncId, SolrClient client) throws IOException, SolrServerException {
+// String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
+// try {
+// if (zkClient().exists(parentPath, true)) {
+// java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
+// if (!children.isEmpty()) {
+// String child = children.get(0);
+// byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true);
+// Map m = (Map) Utils.fromJSON(data);
+// if (m.containsKey("requestid")) {
+// znodeCreated.set(m.get("requestid").equals(asyncId));
+// }
+// }
+// }
+// } catch (Exception e) {
+// throw new RuntimeException(e);
+// }
+// super.processAsync(asyncId, client);
+// return asyncId;
+// }
};
List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST,
@@ -149,7 +152,7 @@ public class ExecutePlanActionTest extends SolrCloudTestCase {
new HashMap<>(Collections.singletonMap("operations", operations)));
action.process(nodeLostEvent, actionContext);
- assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
+// assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
assertNotNull(responses);
assertEquals(2, responses.size());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/185bcb77/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
index 2cd24ae..7824f80 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientClusterDataProvider.java
@@ -71,6 +71,9 @@ public class SolrClientClusterDataProvider implements ClusterDataProvider, MapWr
this.solrClient = solrClient;
this.zkStateReader = solrClient.getZkStateReader();
ClusterState clusterState = zkStateReader.getClusterState();
+ if (clusterState == null) { // zkStateReader still initializing
+ return;
+ }
Map<String, ClusterState.CollectionRef> all = clusterState.getCollectionStates();
all.forEach((collName, ref) -> {
DocCollection coll = ref.get();
@@ -85,7 +88,12 @@ public class SolrClientClusterDataProvider implements ClusterDataProvider, MapWr
}
public Collection<String> getLiveNodes() {
- return zkStateReader.getClusterState().getLiveNodes();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ if (clusterState == null) { // zkStateReader still initializing
+ return Collections.emptySet();
+ } else {
+ return clusterState.getLiveNodes();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/185bcb77/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index b394364..930ff7f 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -139,6 +139,10 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return asyncId;
}
+ public void setAsyncId(String asyncId) {
+ this.asyncId = asyncId;
+ }
+
/**
* Process this request asynchronously, generating and returning a request id
* @param client a Solr client
[17/24] lucene-solr:jira/solr-11285: SOLR-9735: Moving the
CHANGES.txt entries to the right release version
Posted by ab...@apache.org.
SOLR-9735: Moving the CHANGES.txt entries to the right release version
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d58e5fed
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d58e5fed
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d58e5fed
Branch: refs/heads/jira/solr-11285
Commit: d58e5fedea386a1e9c0427ba6c724a1bc91c3296
Parents: b4680e5
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Sep 20 16:28:16 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Sep 20 16:28:16 2017 -0700
----------------------------------------------------------------------
solr/CHANGES.txt | 53 ++++++++++++++++++++++++---------------------------
1 file changed, 25 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d58e5fed/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0fc3056..d2a2213 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -75,6 +75,21 @@ Upgrade Notes
New Features
----------------------
+* SOLR-10339: New set-trigger and remove-trigger APIs for autoscaling. (shalin)
+
+* SOLR-10340: New set-listener and remove-listener API for autoscaling. (shalin)
+
+* SOLR-10358: New suspend-trigger and resume-trigger APIs for autoscaling. (shalin)
+
+* SOLR-10376: Implement autoscaling trigger for nodeAdded event. (shalin)
+
+* SOLR-10396: Implement trigger support for nodeLost event type (Cao Manh Dat, shalin)
+
+* SOLR-10496: New ComputePlanAction for autoscaling which uses the policy framework to compute cluster
+ operations upon a trigger fire. (Noble Paul, shalin)
+
+* SOLR-10965: New ExecutePlanAction for autoscaling which executes the operations computed by ComputePlanAction
+ against the cluster. (shalin)
* SOLR-11019: Add addAll Stream Evaluator (Joel Bernstein)
@@ -104,10 +119,16 @@ New Features
* SOLR-11316: JSON Facet API: min/max aggregations are now supported on single-valued date fields.
(yonik)
-
Bug Fixes
----------------------
+* SOLR-10602: Triggers should be able to restore state from old instances when taking over. (shalin)
+
+* SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling
+ config watcher is fired. (shalin)
+
+* SOLR-10738: TriggerAction is initialised even if the trigger is never scheduled. (shalin)
+
* SOLR-10668: fix NPE at sort=childfield(..) .. on absent values (Mikhail Khludnev)
* SOLR-8984: EnumField's error reporting to now indicate the field name in failure log (Lanny Ripple,
@@ -177,6 +198,9 @@ Optimizations
Other Changes
----------------------
+* SOLR-10643: Throttling strategy for triggers and policy executions. (shalin)
+
+* SOLR-10764: AutoScalingHandler should validate policy and preferences before updating zookeeper. (shalin)
* SOLR-10827: Factor out abstract FilteringSolrMetricReporter class. (Christine Poerschke)
@@ -442,16 +466,6 @@ New Features
* SOLR-10278: A new DSL to set cluster-wide preferences and policies on how to allocate replicas to nodes
(noble, shalin)
-* SOLR-10339: New set-trigger and remove-trigger APIs for autoscaling. (shalin)
-
-* SOLR-10340: New set-listener and remove-listener API for autoscaling. (shalin)
-
-* SOLR-10358: New suspend-trigger and resume-trigger APIs for autoscaling. (shalin)
-
-* SOLR-10376: Implement autoscaling trigger for nodeAdded event. (shalin)
-
-* SOLR-10396: Implement trigger support for nodeLost event type (Cao Manh Dat, shalin)
-
* SOLR-10373: Implement read API for autoscaling configuration at /admin/autoscaling or
/cluster/autoscaling paths. (shalin)
@@ -485,12 +499,6 @@ New Features
multivalued fields, a new JSON request language, and more. DocValues are now required for any field used in the analytics
expression whereas previously docValues was not required. Please see SOLR-10123 for details. (Houston Putman)
-* SOLR-10496: New ComputePlanAction for autoscaling which uses the policy framework to compute cluster
- operations upon a trigger fire. (Noble Paul, shalin)
-
-* SOLR-10965: New ExecutePlanAction for autoscaling which executes the operations computed by ComputePlanAction
- against the cluster. (shalin)
-
* SOLR-10282: bin/solr support for enabling Kerberos authentication (Ishan Chattopadhyaya, Jason Gerlowski)
* SOLR-11093: Add support for PointFields for {!graph} query. (yonik)
@@ -597,13 +605,6 @@ Bug Fixes
* SOLR-10506: Fix memory leak (upon collection reload or ZooKeeper session expiry) in ZkIndexSchemaReader.
(Torsten Bøgh Köster, Christine Poerschke, Jörg Rathlev, Mike Drob)
-* SOLR-10602: Triggers should be able to restore state from old instances when taking over. (shalin)
-
-* SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling
- config watcher is fired. (shalin)
-
-* SOLR-10738: TriggerAction is initialised even if the trigger is never scheduled. (shalin)
-
* SOLR-6807: CloudSolrClient's ZK state version check with the server was ignored when handleSelect=false
(David Smiley)
@@ -807,10 +808,6 @@ Other Changes
* SOLR-10713: Ignore .pid and .out files in solr working directory (Jason Gerlowski via Mike Drob)
-* SOLR-10643: Throttling strategy for triggers and policy executions. (shalin)
-
-* SOLR-10764: AutoScalingHandler should validate policy and preferences before updating zookeeper. (shalin)
-
* SOLR-10782: Improve error handling and tests for Snitch and subclasses and general cleanups. (Noble Paul, shalin)
* SOLR-10419: All collection APIs should use the new Policy framework for replica placement. (Noble Paul, shalin)
[06/24] lucene-solr:jira/solr-11285: SOLR-10451: Remove
contrib/ltr/lib from lib includes in the techproducts example config. Fixes
#249
Posted by ab...@apache.org.
SOLR-10451: Remove contrib/ltr/lib from lib includes in the techproducts example config. Fixes #249
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bc952097
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bc952097
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bc952097
Branch: refs/heads/jira/solr-11285
Commit: bc95209774fa4e1a998497ba224a15123d956ca1
Parents: 40f78dd
Author: Jan Høydahl <ja...@apache.org>
Authored: Tue Sep 19 10:45:23 2017 +0200
Committer: Jan Høydahl <ja...@apache.org>
Committed: Tue Sep 19 10:45:23 2017 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../configsets/sample_techproducts_configs/conf/solrconfig.xml | 1 -
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc952097/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 28fb506..69710a6 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -209,6 +209,8 @@ Other Changes
* SOLR-11351: Make LTRScoringModel model more extensible. (Christine Poerschke)
+* SOLR-10451: Remove contrib/ltr/lib from lib includes in the techproducts example config (sungjunyoung via janhoy)
+
================== 7.0.0 ==================
Versions of Major Components
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc952097/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
index 42b12cf..65750c2 100644
--- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
+++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml
@@ -81,7 +81,6 @@
<lib dir="${solr.install.dir:../../../..}/contrib/langid/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-langid-\d.*\.jar" />
- <lib dir="${solr.install.dir:../../../..}/contrib/ltr/lib/" regex=".*\.jar" />
<lib dir="${solr.install.dir:../../../..}/dist/" regex="solr-ltr-\d.*\.jar" />
<lib dir="${solr.install.dir:../../../..}/contrib/velocity/lib" regex=".*\.jar" />
[21/24] lucene-solr:jira/solr-11285: SOLR-10496: Enabled debug
logging for all overseer messages
Posted by ab...@apache.org.
SOLR-10496: Enabled debug logging for all overseer messages
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d47af823
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d47af823
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d47af823
Branch: refs/heads/jira/solr-11285
Commit: d47af823ed24d039129442256f09ff9e4472cc40
Parents: 5982d87
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Fri Sep 22 14:09:59 2017 -0700
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Fri Sep 22 14:09:59 2017 -0700
----------------------------------------------------------------------
.../org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d47af823/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
index b0bf4e6..d9b88b5 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
@@ -53,7 +53,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.MO
/**
* Test for {@link ComputePlanAction}
*/
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;")
public class ComputePlanActionTest extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
[11/24] lucene-solr:jira/solr-11285: improve exception message
Posted by ab...@apache.org.
improve exception message
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4e559771
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4e559771
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4e559771
Branch: refs/heads/jira/solr-11285
Commit: 4e5597716a100c70ac0066fe70f5b096806d5679
Parents: 7cc9ee6
Author: Mike McCandless <mi...@apache.org>
Authored: Tue Sep 19 14:23:08 2017 -0400
Committer: Mike McCandless <mi...@apache.org>
Committed: Tue Sep 19 14:23:08 2017 -0400
----------------------------------------------------------------------
.../src/java/org/apache/lucene/analysis/synonym/SynonymMap.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4e559771/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
index 7371e23..97db1de 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
@@ -321,7 +321,8 @@ public class SynonymMap {
throw new IllegalArgumentException("term: " + text + " analyzed to a zero-length token");
}
if (posIncAtt.getPositionIncrement() != 1) {
- throw new IllegalArgumentException("term: " + text + " analyzed to a token with posinc != 1");
+ throw new IllegalArgumentException("term: " + text + " analyzed to a token (" + termAtt +
+ ") with position increment != 1 (got: " + posIncAtt.getPositionIncrement() + ")");
}
reuse.grow(reuse.length() + length + 1); /* current + word + separator */
int end = reuse.length();