You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/05/24 14:56:05 UTC
[1/7] lucene-solr:jira/solr-10515: javadocs
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-10515 15d637fe0 -> 7ef545288
javadocs
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cfe5cffd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cfe5cffd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cfe5cffd
Branch: refs/heads/jira/solr-10515
Commit: cfe5cffdddcfb1eb87d4e498e7f24fbaa11f2fd8
Parents: ba670a0
Author: Noble Paul <no...@apache.org>
Authored: Wed May 24 16:26:12 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Wed May 24 16:26:12 2017 +0930
----------------------------------------------------------------------
.../apache/solr/cloud/autoscaling/Policy.java | 29 +++++++++++++++++---
1 file changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cfe5cffd/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index de82fd5..8a1292d 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -46,16 +46,25 @@ import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.stream.Collectors.toList;
+/*The class that reads, parses and applies policies specified in
+ * autoscaling.json
+ *
+ * Create one instance of this class per unique autoscaling.json.
+ * This is immutable and is thread-safe
+ *
+ * Create a fresh new session for each use
+ *
+ */
public class Policy implements MapWriter {
public static final String EACH = "#EACH";
public static final String ANY = "#ANY";
public static final String CLUSTER_POLICY = "cluster-policy";
public static final String CLUSTER_PREFERENCE = "cluster-preferences";
public static final Set<String> GLOBAL_ONLY_TAGS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList("cores")));
- Map<String, List<Clause>> policies = new HashMap<>();
- List<Clause> clusterPolicy;
- List<Preference> clusterPreferences;
- List<String> params = new ArrayList<>();
+ final Map<String, List<Clause>> policies = new HashMap<>();
+ final List<Clause> clusterPolicy;
+ final List<Preference> clusterPreferences;
+ final List<String> params = new ArrayList<>();
public Policy(Map<String, Object> jsonMap) {
@@ -121,6 +130,10 @@ public class Policy implements MapWriter {
}
+ /*This stores the logical state of the system, given a policy and
+ * a cluster state.
+ *
+ */
public class Session implements MapWriter {
final List<String> nodes;
final ClusterDataProvider dataProvider;
@@ -301,6 +314,14 @@ public class Policy implements MapWriter {
}
+ /* A suggester is capable of suggesting a collection operation
+ * given a particular session. Before it suggests a new operation,
+ * it ensures that ,
+ * a) the node that it lightens load on the 'most loaded node' and/or 'lightens load'
+ * on the least loaded node
+ * b) it causes no new violations
+ *
+ */
public static abstract class Suggester {
protected final EnumMap<Hint, Object> hints = new EnumMap<>(Hint.class);
Policy.Session session;
[5/7] lucene-solr:jira/solr-10515: Merge branch 'feature/autoscaling'
into jira/solr-10515
Posted by ab...@apache.org.
Merge branch 'feature/autoscaling' into jira/solr-10515
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a35852e7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a35852e7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a35852e7
Branch: refs/heads/jira/solr-10515
Commit: a35852e726392fbbe971b826780b3c9e4da68828
Parents: 15d637f fe21766
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed May 24 11:12:02 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed May 24 11:12:02 2017 +0200
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../cloud/autoscaling/NodeAddedTrigger.java | 1 -
.../solr/cloud/autoscaling/NodeLostTrigger.java | 1 -
.../cloud/autoscaling/ScheduledTriggers.java | 4 ++
.../autoscaling/TriggerIntegrationTest.java | 6 +--
.../apache/solr/cloud/autoscaling/Policy.java | 57 ++++++++++++++++----
.../solr/cloud/autoscaling/PolicyHelper.java | 8 +--
7 files changed, 56 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a35852e7/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a35852e7/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a35852e7/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a35852e7/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --cc solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index c43fabf,db15f6c..23c1391
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@@ -572,64 -506,8 +571,65 @@@ public class TriggerIntegrationTest ext
@Override
public void init(Map<String, String> args) {
-
+ log.info("TestTriggerAction init");
+ actionCreated.countDown();
}
}
+
+ @Test
+ public void testEventQueue() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger1'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '" + waitForSeconds + "s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestEventQueueAction.class.getName() + "'}]" +
+ "}}";
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ int overseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerLeaderIndex = i;
+ break;
+ }
+ }
+ SolrRequest req = new AutoScalingHandlerTest.AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ if (!actionCreated.await(3, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ // add node to generate the event
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ // we are the first party so we expect index 1
+ int await = actionStarted.await(60, TimeUnit.SECONDS);
+ assertEquals("action started too early", 1, await);
+ // event should be there
+ NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) eventRef.get();
+ assertNotNull(nodeAddedEvent);
+ // but action did not complete yet so the event is still enqueued
+ assertFalse(triggerFired.get());
+ actionStarted.reset();
+ // kill overseer leader
+ cluster.stopJettySolrRunner(overseerLeaderIndex);
+ Thread.sleep(5000);
+ // we are the last party, so we expect index 0
+ await = actionInterrupted.await(3, TimeUnit.SECONDS);
+ assertEquals("action wasn't interrupted", 0, await);
+ // new overseer leader should be elected and run triggers
+ newNode = cluster.startJettySolrRunner();
+ // it should fire again but not complete yet
+ await = actionStarted.await(60, TimeUnit.SECONDS);
+ TriggerEvent replayedEvent = eventRef.get();
+ assertTrue(replayedEvent.getProperty(TriggerEventQueue.ENQUEUE_TIME) != null);
+ assertTrue(replayedEvent.getProperty(TriggerEventQueue.DEQUEUE_TIME) != null);
+ await = actionCompleted.await(10, TimeUnit.SECONDS);
+ assertTrue(triggerFired.get());
+ }
}
[4/7] lucene-solr:jira/solr-10515: Merge remote-tracking branch
'origin/feature/autoscaling' into feature/autoscaling
Posted by ab...@apache.org.
Merge remote-tracking branch 'origin/feature/autoscaling' into feature/autoscaling
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fe217668
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fe217668
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fe217668
Branch: refs/heads/jira/solr-10515
Commit: fe217668cb43ad6e78b135551576e3568efdfaae
Parents: 66a6cff b010ebd
Author: Noble Paul <no...@apache.org>
Authored: Wed May 24 18:38:58 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Wed May 24 18:38:58 2017 +0930
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java | 1 -
.../org/apache/solr/cloud/autoscaling/NodeLostTrigger.java | 1 -
.../org/apache/solr/cloud/autoscaling/ScheduledTriggers.java | 4 ++++
.../apache/solr/cloud/autoscaling/TriggerIntegrationTest.java | 6 +++---
5 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[3/7] lucene-solr:jira/solr-10515: SOLR-10738: TriggerAction is
initialised even if the trigger is never scheduled
Posted by ab...@apache.org.
SOLR-10738: TriggerAction is initialised even if the trigger is never scheduled
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b010ebd5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b010ebd5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b010ebd5
Branch: refs/heads/jira/solr-10515
Commit: b010ebd515a7b97b5efd588d214997e08cb71c95
Parents: cfe5cff
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed May 24 14:38:26 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed May 24 14:38:26 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
.../org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java | 1 -
.../org/apache/solr/cloud/autoscaling/NodeLostTrigger.java | 1 -
.../org/apache/solr/cloud/autoscaling/ScheduledTriggers.java | 4 ++++
.../apache/solr/cloud/autoscaling/TriggerIntegrationTest.java | 6 +++---
5 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b010ebd5/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ecfeedc..5166da7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -107,6 +107,8 @@ Bug Fixes
* SOLR-10714: OverseerTriggerThread does not start triggers on overseer start until autoscaling
config watcher is fired. (shalin)
+* SOLR-10738: TriggerAction is initialised even if the trigger is never scheduled. (shalin)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b010ebd5/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index 5a340be..1754a8b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@ -70,7 +70,6 @@ public class NodeAddedTrigger implements AutoScaling.Trigger<NodeAddedTrigger.No
actions = new ArrayList<>(3);
for (Map<String, String> map : o) {
TriggerAction action = container.getResourceLoader().newInstance(map.get("class"), TriggerAction.class);
- action.init(map);
actions.add(action);
}
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b010ebd5/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 385eca5..8bca93c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -70,7 +70,6 @@ public class NodeLostTrigger implements AutoScaling.Trigger<NodeLostTrigger.Node
actions = new ArrayList<>(3);
for (Map<String, String> map : o) {
TriggerAction action = container.getResourceLoader().newInstance(map.get("class"), TriggerAction.class);
- action.init(map);
actions.add(action);
}
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b010ebd5/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
index 8dfbddf..9c18ac2 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
@@ -143,6 +143,10 @@ public class ScheduledTriggers implements Closeable {
return false;
}
});
+ List<TriggerAction> actions = newTrigger.getActions();
+ for (TriggerAction action : actions) {
+ action.init(newTrigger.getProperties());
+ }
scheduledTrigger.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay(scheduledTrigger, 0, DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS, TimeUnit.SECONDS);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b010ebd5/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index 0ac8183..db15f6c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -473,8 +473,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
public static class TestTriggerAction implements TriggerAction {
public TestTriggerAction() {
- log.info("TestTriggerAction instantiated");
- actionCreated.countDown();
+
}
@Override
@@ -507,7 +506,8 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Override
public void init(Map<String, String> args) {
-
+ log.info("TestTriggerAction init");
+ actionCreated.countDown();
}
}
}
[2/7] lucene-solr:jira/solr-10515: use the hint to add collection to
session
Posted by ab...@apache.org.
use the hint to add collection to session
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/66a6cffd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/66a6cffd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/66a6cffd
Branch: refs/heads/jira/solr-10515
Commit: 66a6cffd6653729439a39d4b8c1bd4d518d49430
Parents: cfe5cff
Author: Noble Paul <no...@apache.org>
Authored: Wed May 24 18:35:54 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Wed May 24 18:35:54 2017 +0930
----------------------------------------------------------------------
.../apache/solr/cloud/autoscaling/Policy.java | 28 +++++++++++++++-----
.../solr/cloud/autoscaling/PolicyHelper.java | 8 +-----
2 files changed, 22 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66a6cffd/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index 8a1292d..614bfa2 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -165,13 +165,7 @@ public class Policy implements MapWriter {
.collect(Collectors.toList());
for (String c : collections) {
- String p = dataProvider.getPolicy(c);
- if (p != null) {
- List<Clause> perCollPolicy = policies.get(p);
- if (perCollPolicy == null)
- throw new RuntimeException(StrUtils.formatString("Policy for collection {0} is {1} . It does not exist", c, p));
- }
- expandedClauses.addAll(mergePolicies(c, policies.getOrDefault(p, emptyList()), clusterPolicy));
+ addClausesForCollection(dataProvider, c);
}
Collections.sort(expandedClauses);
@@ -183,6 +177,16 @@ public class Policy implements MapWriter {
applyRules();
}
+ private void addClausesForCollection(ClusterDataProvider dataProvider, String c) {
+ String p = dataProvider.getPolicy(c);
+ if (p != null) {
+ List<Clause> perCollPolicy = policies.get(p);
+ if (perCollPolicy == null)
+ throw new RuntimeException(StrUtils.formatString("Policy for collection {0} is {1} . It does not exist", c, p));
+ }
+ expandedClauses.addAll(mergePolicies(c, policies.getOrDefault(p, emptyList()), clusterPolicy));
+ }
+
Session copy() {
return new Session(nodes, dataProvider, getMatrixCopy(), expandedClauses, paramsOfInterest);
}
@@ -343,6 +347,16 @@ public class Policy implements MapWriter {
public SolrRequest getOperation() {
if (!isInitialized) {
+ String coll = (String) hints.get(Hint.COLL);
+ if(coll != null){
+ // if this is not a known collection from the existing clusterstate,
+ // then add it
+ if(session.matrix.stream().noneMatch(row -> row.replicaInfo.containsKey(coll))){
+ session.matrix.get(0).replicaInfo.put(coll, new HashMap<>());
+ session.addClausesForCollection(session.dataProvider, coll);
+ Collections.sort(session.expandedClauses);
+ }
+ }
session.applyRules();
originalViolations.addAll(session.getViolations());
this.operation = init();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/66a6cffd/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
index 2012be8..6b199c2 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
@@ -54,13 +54,7 @@ public class PolicyHelper {
@Override
public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- Map<String, Map<String, List<Policy.ReplicaInfo>>> replicaInfo = delegate.getReplicaInfo(node, keys);
- for (String s : optionalPolicyMapping.keySet()) {
- if (!replicaInfo.containsKey(s)) {
- replicaInfo.put(s, new HashMap<>());
- }
- }
- return replicaInfo;
+ return delegate.getReplicaInfo(node, keys);
}
@Override
[6/7] lucene-solr:jira/solr-10515: SOLR-10738: Trigger has an init
method which is called before schedule. Actions are init'ed in this method.
Fixed NodeLostTriggerTest failures.
Posted by ab...@apache.org.
SOLR-10738: Trigger has an init method which is called before schedule. Actions are init'ed in this method. Fixed NodeLostTriggerTest failures.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0214f2ee
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0214f2ee
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0214f2ee
Branch: refs/heads/jira/solr-10515
Commit: 0214f2ee901311aa3c782cf70f715ddc03f0991c
Parents: a35852e
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed May 24 16:46:25 2017 +0530
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed May 24 13:37:48 2017 +0200
----------------------------------------------------------------------
.../org/apache/solr/cloud/autoscaling/AutoScaling.java | 6 ++++++
.../apache/solr/cloud/autoscaling/NodeAddedTrigger.java | 11 +++++++++++
.../apache/solr/cloud/autoscaling/NodeLostTrigger.java | 11 +++++++++++
.../solr/cloud/autoscaling/ScheduledTriggers.java | 5 +----
.../solr/cloud/autoscaling/NodeLostTriggerTest.java | 12 +++++++++++-
.../solr/cloud/autoscaling/TriggerIntegrationTest.java | 2 +-
6 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0214f2ee/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
index cf6a1e4..2236291 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScaling.java
@@ -124,6 +124,12 @@ public class AutoScaling {
/** Restore internal state of this trigger from ZooKeeper. */
void restoreState();
+
+ /**
+ * Called before a trigger is scheduled. Any heavy object creation or initialisation should
+ * be done in this method instead of the Trigger's constructor.
+ */
+ public void init();
}
public static class TriggerFactory implements Closeable {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0214f2ee/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
index c443826..8ebb6d5 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeAddedTrigger.java
@@ -88,6 +88,17 @@ public class NodeAddedTrigger extends TriggerBase {
}
@Override
+ public void init() {
+ List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
+ if (o != null && !o.isEmpty()) {
+ for (int i = 0; i < o.size(); i++) {
+ Map<String, String> map = o.get(i);
+ actions.get(i).init(map);
+ }
+ }
+ }
+
+ @Override
public void setListener(AutoScaling.TriggerListener listener) {
listenerRef.set(listener);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0214f2ee/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
index 981ca3f..525b3c1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/NodeLostTrigger.java
@@ -87,6 +87,17 @@ public class NodeLostTrigger extends TriggerBase {
}
@Override
+ public void init() {
+ List<Map<String, String>> o = (List<Map<String, String>>) properties.get("actions");
+ if (o != null && !o.isEmpty()) {
+ for (int i = 0; i < o.size(); i++) {
+ Map<String, String> map = o.get(i);
+ actions.get(i).init(map);
+ }
+ }
+ }
+
+ @Override
public void setListener(AutoScaling.TriggerListener listener) {
listenerRef.set(listener);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0214f2ee/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
index acc9e97..32e674b 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/ScheduledTriggers.java
@@ -174,10 +174,7 @@ public class ScheduledTriggers implements Closeable {
return false;
}
});
- List<TriggerAction> actions = newTrigger.getActions();
- for (TriggerAction action : actions) {
- action.init(newTrigger.getProperties());
- }
+ newTrigger.init(); // mark as ready for scheduling
scheduledTrigger.scheduledFuture = scheduledThreadPoolExecutor.scheduleWithFixedDelay(scheduledTrigger, 0, DEFAULT_SCHEDULED_TRIGGER_DELAY_SECONDS, TimeUnit.SECONDS);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0214f2ee/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
index e916ab6..3835f2e 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
@@ -196,7 +196,17 @@ public class NodeLostTriggerTest extends SolrCloudTestCase {
NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger", props, container);
trigger.setListener(noFirstRunListener);
trigger.run();
- newNode.stop();
+
+ // stop the newly created node
+ List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
+ for (int i = 0; i < jettySolrRunners.size(); i++) {
+ JettySolrRunner jettySolrRunner = jettySolrRunners.get(i);
+ if (newNode == jettySolrRunner) {
+ cluster.stopJettySolrRunner(i);
+ break;
+ }
+ }
+
trigger.run(); // this run should detect the lost node
trigger.close(); // close the old trigger
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0214f2ee/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index 23c1391..715417a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -375,7 +375,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
NamedList<Object> response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
- if (!actionCreated.await(3, TimeUnit.SECONDS)) {
+ if (!actionCreated.await(10, TimeUnit.SECONDS)) {
fail("The TriggerAction should have been created by now");
}
[7/7] lucene-solr:jira/solr-10515: SOLR-10515 Add unit test for
generating events from saved state. Cleanup.
Posted by ab...@apache.org.
SOLR-10515 Add unit test for generating events from saved state. Cleanup.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7ef54528
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7ef54528
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7ef54528
Branch: refs/heads/jira/solr-10515
Commit: 7ef5452884ab640e2110d82c6df174c337d64c94
Parents: 0214f2e
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed May 24 16:55:27 2017 +0200
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed May 24 16:55:27 2017 +0200
----------------------------------------------------------------------
.../solr/cloud/autoscaling/TriggerBase.java | 3 -
.../autoscaling/TriggerIntegrationTest.java | 140 +++++++++++++------
2 files changed, 98 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7ef54528/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
index 89b3d38..517e051 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerBase.java
@@ -69,11 +69,9 @@ public abstract class TriggerBase implements AutoScaling.Trigger {
byte[] data = Utils.toJSON(map);
// skip saving if identical
if (lastState != null && Arrays.equals(lastState, data)) {
- LOG.debug("--skip saving " + getName());
return;
}
String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + getName();
- LOG.info("--save state: " + path + ": " + state);
try {
if (zkClient.exists(path, true)) {
// update
@@ -101,7 +99,6 @@ public abstract class TriggerBase implements AutoScaling.Trigger {
}
if (data != null) {
Map<String, Object> state = (Map<String, Object>) Utils.fromJSON(data);
- LOG.info("-- restored state of " + path + ": " + state);
setState(state);
lastState = data;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7ef54528/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
index 715417a..ed674da 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerIntegrationTest.java
@@ -23,7 +23,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -61,9 +60,9 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
private static CountDownLatch actionCreated;
private static CountDownLatch triggerFiredLatch;
private static int waitForSeconds = 1;
- private static CyclicBarrier actionStarted;
- private static CyclicBarrier actionInterrupted;
- private static CyclicBarrier actionCompleted;
+ private static CountDownLatch actionStarted;
+ private static CountDownLatch actionInterrupted;
+ private static CountDownLatch actionCompleted;
private static AtomicBoolean triggerFired;
private static AtomicReference<TriggerEvent> eventRef;
@@ -76,15 +75,35 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
.configure();
}
+ private static CountDownLatch getActionCreated() {
+ return actionCreated;
+ }
+
+ private static CountDownLatch getTriggerFiredLatch() {
+ return triggerFiredLatch;
+ }
+
+ private static CountDownLatch getActionStarted() {
+ return actionStarted;
+ }
+
+ private static CountDownLatch getActionInterrupted() {
+ return actionInterrupted;
+ }
+
+ private static CountDownLatch getActionCompleted() {
+ return actionCompleted;
+ }
+
@Before
public void setupTest() throws Exception {
waitForSeconds = 1 + random().nextInt(3);
actionCreated = new CountDownLatch(1);
triggerFiredLatch = new CountDownLatch(1);
triggerFired = new AtomicBoolean(false);
- actionStarted = new CyclicBarrier(2);
- actionInterrupted = new CyclicBarrier(2);
- actionCompleted = new CyclicBarrier(2);
+ actionStarted = new CountDownLatch(1);
+ actionInterrupted = new CountDownLatch(1);
+ actionCompleted = new CountDownLatch(1);
eventRef = new AtomicReference<>();
// clear any persisted auto scaling configuration
Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
@@ -102,7 +121,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Test
public void testTriggerThrottling() throws Exception {
// for this test we want to create two triggers so we must assert that the actions were created twice
- TriggerIntegrationTest.actionCreated = new CountDownLatch(2);
+ actionCreated = new CountDownLatch(2);
// similarly we want both triggers to fire
triggerFiredLatch = new CountDownLatch(2);
@@ -141,13 +160,13 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
JettySolrRunner newNode = cluster.startJettySolrRunner();
- if (!triggerFiredLatch.await(10, TimeUnit.SECONDS)) {
+ if (!triggerFiredLatch.await(20, TimeUnit.SECONDS)) {
fail("Both triggers should have fired by now");
}
// reset shared state
lastActionExecutedAt.set(0);
- TriggerIntegrationTest.actionCreated = new CountDownLatch(2);
+ actionCreated = new CountDownLatch(2);
triggerFiredLatch = new CountDownLatch(2);
setTriggerCommand = "{" +
@@ -212,16 +231,16 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
}
try {
if (lastActionExecutedAt.get() != 0) {
- log.info("last action at " + lastActionExecutedAt.get() + " nano time = " + System.nanoTime());
- if (System.nanoTime() - lastActionExecutedAt.get() < TimeUnit.NANOSECONDS.convert(ScheduledTriggers.DEFAULT_MIN_MS_BETWEEN_ACTIONS - DELTA_MS, TimeUnit.MILLISECONDS)) {
+ log.info("last action at " + lastActionExecutedAt.get() + " time = " + System.currentTimeMillis());
+ if (System.currentTimeMillis() - lastActionExecutedAt.get() < ScheduledTriggers.DEFAULT_MIN_MS_BETWEEN_ACTIONS - DELTA_MS) {
log.info("action executed again before minimum wait time from {}", event.getSource());
fail("TriggerListener was fired before the throttling period");
}
}
if (onlyOnce.compareAndSet(false, true)) {
log.info("action executed from {}", event.getSource());
- lastActionExecutedAt.set(System.nanoTime());
- triggerFiredLatch.countDown();
+ lastActionExecutedAt.set(System.currentTimeMillis());
+ getTriggerFiredLatch().countDown();
} else {
log.info("action executed more than once from {}", event.getSource());
fail("Trigger should not have fired more than once!");
@@ -237,7 +256,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Test
public void testNodeLostTriggerRestoreState() throws Exception {
// for this test we want to update the trigger so we must assert that the actions were created twice
- TriggerIntegrationTest.actionCreated = new CountDownLatch(2);
+ actionCreated = new CountDownLatch(2);
// start a new node
JettySolrRunner newNode = cluster.startJettySolrRunner();
@@ -305,7 +324,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Test
public void testNodeAddedTriggerRestoreState() throws Exception {
// for this test we want to update the trigger so we must assert that the actions were created twice
- TriggerIntegrationTest.actionCreated = new CountDownLatch(2);
+ actionCreated = new CountDownLatch(2);
CloudSolrClient solrClient = cluster.getSolrClient();
waitForSeconds = 5;
@@ -497,10 +516,10 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
try {
if (triggerFired.compareAndSet(false, true)) {
eventRef.set(event);
- if (System.nanoTime() - event.getEventTime() <= TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS)) {
+ if (System.currentTimeMillis() - event.getEventTime() <= TimeUnit.MILLISECONDS.convert(waitForSeconds, TimeUnit.SECONDS)) {
fail("NodeAddedListener was fired before the configured waitFor period");
}
- triggerFiredLatch.countDown();
+ getTriggerFiredLatch().countDown();
} else {
fail("NodeAddedTrigger was fired more than once!");
}
@@ -517,7 +536,8 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Override
public void init(Map<String, String> args) {
-
+ log.info("TestTriggerAction init");
+ actionCreated.countDown();
}
}
@@ -525,7 +545,6 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
public TestEventQueueAction() {
log.info("TestEventQueueAction instantiated");
- actionCreated.countDown();
}
@Override
@@ -541,25 +560,13 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Override
public void process(TriggerEvent event) {
eventRef.set(event);
- try {
- actionStarted.await();
- } catch (InterruptedException | BrokenBarrierException e) {
- throw new RuntimeException("broken barrier", e);
- }
+ getActionStarted().countDown();
try {
Thread.sleep(5000);
triggerFired.compareAndSet(false, true);
- try {
- actionCompleted.await();
- } catch (InterruptedException | BrokenBarrierException e) {
- throw new RuntimeException("broken barrier", e);
- }
+ getActionCompleted().countDown();
} catch (InterruptedException e) {
- try {
- actionInterrupted.await();
- } catch (InterruptedException | BrokenBarrierException e1) {
- throw new RuntimeException("broken barrier", e1);
- }
+ getActionInterrupted().countDown();
return;
}
}
@@ -572,7 +579,7 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
@Override
public void init(Map<String, String> args) {
log.info("TestTriggerAction init");
- actionCreated.countDown();
+ getActionCreated().countDown();
}
}
@@ -607,21 +614,19 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
// add node to generate the event
JettySolrRunner newNode = cluster.startJettySolrRunner();
- // we are the first party so we expect index 1
- int await = actionStarted.await(60, TimeUnit.SECONDS);
- assertEquals("action started too early", 1, await);
+ boolean await = actionStarted.await(60, TimeUnit.SECONDS);
+ assertTrue("action did not start", await);
// event should be there
NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) eventRef.get();
assertNotNull(nodeAddedEvent);
// but action did not complete yet so the event is still enqueued
assertFalse(triggerFired.get());
- actionStarted.reset();
+ actionStarted = new CountDownLatch(1);
// kill overseer leader
cluster.stopJettySolrRunner(overseerLeaderIndex);
Thread.sleep(5000);
- // we are the last party, so we expect index 0
await = actionInterrupted.await(3, TimeUnit.SECONDS);
- assertEquals("action wasn't interrupted", 0, await);
+ assertTrue("action wasn't interrupted", await);
// new overseer leader should be elected and run triggers
newNode = cluster.startJettySolrRunner();
// it should fire again but not complete yet
@@ -632,4 +637,55 @@ public class TriggerIntegrationTest extends SolrCloudTestCase {
await = actionCompleted.await(10, TimeUnit.SECONDS);
assertTrue(triggerFired.get());
}
+
+ @Test
+ public void testEventFromRestoredState() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '10s'," +
+ "'enabled' : true," +
+ "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
+ "}}";
+ SolrRequest req = new AutoScalingHandlerTest.AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ if (!actionCreated.await(10, TimeUnit.SECONDS)) {
+ fail("The TriggerAction should have been created by now");
+ }
+
+ NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
+ String overseerLeader = (String) overSeerStatus.get("leader");
+ int overseerLeaderIndex = 0;
+ for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
+ JettySolrRunner jetty = cluster.getJettySolrRunner(i);
+ if (jetty.getNodeName().equals(overseerLeader)) {
+ overseerLeaderIndex = i;
+ break;
+ }
+ }
+
+ JettySolrRunner newNode = cluster.startJettySolrRunner();
+ boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ // reset
+ triggerFired.set(false);
+ triggerFiredLatch = new CountDownLatch(1);
+ NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) eventRef.get();
+ assertNotNull(nodeAddedEvent);
+ assertEquals("The node added trigger was fired but for a different node",
+ newNode.getNodeName(), nodeAddedEvent.getProperty(NodeAddedTrigger.NodeAddedEvent.NODE_NAME));
+ // add a second node - state of the trigger will change but it won't fire for waitFor sec.
+ JettySolrRunner newNode2 = cluster.startJettySolrRunner();
+ Thread.sleep(10000);
+ // kill overseer leader
+ cluster.stopJettySolrRunner(overseerLeaderIndex);
+ await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
+ assertTrue("The trigger did not fire at all", await);
+ assertTrue(triggerFired.get());
+ }
}