You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/06/07 14:11:52 UTC
[48/50] [abbrv] lucene-solr:feature/autoscaling: Completing merge
from master
Completing merge from master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/2d540259
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/2d540259
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/2d540259
Branch: refs/heads/feature/autoscaling
Commit: 2d540259dc4455850ee4814f8bc2784949272298
Parents: 6a8768e
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Tue Jun 6 19:32:03 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Tue Jun 6 19:32:03 2017 +0530
----------------------------------------------------------------------
.../org/apache/solr/cloud/ZkController.java | 4 +-
.../cloud/autoscaling/AutoScalingHandler.java | 2 +
.../apache/solr/metrics/SolrMetricReporter.java | 11 ----
.../org/apache/solr/cloud/ShardSplitTest.java | 7 ---
.../autoscaling/AutoScalingHandlerTest.java | 34 +++++------
.../org/apache/solr/cloud/autoscaling/Cell.java | 60 ++++++++++++++++++--
.../apache/solr/common/cloud/ZkStateReader.java | 2 +
.../solr/common/params/CollectionParams.java | 1 -
.../apache/solr/util/ReadOnlyCoresLocator.java | 6 --
9 files changed, 79 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/core/src/java/org/apache/solr/cloud/ZkController.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index b859599..086c150 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -665,6 +665,8 @@ public class ZkController {
cmdExecutor.ensureExists(ZkStateReader.LIVE_NODES_ZKNODE, zkClient);
cmdExecutor.ensureExists(ZkStateReader.COLLECTIONS_ZKNODE, zkClient);
cmdExecutor.ensureExists(ZkStateReader.ALIASES, zkClient);
+ cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH, zkClient);
+ cmdExecutor.ensureExists(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH, zkClient);
byte[] emptyJson = "{}".getBytes(StandardCharsets.UTF_8);
cmdExecutor.ensureExists(ZkStateReader.CLUSTER_STATE, emptyJson, CreateMode.PERSISTENT, zkClient);
cmdExecutor.ensureExists(ZkStateReader.SOLR_SECURITY_CONF_PATH, emptyJson, CreateMode.PERSISTENT, zkClient);
@@ -1842,7 +1844,7 @@ public class ZkController {
}
}
- CoreContainer getCoreContainer() {
+ public CoreContainer getCoreContainer() {
return cc;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
index ff101f3..f1effd3 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/AutoScalingHandler.java
@@ -24,7 +24,9 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/core/src/java/org/apache/solr/metrics/SolrMetricReporter.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/metrics/SolrMetricReporter.java b/solr/core/src/java/org/apache/solr/metrics/SolrMetricReporter.java
index 78931a2..222377b 100644
--- a/solr/core/src/java/org/apache/solr/metrics/SolrMetricReporter.java
+++ b/solr/core/src/java/org/apache/solr/metrics/SolrMetricReporter.java
@@ -86,17 +86,6 @@ public abstract class SolrMetricReporter implements Closeable, PluginInfoInitial
}
/**
- * Enable reporting, defaults to true. Implementations should check this flag in
- * {@link #validate()} and accordingly enable or disable reporting.
- * @param enabled enable, defaults to true when null or not set.
- */
- public void setEnabled(Boolean enabled) {
- if (enabled != null) {
- this.enabled = enabled;
- }
- }
-
- /**
* Get the effective {@link PluginInfo} instance that was used for
* initialization of this plugin.
* @return plugin info, or null if not yet initialized.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
index 4de011b..e23a8bf 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
@@ -86,12 +86,6 @@ public class ShardSplitTest extends BasicDistributedZkTest {
useFactory(null);
}
- //TODO for now, onlyLeaderIndexes do not work with ShardSplitTest
- @Override
- protected int getRealtimeReplicas() {
- return -1;
- }
-
@Test
public void test() throws Exception {
@@ -1031,4 +1025,3 @@ public class ShardSplitTest extends BasicDistributedZkTest {
return client;
}
}
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 2f02df9..25e42eb 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -76,11 +76,11 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"\t}\n" +
"}";
// these should be no-ops because there are no triggers, and it should succeed
- SolrRequest req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, suspendEachCommand);
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendEachCommand);
NamedList<Object> response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
assertEquals(response.get("changed").toString(), "[]");
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, resumeEachCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeEachCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
assertEquals(response.get("changed").toString(), "[]");
@@ -91,7 +91,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'event' : 'nodeLost'," +
"'waitFor' : '10m'," +
"'enabled' : true}}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -103,7 +103,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'enabled' : true" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -112,7 +112,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"\t\t\"name\" : \"node_lost_trigger\"\n" +
"\t}\n" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, suspendTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
assertEquals(response.get("changed").toString(), "[node_lost_trigger]");
@@ -136,7 +136,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'name' : '" + Policy.EACH + "'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, suspendTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
List<String> changed = (List<String>)response.get("changed");
@@ -159,7 +159,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'name' : 'node_added_trigger'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, resumeTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
changed = (List<String>)response.get("changed");
@@ -182,7 +182,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'name' : '" + Policy.EACH + "'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, resumeTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
changed = (List<String>)response.get("changed");
@@ -206,7 +206,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'timeout' : '1h'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, suspendTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
changed = (List<String>)response.get("changed");
@@ -242,7 +242,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'class' : 'solr.LogPlanAction'," +
"'collection' : '.system'" +
"}]}}";
- SolrRequest req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
NamedList<Object> response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
@@ -267,7 +267,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'waitFor' : '20m'," +
"'enabled' : false" +
"}}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
@@ -295,7 +295,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setListenerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
@@ -313,7 +313,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'name' : 'node_lost_trigger'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, removeTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, removeTriggerCommand);
try {
response = solrClient.request(req);
fail("Trying to remove trigger which has listeners registered should have failed");
@@ -326,7 +326,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"\t\t\"name\" : \"xyz\"\n" +
"\t}\n" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, removeListenerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, removeListenerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
@@ -340,7 +340,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'name' : 'node_lost_trigger'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, removeTriggerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, removeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
@@ -359,7 +359,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" +
"}" +
"}";
- req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setListenerCommand);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
try {
response = solrClient.request(req);
fail("Adding a listener on a non-existent trigger should have failed");
@@ -504,7 +504,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
"'waitFor' : '0s'," +
"'enabled' : true" +
"}}";
- SolrRequest req = new AutoScalingRequest(SolrRequest.METHOD.POST, path, setTriggerCommand);
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
NamedList<Object> response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
index d1a3626..3590f52 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Cell.java
@@ -1,7 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.solr.cloud.autoscaling;
-/**
- * Created by shalin on 6/6/17.
- */
-public class Cell {
-}
+import java.io.IOException;
+import java.util.HashMap;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.util.Utils;
+
+class Cell implements MapWriter {
+ final int index;
+ final String name;
+ Object val, approxVal;
+
+ Cell(int index, String name, Object val) {
+ this.index = index;
+ this.name = name;
+ this.val = val;
+ }
+
+ Cell(int index, String name, Object val, Object approxVal) {
+ this.index = index;
+ this.name = name;
+ this.val = val;
+ this.approxVal = approxVal;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, val);
+ }
+
+ @Override
+ public String toString() {
+ return Utils.toJSONString(this.toMap(new HashMap<>()));
+ }
+
+ public Cell copy() {
+ return new Cell(index, name, val, approxVal);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index 1b03534..c1bfd87 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -94,6 +94,8 @@ public class ZkStateReader implements Closeable {
public static final String REJOIN_AT_HEAD_PROP = "rejoinAtHead";
public static final String SOLR_SECURITY_CONF_PATH = "/security.json";
public static final String SOLR_AUTOSCALING_CONF_PATH = "/autoscaling.json";
+ public static final String SOLR_AUTOSCALING_EVENTS_PATH = "/autoscaling/events";
+ public static final String SOLR_AUTOSCALING_TRIGGER_STATE_PATH = "/autoscaling/triggerState";
public static final String REPLICATION_FACTOR = "replicationFactor";
public static final String MAX_SHARDS_PER_NODE = "maxShardsPerNode";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index 285edf5..d79fafa 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -96,7 +96,6 @@ public interface CollectionParams {
CREATESNAPSHOT(true, LockLevel.COLLECTION),
DELETESNAPSHOT(true, LockLevel.COLLECTION),
LISTSNAPSHOTS(false, LockLevel.NONE),
- MOVEREPLICA(false, LockLevel.SHARD),
//only for testing. it just waits for specified time
// these are not exposed via collection API commands
// but the overseer is aware of these tasks
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/2d540259/solr/test-framework/src/java/org/apache/solr/util/ReadOnlyCoresLocator.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/util/ReadOnlyCoresLocator.java b/solr/test-framework/src/java/org/apache/solr/util/ReadOnlyCoresLocator.java
index 3ad3ce2..3d11ff7 100644
--- a/solr/test-framework/src/java/org/apache/solr/util/ReadOnlyCoresLocator.java
+++ b/solr/test-framework/src/java/org/apache/solr/util/ReadOnlyCoresLocator.java
@@ -47,10 +47,4 @@ public abstract class ReadOnlyCoresLocator implements CoresLocator {
// no-op
}
- @Override
- public CoreDescriptor reload(CoreContainer cc, CoreDescriptor cd) {
- return null; // no-op
- }
-
-
}