You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2020/10/14 07:02:57 UTC
[lucene-solr] 02/02: SOLR-14930: Deprecate rulebased replica
placement starategy
This is an automated email from the ASF dual-hosted git repository.
noble pushed a commit to branch jira/solr14930
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 96966da0cc2f1f343c42640c287e32168b2a74bd
Author: Noble Paul <no...@gmail.com>
AuthorDate: Wed Oct 14 18:02:24 2020 +1100
SOLR-14930: Deprecate rulebased replica placement starategy
---
.../apache/solr/cloud/rule/ImplicitSnitchTest.java | 244 ---------------
.../org/apache/solr/cloud/rule/RuleEngineTest.java | 323 --------------------
.../test/org/apache/solr/cloud/rule/RulesTest.java | 328 ---------------------
3 files changed, 895 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java
deleted file mode 100644
index 175c373..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.rule;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.google.common.collect.Sets;
-import org.apache.lucene.util.TestRuleLimitSysouts;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
-import org.apache.solr.common.cloud.rule.SnitchContext;
-import org.apache.zookeeper.KeeperException;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import static org.hamcrest.core.Is.is;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.when;
-
-@TestRuleLimitSysouts.Limit(bytes=32000)
-public class ImplicitSnitchTest extends SolrTestCaseJ4 {
-
- private ImplicitSnitch snitch;
- private SnitchContext context;
-
- private static final String IP_1 = "ip_1";
- private static final String IP_2 = "ip_2";
- private static final String IP_3 = "ip_3";
- private static final String IP_4 = "ip_4";
-
- @Before
- public void beforeImplicitSnitchTest() {
- snitch = new ImplicitSnitch();
- context = new ServerSnitchContext(null, null, new HashMap<>(),null);
- }
-
-
- @Test
- public void testGetTags_withAllIPv4RequestedTags_with_omitted_zeros_returns_four_tags() throws Exception {
- String node = "5:8983_solr";
-
- snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(4));
- assertThat(tags.get(IP_1), is("5"));
- assertThat(tags.get(IP_2), is("0"));
- assertThat(tags.get(IP_3), is("0"));
- assertThat(tags.get(IP_4), is("0"));
- }
-
-
- @Test
- public void testGetTags_withAllIPv4RequestedTags_returns_four_tags() throws Exception {
- String node = "192.168.1.2:8983_solr";
-
- snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(4));
- assertThat(tags.get(IP_1), is("2"));
- assertThat(tags.get(IP_2), is("1"));
- assertThat(tags.get(IP_3), is("168"));
- assertThat(tags.get(IP_4), is("192"));
- }
-
- @Test
- public void testGetTags_withIPv4RequestedTags_ip2_and_ip4_returns_two_tags() throws Exception {
- String node = "192.168.1.2:8983_solr";
-
- SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
- snitch.getTags(node, Sets.newHashSet(IP_2, IP_4), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(2));
- assertThat(tags.get(IP_2), is("1"));
- assertThat(tags.get(IP_4), is("192"));
- }
-
- @Test
- @Ignore("SOLR-12360: local DNS resolver may return some address for a host named '192.168.1.2.1'")
- public void testGetTags_with_wrong_ipv4_format_ip_returns_nothing() throws Exception {
- String node = "192.168.1.2.1:8983_solr";
-
- SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
- snitch.getTags(node, Sets.newHashSet(IP_1), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(0));
- }
-
-
- @Test
- public void testGetTags_with_correct_ipv6_format_ip_returns_nothing() throws Exception {
- String node = "[0:0:0:0:0:0:0:1]:8983_solr";
-
- SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
- snitch.getTags(node, Sets.newHashSet(IP_1), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(0)); //This will fail when IPv6 is implemented
- }
-
-
- @Test
- public void testGetTags_withEmptyRequestedTag_returns_nothing() throws Exception {
- String node = "192.168.1.2:8983_solr";
-
- snitch.getTags(node, Sets.newHashSet(), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(0));
- }
-
-
- @Test
- public void testGetTags_withAllHostNameRequestedTags_returns_all_Tags() throws Exception {
- SolrTestCaseJ4.assumeWorkingMockito();
-
- String node = "serv01.dc01.london.uk.apache.org:8983_solr";
-
- SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
- //We need mocking here otherwise, we would need proper DNS entry for this test to pass
- ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
- when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
-
- mockedSnitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(4));
- assertThat(tags.get(IP_1), is("13"));
- assertThat(tags.get(IP_2), is("12"));
- assertThat(tags.get(IP_3), is("11"));
- assertThat(tags.get(IP_4), is("10"));
- }
-
- @Test
- public void testGetTags_withHostNameRequestedTag_ip3_returns_1_tag() throws Exception {
- SolrTestCaseJ4.assumeWorkingMockito();
-
- String node = "serv01.dc01.london.uk.apache.org:8983_solr";
-
- SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
- //We need mocking here otherwise, we would need proper DNS entry for this test to pass
- ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
- when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
- mockedSnitch.getTags(node, Sets.newHashSet(IP_3), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(1));
- assertThat(tags.get(IP_3), is("11"));
- }
-
- @Test
- public void testGetTags_withHostNameRequestedTag_ip99999_returns_nothing() throws Exception {
- SolrTestCaseJ4.assumeWorkingMockito();
-
- String node = "serv01.dc01.london.uk.apache.org:8983_solr";
-
- SnitchContext context = new ServerSnitchContext(null, node, new HashMap<>(),null);
- //We need mocking here otherwise, we would need proper DNS entry for this test to pass
- ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
- when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
- mockedSnitch.getTags(node, Sets.newHashSet("ip_99999"), context);
-
- Map<String, Object> tags = context.getTags();
- assertThat(tags.entrySet().size(), is(0));
- }
-
- @Test
- public void testIsKnownTag_ip1() throws Exception {
- assertFalse(snitch.isKnownTag("ip_0"));
- assertTrue(snitch.isKnownTag(IP_1));
- assertTrue(snitch.isKnownTag(IP_2));
- assertTrue(snitch.isKnownTag(IP_3));
- assertTrue(snitch.isKnownTag(IP_4));
- assertFalse(snitch.isKnownTag("ip_5"));
- }
-
- @Test
- public void testExceptions() throws Exception {
- ImplicitSnitch implicitSnitch = new ImplicitSnitch();
- ServerSnitchContext noNodeExceptionSnitch = new ServerSnitchContext(null, null, new HashMap<>(), null) {
- @Override
- @SuppressWarnings({"rawtypes"})
- public Map getZkJson(String path) throws KeeperException, InterruptedException {
- throw new KeeperException.NoNodeException();
- }
- };
- implicitSnitch.getTags("", Collections.singleton(ImplicitSnitch.ROLE), noNodeExceptionSnitch);
- @SuppressWarnings({"rawtypes"})
- Map map = (Map) noNodeExceptionSnitch.retrieve(ZkStateReader.ROLES); // todo it the key really supposed to /roles.json?
- assertNotNull(map);
- assertEquals(0, map.size());
-
- implicitSnitch.getTags("", Collections.singleton(ImplicitSnitch.NODEROLE), noNodeExceptionSnitch);
- map = (Map) noNodeExceptionSnitch.retrieve(ZkStateReader.ROLES); // todo it the key really supposed to /roles.json?
- assertNotNull(map);
- assertEquals(0, map.size());
-
- ServerSnitchContext keeperExceptionSnitch = new ServerSnitchContext(null, null, new HashMap<>(), null) {
- @Override
- @SuppressWarnings({"rawtypes"})
- public Map getZkJson(String path) throws KeeperException, InterruptedException {
- throw new KeeperException.ConnectionLossException();
- }
- };
- expectThrows(SolrException.class, KeeperException.ConnectionLossException.class, () -> implicitSnitch.getTags("", Collections.singleton(ImplicitSnitch.ROLE), keeperExceptionSnitch));
- expectThrows(SolrException.class, KeeperException.ConnectionLossException.class, () -> implicitSnitch.getTags("", Collections.singleton(ImplicitSnitch.NODEROLE), keeperExceptionSnitch));
-
- ServerSnitchContext remoteExceptionSnitch = new ServerSnitchContext(null, null, new HashMap<>(), null) {
-
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- throw new RuntimeException();
- }
- };
- expectThrows(SolrException.class, RuntimeException.class, () -> implicitSnitch.getTags("", Collections.singleton(ImplicitSnitch.CORES), remoteExceptionSnitch));
- expectThrows(SolrException.class, RuntimeException.class, () -> implicitSnitch.getTags("", Collections.singleton(ImplicitSnitch.DISK), remoteExceptionSnitch));
- expectThrows(SolrException.class, RuntimeException.class, () -> implicitSnitch.getTags("", Collections.singleton(ImplicitSnitch.SYSPROP + "xyz"), remoteExceptionSnitch));
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java
deleted file mode 100644
index 0d05e3e..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/rule/RuleEngineTest.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.rule;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.cloud.DelegatingCloudManager;
-import org.apache.solr.client.solrj.cloud.NodeStateProvider;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ReplicaPosition;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.rule.Snitch;
-import org.apache.solr.common.cloud.rule.SnitchContext;
-import org.apache.solr.common.util.Utils;
-import org.junit.Test;
-
-import static java.util.Collections.singletonList;
-import static org.apache.solr.cloud.rule.Rule.parseRule;
-import static org.apache.solr.common.util.Utils.makeMap;
-
-public class RuleEngineTest extends SolrTestCaseJ4{
- @Test
- @SuppressWarnings({"unchecked", "rawtypes"})
- public void testPlacement2(){
-
-
- String s = "{" +
- " '127.0.0.1:49961_':{" +
- " 'node':'127.0.0.1:49961_'," +
- " 'freedisk':992," +
- " 'cores':1}," +
- " '127.0.0.1:49955_':{" +
- " 'node':'127.0.0.1:49955_'," +
- " 'freedisk':992," +
- " 'cores':1}," +
- " '127.0.0.1:49952_':{" +
- " 'node':'127.0.0.1:49952_'," +
- " 'freedisk':992," +
- " 'cores':1}," +
- " '127.0.0.1:49947_':{" +
- " 'node':'127.0.0.1:49947_'," +
- " 'freedisk':992," +
- " 'cores':1}," +
- " '127.0.0.1:49958_':{" +
- " 'node':'127.0.0.1:49958_'," +
- " 'freedisk':992," +
- " 'cores':1}}";
- MockSnitch.nodeVsTags = (Map) Utils.fromJSON(s.getBytes(StandardCharsets.UTF_8));
- Map shardVsReplicaCount = makeMap("shard1", 2, "shard2", 2);
-
- List<Rule> rules = parseRules("[{'cores':'<4'}, {" +
- "'replica':'1',shard:'*','node':'*'}," +
- " {'freedisk':'>1'}]");
-
- Map<ReplicaPosition, String> mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null ).getNodeMappings();
- assertNotNull(mapping);
-
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null ).getNodeMappings();
- assertNotNull(mapping);
-
- rules = parseRules("[{role:'!overseer'}, {'freedisk':'>1'}]" );
- Map<String, Object> snitchSession = new HashMap<>();
- List<String> preferredOverseerNodes = ImmutableList.of("127.0.0.1:49947_", "127.0.0.1:49952_");
- ReplicaAssigner replicaAssigner = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null) {
-
- @Override
- protected SnitchContext getSnitchCtx(String node, SnitchInfoImpl info, SolrCloudManager cloudManager) {
- return new ServerSnitchContext(info, node, snitchSession,cloudManager){
- @Override
- @SuppressWarnings({"rawtypes"})
- public Map getZkJson(String path) {
- if(ZkStateReader.ROLES.equals(path)){
- return Collections.singletonMap("overseer", preferredOverseerNodes);
- }
- return null;
- }
-
- };
- }
-
- };
- mapping = replicaAssigner.getNodeMappings();
- assertNotNull(mapping);
-
- for (String nodeName : mapping.values()) {
- assertFalse(preferredOverseerNodes.contains(nodeName));
- }
-
- }
-
- @SuppressWarnings({"unchecked", "rawtypes"})
- public void testPlacement3(){
- String s = "{" +
- " '127.0.0.1:49961_':{" +
- " 'node':'127.0.0.1:49961_'," +
- " 'freedisk':992," +
- " 'cores':1}," +
- " '127.0.0.2:49955_':{" +
- " 'node':'127.0.0.1:49955_'," +
- " 'freedisk':995," +
- " 'cores':1}," +
- " '127.0.0.3:49952_':{" +
- " 'node':'127.0.0.1:49952_'," +
- " 'freedisk':990," +
- " 'cores':1}," +
- " '127.0.0.1:49947_':{" +
- " 'node':'127.0.0.1:49947_'," +
- " 'freedisk':980," +
- " 'cores':1}," +
- " '127.0.0.2:49958_':{" +
- " 'node':'127.0.0.1:49958_'," +
- " 'freedisk':970," +
- " 'cores':1}}";
- MockSnitch.nodeVsTags = (Map) Utils.fromJSON(s.getBytes(StandardCharsets.UTF_8));
- //test not
- List<Rule> rules = parseRules(
- "[{cores:'<4'}, " +
- "{replica:'1',shard:'*',node:'*'}," +
- "{node:'!127.0.0.1:49947_'}," +
- "{freedisk:'>1'}]");
- Map shardVsReplicaCount = makeMap("shard1", 2, "shard2", 2);
- Map<ReplicaPosition, String> mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null).getNodeMappings();
- assertNotNull(mapping);
- assertFalse(mapping.containsValue("127.0.0.1:49947_"));
-
- rules = parseRules(
- "[{cores:'<4'}, " +
- "{replica:'1',node:'*'}," +
- "{freedisk:'>980'}]");
- shardVsReplicaCount = makeMap("shard1", 2, "shard2", 2);
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null).getNodeMappings0();
- assertNull(mapping);
-
-
- rules = parseRules(
- "[{cores:'<4'}, " +
- "{replica:'1',node:'*'}," +
- "{freedisk:'>980~'}]");
- shardVsReplicaCount = makeMap("shard1", 2, "shard2", 2);
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null).getNodeMappings0();
- assertNotNull(mapping);
- assertFalse(mapping.containsValue("127.0.0.2:49958_"));
-
- rules = parseRules(
- "[{cores:'<4'}, " +
- "{replica:'1',shard:'*',host:'*'}]"
- );
- shardVsReplicaCount = makeMap("shard1", 2, "shard2", 2);
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null).getNodeMappings();
- assertNotNull(mapping);
-
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, Collections.emptyList(),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), new DelegatingCloudManager(null){
- @Override
- public NodeStateProvider getNodeStateProvider() {
- return new NodeStateProvider() {
- @Override
- public void close() throws IOException { }
-
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- return (Map<String, Object>) MockSnitch.nodeVsTags.get(node);
- }
-
- @Override
- public Map<String, Map<String, List<Replica>>> getReplicaInfo(String node, Collection<String> keys) {
- return null;
- }
- };
- }
- }, null).getNodeMappings();
- assertNotNull(mapping);
-
-
-
- rules = parseRules(
- "[{cores:'<4'}, " +
- "{replica:'1',shard:'**',host:'*'}]"
- );
- shardVsReplicaCount = makeMap("shard1", 2, "shard2", 2);
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null).getNodeMappings0();
- assertNull(mapping);
-
- rules = parseRules(
- "[{cores:'<4'}, " +
- "{replica:'1~',shard:'**',host:'*'}]"
- );
- shardVsReplicaCount = makeMap("shard1", 2, "shard2", 2);
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null).getNodeMappings();
- assertNotNull(mapping);
-
-
- }
-
- @SuppressWarnings({"rawtypes"})
- private List<Rule> parseRules(String s) {
-
- List maps = (List) Utils.fromJSON(s.getBytes(StandardCharsets.UTF_8));
-
- List<Rule> rules = new ArrayList<>();
- for (Object map : maps) rules.add(new Rule((Map) map));
- return rules;
- }
-
- @Test
- @SuppressWarnings({"unchecked", "rawtypes"})
- public void testPlacement() throws Exception {
- String rulesStr = "rack:*,replica:<2";
- List<Rule> rules = parse(Arrays.asList(rulesStr));
- Map shardVsReplicaCount = makeMap("shard1", 3, "shard2", 3);
- Map nodeVsTags = makeMap(
- "node1:80", makeMap("rack", "178"),
- "node2:80", makeMap("rack", "179"),
- "node3:80", makeMap("rack", "180"),
- "node4:80", makeMap("rack", "181"),
- "node5:80", makeMap("rack", "182")
- );
- MockSnitch.nodeVsTags = nodeVsTags;
- Map<ReplicaPosition, String> mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null, null).getNodeMappings0();
- assertNull(mapping);
- rulesStr = "rack:*,replica:<2~";
- rules = parse(Arrays.asList(rulesStr));
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null ,null).getNodeMappings();
- assertNotNull(mapping);
-
- rulesStr = "rack:*,shard:*,replica:<2";//for each shard there can be a max of 1 replica
- rules = parse(Arrays.asList(rulesStr));
- mapping = new ReplicaAssigner(
- rules,
- shardVsReplicaCount, singletonList(MockSnitch.class.getName()),
- new HashMap(), new ArrayList<>(MockSnitch.nodeVsTags.keySet()), null,null ).getNodeMappings();
- assertNotNull(mapping);
- }
-
- public static class MockSnitch extends Snitch {
- @SuppressWarnings({"rawtypes"})
- static Map nodeVsTags = Collections.emptyMap();
-
- @Override
- @SuppressWarnings({"unchecked"})
- public void getTags(String solrNode, Set<String> requestedTags, SnitchContext ctx) {
- ctx.getTags().putAll((Map<? extends String, ?>) nodeVsTags.get(solrNode));
- }
-
- @Override
- public boolean isKnownTag(String tag) {
- @SuppressWarnings({"rawtypes"})
- Map next = (Map) nodeVsTags.values().iterator().next();
- return next.containsKey(tag);
- }
- }
-
- public static List<Rule> parse(List<String> rules) throws IOException {
- assert rules != null && !rules.isEmpty();
- ArrayList<Rule> result = new ArrayList<>();
- for (String s : rules) {
- if (s == null || s.trim().isEmpty()) continue;
- result.add(new Rule(parseRule(s)));
- }
- return result;
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
deleted file mode 100644
index e9586a0..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.rule;
-
-import java.lang.invoke.MethodHandles;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.GenericSolrRequest;
-import org.apache.solr.client.solrj.response.SimpleSolrResponse;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.junit.After;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
-import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
-import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
-import static org.junit.matchers.JUnitMatchers.containsString;
-
-@LuceneTestCase.Slow
-public class RulesTest extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(5)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @org.junit.Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- @After
- public void removeCollections() throws Exception {
- cluster.deleteAllCollections();
- }
-
- @Test
- public void doIntegrationTest() throws Exception {
- assertEquals("Sanity Check: someone changed the cluster; " +
- "test logic requires specific number of jetty nodes",
- 5, cluster.getJettySolrRunners().size());
-
- final long minGB = (random().nextBoolean() ? 1 : 0);
- final Path toTest = Paths.get("").toAbsolutePath();
- assumeTrue("doIntegrationTest needs minGB="+minGB+" usable disk space",
- ImplicitSnitch.getUsableSpaceInGB(toTest) > minGB);
-
- String rulesColl = "rulesColl";
- CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
- .setRule("cores:<4", "node:*,replica:<2", "freedisk:>"+minGB)
- .setSnitch("class:ImplicitSnitch")
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(rulesColl, 1, 2);
-
- DocCollection rulesCollection = getCollectionState(rulesColl);
-
- @SuppressWarnings({"rawtypes"})
- List list = (List) rulesCollection.get("rule");
- assertEquals(3, list.size());
- assertEquals ( "<4", ((Map)list.get(0)).get("cores"));
- assertEquals("<2", ((Map) list.get(1)).get("replica"));
- assertEquals(">"+minGB, ((Map) list.get(2)).get("freedisk"));
- list = (List) rulesCollection.get("snitch");
- assertEquals(1, list.size());
- assertEquals ( "ImplicitSnitch", ((Map)list.get(0)).get("class"));
-
- CollectionAdminRequest.createShard(rulesColl, "shard2").process(cluster.getSolrClient());
- CollectionAdminRequest.addReplicaToShard(rulesColl, "shard2").process(cluster.getSolrClient());
-
- waitForState("Should have found shard1 w/2active replicas + shard2 w/1active replica",
- rulesColl, (liveNodes, collection) -> {
- // short circut if collection is deleted
- // or we don't yet have the correct number of slices
- if (null == collection || 2 != collection.getSlices().size()) {
- return false;
- }
- final Set<String> replicaNodes = new HashSet<>();
- for (Slice slice : collection.getSlices()) {
- // short circut if our slice isn't active
- if (Slice.State.ACTIVE != slice.getState()) {
- return false;
- }
- if (slice.getName().equals("shard1")) {
- // for shard1, we should have 2 fully live replicas
- final List<Replica> liveReplicas = slice.getReplicas
- ((r) -> r.isActive(liveNodes));
- if (2 != liveReplicas.size()) {
- return false;
- }
- replicaNodes.addAll(liveReplicas.stream().map
- (Replica::getNodeName).collect(Collectors.toList()));
- } else if (slice.getName().equals("shard2")) {
- // for shard2, we should have 3 fully live replicas
- final List<Replica> liveReplicas = slice.getReplicas
- ((r) -> r.isActive(liveNodes));
- if (3 != liveReplicas.size()) {
- return false;
- }
- replicaNodes.addAll(liveReplicas.stream().map
- (Replica::getNodeName).collect(Collectors.toList()));
- } else {
- // WTF?
- return false;
- }
- }
- // now sanity check that the rules were *obeyed* and
- // each replica is on a unique node
- return 5 == replicaNodes.size();
- });
-
- // adding an additional replica should fail since our rule says at most one replica
- // per node, and we know every node already has one replica
- expectedException.expect(BaseHttpSolrClient.RemoteSolrException.class);
- expectedException.expectMessage(containsString("Could not identify nodes matching the rules"));
- CollectionAdminRequest.addReplicaToShard(rulesColl, "shard2").process(cluster.getSolrClient());
-
- }
-
- @Test
- public void testPortRule() throws Exception {
-
- JettySolrRunner jetty = cluster.getRandomJetty(random());
- String port = Integer.toString(jetty.getLocalPort());
-
- String rulesColl = "portRuleColl";
- CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
- .setRule("port:" + port)
- .setSnitch("class:ImplicitSnitch")
- .process(cluster.getSolrClient());
-
- waitForState("Collection should have followed port rule w/ImplicitSnitch, not cluster policy",
- rulesColl, (liveNodes, rulesCollection) -> {
- // first sanity check that the collection exists & the rules/snitch are listed
- if (null == rulesCollection) {
- return false;
- } else {
- @SuppressWarnings({"rawtypes"})
- List list = (List) rulesCollection.get("rule");
- if (null == list || 1 != list.size()) {
- return false;
- }
- if (! port.equals(((Map) list.get(0)).get("port"))) {
- return false;
- }
- list = (List) rulesCollection.get("snitch");
- if (null == list || 1 != list.size()) {
- return false;
- }
- if (! "ImplicitSnitch".equals(((Map)list.get(0)).get("class"))) {
- return false;
- }
- }
- if (2 != rulesCollection.getReplicas().size()) {
- return false;
- }
- // now sanity check that the rules were *obeyed*
- return rulesCollection.getReplicas().stream().allMatch
- (replica -> (replica.getNodeName().contains(port) &&
- replica.isActive(liveNodes)));
- });
- }
-
- @Test
- @SuppressWarnings({"unchecked"})
- public void testHostFragmentRule() throws Exception {
-
- String rulesColl = "hostFragment";
-
- JettySolrRunner jetty = cluster.getRandomJetty(random());
- String host = jetty.getBaseUrl().getHost();
- String[] ipFragments = host.split("\\.");
- String ip_1 = ipFragments[ipFragments.length - 1];
- String ip_2 = ipFragments[ipFragments.length - 2];
-
- CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
- .setRule("ip_2:" + ip_2, "ip_1:" + ip_1)
- .setSnitch("class:ImplicitSnitch")
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(rulesColl, 1, 2);
-
- DocCollection rulesCollection = getCollectionState(rulesColl);
- @SuppressWarnings({"rawtypes"})
- List<Map> list = (List<Map>) rulesCollection.get("rule");
- assertEquals(2, list.size());
- assertEquals(ip_2, list.get(0).get("ip_2"));
- assertEquals(ip_1, list.get(1).get("ip_1"));
-
- list = (List) rulesCollection.get("snitch");
- assertEquals(1, list.size());
- assertEquals("ImplicitSnitch", list.get(0).get("class"));
-
- }
-
-
- @Test
- public void testHostFragmentRuleThrowsExceptionWhenIpDoesNotMatch() throws Exception {
-
- String rulesColl = "ipRuleColl";
-
- JettySolrRunner jetty = cluster.getRandomJetty(random());
- String host = jetty.getBaseUrl().getHost();
- String[] ipFragments = host.split("\\.");
- String ip_1 = ipFragments[ipFragments.length - 1];
- String ip_2 = ipFragments[ipFragments.length - 2];
-
- expectedException.expect(BaseHttpSolrClient.RemoteSolrException.class);
- expectedException.expectMessage(containsString("ip_1"));
-
- CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
- .setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999")
- .setSnitch("class:ImplicitSnitch")
- .process(cluster.getSolrClient());
-
- }
-
- @Test
- public void testInvokeApi() throws Exception {
- JettySolrRunner jetty = cluster.getRandomJetty(random());
- try (SolrClient client = getHttpSolrClient(jetty.getBaseUrl().toString())) {
- GenericSolrRequest req = new GenericSolrRequest(GET, "/____v2/node/invoke", new ModifiableSolrParams()
- .add("class", ImplicitSnitch.class.getName())
- .add("cores", "1")
- .add("freedisk", "1")
- );
- SimpleSolrResponse rsp = req.process(client);
- assertNotNull(((Map) rsp.getResponse().get(ImplicitSnitch.class.getName())).get("cores"));
- assertNotNull(((Map) rsp.getResponse().get(ImplicitSnitch.class.getName())).get("freedisk"));
- }
- }
-
-
- @Test
- public void testModifyColl() throws Exception {
- final Path toTest = Paths.get("").toAbsolutePath();
-
- final long minGB1 = (random().nextBoolean() ? 1 : 0);
- final long minGB2 = 5;
- assumeTrue("testModifyColl needs minGB1="+minGB1+" usable disk space",
- ImplicitSnitch.getUsableSpaceInGB(toTest) > minGB1);
- assumeTrue("testModifyColl needs minGB2="+minGB2+" usable disk space",
- ImplicitSnitch.getUsableSpaceInGB(toTest) > minGB2);
-
- String rulesColl = "modifyColl";
- CollectionAdminRequest.createCollection(rulesColl, "conf", 1, 2)
- .setRule("cores:<4", "node:*,replica:1", "freedisk:>" + minGB1)
- .setSnitch("class:ImplicitSnitch")
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(rulesColl, 1, 2);
-
-
- // TODO: Make a MODIFYCOLLECTION SolrJ class
- ModifiableSolrParams p = new ModifiableSolrParams();
- p.add("collection", rulesColl);
- p.add("action", "MODIFYCOLLECTION");
- p.add("rule", "cores:<5");
- p.add("rule", "node:*,replica:1");
- p.add("rule", "freedisk:>"+minGB2);
- cluster.getSolrClient().request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
-
- waitForState("Should have found updated rules in DocCollection",
- rulesColl, (liveNodes, rulesCollection) -> {
- if (null == rulesCollection) {
- return false;
- }
- @SuppressWarnings({"rawtypes"})
- List list = (List) rulesCollection.get("rule");
- if (null == list || 3 != list.size()) {
- return false;
- }
- if (! "<5".equals(((Map) list.get(0)).get("cores"))) {
- return false;
- }
- if (! "1".equals(((Map) list.get(1)).get("replica"))) {
- return false;
- }
- if (! (">"+minGB2).equals(((Map) list.get(2)).get("freedisk"))) {
- return false;
- }
- list = (List) rulesCollection.get("snitch");
- if (null == list || 1 != list.size()) {
- return false;
- }
- if (! "ImplicitSnitch".equals(((Map) list.get(0)).get("class"))) {
- return false;
- }
- return true;
- });
-
- }
-}