You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2016/02/25 02:57:54 UTC
[34/50] [abbrv] lucene-solr git commit: SOLR-8522: Make it possible
to use ip fragments in replica placement rules , such as ip_1, ip_2 etc
SOLR-8522: Make it possible to use ip fragments in replica placement rules , such as ip_1, ip_2 etc
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/cf964326
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/cf964326
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/cf964326
Branch: refs/heads/jira/SOLR-445
Commit: cf964326309feb7a5a41a3e4f22cad073807a097
Parents: e069d31
Author: Noble Paul <no...@apache.org>
Authored: Tue Feb 23 13:31:08 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Tue Feb 23 13:31:08 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
solr/core/ivy.xml | 2 +
.../apache/solr/cloud/rule/ImplicitSnitch.java | 80 +++++++-
.../solr/cloud/rule/ImplicitSnitchTest.java | 186 +++++++++++++++++++
.../org/apache/solr/cloud/rule/RulesTest.java | 77 ++++++++
5 files changed, 343 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf964326/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index a3e59b4..525e8d5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -153,6 +153,8 @@ New Features
a connection to zookeeper has been lost and there is a possibility of stale data on the node the request is coming
from. (Keith Laban, Dennis Gove)
+* SOLR-8522: Make it possible to use ip fragments in replica placement rules , such as ip_1, ip_2 etc (Arcadius Ahouansou, noble)
+
Bug Fixes
----------------------
* SOLR-8386: Add field option in the new admin UI schema page loads up even when no schemaFactory has been
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf964326/solr/core/ivy.xml
----------------------------------------------------------------------
diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml
index f6b2cac..2936c5b 100644
--- a/solr/core/ivy.xml
+++ b/solr/core/ivy.xml
@@ -54,6 +54,8 @@
<dependency org="cglib" name="cglib-nodep" rev="${/cglib/cglib-nodep}" conf="test"/>
<dependency org="org.objenesis" name="objenesis" rev="${/org.objenesis/objenesis}" conf="test"/>
+ <dependency org="org.mockito" name="mockito-core" rev="${/org.mockito/mockito-core}" conf="test"/>
+
<dependency org="com.fasterxml.jackson.core" name="jackson-core" rev="${/com.fasterxml.jackson.core/jackson-core}" conf="compile"/>
<dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="${/com.fasterxml.jackson.core/jackson-databind}" conf="test"/>
<dependency org="com.fasterxml.jackson.core" name="jackson-annotations" rev="${/com.fasterxml.jackson.core/jackson-annotations}" conf="test"/>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf964326/solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java b/solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java
index cbaa90f..d089aa0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java
+++ b/solr/core/src/java/org/apache/solr/cloud/rule/ImplicitSnitch.java
@@ -17,21 +17,29 @@
package org.apache.solr.cloud.rule;
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.InetAddress;
import java.nio.file.Files;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.request.SolrQueryRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
public static final Pattern hostAndPortPattern = Pattern.compile("(?:https?://)?([^:]+):(\\d+)");
@@ -42,8 +50,10 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
public static final String CORES = "cores";
public static final String DISK = "freedisk";
public static final String SYSPROP = "sysprop.";
+ public static final List<String> IP_SNITCHES = ImmutableList.of("ip_1", "ip_2", "ip_3", "ip_4");
+
+ public static final Set<String> tags = ImmutableSet.<String>builder().add(NODE, PORT, HOST, CORES, DISK).addAll(IP_SNITCHES).build();
- public static final Set<String> tags = ImmutableSet.of(NODE, PORT, HOST, CORES, DISK);
@Override
@@ -57,6 +67,9 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
if (hostAndPortMatcher.find()) ctx.getTags().put(PORT, hostAndPortMatcher.group(2));
}
+
+ addIpTags(solrNode, requestedTags, ctx);
+
ModifiableSolrParams params = new ModifiableSolrParams();
if (requestedTags.contains(CORES)) params.add(CORES, "1");
if (requestedTags.contains(DISK)) params.add(DISK, "1");
@@ -71,7 +84,7 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
long spaceInGB = space / 1024 / 1024 / 1024;
return spaceInGB;
}
-
+
public Map<String, Object> invoke(SolrQueryRequest req) {
Map<String, Object> result = new HashMap<>();
if (req.getParams().getInt(CORES, -1) == 1) {
@@ -88,16 +101,75 @@ public class ImplicitSnitch extends Snitch implements CoreAdminHandler.Invocable
}
String[] sysProps = req.getParams().getParams(SYSPROP);
if (sysProps != null && sysProps.length > 0) {
- for (String prop : sysProps) result.put(SYSPROP+prop, System.getProperty(prop));
+ for (String prop : sysProps) result.put(SYSPROP + prop, System.getProperty(prop));
}
return result;
}
+ private static final String HOST_FRAG_SEPARATOR_REGEX = "\\.";
@Override
public boolean isKnownTag(String tag) {
return tags.contains(tag) ||
- tag.startsWith(SYSPROP);//a system property
+ tag.startsWith(SYSPROP);
+ }
+
+ private void addIpTags(String solrNode, Set<String> requestedTags, SnitchContext context) {
+
+ List<String> requestedHostTags = new ArrayList<>();
+ for (String tag : requestedTags) {
+ if (IP_SNITCHES.contains(tag)) {
+ requestedHostTags.add(tag);
+ }
+ }
+
+ if (requestedHostTags.isEmpty()) {
+ return;
+ }
+
+ String[] ipFragments = getIpFragments(solrNode);
+
+ if (ipFragments == null) {
+ return;
+ }
+
+ int ipSnitchCount = IP_SNITCHES.size();
+ for (int i = 0; i < ipSnitchCount; i++) {
+ String currentTagValue = ipFragments[i];
+ String currentTagKey = IP_SNITCHES.get(ipSnitchCount - i - 1);
+
+ if (requestedHostTags.contains(currentTagKey)) {
+ context.getTags().put(currentTagKey, currentTagValue);
+ }
+
+ }
+
+ }
+
+ private String[] getIpFragments(String solrNode) {
+ Matcher hostAndPortMatcher = hostAndPortPattern.matcher(solrNode);
+ if (hostAndPortMatcher.find()) {
+ String host = hostAndPortMatcher.group(1);
+ if (host != null) {
+ String ip = getHostIp(host);
+ if (ip != null) {
+ return ip.split(HOST_FRAG_SEPARATOR_REGEX); //IPv6 support will be provided by SOLR-8523
+ }
+ }
+ }
+
+ log.warn("Failed to match host IP address from node URL [{}] using regex [{}]", solrNode, hostAndPortPattern.pattern());
+ return null;
+ }
+
+ protected String getHostIp(String host) {
+ try {
+ InetAddress address = InetAddress.getByName(host);
+ return address.getHostAddress();
+ } catch (Exception e) {
+ log.warn("Failed to get IP address from host [{}], with exception [{}] ", host, e);
+ return null;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf964326/solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java
new file mode 100644
index 0000000..a5abb16
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/rule/ImplicitSnitchTest.java
@@ -0,0 +1,186 @@
+package org.apache.solr.cloud.rule;
+
+import java.util.Map;
+
+import com.google.common.collect.Sets;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.when;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class ImplicitSnitchTest {
+
+ private ImplicitSnitch snitch;
+ private SnitchContext context;
+
+ private static final String IP_1 = "ip_1";
+ private static final String IP_2 = "ip_2";
+ private static final String IP_3 = "ip_3";
+ private static final String IP_4 = "ip_4";
+
+ @Before
+ public void beforeImplicitSnitchTest() {
+ snitch = new ImplicitSnitch();
+ context = new SnitchContext(null, null);
+ }
+
+
+ @Test
+ public void testGetTags_withAllIPv4RequestedTags_with_omitted_zeros_returns_four_tags() throws Exception {
+ String node = "5:8983_solr";
+
+ snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(4));
+ assertThat(tags.get(IP_1), is("5"));
+ assertThat(tags.get(IP_2), is("0"));
+ assertThat(tags.get(IP_3), is("0"));
+ assertThat(tags.get(IP_4), is("0"));
+ }
+
+
+ @Test
+ public void testGetTags_withAllIPv4RequestedTags_returns_four_tags() throws Exception {
+ String node = "192.168.1.2:8983_solr";
+
+ snitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(4));
+ assertThat(tags.get(IP_1), is("2"));
+ assertThat(tags.get(IP_2), is("1"));
+ assertThat(tags.get(IP_3), is("168"));
+ assertThat(tags.get(IP_4), is("192"));
+ }
+
+ @Test
+ public void testGetTags_withIPv4RequestedTags_ip2_and_ip4_returns_two_tags() throws Exception {
+ String node = "192.168.1.2:8983_solr";
+
+ SnitchContext context = new SnitchContext(null, node);
+ snitch.getTags(node, Sets.newHashSet(IP_2, IP_4), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(2));
+ assertThat(tags.get(IP_2), is("1"));
+ assertThat(tags.get(IP_4), is("192"));
+ }
+
+ @Test
+ public void testGetTags_with_wrong_ipv4_format_ip_returns_nothing() throws Exception {
+ String node = "192.168.1.2.1:8983_solr";
+
+ SnitchContext context = new SnitchContext(null, node);
+ snitch.getTags(node, Sets.newHashSet(IP_1), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(0));
+ }
+
+
+ @Test
+ public void testGetTags_with_correct_ipv6_format_ip_returns_nothing() throws Exception {
+ String node = "[0:0:0:0:0:0:0:1]:8983_solr";
+
+ SnitchContext context = new SnitchContext(null, node);
+ snitch.getTags(node, Sets.newHashSet(IP_1), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(0)); //This will fail when IPv6 is implemented
+ }
+
+
+ @Test
+ public void testGetTags_withEmptyRequestedTag_returns_nothing() throws Exception {
+ String node = "192.168.1.2:8983_solr";
+
+ snitch.getTags(node, Sets.newHashSet(), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(0));
+ }
+
+
+ @Test
+ public void testGetTags_withAllHostNameRequestedTags_returns_all_Tags() throws Exception {
+ String node = "serv01.dc01.london.uk.apache.org:8983_solr";
+
+ SnitchContext context = new SnitchContext(null, node);
+ //We need mocking here otherwise, we would need proper DNS entry for this test to pass
+ ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
+ when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
+
+ mockedSnitch.getTags(node, Sets.newHashSet(IP_1, IP_2, IP_3, IP_4), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(4));
+ assertThat(tags.get(IP_1), is("13"));
+ assertThat(tags.get(IP_2), is("12"));
+ assertThat(tags.get(IP_3), is("11"));
+ assertThat(tags.get(IP_4), is("10"));
+ }
+
+ @Test
+ public void testGetTags_withHostNameRequestedTag_ip3_returns_1_tag() throws Exception {
+ String node = "serv01.dc01.london.uk.apache.org:8983_solr";
+
+ SnitchContext context = new SnitchContext(null, node);
+ //We need mocking here otherwise, we would need proper DNS entry for this test to pass
+ ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
+ when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
+ mockedSnitch.getTags(node, Sets.newHashSet(IP_3), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(1));
+ assertThat(tags.get(IP_3), is("11"));
+ }
+
+ @Test
+ public void testGetTags_withHostNameRequestedTag_ip99999_returns_nothing() throws Exception {
+ String node = "serv01.dc01.london.uk.apache.org:8983_solr";
+
+ SnitchContext context = new SnitchContext(null, node);
+ //We need mocking here otherwise, we would need proper DNS entry for this test to pass
+ ImplicitSnitch mockedSnitch = Mockito.spy(snitch);
+ when(mockedSnitch.getHostIp(anyString())).thenReturn("10.11.12.13");
+ mockedSnitch.getTags(node, Sets.newHashSet("ip_99999"), context);
+
+ Map<String, Object> tags = context.getTags();
+ assertThat(tags.entrySet().size(), is(0));
+ }
+
+ @Test
+ public void testIsKnownTag_ip1() throws Exception {
+ assertFalse(snitch.isKnownTag("ip_0"));
+ assertTrue(snitch.isKnownTag(IP_1));
+ assertTrue(snitch.isKnownTag(IP_2));
+ assertTrue(snitch.isKnownTag(IP_3));
+ assertTrue(snitch.isKnownTag(IP_4));
+ assertFalse(snitch.isKnownTag("ip_5"));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/cf964326/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
index cf8cfd7..f23d475 100644
--- a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
@@ -34,15 +34,20 @@ import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
+import static org.junit.matchers.JUnitMatchers.containsString;
public class RulesTest extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ @org.junit.Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
@Test
@ShardsFixed(num = 5)
public void doIntegrationTest() throws Exception {
@@ -128,6 +133,78 @@ public class RulesTest extends AbstractFullDistribZkTestBase {
}
@Test
+ public void testHostFragmentRule() throws Exception {
+ String rulesColl = "ipRuleColl";
+ String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
+ String ip_1 = "-1";
+ String ip_2 = "-1";
+ Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
+ if (hostAndPortMatcher.find()) {
+ String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
+ ip_1 = ipFragments[ipFragments.length - 1];
+ ip_2 = ipFragments[ipFragments.length - 2];
+ }
+
+ try (SolrClient client = createNewSolrClient("", baseUrl)) {
+ CollectionAdminResponse rsp;
+ CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
+ create.setCollectionName(rulesColl);
+ create.setShards("shard1");
+ create.setRouterName(ImplicitDocRouter.NAME);
+ create.setReplicationFactor(2);
+ create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1);
+ create.setSnitch("class:ImplicitSnitch");
+ rsp = create.process(client);
+ assertEquals(0, rsp.getStatus());
+ assertTrue(rsp.isSuccess());
+
+ }
+
+ DocCollection rulesCollection = cloudClient.getZkStateReader().getClusterState().getCollection(rulesColl);
+ List<Map> list = (List<Map>) rulesCollection.get("rule");
+ assertEquals(2, list.size());
+ assertEquals(ip_2, list.get(0).get("ip_2"));
+ assertEquals(ip_1, list.get(1).get("ip_1"));
+
+ list = (List) rulesCollection.get("snitch");
+ assertEquals(1, list.size());
+ assertEquals("ImplicitSnitch", list.get(0).get("class"));
+ }
+
+
+ @Test
+ public void testHostFragmentRuleThrowsExceptionWhenIpDoesNotMatch() throws Exception {
+ String rulesColl = "ipRuleColl";
+ String baseUrl = getBaseUrl((HttpSolrClient) clients.get(0));
+ String ip_1 = "-1";
+ String ip_2 = "-1";
+ Matcher hostAndPortMatcher = Pattern.compile("(?:https?://)?([^:]+):(\\d+)").matcher(baseUrl);
+ if (hostAndPortMatcher.find()) {
+ String[] ipFragments = hostAndPortMatcher.group(1).split("\\.");
+ ip_1 = ipFragments[ipFragments.length - 1];
+ ip_2 = ipFragments[ipFragments.length - 2];
+ }
+
+ try (SolrClient client = createNewSolrClient("", baseUrl)) {
+ CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
+ create.setCollectionName(rulesColl);
+ create.setShards("shard1");
+ create.setRouterName(ImplicitDocRouter.NAME);
+ create.setReplicationFactor(2);
+
+ create.setRule("ip_2:" + ip_2, "ip_1:" + ip_1 + "9999");
+ create.setSnitch("class:ImplicitSnitch");
+
+ expectedException.expect(HttpSolrClient.RemoteSolrException.class);
+ expectedException.expectMessage(containsString("ip_1"));
+
+ create.process(client);
+ }
+
+ }
+
+
+ @Test
public void testModifyColl() throws Exception {
final long minGB1 = (random().nextBoolean() ? 1 : 0);
final long minGB2 = 5;