You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/10/19 10:36:54 UTC
[47/50] [abbrv] lucene-solr:jira/solr-11320: SOLR-11444: Improve
consistency of collection alias handling and collection list references.
Other refactorings of nearby code too.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
index e927ddb..4ff748c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
@@ -16,47 +16,132 @@
*/
package org.apache.solr.common.cloud;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.common.util.Utils;
+
+/**
+ * Holds collection aliases -- virtual collections that point to one or more other collections.
+ * We might add other types of aliases here some day.
+ * Immutable.
+ */
public class Aliases {
- private Map<String,Map<String,String>> aliasMap;
+ public static final Aliases EMPTY = new Aliases(Collections.emptyMap());
+
+ /** Map of "collection" string constant to ->
+ * alias name -> comma delimited list of collections */
+ private final Map<String,Map<String,String>> aliasMap; // not-null
+
+ private final Map<String, List<String>> collectionAliasListMap; // not-null; computed from aliasMap
+
+ public static Aliases fromJSON(byte[] bytes) {
+ if (bytes == null || bytes.length == 0) {
+ return EMPTY;
+ }
+ return new Aliases((Map<String,Map<String,String>>) Utils.fromJSON(bytes));
+ }
- public Aliases(Map<String,Map<String,String>> aliasMap) {
+ private Aliases(Map<String, Map<String,String>> aliasMap) {
this.aliasMap = aliasMap;
+ collectionAliasListMap = convertMapOfCommaDelimitedToMapOfList(getCollectionAliasMap());
}
- public Aliases() {
- this.aliasMap = new HashMap<>();
+ public static Map<String, List<String>> convertMapOfCommaDelimitedToMapOfList(Map<String, String> collectionAliasMap) {
+ Map<String, List<String>> collectionAliasListMap = new LinkedHashMap<>(collectionAliasMap.size());
+ for (Map.Entry<String, String> entry : collectionAliasMap.entrySet()) {
+ collectionAliasListMap.put(entry.getKey(), StrUtils.splitSmart(entry.getValue(), ",", true));
+ }
+ return collectionAliasListMap;
}
-
+
+ /**
+ * Returns an unmodifiable Map of collection aliases mapped to a comma delimited list of what the alias maps to.
+ * Does not return null.
+ * Prefer use of {@link #getCollectionAliasListMap()} instead, where appropriate.
+ */
public Map<String,String> getCollectionAliasMap() {
Map<String,String> cam = aliasMap.get("collection");
- if (cam == null) return null;
- return Collections.unmodifiableMap(cam);
+ return cam == null ? Collections.emptyMap() : Collections.unmodifiableMap(cam);
+ }
+
+ /**
+ * Returns an unmodifiable Map of collection aliases mapped to a list of what the alias maps to.
+ * Does not return null.
+ */
+ public Map<String,List<String>> getCollectionAliasListMap() {
+ return Collections.unmodifiableMap(collectionAliasListMap);
}
- public Map<String,Map<String,String>> getAliasMap() {
- return Collections.unmodifiableMap(aliasMap);
+ public boolean hasCollectionAliases() {
+ return !collectionAliasListMap.isEmpty();
}
- public int collectionAliasSize() {
- Map<String,String> cam = aliasMap.get("collection");
- if (cam == null) return 0;
- return cam.size();
+ /**
+ * Returns a list of collections that the input alias name maps to. If there
+ * are none, the input is returned. One level of alias indirection is supported (alias to alias to collection).
+ * Treat the result as unmodifiable.
+ */
+ public List<String> resolveAliases(String aliasName) {
+ return resolveAliasesGivenAliasMap(collectionAliasListMap, aliasName);
}
-
- @Override
- public String toString() {
- return "Aliases [aliasMap=" + aliasMap + "]";
+
+ /** @lucene.internal */
+ public static List<String> resolveAliasesGivenAliasMap(Map<String, List<String>> collectionAliasListMap, String aliasName) {
+ //return collectionAliasListMap.getOrDefault(aliasName, Collections.singletonList(aliasName));
+ // TODO deprecate and remove this dubious feature?
+ // Due to another level of indirection, this is more complicated...
+ List<String> level1 = collectionAliasListMap.get(aliasName);
+ if (level1 == null) {
+ return Collections.singletonList(aliasName);// is a collection
+ }
+ List<String> result = new ArrayList<>(level1.size());
+ for (String level1Alias : level1) {
+ List<String> level2 = collectionAliasListMap.get(level1Alias);
+ if (level2 == null) {
+ result.add(level1Alias);
+ } else {
+ result.addAll(level2);
+ }
+ }
+ return result;
}
- public String getCollectionAlias(String collectionName) {
- Map<String,String> cam = aliasMap.get("collection");
- if (cam == null) return null;
- return cam.get(collectionName);
+ /**
+ * Creates a new Aliases instance with the same data as the current one but with a modification based on the
+ * parameters. If {@code collections} is null, then the {@code alias} is removed, otherwise it is added/updated.
+ */
+ public Aliases cloneWithCollectionAlias(String alias, String collections) {
+ Map<String,String> newCollectionMap = new HashMap<>(getCollectionAliasMap());
+ if (collections == null) {
+ newCollectionMap.remove(alias);
+ } else {
+ newCollectionMap.put(alias, collections);
+ }
+ if (newCollectionMap.isEmpty()) {
+ return EMPTY;
+ } else {
+ return new Aliases(Collections.singletonMap("collection", newCollectionMap));
+ }
}
+ /** Serialize to ZooKeeper. */
+ public byte[] toJSON() {
+ if (collectionAliasListMap.isEmpty()) {
+ return null;
+ } else {
+ return Utils.toJSON(aliasMap);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Aliases [aliasMap=" + aliasMap + "]";
+ }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
index 8e52e41..bfe33c5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ClusterState.java
@@ -232,15 +232,6 @@ public class ClusterState implements JSONWriter.Writable {
return new ClusterState( liveNodes, collections,version);
}
- public static Aliases load(byte[] bytes) {
- if (bytes == null || bytes.length == 0) {
- return new Aliases();
- }
- Map<String,Map<String,String>> aliasMap = (Map<String,Map<String,String>>) Utils.fromJSON(bytes);
-
- return new Aliases(aliasMap);
- }
-
// TODO move to static DocCollection.loadFromMap
private static DocCollection collectionFromObjects(String name, Map<String, Object> objs, Integer version, String znode) {
Map<String,Object> props;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index e45565f..4695d7d 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -257,7 +257,7 @@ public class ZkStateReader implements Closeable {
private final boolean closeClient;
- private volatile Aliases aliases = new Aliases();
+ private volatile Aliases aliases = Aliases.EMPTY;
private volatile boolean closed = false;
@@ -385,8 +385,10 @@ public class ZkStateReader implements Closeable {
public void updateLiveNodes() throws KeeperException, InterruptedException {
refreshLiveNodes(null);
}
-
+
+ /** Never null. */
public Aliases getAliases() {
+ assert aliases != null;
return aliases;
}
@@ -455,7 +457,7 @@ public class ZkStateReader implements Closeable {
final Watcher thisWatch = this;
final Stat stat = new Stat();
final byte[] data = zkClient.getData(ALIASES, thisWatch, stat, true);
- ZkStateReader.this.aliases = ClusterState.load(data);
+ ZkStateReader.this.aliases = Aliases.fromJSON(data);
LOG.debug("New alias definition is: " + ZkStateReader.this.aliases.toString());
}
} catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
@@ -896,7 +898,7 @@ public class ZkStateReader implements Closeable {
public void updateAliases() throws KeeperException, InterruptedException {
final byte[] data = zkClient.getData(ALIASES, null, null, true);
- this.aliases = ClusterState.load(data);
+ this.aliases = Aliases.fromJSON(data);
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
index a20c6e2..117de57 100644
--- a/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
+++ b/solr/solrj/src/java/org/apache/solr/common/util/StrUtils.java
@@ -88,6 +88,7 @@ public class StrUtils {
* @param s the string to split
* @param separator the separator to split on
* @param decode decode backslash escaping
+ * @return not null
*/
public static List<String> splitSmart(String s, String separator, boolean decode) {
ArrayList<String> lst = new ArrayList<>(2);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
index a3dd436..cfcfcd6 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudSolrClientCacheTest.java
@@ -22,6 +22,7 @@ import java.net.SocketException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
@@ -130,13 +131,8 @@ public class CloudSolrClientCacheTest extends SolrTestCaseJ4 {
}
@Override
- public String getAlias(String collection) {
- return collection;
- }
-
- @Override
- public String getCollectionName(String name) {
- return name;
+ public List<String> resolveAlias(String collection) {
+ return Collections.singletonList(collection);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
index ce14907..bb07c45 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java
@@ -28,7 +28,6 @@ import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.SortedSet;
@@ -555,13 +554,7 @@ public class JdbcTest extends SolrCloudTestCase {
tables.addAll(collectionsSet);
Aliases aliases = zkStateReader.getAliases();
- if(aliases != null) {
- Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap();
- if(collectionAliasMap != null) {
- Set<String> aliasesSet = collectionAliasMap.keySet();
- tables.addAll(aliasesSet);
- }
- }
+ tables.addAll(aliases.getCollectionAliasListMap().keySet());
try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) {
for(String table : tables) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e001f352/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 8e129c7..9d9a9dc 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -84,6 +84,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.impl.LBHttpSolrClient;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.cloud.IpTables;
+import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@@ -2224,6 +2225,14 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
super();
}
+ public CloudSolrClient.Builder withCluster(MiniSolrCloudCluster cluster) {
+ if (random().nextBoolean()) {
+ return withZkHost(cluster.getZkServer().getZkAddress());
+ } else {
+ return withSolrUrl(cluster.getRandomJetty(random()).getBaseUrl().toString());
+ }
+ }
+
@Override
public CloudSolrClient.Builder sendDirectUpdatesToShardLeadersOnly() {
configuredDUTflag = true;
@@ -2247,7 +2256,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
@Override
public CloudSolrClient build() {
if (configuredDUTflag == false) {
- // flag value not explicity configured
+ // flag value not explicitly configured
if (random().nextBoolean()) {
// so randomly choose a value
randomlyChooseDirectUpdatesToLeadersOnly();
@@ -2262,7 +2271,7 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
/**
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
- * Tests that do not wish to have any randomized behavior should use the
+ * Tests that do not wish to have any randomized behavior should use the
* {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly
*/
public static CloudSolrClient getCloudSolrClient(String zkHost) {
@@ -2270,7 +2279,18 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
.withZkHost(zkHost)
.build();
}
-
+
+ /**
+ * This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
+ * Tests that do not wish to have any randomized behavior should use the
+ * {@link org.apache.solr.client.solrj.impl.CloudSolrClient.Builder} class directly
+ */
+ public static CloudSolrClient getCloudSolrClient(MiniSolrCloudCluster cluster) {
+ return new CloudSolrClientBuilder()
+ .withCluster(cluster)
+ .build();
+ }
+
/**
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
* Tests that do not wish to have any randomized behavior should use the
@@ -2300,7 +2320,11 @@ public abstract class SolrTestCaseJ4 extends LuceneTestCase {
.sendUpdatesToAllReplicasInShard()
.build();
}
-
+
+ public static CloudSolrClientBuilder newCloudSolrClient(String zkHost) {
+ return (CloudSolrClientBuilder) new CloudSolrClientBuilder().withZkHost(zkHost);
+ }
+
/**
* This method <i>may</i> randomize unspecified aspects of the resulting SolrClient.
* Tests that do not wish to have any randomized behavior should use the