You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2017/11/20 14:40:56 UTC
[3/9] lucene-solr:jira/solr-11285-sim: SOLR-11487: Collection Aliases
may now have metadata
SOLR-11487: Collection Aliases may now have metadata
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fd1820a4
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fd1820a4
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fd1820a4
Branch: refs/heads/jira/solr-11285-sim
Commit: fd1820a430c321e6a2b2910004d7d2be60d3db4a
Parents: c561ffe
Author: David Smiley <ds...@apache.org>
Authored: Thu Nov 16 13:57:22 2017 -0500
Committer: David Smiley <ds...@apache.org>
Committed: Thu Nov 16 13:57:22 2017 -0500
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../org/apache/solr/cloud/CreateAliasCmd.java | 37 +---
.../org/apache/solr/cloud/DeleteAliasCmd.java | 40 +---
.../org/apache/solr/servlet/HttpSolrCall.java | 4 +-
.../apache/solr/cloud/AliasIntegrationTest.java | 133 ++++++++++++
.../org/apache/solr/common/cloud/Aliases.java | 216 ++++++++++++++-----
.../apache/solr/common/cloud/ZkStateReader.java | 209 +++++++++++++-----
7 files changed, 464 insertions(+), 178 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd1820a4/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 1130f52..5615427 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -69,6 +69,9 @@ New Features
* SOLR-11538: Implement suggestions for port,ip_*, nodeRole,sysprop.*, metrics:* (noble)
+* SOLR-11487: Collection Aliases may now have metadata (currently an internal feature).
+ (Gus Heck, David Smiley)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd1820a4/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java
index 16a4266..598764e 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateAliasCmd.java
@@ -17,13 +17,10 @@
*/
package org.apache.solr.cloud;
-import java.lang.invoke.MethodHandles;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
-import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
import org.apache.solr.common.SolrException;
@@ -32,16 +29,11 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
-import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.NAME;
public class CreateAliasCmd implements Cmd {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final OverseerCollectionMessageHandler ocmh;
public CreateAliasCmd(OverseerCollectionMessageHandler ocmh) {
@@ -57,20 +49,7 @@ public class CreateAliasCmd implements Cmd {
ZkStateReader zkStateReader = ocmh.zkStateReader;
validateAllCollectionsExistAndNoDups(collections, zkStateReader);
- byte[] jsonBytes = zkStateReader.getAliases().cloneWithCollectionAlias(aliasName, collections).toJSON();
- try {
- zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true);
-
- checkForAlias(aliasName, collections);
- // some fudge for other nodes
- Thread.sleep(100);
- } catch (KeeperException e) {
- log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } catch (InterruptedException e) {
- log.warn("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
+ zkStateReader.aliasesHolder.applyModificationAndExportToZk(aliases -> aliases.cloneWithCollectionAlias(aliasName, collections));
}
private void validateAllCollectionsExistAndNoDups(String collections, ZkStateReader zkStateReader) {
@@ -89,18 +68,4 @@ public class CreateAliasCmd implements Cmd {
}
}
- private void checkForAlias(String name, String value) {
- TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
- boolean success = false;
- while (!timeout.hasTimedOut()) {
- String collections = ocmh.zkStateReader.getAliases().getCollectionAliasMap().get(name);
- if (Objects.equals(collections, value)) {
- success = true;
- break;
- }
- }
- if (!success) {
- log.warn("Timeout waiting to be notified of Alias change...");
- }
- }
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd1820a4/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java
index 4743cb9..9c9f1c6 100644
--- a/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/DeleteAliasCmd.java
@@ -18,23 +18,14 @@
package org.apache.solr.cloud;
-import java.lang.invoke.MethodHandles;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
-import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.KeeperException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.NAME;
public class DeleteAliasCmd implements OverseerCollectionMessageHandler.Cmd {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private final OverseerCollectionMessageHandler ocmh;
public DeleteAliasCmd(OverseerCollectionMessageHandler ocmh) {
@@ -46,34 +37,7 @@ public class DeleteAliasCmd implements OverseerCollectionMessageHandler.Cmd {
String aliasName = message.getStr(NAME);
ZkStateReader zkStateReader = ocmh.zkStateReader;
- byte[] jsonBytes = zkStateReader.getAliases().cloneWithCollectionAlias(aliasName, null).toJSON();
- try {
- zkStateReader.getZkClient().setData(ZkStateReader.ALIASES, jsonBytes, true);
-
- checkForAliasAbsence(aliasName);
- // some fudge for other nodes
- Thread.sleep(100);
- } catch (KeeperException e) {
- log.error("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- } catch (InterruptedException e) {
- log.warn("", e);
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
- }
-
- }
- private void checkForAliasAbsence(String name) {
- TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS);
- boolean success = false;
- while (! timeout.hasTimedOut()) {
- String collections = ocmh.zkStateReader.getAliases().getCollectionAliasMap().get(name);
- if (collections == null) {
- success = true;
- break;
- }
- }
- if (!success) {
- log.warn("Timeout waiting to be notified of Alias change...");
- }
+ zkStateReader.aliasesHolder.applyModificationAndExportToZk(a -> a.cloneWithCollectionAlias(aliasName, null));
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd1820a4/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
index 5022817..0e575d6 100644
--- a/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
+++ b/solr/core/src/java/org/apache/solr/servlet/HttpSolrCall.java
@@ -443,9 +443,7 @@ public class HttpSolrCall {
if (!retry) {
// we couldn't find a core to work with, try reloading aliases
// TODO: it would be nice if admin ui elements skipped this...
- ZkStateReader reader = cores.getZkController()
- .getZkStateReader();
- reader.updateAliases();
+ cores.getZkController().getZkStateReader().aliasesHolder.update();
action = RETRY;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd1820a4/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
index b3568a1..542a5f4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/AliasIntegrationTest.java
@@ -17,7 +17,10 @@
package org.apache.solr.cloud;
import java.io.IOException;
+import java.util.List;
+import java.util.Map;
import java.util.function.Consumer;
+import java.util.function.UnaryOperator;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
@@ -28,11 +31,17 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.Utils;
import org.junit.BeforeClass;
import org.junit.Test;
+import static org.apache.solr.common.cloud.ZkStateReader.ALIASES;
+
public class AliasIntegrationTest extends SolrCloudTestCase {
@BeforeClass
@@ -41,6 +50,130 @@ public class AliasIntegrationTest extends SolrCloudTestCase {
.addConfig("conf", configset("cloud-minimal"))
.configure();
}
+
+ @Test
+ public void testMetadata() throws Exception {
+ CollectionAdminRequest.createCollection("collection1meta", "conf", 2, 1).process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection("collection2meta", "conf", 1, 1).process(cluster.getSolrClient());
+ waitForState("Expected collection1 to be created with 2 shards and 1 replica", "collection1meta", clusterShape(2, 1));
+ waitForState("Expected collection2 to be created with 1 shard and 1 replica", "collection2meta", clusterShape(1, 1));
+ ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
+ zkStateReader.createClusterStateWatchersAndUpdate();
+ List<String> aliases = zkStateReader.getAliases().resolveAliases("meta1");
+ assertEquals(1, aliases.size());
+ assertEquals("meta1", aliases.get(0));
+ UnaryOperator<Aliases> op6 = a -> a.cloneWithCollectionAlias("meta1", "collection1meta,collection2meta");
+ final ZkStateReader.AliasesManager aliasesHolder = zkStateReader.aliasesHolder;
+
+ aliasesHolder.applyModificationAndExportToZk(op6);
+ aliases = zkStateReader.getAliases().resolveAliases("meta1");
+ assertEquals(2, aliases.size());
+ assertEquals("collection1meta", aliases.get(0));
+ assertEquals("collection2meta", aliases.get(1));
+ //ensure we have the back-compat format in ZK:
+ final byte[] rawBytes = zkStateReader.getZkClient().getData(ALIASES, null, null, true);
+ assertTrue(((Map<String,Map<String,?>>)Utils.fromJSON(rawBytes)).get("collection").get("meta1") instanceof String);
+
+ // set metadata
+ UnaryOperator<Aliases> op5 = a -> a.cloneWithCollectionAliasMetadata("meta1", "foo", "bar");
+ aliasesHolder.applyModificationAndExportToZk(op5);
+ Map<String, String> meta = zkStateReader.getAliases().getCollectionAliasMetadata("meta1");
+ assertNotNull(meta);
+ assertTrue(meta.containsKey("foo"));
+ assertEquals("bar", meta.get("foo"));
+
+ // set more metadata
+ UnaryOperator<Aliases> op4 = a -> a.cloneWithCollectionAliasMetadata("meta1", "foobar", "bazbam");
+ aliasesHolder.applyModificationAndExportToZk(op4);
+ meta = zkStateReader.getAliases().getCollectionAliasMetadata("meta1");
+ assertNotNull(meta);
+
+ // old metadata still there
+ assertTrue(meta.containsKey("foo"));
+ assertEquals("bar", meta.get("foo"));
+
+ // new metadata added
+ assertTrue(meta.containsKey("foobar"));
+ assertEquals("bazbam", meta.get("foobar"));
+
+ // remove metadata
+ UnaryOperator<Aliases> op3 = a -> a.cloneWithCollectionAliasMetadata("meta1", "foo", null);
+ aliasesHolder.applyModificationAndExportToZk(op3);
+ meta = zkStateReader.getAliases().getCollectionAliasMetadata("meta1");
+ assertNotNull(meta);
+
+ // verify key was removed
+ assertFalse(meta.containsKey("foo"));
+
+ // but only the specified key was removed
+ assertTrue(meta.containsKey("foobar"));
+ assertEquals("bazbam", meta.get("foobar"));
+
+ // removal of non existent key should succeed.
+ UnaryOperator<Aliases> op2 = a -> a.cloneWithCollectionAliasMetadata("meta1", "foo", null);
+ aliasesHolder.applyModificationAndExportToZk(op2);
+
+ // chained invocations
+ UnaryOperator<Aliases> op1 = a ->
+ a.cloneWithCollectionAliasMetadata("meta1", "foo2", "bazbam")
+ .cloneWithCollectionAliasMetadata("meta1", "foo3", "bazbam2");
+ aliasesHolder.applyModificationAndExportToZk(op1);
+
+ // some other independent update (not overwritten)
+ UnaryOperator<Aliases> op = a -> a.cloneWithCollectionAlias("meta3", "collection1meta,collection2meta");
+ aliasesHolder.applyModificationAndExportToZk(op);
+
+ // competing went through
+ assertEquals("collection1meta,collection2meta", zkStateReader.getAliases().getCollectionAliasMap().get("meta3"));
+
+ meta = zkStateReader.getAliases().getCollectionAliasMetadata("meta1");
+ assertNotNull(meta);
+
+ // old metadata still there
+ assertTrue(meta.containsKey("foobar"));
+ assertEquals("bazbam", meta.get("foobar"));
+
+ // competing update not overwritten
+ assertEquals("collection1meta,collection2meta", zkStateReader.getAliases().getCollectionAliasMap().get("meta3"));
+
+ // new metadata added
+ assertTrue(meta.containsKey("foo2"));
+ assertEquals("bazbam", meta.get("foo2"));
+ assertTrue(meta.containsKey("foo3"));
+ assertEquals("bazbam2", meta.get("foo3"));
+
+ // now check that an independently constructed ZkStateReader can see what we've done.
+ // i.e. the data is really in zookeeper
+ String zkAddress = cluster.getZkServer().getZkAddress();
+ boolean createdZKSR = false;
+ try(SolrZkClient zkClient = new SolrZkClient(zkAddress, 30000)) {
+
+ ZkController.createClusterZkNodes(zkClient);
+
+ zkStateReader = new ZkStateReader(zkClient);
+ createdZKSR = true;
+ zkStateReader.createClusterStateWatchersAndUpdate();
+
+ meta = zkStateReader.getAliases().getCollectionAliasMetadata("meta1");
+ assertNotNull(meta);
+
+ // verify key was removed in independent view
+ assertFalse(meta.containsKey("foo"));
+
+ // but only the specified key was removed
+ assertTrue(meta.containsKey("foobar"));
+ assertEquals("bazbam", meta.get("foobar"));
+
+ Aliases a = zkStateReader.getAliases();
+ Aliases clone = a.cloneWithCollectionAlias("meta1", null);
+ meta = clone.getCollectionAliasMetadata("meta1");
+ assertEquals(0,meta.size());
+ } finally {
+ if (createdZKSR) {
+ zkStateReader.close();
+ }
+ }
+ }
@Test
public void test() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd1820a4/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
index 4ff748c..d77b544 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/Aliases.java
@@ -18,10 +18,13 @@ package org.apache.solr.common.cloud;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
+import java.util.function.UnaryOperator;
+import java.util.stream.Collectors;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
@@ -33,66 +36,148 @@ import org.apache.solr.common.util.Utils;
*/
public class Aliases {
- public static final Aliases EMPTY = new Aliases(Collections.emptyMap());
+ /**
+ * An empty, minimal Aliases primarily used to support the non-cloud solr use cases. Not normally useful
+ * in cloud situations where the version of the node needs to be tracked even if all aliases are removed.
+ * A version of 0 is provided rather than -1 to minimize the possibility that if this is used in a cloud
+ * instance data is written without version checking.
+ */
+ public static final Aliases EMPTY = new Aliases(Collections.emptyMap(), Collections.emptyMap(), 0);
+
+ private static final String COLLECTION = "collection";
+ private static final String COLLECTION_METADATA = "collection_metadata";
+
+ // aliasName -> list of collections. (note: the Lists here should be unmodifiable)
+ private final Map<String, List<String>> collectionAliases; // not null
+
+ // aliasName --> metadataKey --> metadataValue (note: the inner Map here should be unmodifiable)
+ private final Map<String, Map<String, String>> collectionAliasMetadata; // notnull
- /** Map of "collection" string constant to ->
- * alias name -> comma delimited list of collections */
- private final Map<String,Map<String,String>> aliasMap; // not-null
+ private final int zNodeVersion;
- private final Map<String, List<String>> collectionAliasListMap; // not-null; computed from aliasMap
+ /** Construct aliases directly with this information -- caller should not retain.
+ * Any deeply nested collections are assumed to already be unmodifiable. */
+ private Aliases(Map<String, List<String>> collectionAliases,
+ Map<String, Map<String, String>> collectionAliasMetadata,
+ int zNodeVersion) {
+ this.collectionAliases = Objects.requireNonNull(collectionAliases);
+ this.collectionAliasMetadata = Objects.requireNonNull(collectionAliasMetadata);
+ this.zNodeVersion = zNodeVersion;
+ }
- public static Aliases fromJSON(byte[] bytes) {
+ /**
+ * Create an instance from the JSON bytes read from zookeeper. Generally this should
+ * only be done by a ZkStateReader.
+ *
+ * @param bytes The bytes read via a getData request to zookeeper (possibly null)
+ * @param zNodeVersion the version of the data in zookeeper that this instance corresponds to
+ * @return A new immutable Aliases object
+ */
+ @SuppressWarnings("unchecked")
+ public static Aliases fromJSON(byte[] bytes, int zNodeVersion) {
+ Map<String, Map> aliasMap;
if (bytes == null || bytes.length == 0) {
- return EMPTY;
+ aliasMap = Collections.emptyMap();
+ } else {
+ aliasMap = (Map<String, Map>) Utils.fromJSON(bytes);
}
- return new Aliases((Map<String,Map<String,String>>) Utils.fromJSON(bytes));
+
+ Map colAliases = aliasMap.getOrDefault(COLLECTION, Collections.emptyMap());
+ colAliases = convertMapOfCommaDelimitedToMapOfList(colAliases); // also unmodifiable
+
+ Map<String, Map<String, String>> colMeta = aliasMap.getOrDefault(COLLECTION_METADATA, Collections.emptyMap());
+ colMeta.replaceAll((k, metaMap) -> Collections.unmodifiableMap(metaMap));
+
+ return new Aliases(colAliases, colMeta, zNodeVersion);
}
- private Aliases(Map<String, Map<String,String>> aliasMap) {
- this.aliasMap = aliasMap;
- collectionAliasListMap = convertMapOfCommaDelimitedToMapOfList(getCollectionAliasMap());
+ /**
+ * Serialize our state.
+ */
+ public byte[] toJSON() {
+ if (collectionAliases.isEmpty()) {
+ assert collectionAliasMetadata.isEmpty();
+ return null;
+ } else {
+ Map<String,Map> tmp = new LinkedHashMap<>();
+ tmp.put(COLLECTION, convertMapOfListToMapOfCommaDelimited(collectionAliases));
+ if (!collectionAliasMetadata.isEmpty()) {
+ tmp.put(COLLECTION_METADATA, collectionAliasMetadata);
+ }
+ return Utils.toJSON(tmp);
+ }
}
public static Map<String, List<String>> convertMapOfCommaDelimitedToMapOfList(Map<String, String> collectionAliasMap) {
- Map<String, List<String>> collectionAliasListMap = new LinkedHashMap<>(collectionAliasMap.size());
- for (Map.Entry<String, String> entry : collectionAliasMap.entrySet()) {
- collectionAliasListMap.put(entry.getKey(), StrUtils.splitSmart(entry.getValue(), ",", true));
- }
- return collectionAliasListMap;
+ return collectionAliasMap.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> splitCollections(e.getValue()),
+ (a, b) -> {throw new IllegalStateException(String.format(Locale.ROOT, "Duplicate key %s", b));},
+ LinkedHashMap::new));
+ }
+
+ private static List<String> splitCollections(String collections) {
+ return Collections.unmodifiableList(StrUtils.splitSmart(collections, ",", true));
+ }
+
+ public static Map<String,String> convertMapOfListToMapOfCommaDelimited(Map<String,List<String>> collectionAliasMap) {
+ return collectionAliasMap.entrySet().stream()
+ .collect(Collectors.toMap(Map.Entry::getKey, e -> String.join(",", e.getValue()),
+ (a, b) -> {throw new IllegalStateException(String.format(Locale.ROOT, "Duplicate key %s", b));},
+ LinkedHashMap::new));
+ }
+
+ public int getZNodeVersion() {
+ return zNodeVersion;
}
/**
- * Returns an unmodifiable Map of collection aliases mapped to a comma delimited list of what the alias maps to.
- * Does not return null.
- * Prefer use of {@link #getCollectionAliasListMap()} instead, where appropriate.
+ * Get a map similar to the JSON data as stored in zookeeper. Callers may prefer use of
+ * {@link #getCollectionAliasListMap()} instead, if collection names will be iterated.
+ *
+ * @return an unmodifiable Map of collection aliases mapped to a comma delimited string of the collection(s) the
+ * alias maps to. Does not return null.
*/
- public Map<String,String> getCollectionAliasMap() {
- Map<String,String> cam = aliasMap.get("collection");
- return cam == null ? Collections.emptyMap() : Collections.unmodifiableMap(cam);
+ @SuppressWarnings("unchecked")
+ public Map<String, String> getCollectionAliasMap() {
+ return Collections.unmodifiableMap(convertMapOfListToMapOfCommaDelimited(collectionAliases));
}
/**
- * Returns an unmodifiable Map of collection aliases mapped to a list of what the alias maps to.
+ * Get a fully parsed map of collection aliases.
+ *
+ * @return an unmodifiable Map of collection aliases mapped to a list of the collection(s) the alias maps to.
* Does not return null.
*/
public Map<String,List<String>> getCollectionAliasListMap() {
- return Collections.unmodifiableMap(collectionAliasListMap);
+ // Note: Lists contained by this map are already unmodifiable and can be shared safely
+ return Collections.unmodifiableMap(collectionAliases);
}
-
- public boolean hasCollectionAliases() {
- return !collectionAliasListMap.isEmpty();
+
+ /**
+ * Returns an unmodifiable Map of metadata for a given alias. If an alias by the given name
+ * exists, this method will never return null.
+ *
+ * @param alias the name of an alias also found as a key in {@link #getCollectionAliasListMap()}
+ * @return The metadata for the alias (possibly empty) or null if the alias does not exist.
+ */
+ public Map<String,String> getCollectionAliasMetadata(String alias) {
+ // Note: map is already unmodifiable; it can be shared safely
+ return collectionAliasMetadata.getOrDefault(alias, Collections.emptyMap());
}
/**
- * Returns a list of collections that the input alias name maps to. If there
- * are none, the input is returned. One level of alias indirection is supported (alias to alias to collection).
- * Treat the result as unmodifiable.
+ * List the collections associated with a particular alias. One level of alias indirection is supported
+ * (alias to alias to collection). Such indirection may be deprecated in the future, use with caution.
+ *
+ * @return An unmodifiable list of collections names that the input alias name maps to. If there
+ * are none, the input is returned.
*/
public List<String> resolveAliases(String aliasName) {
- return resolveAliasesGivenAliasMap(collectionAliasListMap, aliasName);
+ return resolveAliasesGivenAliasMap(collectionAliases, aliasName);
}
/** @lucene.internal */
+ @SuppressWarnings("JavaDoc")
public static List<String> resolveAliasesGivenAliasMap(Map<String, List<String>> collectionAliasListMap, String aliasName) {
//return collectionAliasListMap.getOrDefault(aliasName, Collections.singletonList(aliasName));
// TODO deprecate and remove this dubious feature?
@@ -110,38 +195,73 @@ public class Aliases {
result.addAll(level2);
}
}
- return result;
+ return Collections.unmodifiableList(result);
}
/**
* Creates a new Aliases instance with the same data as the current one but with a modification based on the
- * parameters. If {@code collections} is null, then the {@code alias} is removed, otherwise it is added/updated.
+ * parameters.
+ * <p>
+ * Note that the state in zookeeper is unaffected by this method and the change must still be persisted via
+ * {@link ZkStateReader.AliasesManager#applyModificationAndExportToZk(UnaryOperator)}
+ *
+ * @param alias the alias to update, must not be null
+ * @param collections the comma separated list of collections for the alias, null to remove the alias
*/
public Aliases cloneWithCollectionAlias(String alias, String collections) {
- Map<String,String> newCollectionMap = new HashMap<>(getCollectionAliasMap());
- if (collections == null) {
- newCollectionMap.remove(alias);
- } else {
- newCollectionMap.put(alias, collections);
+ if (alias == null) {
+ throw new NullPointerException("Alias name cannot be null");
}
- if (newCollectionMap.isEmpty()) {
- return EMPTY;
+ Map<String, Map<String, String>> newColMetadata;
+ Map<String, List<String>> newColAliases = new LinkedHashMap<>(this.collectionAliases);//clone to modify
+ if (collections == null) { // REMOVE:
+ newColMetadata = new LinkedHashMap<>(this.collectionAliasMetadata);//clone to modify
+ newColMetadata.remove(alias);
+ newColAliases.remove(alias);
} else {
- return new Aliases(Collections.singletonMap("collection", newCollectionMap));
+ newColMetadata = this.collectionAliasMetadata;// no changes
+ // java representation is a list, so split before adding to maintain consistency
+ newColAliases.put(alias, splitCollections(collections)); // note: unmodifiableList
}
+ return new Aliases(newColAliases, newColMetadata, zNodeVersion);
}
- /** Serialize to ZooKeeper. */
- public byte[] toJSON() {
- if (collectionAliasListMap.isEmpty()) {
- return null;
+ /**
+ * Set the value for some metadata on a collection alias. This is done by creating a new Aliases instance
+ * with the same data as the current one but with a modification based on the parameters.
+ * <p>
+ * Note that the state in zookeeper is unaffected by this method and the change must still be persisted via
+ * {@link ZkStateReader.AliasesManager#applyModificationAndExportToZk(UnaryOperator)}
+ *
+ * @param alias the alias to update
+ * @param metadataKey the key for the metadata
+ * @param metadataValue the metadata to add/replace, null to remove the key.
+ * @return An immutable copy of the aliases with the new metadata.
+ */
+ public Aliases cloneWithCollectionAliasMetadata(String alias, String metadataKey, String metadataValue){
+ if (!collectionAliases.containsKey(alias)) {
+ throw new IllegalArgumentException(alias + " is not a valid alias");
+ }
+ if (metadataKey == null) {
+ throw new IllegalArgumentException("Null is not a valid metadata key");
+ }
+ Map<String,Map<String,String>> newColMetadata = new LinkedHashMap<>(this.collectionAliasMetadata);//clone to modify
+ Map<String, String> newMetaMap = new LinkedHashMap<>(newColMetadata.getOrDefault(alias, Collections.emptyMap()));
+ if (metadataValue != null) {
+ newMetaMap.put(metadataKey, metadataValue);
} else {
- return Utils.toJSON(aliasMap);
+ newMetaMap.remove(metadataKey);
}
+ newColMetadata.put(alias, Collections.unmodifiableMap(newMetaMap));
+ return new Aliases(collectionAliases, newColMetadata, zNodeVersion);
}
@Override
public String toString() {
- return "Aliases [aliasMap=" + aliasMap + "]";
+ return "Aliases{" +
+ "collectionAliases=" + collectionAliases +
+ ", collectionAliasMetadata=" + collectionAliasMetadata +
+ ", zNodeVersion=" + zNodeVersion +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fd1820a4/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index b70d2f1..e778151 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -21,6 +21,7 @@ import java.io.UnsupportedEncodingException;
import java.lang.invoke.MethodHandles;
import java.net.URLDecoder;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
@@ -42,6 +43,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.UnaryOperator;
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
@@ -124,7 +126,6 @@ public class ZkStateReader implements Closeable {
public static final String REPLICA_TYPE = "type";
-
/** A view of the current state of all collections; combines all the different state sources into a single view. */
protected volatile ClusterState clusterState;
@@ -257,8 +258,6 @@ public class ZkStateReader implements Closeable {
private final boolean closeClient;
- private volatile Aliases aliases = Aliases.EMPTY;
-
private volatile boolean closed = false;
public ZkStateReader(SolrZkClient zkClient) {
@@ -386,12 +385,6 @@ public class ZkStateReader implements Closeable {
refreshLiveNodes(null);
}
- /** Never null. */
- public Aliases getAliases() {
- assert aliases != null;
- return aliases;
- }
-
public Integer compareStateVersions(String coll, int version) {
DocCollection collection = clusterState.getCollectionOrNull(coll);
if (collection == null) return null;
@@ -436,45 +429,7 @@ public class ZkStateReader implements Closeable {
refreshLegacyClusterState(new LegacyClusterStateWatcher());
refreshStateFormat2Collections();
refreshCollectionList(new CollectionsChildWatcher());
-
- synchronized (ZkStateReader.this.getUpdateLock()) {
- constructState(Collections.emptySet());
-
- zkClient.exists(ALIASES,
- new Watcher() {
-
- @Override
- public void process(WatchedEvent event) {
- // session events are not change events, and do not remove the watcher
- if (EventType.None.equals(event.getType())) {
- return;
- }
- try {
- synchronized (ZkStateReader.this.getUpdateLock()) {
- LOG.debug("Updating aliases... ");
-
- // remake watch
- final Watcher thisWatch = this;
- final Stat stat = new Stat();
- final byte[] data = zkClient.getData(ALIASES, thisWatch, stat, true);
- ZkStateReader.this.aliases = Aliases.fromJSON(data);
- LOG.debug("New alias definition is: " + ZkStateReader.this.aliases.toString());
- }
- } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
- LOG.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage());
- } catch (KeeperException e) {
- LOG.error("A ZK error has occurred", e);
- throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
- } catch (InterruptedException e) {
- // Restore the interrupted status
- Thread.currentThread().interrupt();
- LOG.warn("Interrupted", e);
- }
- }
-
- }, true);
- }
- updateAliases();
+ refreshAliases(aliasesHolder);
if (securityNodeListener != null) {
addSecuritynodeWatcher(pair -> {
@@ -896,11 +851,6 @@ public class ZkStateReader implements Closeable {
return zkClient;
}
- public void updateAliases() throws KeeperException, InterruptedException {
- final byte[] data = zkClient.getData(ALIASES, null, null, true);
- this.aliases = Aliases.fromJSON(data);
- }
-
/**
* Get a cluster property
*
@@ -1473,4 +1423,157 @@ public class ZkStateReader implements Closeable {
}
+ //
+ // Aliases related
+ //
+
+ /** Access to the {@link Aliases}. */
+ public final AliasesManager aliasesHolder = new AliasesManager();
+
+ /**
+ * Get an immutable copy of the present state of the aliases. References to this object should not be retained
+ * in any context where it will be important to know if aliases have changed.
+ *
+ * @return The current aliases, Aliases.EMPTY if not solr cloud, or no aliases have existed yet. Never returns null.
+ */
+ public Aliases getAliases() {
+ return aliasesHolder.getAliases();
+ }
+
+ // called by createClusterStateWatchersAndUpdate()
+ private void refreshAliases(AliasesManager watcher) throws KeeperException, InterruptedException {
+ synchronized (getUpdateLock()) {
+ constructState(Collections.emptySet());
+ zkClient.exists(ALIASES, watcher, true);
+ }
+ aliasesHolder.update();
+ }
+
+ /**
+ * A class to manage the aliases instance, including watching for changes.
+ * There should only ever be one instance of this class
+ * per instance of ZkStateReader. Normally it will not be useful to create a new instance since
+ * this watcher automatically re-registers itself every time it is updated.
+ */
+ public class AliasesManager implements Watcher { // the holder is a Zk watcher
+ // note: as of this writing, this class if very generic. Is it useful to use for other ZK managed things?
+ private final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private volatile Aliases aliases = Aliases.EMPTY;
+
+ public Aliases getAliases() {
+ return aliases; // volatile read
+ }
+
+ /**
+ * Writes an updated {@link Aliases} to zk.
+ * It will retry if there are races with other modifications, giving up after 30 seconds with a SolrException.
+ * The caller should understand it's possible the aliases has further changed if it examines it.
+ */
+ public void applyModificationAndExportToZk(UnaryOperator<Aliases> op) {
+ final long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
+ int triesLeft = 5;
+ while (triesLeft > 0) {
+ triesLeft--;
+ // we could synchronize on "this" but there doesn't seem to be a point; we have a retry loop.
+ Aliases curAliases = getAliases();
+ Aliases modAliases = op.apply(curAliases);
+ final byte[] modAliasesJson = modAliases.toJSON();
+ if (curAliases == modAliases) {
+ LOG.debug("Current aliases has the desired modification; no further ZK interaction needed.");
+ return;
+ }
+
+ try {
+ try {
+ final Stat stat = getZkClient().setData(ALIASES, modAliasesJson, curAliases.getZNodeVersion(), true);
+ setIfNewer(Aliases.fromJSON(modAliasesJson, stat.getVersion()));
+ return;
+ } catch (KeeperException.BadVersionException e) {
+ LOG.debug(e.toString(), e);
+ LOG.warn("Couldn't save aliases due to race with another modification; will update and retry until timeout");
+ // considered a backoff here, but we really do want to compete strongly since the normal case is
+ // that we will do one update and succeed. This is left as a hot loop for 5 tries intentionally.
+ // More failures than that here probably indicate a bug or a very strange high write frequency usage for
+ // aliases.json, timeouts mean zk is being very slow to respond, or this node is being crushed
+ // by other processing and just can't find any cpu cycles at all.
+ update();
+ if (deadlineNanos < System.nanoTime()) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Timed out trying to update aliases! " +
+ "Either zookeeper or this node may be overloaded.");
+ }
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new ZooKeeperException(ErrorCode.SERVER_ERROR, e.toString(), e);
+ } catch (KeeperException e) {
+ throw new ZooKeeperException(ErrorCode.SERVER_ERROR, e.toString(), e);
+ }
+ }
+ if (triesLeft == 0) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Too many successive version failures trying to update aliases");
+ }
+ }
+
+ /**
+ * Ensures the internal aliases is up to date. If there is a change, return true.
+ *
+ * @return true if an update was performed
+ */
+ public boolean update() throws KeeperException, InterruptedException {
+ Stat stat = new Stat();
+ final byte[] data = zkClient.getData(ALIASES, null, stat, true);
+ return setIfNewer(Aliases.fromJSON(data, stat.getVersion()));
+ }
+
+ // ZK Watcher interface
+ @Override
+ public void process(WatchedEvent event) {
+ // session events are not change events, and do not remove the watcher
+ if (EventType.None.equals(event.getType())) {
+ return;
+ }
+ try {
+ LOG.debug("Aliases: updating");
+
+ // re-register the watch
+ Stat stat = new Stat();
+ final byte[] data = zkClient.getData(ALIASES, this, stat, true);
+ // note: it'd be nice to avoid possibly needlessly parsing if we don't update aliases but not a big deal
+ setIfNewer(Aliases.fromJSON(data, stat.getVersion()));
+ } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) {
+ LOG.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage());
+ } catch (KeeperException e) {
+ LOG.error("A ZK error has occurred", e);
+ throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "A ZK error has occurred", e);
+ } catch (InterruptedException e) {
+ // Restore the interrupted status
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted", e);
+ }
+ }
+
+ /**
+ * Update the internal aliases reference with a new one, provided that its ZK version has increased.
+ *
+ * @param newAliases the potentially newer version of Aliases
+ */
+ private boolean setIfNewer(Aliases newAliases) {
+ synchronized (this) {
+ int cmp = Integer.compare(aliases.getZNodeVersion(), newAliases.getZNodeVersion());
+ if (cmp < 0) {
+ LOG.debug("Aliases: cmp={}, new definition is: {}", cmp, newAliases);
+ aliases = newAliases;
+ this.notifyAll();
+ return true;
+ } else {
+ LOG.debug("Aliases: cmp={}, not overwriting ZK version.", cmp);
+ assert cmp != 0 || Arrays.equals(aliases.toJSON(), newAliases.toJSON()) : aliases + " != " + newAliases;
+ return false;
+ }
+ }
+ }
+
+ }
+
}