You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/03/28 13:54:48 UTC
svn commit: r1582734 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/cloud/
core/src/java/org/apache/solr/handler/admin/
core/src/test/org/apache/solr/cloud/
solrj/src/java/org/apache/solr/common/params/
Author: shalin
Date: Fri Mar 28 12:54:47 2014
New Revision: 1582734
URL: http://svn.apache.org/r1582734
Log:
SOLR-5466: A new List collections and cluster status API which clients can use to read collection and shard information instead of reading data directly from ZooKeeper
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1582734&r1=1582733&r2=1582734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri Mar 28 12:54:47 2014
@@ -159,6 +159,10 @@ New Features
* SOLR-4478: Allow cores to use configuration from a configsets directory
outside their instance directory. (Alan Woodward, Erick Erickson)
+* SOLR-5466: A new List collections and cluster status API which clients can use
+ to read collection and shard information instead of reading data directly from ZooKeeper.
+ (Dave Seltzer, Varun Thacker, Vitaliy Zhovtyuk, Erick Erickson, shalin)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1582734&r1=1582733&r2=1582734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Fri Mar 28 12:54:47 2014
@@ -88,6 +88,8 @@ import static org.apache.solr.common.clo
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERSTATUS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.LIST;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
@@ -136,7 +138,6 @@ public class OverseerCollectionProcessor
public static final String COLL_PROP_PREFIX = "property.";
-
public static final Set<String> KNOWN_CLUSTER_PROPS = ImmutableSet.of(ZkStateReader.LEGACY_CLOUD, ZkStateReader.URL_SCHEME);
public static final Map<String,Object> COLL_PROPS = ZkNodeProps.makeMap(
@@ -474,9 +475,11 @@ public class OverseerCollectionProcessor
requestStatus(message, results);
} else if (OVERSEERSTATUS.isEqual(operation)) {
getOverseerStatus(message, results);
- }
-
- else {
+ } else if(LIST.isEqual(operation)) {
+ listCollections(zkStateReader.getClusterState(), results);
+ } else if (CLUSTERSTATUS.isEqual(operation)) {
+ getClusterStatus(zkStateReader.getClusterState(), message, results);
+ } else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ operation);
}
@@ -567,6 +570,131 @@ public class OverseerCollectionProcessor
}
+ private void getClusterStatus(ClusterState clusterState, ZkNodeProps message, NamedList results) {
+ String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
+
+ // read aliases
+ Aliases aliases = zkStateReader.getAliases();
+ Map<String, List<String>> collectionVsAliases = new HashMap<>();
+ Map<String, String> aliasVsCollections = aliases.getCollectionAliasMap();
+ if (aliasVsCollections != null) {
+ for (Map.Entry<String, String> entry : aliasVsCollections.entrySet()) {
+ List<String> colls = StrUtils.splitSmart(entry.getValue(), ',');
+ String alias = entry.getKey();
+ for (String coll : colls) {
+ if (collection == null || collection.equals(coll)) {
+ List<String> list = collectionVsAliases.get(coll);
+ if (list == null) {
+ list = new ArrayList<>();
+ collectionVsAliases.put(coll, list);
+ }
+ list.add(alias);
+ }
+ }
+ }
+ }
+
+ // convert cluster state into a map of writable types
+ byte[] bytes = ZkStateReader.toJSON(clusterState);
+ Map<String, Object> stateMap = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
+
+ String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
+ NamedList<Object> collectionProps = new SimpleOrderedMap<Object>();
+ if (collection == null) {
+ Set<String> collections = clusterState.getCollections();
+ for (String name : collections) {
+ Map<String, Object> collectionStatus = getCollectionStatus(stateMap, name, shard);
+ if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(name));
+ }
+ collectionProps.add(name, collectionStatus);
+ }
+ } else {
+ String routeKey = message.getStr(ShardParams._ROUTE_);
+ if (routeKey == null) {
+ Map<String, Object> collectionStatus = getCollectionStatus(stateMap, collection, shard);
+ if (collectionVsAliases.containsKey(collection) && !collectionVsAliases.get(collection).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(collection));
+ }
+ collectionProps.add(collection, collectionStatus);
+ } else {
+ DocCollection docCollection = clusterState.getCollection(collection);
+ DocRouter router = docCollection.getRouter();
+ Collection<Slice> slices = router.getSearchSlices(routeKey, null, docCollection);
+ String s = "";
+ for (Slice slice : slices) {
+ s += slice.getName() + ",";
+ }
+ if (shard != null) {
+ s += shard;
+ }
+ Map<String, Object> collectionStatus = getCollectionStatus(stateMap, collection, s);
+ if (collectionVsAliases.containsKey(collection) && !collectionVsAliases.get(collection).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(collection));
+ }
+ collectionProps.add(collection, collectionStatus);
+ }
+ }
+
+
+ NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
+ clusterStatus.add("collections", collectionProps);
+
+ // read cluster properties
+ Map clusterProps = zkStateReader.getClusterProps();
+ if (clusterProps != null && !clusterProps.isEmpty()) {
+ clusterStatus.add("properties", clusterProps);
+ }
+
+ // add the alias map too
+ if (aliasVsCollections != null && !aliasVsCollections.isEmpty()) {
+ clusterStatus.add("aliases", aliasVsCollections);
+ }
+
+ results.add("cluster", clusterStatus);
+ }
+
+ /**
+ * Get collection status from cluster state.
+ * Can return collection status by given shard name.
+ *
+ *
+ * @param clusterState
+ * @param name collection name
+ * @param shardStr comma separated shard names
+ * @return map of collection properties
+ */
+ private Map<String, Object> getCollectionStatus(Map<String, Object> clusterState, String name, String shardStr) {
+ Map<String, Object> docCollection = (Map<String, Object>) clusterState.get(name);
+ if (docCollection == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection: " + name + " not found");
+ }
+ if (shardStr == null) {
+ return docCollection;
+ } else {
+ Map<String, Object> shards = (Map<String, Object>) docCollection.get("shards");
+ Map<String, Object> selected = new HashMap<>();
+ List<String> selectedShards = Arrays.asList(shardStr.split(","));
+ for (String selectedShard : selectedShards) {
+ if (!shards.containsKey(selectedShard)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection: " + name + " shard: " + selectedShard + " not found");
+ }
+ selected.put(selectedShard, shards.get(selectedShard));
+ docCollection.put("shards", selected);
+ }
+ return docCollection;
+ }
+ }
+
+ private void listCollections(ClusterState clusterState, NamedList results) {
+ Set<String> collections = clusterState.getCollections();
+ List<String> collectionList = new ArrayList<String>();
+ for (String collection : collections) {
+ collectionList.add(collection);
+ }
+ results.add("collections", collectionList);
+ }
+
private void processRoleCommand(ZkNodeProps message, String operation) throws KeeperException, InterruptedException {
SolrZkClient zkClient = zkStateReader.getZkClient();
Map roles = null;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1582734&r1=1582733&r2=1582734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Fri Mar 28 12:54:47 2014
@@ -211,6 +211,14 @@ public class CollectionsHandler extends
this.handleOverseerStatus(req, rsp);
break;
}
+ case LIST: {
+ this.handleListAction(req, rsp);
+ break;
+ }
+ case CLUSTERSTATUS: {
+ this.handleClusterStatus(req, rsp);
+ break;
+ }
default: {
throw new RuntimeException("Unknown action: " + action);
}
@@ -574,6 +582,36 @@ public class CollectionsHandler extends
handleResponse(CollectionAction.ADDREPLICA.toString(), m, rsp);
}
+ /**
+ * Handle cluster status request.
+ * Can return status per specific collection/shard or per all collections.
+ *
+ * @param req solr request
+ * @param rsp solr response
+ */
+ private void handleClusterStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ Map<String,Object> props = new HashMap<>();
+ props.put(Overseer.QUEUE_OPERATION, CollectionAction.CLUSTERSTATUS.toLower());
+ copyIfNotNull(req.getParams(), props, COLLECTION_PROP, SHARD_ID_PROP, ShardParams._ROUTE_);
+ handleResponse(CollectionAction.CLUSTERSTATUS.toString(), new ZkNodeProps(props), rsp);
+ }
+
+ /**
+ * Handled list collection request.
+ * Do list collection request to zk host
+ *
+ * @param req solr request
+ * @param rsp solr response
+ * @throws KeeperException zk connection failed
+ * @throws InterruptedException connection interrupted
+ */
+ private void handleListAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ Map<String, Object> props = ZkNodeProps.makeMap(
+ Overseer.QUEUE_OPERATION, CollectionAction.LIST.toString().toLowerCase(Locale.ROOT));
+ handleResponse(CollectionAction.LIST.toString(), new ZkNodeProps(props), rsp);
+ }
+
+
public static ModifiableSolrParams params(String... params) {
ModifiableSolrParams msp = new ModifiableSolrParams();
for (int i=0; i<params.length; i+=2) {
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java?rev=1582734&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java Fri Mar 28 12:54:47 2014
@@ -0,0 +1,241 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import com.google.common.collect.Lists;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrServer;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.NamedList;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+public class TestCollectionAPI extends AbstractFullDistribZkTestBase {
+
+ public static final String COLLECTION_NAME = "testcollection";
+ public static final String COLLECTION_NAME1 = "testcollection1";
+
+ public TestCollectionAPI() {
+ schemaString = "schema15.xml"; // we need a string id
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ fixShardCount = true;
+ sliceCount = 2;
+ shardCount = 2;
+ super.setUp();
+ }
+
+ @Override
+ public void doTest() throws Exception {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ createCollection(null, COLLECTION_NAME, 2, 1, 1, client, null, "conf1");
+ createCollection(null, COLLECTION_NAME1, 1, 1, 1, client, null, "conf1");
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+
+ listCollection();
+ clusterStatusNoCollection();
+ clusterStatusWithCollection();
+ clusterStatusWithCollectionAndShard();
+ clusterStatusWithRouteKey();
+ clusterStatusAliasTest();
+ }
+
+ private void clusterStatusWithCollectionAndShard() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", COLLECTION_NAME);
+ params.set("shard", SHARD1);
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList<Object> rsp = client.request(request);
+ NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(COLLECTION_NAME));
+ assertEquals(1, collections.size());
+ Map<String, Object> collection = (Map<String, Object>) collections.get(COLLECTION_NAME);
+ Map<String, Object> shardStatus = (Map<String, Object>) collection.get("shards");
+ assertEquals(1, shardStatus.size());
+ Map<String, Object> selectedShardStatus = (Map<String, Object>) shardStatus.get(SHARD1);
+ assertNotNull(selectedShardStatus);
+
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+
+
+ private void listCollection() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.LIST.toString());
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList<Object> rsp = client.request(request);
+ List<String> collections = (List<String>) rsp.get("collections");
+ assertTrue("control_collection was not found in list", collections.contains("control_collection"));
+ assertTrue(DEFAULT_COLLECTION + " was not found in list", collections.contains(DEFAULT_COLLECTION));
+ assertTrue(COLLECTION_NAME + " was not found in list", collections.contains(COLLECTION_NAME));
+ assertTrue(COLLECTION_NAME1 + " was not found in list", collections.contains(COLLECTION_NAME1));
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+
+
+ }
+
+ private void clusterStatusNoCollection() throws Exception {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList<Object> rsp = client.request(request);
+ NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(COLLECTION_NAME1));
+ assertEquals(4, collections.size());
+
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+
+ }
+
+ private void clusterStatusWithCollection() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", COLLECTION_NAME);
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList<Object> rsp = client.request(request);
+ NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(COLLECTION_NAME));
+ assertEquals(1, collections.size());
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+
+ private void clusterStatusWithRouteKey() throws IOException, SolrServerException {
+ CloudSolrServer client = createCloudClient(DEFAULT_COLLECTION);
+ try {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", "a!123"); // goes to shard2. see ShardRoutingTest for details
+ client.add(doc);
+ client.commit();
+
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", DEFAULT_COLLECTION);
+ params.set(ShardParams._ROUTE_, "a!");
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList<Object> rsp = client.request(request);
+ NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(DEFAULT_COLLECTION));
+ assertEquals(1, collections.size());
+ Map<String, Object> collection = (Map<String, Object>) collections.get(DEFAULT_COLLECTION);
+ Map<String, Object> shardStatus = (Map<String, Object>) collection.get("shards");
+ assertEquals(1, shardStatus.size());
+ Map<String, Object> selectedShardStatus = (Map<String, Object>) shardStatus.get(SHARD2);
+ assertNotNull(selectedShardStatus);
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+
+ private void clusterStatusAliasTest() throws Exception {
+ CloudSolrServer client = createCloudClient(null);
+ try {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CREATEALIAS.toString());
+ params.set("name", "myalias");
+ params.set("collections", DEFAULT_COLLECTION + "," + COLLECTION_NAME);
+ SolrRequest request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+ client.request(request);
+ params = new ModifiableSolrParams();
+ params.set("action", CollectionParams.CollectionAction.CLUSTERSTATUS.toString());
+ params.set("collection", DEFAULT_COLLECTION);
+ request = new QueryRequest(params);
+ request.setPath("/admin/collections");
+
+ NamedList<Object> rsp = client.request(request);
+
+
+ NamedList<Object> cluster = (NamedList<Object>) rsp.get("cluster");
+ assertNotNull("Cluster state should not be null", cluster);
+ Map<String, String> aliases = (Map<String, String>) cluster.get("aliases");
+ assertNotNull("Aliases should not be null", aliases);
+ assertEquals("Alias: myalias not found in cluster status",
+ DEFAULT_COLLECTION + "," + COLLECTION_NAME, aliases.get("myalias"));
+
+ NamedList<Object> collections = (NamedList<Object>) cluster.get("collections");
+ assertNotNull("Collections should not be null in cluster state", collections);
+ assertNotNull(collections.get(DEFAULT_COLLECTION));
+ Map<String, Object> collection = (Map<String, Object>) collections.get(DEFAULT_COLLECTION);
+ List<String> collAlias = (List<String>) collection.get("aliases");
+ assertEquals("Aliases not found", Lists.newArrayList("myalias"), collAlias);
+ } finally {
+ //remove collections
+ client.shutdown();
+ }
+ }
+}
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java?rev=1582734&r1=1582733&r2=1582734&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java Fri Mar 28 12:54:47 2014
@@ -44,7 +44,9 @@ public interface CollectionParams
CLUSTERPROP,
REQUESTSTATUS,
ADDREPLICA,
- OVERSEERSTATUS;
+ OVERSEERSTATUS,
+ LIST,
+ CLUSTERSTATUS;
public static CollectionAction get( String p )
{