You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2014/03/28 13:56:06 UTC
svn commit: r1582736 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/cloud/
solr/core/src/java/org/apache/solr/handler/admin/
solr/core/src/test/org/apache/solr/cloud/ solr/solrj/
solr/solrj/src/java/org/apac...
Author: shalin
Date: Fri Mar 28 12:56:06 2014
New Revision: 1582736
URL: http://svn.apache.org/r1582736
Log:
SOLR-5466: A new List collections and cluster status API which clients can use to read collection and shard information instead of reading data directly from ZooKeeper
Added:
lucene/dev/branches/branch_4x/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
- copied unchanged from r1582734, lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/TestCollectionAPI.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1582736&r1=1582735&r2=1582736&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Fri Mar 28 12:56:06 2014
@@ -106,6 +106,10 @@ New Features
* SOLR-4478: Allow cores to use configuration from a configsets directory
outside their instance directory. (Alan Woodward, Erick Erickson)
+* SOLR-5466: A new List collections and cluster status API which clients can use
+ to read collection and shard information instead of reading data directly from ZooKeeper.
+ (Dave Seltzer, Varun Thacker, Vitaliy Zhovtyuk, Erick Erickson, shalin)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1582736&r1=1582735&r2=1582736&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Fri Mar 28 12:56:06 2014
@@ -93,6 +93,8 @@ import static org.apache.solr.common.clo
import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERSTATUS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.LIST;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.OVERSEERSTATUS;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
@@ -141,7 +143,6 @@ public class OverseerCollectionProcessor
public static final String COLL_PROP_PREFIX = "property.";
-
public static final Set<String> KNOWN_CLUSTER_PROPS = ImmutableSet.of(ZkStateReader.LEGACY_CLOUD, ZkStateReader.URL_SCHEME);
public static final Map<String,Object> COLL_PROPS = ZkNodeProps.makeMap(
@@ -479,9 +480,11 @@ public class OverseerCollectionProcessor
requestStatus(message, results);
} else if (OVERSEERSTATUS.isEqual(operation)) {
getOverseerStatus(message, results);
- }
-
- else {
+ } else if(LIST.isEqual(operation)) {
+ listCollections(zkStateReader.getClusterState(), results);
+ } else if (CLUSTERSTATUS.isEqual(operation)) {
+ getClusterStatus(zkStateReader.getClusterState(), message, results);
+ } else {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ operation);
}
@@ -572,6 +575,131 @@ public class OverseerCollectionProcessor
}
+ private void getClusterStatus(ClusterState clusterState, ZkNodeProps message, NamedList results) {
+ String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
+
+ // read aliases
+ Aliases aliases = zkStateReader.getAliases();
+ Map<String, List<String>> collectionVsAliases = new HashMap<>();
+ Map<String, String> aliasVsCollections = aliases.getCollectionAliasMap();
+ if (aliasVsCollections != null) {
+ for (Map.Entry<String, String> entry : aliasVsCollections.entrySet()) {
+ List<String> colls = StrUtils.splitSmart(entry.getValue(), ',');
+ String alias = entry.getKey();
+ for (String coll : colls) {
+ if (collection == null || collection.equals(coll)) {
+ List<String> list = collectionVsAliases.get(coll);
+ if (list == null) {
+ list = new ArrayList<>();
+ collectionVsAliases.put(coll, list);
+ }
+ list.add(alias);
+ }
+ }
+ }
+ }
+
+ // convert cluster state into a map of writable types
+ byte[] bytes = ZkStateReader.toJSON(clusterState);
+ Map<String, Object> stateMap = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
+
+ String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
+ NamedList<Object> collectionProps = new SimpleOrderedMap<Object>();
+ if (collection == null) {
+ Set<String> collections = clusterState.getCollections();
+ for (String name : collections) {
+ Map<String, Object> collectionStatus = getCollectionStatus(stateMap, name, shard);
+ if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(name));
+ }
+ collectionProps.add(name, collectionStatus);
+ }
+ } else {
+ String routeKey = message.getStr(ShardParams._ROUTE_);
+ if (routeKey == null) {
+ Map<String, Object> collectionStatus = getCollectionStatus(stateMap, collection, shard);
+ if (collectionVsAliases.containsKey(collection) && !collectionVsAliases.get(collection).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(collection));
+ }
+ collectionProps.add(collection, collectionStatus);
+ } else {
+ DocCollection docCollection = clusterState.getCollection(collection);
+ DocRouter router = docCollection.getRouter();
+ Collection<Slice> slices = router.getSearchSlices(routeKey, null, docCollection);
+ String s = "";
+ for (Slice slice : slices) {
+ s += slice.getName() + ",";
+ }
+ if (shard != null) {
+ s += shard;
+ }
+ Map<String, Object> collectionStatus = getCollectionStatus(stateMap, collection, s);
+ if (collectionVsAliases.containsKey(collection) && !collectionVsAliases.get(collection).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(collection));
+ }
+ collectionProps.add(collection, collectionStatus);
+ }
+ }
+
+
+ NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
+ clusterStatus.add("collections", collectionProps);
+
+ // read cluster properties
+ Map clusterProps = zkStateReader.getClusterProps();
+ if (clusterProps != null && !clusterProps.isEmpty()) {
+ clusterStatus.add("properties", clusterProps);
+ }
+
+ // add the alias map too
+ if (aliasVsCollections != null && !aliasVsCollections.isEmpty()) {
+ clusterStatus.add("aliases", aliasVsCollections);
+ }
+
+ results.add("cluster", clusterStatus);
+ }
+
+ /**
+ * Get collection status from cluster state.
+ * Can return collection status by given shard name.
+ *
+ *
+ * @param clusterState
+ * @param name collection name
+ * @param shardStr comma separated shard names
+ * @return map of collection properties
+ */
+ private Map<String, Object> getCollectionStatus(Map<String, Object> clusterState, String name, String shardStr) {
+ Map<String, Object> docCollection = (Map<String, Object>) clusterState.get(name);
+ if (docCollection == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection: " + name + " not found");
+ }
+ if (shardStr == null) {
+ return docCollection;
+ } else {
+ Map<String, Object> shards = (Map<String, Object>) docCollection.get("shards");
+ Map<String, Object> selected = new HashMap<>();
+ List<String> selectedShards = Arrays.asList(shardStr.split(","));
+ for (String selectedShard : selectedShards) {
+ if (!shards.containsKey(selectedShard)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Collection: " + name + " shard: " + selectedShard + " not found");
+ }
+ selected.put(selectedShard, shards.get(selectedShard));
+ docCollection.put("shards", selected);
+ }
+ return docCollection;
+ }
+ }
+
+ private void listCollections(ClusterState clusterState, NamedList results) {
+ Set<String> collections = clusterState.getCollections();
+ List<String> collectionList = new ArrayList<String>();
+ for (String collection : collections) {
+ collectionList.add(collection);
+ }
+ results.add("collections", collectionList);
+ }
+
private void processRoleCommand(ZkNodeProps message, String operation) throws KeeperException, InterruptedException {
SolrZkClient zkClient = zkStateReader.getZkClient();
Map roles = null;
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1582736&r1=1582735&r2=1582736&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Fri Mar 28 12:56:06 2014
@@ -211,6 +211,14 @@ public class CollectionsHandler extends
this.handleOverseerStatus(req, rsp);
break;
}
+ case LIST: {
+ this.handleListAction(req, rsp);
+ break;
+ }
+ case CLUSTERSTATUS: {
+ this.handleClusterStatus(req, rsp);
+ break;
+ }
default: {
throw new RuntimeException("Unknown action: " + action);
}
@@ -574,6 +582,36 @@ public class CollectionsHandler extends
handleResponse(CollectionAction.ADDREPLICA.toString(), m, rsp);
}
+ /**
+ * Handle cluster status request.
+ * Can return status per specific collection/shard or per all collections.
+ *
+ * @param req solr request
+ * @param rsp solr response
+ */
+ private void handleClusterStatus(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ Map<String,Object> props = new HashMap<>();
+ props.put(Overseer.QUEUE_OPERATION, CollectionAction.CLUSTERSTATUS.toLower());
+ copyIfNotNull(req.getParams(), props, COLLECTION_PROP, SHARD_ID_PROP, ShardParams._ROUTE_);
+ handleResponse(CollectionAction.CLUSTERSTATUS.toString(), new ZkNodeProps(props), rsp);
+ }
+
+ /**
+ * Handled list collection request.
+ * Do list collection request to zk host
+ *
+ * @param req solr request
+ * @param rsp solr response
+ * @throws KeeperException zk connection failed
+ * @throws InterruptedException connection interrupted
+ */
+ private void handleListAction(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ Map<String, Object> props = ZkNodeProps.makeMap(
+ Overseer.QUEUE_OPERATION, CollectionAction.LIST.toString().toLowerCase(Locale.ROOT));
+ handleResponse(CollectionAction.LIST.toString(), new ZkNodeProps(props), rsp);
+ }
+
+
public static ModifiableSolrParams params(String... params) {
ModifiableSolrParams msp = new ModifiableSolrParams();
for (int i=0; i<params.length; i+=2) {
Modified: lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java?rev=1582736&r1=1582735&r2=1582736&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java (original)
+++ lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java Fri Mar 28 12:56:06 2014
@@ -44,7 +44,9 @@ public interface CollectionParams
CLUSTERPROP,
REQUESTSTATUS,
ADDREPLICA,
- OVERSEERSTATUS;
+ OVERSEERSTATUS,
+ LIST,
+ CLUSTERSTATUS;
public static CollectionAction get( String p )
{