You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2015/06/04 12:54:59 UTC
svn commit: r1683514 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/cloud/
core/src/java/org/apache/solr/handler/admin/
Author: noble
Date: Thu Jun 4 10:54:59 2015
New Revision: 1683514
URL: http://svn.apache.org/r1683514
Log:
SOLR-7636: CLUSTERSTATUS API is executed at CollectionsHandler
Added:
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1683514&r1=1683513&r2=1683514&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Thu Jun 4 10:54:59 2015
@@ -118,6 +118,8 @@ Other Changes
SolrDispatchFilter#execute() and SolrDispatchFilter#sendError(). You can now override these
functions in HttpSolrCall. (ryan)
+* SOLR-7636: CLUSTERSTATUS API is executed at CollectionsHandler (noble)
+
================== 5.2.0 ==================
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1683514&r1=1683513&r2=1683514&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Thu Jun 4 10:54:59 2015
@@ -17,14 +17,6 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
-import static org.apache.solr.cloud.Assign.*;
-import static org.apache.solr.common.cloud.DocCollection.SNITCH;
-import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
-import static org.apache.solr.common.cloud.ZkStateReader.*;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
-import static org.apache.solr.common.params.CommonParams.*;
-import static org.apache.solr.common.util.StrUtils.formatString;
-
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
@@ -55,11 +47,11 @@ import org.apache.solr.client.solrj.resp
import org.apache.solr.cloud.Assign.Node;
import org.apache.solr.cloud.DistributedQueue.QueueEvent;
import org.apache.solr.cloud.Overseer.LeaderStatus;
-import org.apache.solr.cloud.rule.Rule;
-import org.apache.solr.cloud.rule.ReplicaAssigner;
-import org.apache.solr.cloud.rule.ReplicaAssigner.Position;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
import org.apache.solr.cloud.overseer.OverseerAction;
+import org.apache.solr.cloud.rule.ReplicaAssigner;
+import org.apache.solr.cloud.rule.ReplicaAssigner.Position;
+import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.Aliases;
@@ -87,11 +79,11 @@ import org.apache.solr.common.util.Execu
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.handler.admin.ClusterStatus;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
-import org.apache.solr.logging.MDCLoggingContext;
import org.apache.solr.update.SolrIndexSplitter;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.stats.Snapshot;
@@ -102,7 +94,35 @@ import org.apache.zookeeper.KeeperExcept
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
+
+import static org.apache.solr.cloud.Assign.getNodesForNewShard;
+import static org.apache.solr.common.cloud.DocCollection.SNITCH;
+import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
+import static org.apache.solr.common.cloud.ZkStateReader.BASE_URL_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICAPROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDROLE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.BALANCESHARDUNIQUE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CLUSTERSTATUS;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.common.util.StrUtils.formatString;
public class OverseerCollectionProcessor implements Runnable, Closeable {
@@ -367,6 +387,7 @@ public class OverseerCollectionProcessor
return true;
// CLUSTERSTATUS is always mutually exclusive
+ //TODO deprecated remove this check .
if(CLUSTERSTATUS.isEqual(message.getStr(Overseer.QUEUE_OPERATION)))
return true;
@@ -598,8 +619,8 @@ public class OverseerCollectionProcessor
case OVERSEERSTATUS:
getOverseerStatus(message, results);
break;
- case CLUSTERSTATUS:
- getClusterStatus(zkStateReader.getClusterState(), message, results);
+ case CLUSTERSTATUS://TODO . deprecated. OCP does not need to do it .remove in a later release
+ new ClusterStatus(zkStateReader, message).getClusterStatus(results);
break;
case ADDREPLICAPROP:
processReplicaAddPropertyCommand(message);
@@ -2782,7 +2803,8 @@ public class OverseerCollectionProcessor
synchronized (runningTasks) {
runningTasks.add(head.getId());
}
- if(!CLUSTERSTATUS.isEqual(message.getStr(Overseer.QUEUE_OPERATION)) && collectionName != null) {
+ //TODO deprecated remove this check .
+ if (!CLUSTERSTATUS.isEqual(message.getStr(Overseer.QUEUE_OPERATION)) && collectionName != null) {
synchronized (collectionWip) {
collectionWip.add(collectionName);
}
Added: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java?rev=1683514&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java (added)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java Thu Jun 4 10:54:59 2015
@@ -0,0 +1,236 @@
+package org.apache.solr.handler.admin;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.Aliases;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.zookeeper.KeeperException;
+
+public class ClusterStatus {
+ private final ZkStateReader zkStateReader;
+ private final String collection;
+ private ZkNodeProps message;
+
+ public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) {
+ this.zkStateReader = zkStateReader;
+ this.message = props;
+ collection = props.getStr(ZkStateReader.COLLECTION_PROP);
+
+ }
+
+ @SuppressWarnings("unchecked")
+ public void getClusterStatus(NamedList results)
+ throws KeeperException, InterruptedException {
+
+
+ // read aliases
+ Aliases aliases = zkStateReader.getAliases();
+ Map<String, List<String>> collectionVsAliases = new HashMap<>();
+ Map<String, String> aliasVsCollections = aliases.getCollectionAliasMap();
+ if (aliasVsCollections != null) {
+ for (Map.Entry<String, String> entry : aliasVsCollections.entrySet()) {
+ List<String> colls = StrUtils.splitSmart(entry.getValue(), ',');
+ String alias = entry.getKey();
+ for (String coll : colls) {
+ if (collection == null || collection.equals(coll)) {
+ List<String> list = collectionVsAliases.get(coll);
+ if (list == null) {
+ list = new ArrayList<>();
+ collectionVsAliases.put(coll, list);
+ }
+ list.add(alias);
+ }
+ }
+ }
+ }
+
+ Map roles = null;
+ if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {
+ roles = (Map) ZkStateReader.fromJSON(zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true));
+ }
+
+ ClusterState clusterState = zkStateReader.getClusterState();
+
+ // convert cluster state into a map of writable types
+ byte[] bytes = ZkStateReader.toJSON(clusterState);
+ Map<String, Object> stateMap = (Map<String,Object>) ZkStateReader.fromJSON(bytes);
+
+ Set<String> collections = new HashSet<>();
+ String routeKey = message.getStr(ShardParams._ROUTE_);
+ String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
+ if (collection == null) {
+ collections = new HashSet<>(clusterState.getCollections());
+ } else {
+ collections = Collections.singleton(collection);
+ }
+
+ NamedList<Object> collectionProps = new SimpleOrderedMap<Object>();
+
+ for (String name : collections) {
+ Map<String, Object> collectionStatus = null;
+ DocCollection clusterStateCollection = clusterState.getCollection(name);
+
+ Set<String> requestedShards = new HashSet<>();
+ if (routeKey != null) {
+ DocRouter router = clusterStateCollection.getRouter();
+ Collection<Slice> slices = router.getSearchSlices(routeKey, null, clusterStateCollection);
+ for (Slice slice : slices) {
+ requestedShards.add(slice.getName());
+ }
+ }
+ if (shard != null) {
+ requestedShards.add(shard);
+ }
+
+ if (clusterStateCollection.getStateFormat() > 1) {
+ bytes = ZkStateReader.toJSON(clusterStateCollection);
+ Map<String, Object> docCollection = (Map<String, Object>) ZkStateReader.fromJSON(bytes);
+ collectionStatus = getCollectionStatus(docCollection, name, requestedShards);
+ } else {
+ collectionStatus = getCollectionStatus((Map<String, Object>) stateMap.get(name), name, requestedShards);
+ }
+
+ collectionStatus.put("znodeVersion", clusterStateCollection.getZNodeVersion());
+ if (collectionVsAliases.containsKey(name) && !collectionVsAliases.get(name).isEmpty()) {
+ collectionStatus.put("aliases", collectionVsAliases.get(name));
+ }
+ String configName = zkStateReader.readConfigName(name);
+ collectionStatus.put("configName", configName);
+ collectionProps.add(name, collectionStatus);
+ }
+
+ List<String> liveNodes = zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true);
+
+ // now we need to walk the collectionProps tree to cross-check replica state with live nodes
+ crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps);
+
+ NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
+ clusterStatus.add("collections", collectionProps);
+
+ // read cluster properties
+ Map clusterProps = zkStateReader.getClusterProps();
+ if (clusterProps != null && !clusterProps.isEmpty()) {
+ clusterStatus.add("properties", clusterProps);
+ }
+
+ // add the alias map too
+ if (aliasVsCollections != null && !aliasVsCollections.isEmpty()) {
+ clusterStatus.add("aliases", aliasVsCollections);
+ }
+
+ // add the roles map
+ if (roles != null) {
+ clusterStatus.add("roles", roles);
+ }
+
+ // add live_nodes
+ clusterStatus.add("live_nodes", liveNodes);
+
+ results.add("cluster", clusterStatus);
+ }
+ /**
+ * Get collection status from cluster state.
+ * Can return collection status by given shard name.
+ *
+ *
+ * @param collection collection map parsed from JSON-serialized {@link ClusterState}
+ * @param name collection name
+ * @param requestedShards a set of shards to be returned in the status.
+ * An empty or null values indicates <b>all</b> shards.
+ * @return map of collection properties
+ */
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> getCollectionStatus(Map<String, Object> collection, String name, Set<String> requestedShards) {
+ if (collection == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + name + " not found");
+ }
+ if (requestedShards == null || requestedShards.isEmpty()) {
+ return collection;
+ } else {
+ Map<String, Object> shards = (Map<String, Object>) collection.get("shards");
+ Map<String, Object> selected = new HashMap<>();
+ for (String selectedShard : requestedShards) {
+ if (!shards.containsKey(selectedShard)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection: " + name + " shard: " + selectedShard + " not found");
+ }
+ selected.put(selectedShard, shards.get(selectedShard));
+ collection.put("shards", selected);
+ }
+ return collection;
+ }
+ }
+
+
+
+ /**
+ * Walks the tree of collection status to verify that any replicas not reporting a "down" status is
+ * on a live node, if any replicas reporting their status as "active" but the node is not live is
+ * marked as "down"; used by CLUSTERSTATUS.
+ * @param liveNodes List of currently live node names.
+ * @param collectionProps Map of collection status information pulled directly from ZooKeeper.
+ */
+
+ @SuppressWarnings("unchecked")
+ protected void crossCheckReplicaStateWithLiveNodes(List<String> liveNodes, NamedList<Object> collectionProps) {
+ Iterator<Map.Entry<String,Object>> colls = collectionProps.iterator();
+ while (colls.hasNext()) {
+ Map.Entry<String,Object> next = colls.next();
+ Map<String,Object> collMap = (Map<String,Object>)next.getValue();
+ Map<String,Object> shards = (Map<String,Object>)collMap.get("shards");
+ for (Object nextShard : shards.values()) {
+ Map<String,Object> shardMap = (Map<String,Object>)nextShard;
+ Map<String,Object> replicas = (Map<String,Object>)shardMap.get("replicas");
+ for (Object nextReplica : replicas.values()) {
+ Map<String,Object> replicaMap = (Map<String,Object>)nextReplica;
+ if (Replica.State.getState((String) replicaMap.get(ZkStateReader.STATE_PROP)) != Replica.State.DOWN) {
+ // not down, so verify the node is live
+ String node_name = (String)replicaMap.get(ZkStateReader.NODE_NAME_PROP);
+ if (!liveNodes.contains(node_name)) {
+ // node is not live, so this replica is actually down
+ replicaMap.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1683514&r1=1683513&r2=1683514&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Thu Jun 4 10:54:59 2015
@@ -20,6 +20,10 @@ package org.apache.solr.handler.admin;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -44,9 +48,13 @@ import org.apache.solr.cloud.rule.Replic
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.ImplicitDocRouter;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -55,9 +63,11 @@ import org.apache.solr.common.cloud.ZkSt
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.BlobHandler;
import org.apache.solr.handler.RequestHandlerBase;
@@ -597,10 +607,13 @@ public class CollectionsHandler extends
@Override
Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler handler)
throws KeeperException, InterruptedException {
- return req.getParams().getAll(null,
+ Map<String, Object> all = req.getParams().getAll(null,
COLLECTION_PROP,
SHARD_ID_PROP,
_ROUTE_);
+ new ClusterStatus(handler.coreContainer.getZkController().getZkStateReader(),
+ new ZkNodeProps(all)).getClusterStatus(rsp.getValues());
+ return null;
}
},
ADDREPLICAPROP_OP(ADDREPLICAPROP) {
@@ -759,4 +772,4 @@ public class CollectionsHandler extends
MAX_SHARDS_PER_NODE,
AUTO_ADD_REPLICAS);
-}
+ }