You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/08/13 08:33:34 UTC
svn commit: r1513364 [2/5] - in /lucene/dev/branches/lucene3069: ./
dev-tools/ dev-tools/idea/solr/core/src/java/
dev-tools/idea/solr/core/src/test/ dev-tools/maven/
dev-tools/maven/solr/core/src/java/ dev-tools/maven/solr/core/src/test/
dev-tools/scri...
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/Overseer.java Tue Aug 13 06:33:29 2013
@@ -17,16 +17,6 @@ package org.apache.solr.cloud;
* the License.
*/
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClosableThread;
@@ -46,6 +36,16 @@ import org.apache.zookeeper.KeeperExcept
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
/**
* Cluster leader. Responsible node assignments, cluster state file?
*/
@@ -54,7 +54,7 @@ public class Overseer {
public static final String DELETECORE = "deletecore";
public static final String REMOVECOLLECTION = "removecollection";
public static final String REMOVESHARD = "removeshard";
-
+
private static final int STATE_UPDATE_DELAY = 1500; // delay between cloud state updates
@@ -203,13 +203,36 @@ public class Overseer {
clusterState = createShard(clusterState, message);
} else if ("updateshardstate".equals(operation)) {
clusterState = updateShardState(clusterState, message);
+ } else if (OverseerCollectionProcessor.CREATECOLLECTION.equals(operation)) {
+ clusterState = buildCollection(clusterState, message);
} else {
throw new RuntimeException("unknown operation:" + operation
+ " contents:" + message.getProperties());
}
return clusterState;
}
-
+
+ private ClusterState buildCollection(ClusterState clusterState, ZkNodeProps message) {
+ String collection = message.getStr("name");
+ log.info("building a new collection: " + collection);
+ if(clusterState.getCollections().contains(collection) ){
+ log.warn("Collection {} already exists. exit" ,collection);
+ return clusterState;
+ }
+
+ ArrayList<String> shardNames = new ArrayList<String>();
+
+ if(ImplicitDocRouter.NAME.equals( message.getStr("router",DocRouter.DEFAULT_NAME))){
+ getShardNames(shardNames,message.getStr("shards",DocRouter.DEFAULT_NAME));
+ } else {
+ int numShards = message.getInt(ZkStateReader.NUM_SHARDS_PROP, -1);
+ if(numShards<1) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,"numShards is a required parameter for 'compositeId' router");
+ getShardNames(numShards, shardNames);
+ }
+
+ return createCollection(clusterState,collection,shardNames,message);
+ }
+
private ClusterState updateShardState(ClusterState clusterState, ZkNodeProps message) {
String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
log.info("Update shard state invoked for collection: " + collection);
@@ -294,12 +317,22 @@ public class Overseer {
}
message.getProperties().put(ZkStateReader.CORE_NODE_NAME_PROP, coreNodeName);
}
- Integer numShards = message.getStr(ZkStateReader.NUM_SHARDS_PROP)!=null?Integer.parseInt(message.getStr(ZkStateReader.NUM_SHARDS_PROP)):null;
+ Integer numShards = message.getInt(ZkStateReader.NUM_SHARDS_PROP, null);
log.info("Update state numShards={} message={}", numShards, message);
+
+ String router = message.getStr(OverseerCollectionProcessor.ROUTER,DocRouter.DEFAULT_NAME);
+ List<String> shardNames = new ArrayList<String>();
+
//collection does not yet exist, create placeholders if num shards is specified
boolean collectionExists = state.getCollections().contains(collection);
if (!collectionExists && numShards!=null) {
- state = createCollection(state, collection, numShards);
+ if(ImplicitDocRouter.NAME.equals(router)){
+ getShardNames(shardNames, message.getStr("shards",null));
+ numShards = shardNames.size();
+ }else {
+ getShardNames(numShards, shardNames);
+ }
+ state = createCollection(state, collection, shardNames, message);
}
// use the provided non null shardId
@@ -391,34 +424,42 @@ public class Overseer {
return newClusterState;
}
- private Map<String,Object> defaultCollectionProps() {
- HashMap<String,Object> props = new HashMap<String, Object>(2);
- props.put(DocCollection.DOC_ROUTER, DocRouter.DEFAULT_NAME);
- return props;
- }
+ private ClusterState createCollection(ClusterState state, String collectionName, List<String> shards , ZkNodeProps message) {
+ log.info("Create collection {} with shards {}", collectionName, shards);;
- private ClusterState createCollection(ClusterState state, String collectionName, int numShards) {
- log.info("Create collection {} with numShards {}", collectionName, numShards);
+ String routerName = message.getStr(OverseerCollectionProcessor.ROUTER,DocRouter.DEFAULT_NAME);
+ DocRouter router = DocRouter.getDocRouter(routerName);
- DocRouter router = DocRouter.DEFAULT;
- List<DocRouter.Range> ranges = router.partitionRange(numShards, router.fullRange());
+ List<DocRouter.Range> ranges = router.partitionRange(shards.size(), router.fullRange());
Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>();
Map<String, Slice> newSlices = new LinkedHashMap<String,Slice>();
newCollections.putAll(state.getCollectionStates());
+ for (int i = 0; i < shards.size(); i++) {
+ String sliceName = shards.get(i);
+ /*}
for (int i = 0; i < numShards; i++) {
- final String sliceName = "shard" + (i+1);
+ final String sliceName = "shard" + (i+1);*/
- Map<String,Object> sliceProps = new LinkedHashMap<String,Object>(1);
- sliceProps.put(Slice.RANGE, ranges.get(i));
+ Map<String, Object> sliceProps = new LinkedHashMap<String, Object>(1);
+ sliceProps.put(Slice.RANGE, ranges == null? null: ranges.get(i));
newSlices.put(sliceName, new Slice(sliceName, null, sliceProps));
}
// TODO: fill in with collection properties read from the /collections/<collectionName> node
- Map<String,Object> collectionProps = defaultCollectionProps();
+ Map<String,Object> collectionProps = new HashMap<String,Object>();
+
+ for (Entry<String, Object> e : OverseerCollectionProcessor.COLL_PROPS.entrySet()) {
+ Object val = message.get(e.getKey());
+ if(val == null){
+ val = OverseerCollectionProcessor.COLL_PROPS.get(e.getKey());
+ }
+ if(val != null) collectionProps.put(e.getKey(),val);
+ }
+ collectionProps.put(DocCollection.DOC_ROUTER, routerName);
DocCollection newCollection = new DocCollection(collectionName, newSlices, collectionProps, router);
@@ -466,7 +507,6 @@ public class Overseer {
private ClusterState updateSlice(ClusterState state, String collectionName, Slice slice) {
// System.out.println("###!!!### OLD CLUSTERSTATE: " + JSONUtil.toJSON(state.getCollectionStates()));
// System.out.println("Updating slice:" + slice);
-
Map<String, DocCollection> newCollections = new LinkedHashMap<String,DocCollection>(state.getCollectionStates()); // make a shallow copy
DocCollection coll = newCollections.get(collectionName);
Map<String,Slice> slices;
@@ -681,6 +721,28 @@ public class Overseer {
}
+ static void getShardNames(Integer numShards, List<String> shardNames) {
+ if(numShards == null)
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "numShards" + " is a required param");
+ for (int i = 0; i < numShards; i++) {
+ final String sliceName = "shard" + (i + 1);
+ shardNames.add(sliceName);
+ }
+
+ }
+
+ static void getShardNames(List<String> shardNames, String shards) {
+ if(shards ==null)
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "shards" + " is a required param");
+ for (String s : shards.split(",")) {
+ if(s ==null || s.trim().isEmpty()) continue;
+ shardNames.add(s.trim());
+ }
+ if(shardNames.isEmpty())
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "shards" + " is a required param");
+
+ }
+
class OverseerThread extends Thread implements ClosableThread {
private volatile boolean isClosed;
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Tue Aug 13 06:33:29 2013
@@ -17,14 +17,6 @@ package org.apache.solr.cloud;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -40,6 +32,7 @@ import org.apache.solr.common.cloud.Clos
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.DocRouter;
+import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.PlainIdRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
@@ -61,6 +54,21 @@ import org.apache.zookeeper.KeeperExcept
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.solr.cloud.Assign.Node;
+import static org.apache.solr.cloud.Assign.getNodesForNewShard;
+import static org.apache.solr.common.cloud.DocRouter.ROUTE_FIELD;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+
+
public class OverseerCollectionProcessor implements Runnable, ClosableThread {
public static final String NUM_SLICES = "numShards";
@@ -85,6 +93,22 @@ public class OverseerCollectionProcessor
public static final String DELETESHARD = "deleteshard";
+ public static final String ROUTER = "router";
+
+ public static final String SHARDS_PROP = "shards";
+
+ public static final String CREATESHARD = "createshard";
+
+ public static final String COLL_CONF = "collection.configName";
+
+
+ public static final Map<String,Object> COLL_PROPS = asMap(
+ ROUTER,DocRouter.DEFAULT_NAME,
+ REPLICATION_FACTOR, "1",
+ MAX_SHARDS_PER_NODE,"1",
+ ROUTE_FIELD,null);
+
+
// TODO: use from Overseer?
private static final String QUEUE_OPERATION = "operation";
@@ -168,7 +192,8 @@ public class OverseerCollectionProcessor
protected SolrResponse processMessage(ZkNodeProps message, String operation) {
-
+ log.warn("OverseerCollectionProcessor.processMessage : "+ operation + " , "+ message.toString());
+
NamedList results = new NamedList();
try {
if (CREATECOLLECTION.equals(operation)) {
@@ -185,6 +210,8 @@ public class OverseerCollectionProcessor
deleteAlias(zkStateReader.getAliases(), message);
} else if (SPLITSHARD.equals(operation)) {
splitShard(zkStateReader.getClusterState(), message, results);
+ } else if (CREATESHARD.equals(operation)) {
+ createShard(zkStateReader.getClusterState(), message, results);
} else if (DELETESHARD.equals(operation)) {
deleteShard(zkStateReader.getClusterState(), message, results);
} else {
@@ -333,7 +360,84 @@ public class OverseerCollectionProcessor
}
}
-
+
+ private boolean createShard(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
+ log.info("create shard invoked");
+ String collectionName = message.getStr(COLLECTION_PROP);
+ String shard = message.getStr(SHARD_ID_PROP);
+ if(collectionName == null || shard ==null)
+ throw new SolrException(ErrorCode.BAD_REQUEST, "'collection' and 'shard' are required parameters" );
+ int numSlices = 1;
+
+ DocCollection collection = clusterState.getCollection(collectionName);
+ int maxShardsPerNode = collection.getInt(MAX_SHARDS_PER_NODE, 1);
+ int repFactor = message.getInt(REPLICATION_FACTOR, collection.getInt(MAX_SHARDS_PER_NODE, 1));
+// int minReplicas = message.getInt("minReplicas",repFactor);
+ String createNodeSetStr =message.getStr(CREATE_NODE_SET);
+
+ ArrayList<Node> sortedNodeList = getNodesForNewShard(clusterState, collectionName, numSlices, maxShardsPerNode, repFactor, createNodeSetStr);
+
+ Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(message));
+ // wait for a while until we don't see the collection
+ long waitUntil = System.currentTimeMillis() + 30000;
+ boolean created = false;
+ while (System.currentTimeMillis() < waitUntil) {
+ Thread.sleep(100);
+ created = zkStateReader.getClusterState().getCollection(collectionName).getSlice(shard) !=null;
+ if (created) break;
+ }
+ if (!created)
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully create shard: " + message.getStr("name"));
+
+
+ String configName = message.getStr(COLL_CONF);
+ String sliceName = shard;
+ for (int j = 1; j <= repFactor; j++) {
+ String nodeName = sortedNodeList.get(((j - 1)) % sortedNodeList.size()).nodeName;
+ String shardName = collectionName + "_" + sliceName + "_replica" + j;
+ log.info("Creating shard " + shardName + " as part of slice "
+ + sliceName + " of collection " + collectionName + " on "
+ + nodeName);
+
+ // Need to create new params for each request
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString());
+
+ params.set(CoreAdminParams.NAME, shardName);
+ params.set(COLL_CONF, configName);
+ params.set(CoreAdminParams.COLLECTION, collectionName);
+ params.set(CoreAdminParams.SHARD, sliceName);
+ params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices);
+
+ ShardRequest sreq = new ShardRequest();
+ params.set("qt", adminPath);
+ sreq.purpose = 1;
+ String replica = zkStateReader.getZkClient()
+ .getBaseUrlForNodeName(nodeName);
+ if (replica.startsWith("http://")) replica = replica.substring(7);
+ sreq.shards = new String[]{replica};
+ sreq.actualShards = sreq.shards;
+ sreq.params = params;
+
+ shardHandler.submit(sreq, replica, sreq.params);
+
+ }
+
+ ShardResponse srsp;
+ do {
+ srsp = shardHandler.takeCompletedOrError();
+ if (srsp != null) {
+ processResponse(results, srsp);
+ }
+ } while (srsp != null);
+
+ log.info("Finished create command on all shards for collection: "
+ + collectionName);
+
+ return true;
+ }
+
+
private boolean splitShard(ClusterState clusterState, ZkNodeProps message, NamedList results) {
log.info("Split shard invoked");
String collectionName = message.getStr("collection");
@@ -674,7 +778,7 @@ public class OverseerCollectionProcessor
"The slice: " + slice.getName() + " is currently "
+ slice.getState() + ". Only INACTIVE (or custom-hashed) slices can be deleted.");
}
-
+
try {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.UNLOAD.toString());
@@ -732,7 +836,7 @@ public class OverseerCollectionProcessor
shardHandler.submit(sreq, replica, sreq.params);
}
- private void createCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) {
+ private void createCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
String collectionName = message.getStr("name");
if (clusterState.getCollections().contains(collectionName)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName);
@@ -742,14 +846,22 @@ public class OverseerCollectionProcessor
// look at the replication factor and see if it matches reality
// if it does not, find best nodes to create more cores
- int repFactor = msgStrToInt(message, REPLICATION_FACTOR, 1);
- Integer numSlices = msgStrToInt(message, NUM_SLICES, null);
-
- if (numSlices == null) {
+ int repFactor = message.getInt( REPLICATION_FACTOR, 1);
+ Integer numSlices = message.getInt(NUM_SLICES, null);
+ String router = message.getStr(ROUTER, DocRouter.DEFAULT_NAME);
+ List<String> shardNames = new ArrayList<>();
+ if(ImplicitDocRouter.NAME.equals(router)){
+ Overseer.getShardNames(shardNames, message.getStr("shards",null));
+ numSlices = shardNames.size();
+ } else {
+ Overseer.getShardNames(numSlices,shardNames);
+ }
+
+ if (numSlices == null ) {
throw new SolrException(ErrorCode.BAD_REQUEST, NUM_SLICES + " is a required param");
}
-
- int maxShardsPerNode = msgStrToInt(message, MAX_SHARDS_PER_NODE, 1);
+
+ int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
String createNodeSetStr;
List<String> createNodeList = ((createNodeSetStr = message.getStr(CREATE_NODE_SET)) == null)?null:StrUtils.splitSmart(createNodeSetStr, ",", true);
@@ -761,8 +873,6 @@ public class OverseerCollectionProcessor
throw new SolrException(ErrorCode.BAD_REQUEST, NUM_SLICES + " must be > 0");
}
- String configName = message.getStr("collection.configName");
-
// we need to look at every node and see how many cores it serves
// add our new cores to existing nodes serving the least number of cores
// but (for now) require that each core goes on a distinct node.
@@ -805,26 +915,44 @@ public class OverseerCollectionProcessor
+ ". This requires " + requestedShardsToCreate
+ " shards to be created (higher than the allowed number)");
}
-
- for (int i = 1; i <= numSlices; i++) {
+
+// ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
+// Overseer.CREATECOLLECTION, "name", message.getStr("name"));
+ Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(message));
+
+ // wait for a while until we don't see the collection
+ long waitUntil = System.currentTimeMillis() + 30000;
+ boolean created = false;
+ while (System.currentTimeMillis() < waitUntil) {
+ Thread.sleep(100);
+ created = zkStateReader.getClusterState().getCollections().contains(message.getStr("name"));
+ if(created) break;
+ }
+ if (!created)
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully createcollection: " + message.getStr("name"));
+
+
+ String configName = message.getStr(COLL_CONF);
+ log.info("going to create cores replicas shardNames {} , repFactor : {}", shardNames, repFactor);
+ for (int i = 1; i <= shardNames.size(); i++) {
+ String sliceName = shardNames.get(i-1);
for (int j = 1; j <= repFactor; j++) {
String nodeName = nodeList.get((repFactor * (i - 1) + (j - 1)) % nodeList.size());
- String sliceName = "shard" + i;
String shardName = collectionName + "_" + sliceName + "_replica" + j;
log.info("Creating shard " + shardName + " as part of slice "
+ sliceName + " of collection " + collectionName + " on "
+ nodeName);
-
+
// Need to create new params for each request
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString());
-
+
params.set(CoreAdminParams.NAME, shardName);
- params.set("collection.configName", configName);
+ params.set(COLL_CONF, configName);
params.set(CoreAdminParams.COLLECTION, collectionName);
params.set(CoreAdminParams.SHARD, sliceName);
params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices);
-
+
ShardRequest sreq = new ShardRequest();
params.set("qt", adminPath);
sreq.purpose = 1;
@@ -834,12 +962,12 @@ public class OverseerCollectionProcessor
sreq.shards = new String[] {replica};
sreq.actualShards = sreq.shards;
sreq.params = params;
-
+
shardHandler.submit(sreq, replica, sreq.params);
-
+
}
}
-
+
ShardResponse srsp;
do {
srsp = shardHandler.takeCompletedOrError();
@@ -857,7 +985,7 @@ public class OverseerCollectionProcessor
throw new SolrException(ErrorCode.SERVER_ERROR, null, ex);
}
}
-
+
private void collectionCmd(ClusterState clusterState, ZkNodeProps message, ModifiableSolrParams params, NamedList results, String stateMatcher) {
log.info("Executing Collection Cmd : " + params);
String collectionName = message.getStr("name");
@@ -947,19 +1075,16 @@ public class OverseerCollectionProcessor
}
}
- private Integer msgStrToInt(ZkNodeProps message, String key, Integer def)
- throws Exception {
- String str = message.getStr(key);
- try {
- return str == null ? def : Integer.valueOf(str);
- } catch (Exception ex) {
- SolrException.log(log, "Could not parse " + key, ex);
- throw ex;
- }
- }
-
@Override
public boolean isClosed() {
return isClosed;
}
+
+ public static Map<String, Object> asMap(Object... vals) {
+ HashMap<String, Object> m = new HashMap<String, Object>();
+ for(int i=0; i<vals.length; i+=2) {
+ m.put(String.valueOf(vals[i]), vals[i+1]);
+ }
+ return m;
+ }
}
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/SyncStrategy.java Tue Aug 13 06:33:29 2013
@@ -67,7 +67,7 @@ public class SyncStrategy {
ModifiableSolrParams params = new ModifiableSolrParams();
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS, 10000);
params.set(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, 20);
- params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 30000);
+ params.set(HttpClientUtil.PROP_CONNECTION_TIMEOUT, 15000);
params.set(HttpClientUtil.PROP_SO_TIMEOUT, 30000);
params.set(HttpClientUtil.PROP_USE_RETRY, false);
client = HttpClientUtil.createClient(params);
@@ -285,8 +285,8 @@ public class SyncStrategy {
recoverRequestCmd.setCoreName(coreName);
HttpSolrServer server = new HttpSolrServer(baseUrl, client);
- server.setConnectionTimeout(45000);
- server.setSoTimeout(45000);
+ server.setConnectionTimeout(15000);
+ server.setSoTimeout(30000);
try {
server.request(recoverRequestCmd);
} catch (Throwable t) {
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java Tue Aug 13 06:33:29 2013
@@ -129,7 +129,7 @@ public class ZkCLI {
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + UPCONFIG + " -" + CONFDIR + " /opt/solr/collection1/conf" + " -" + CONFNAME + " myconf");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + DOWNCONFIG + " -" + CONFDIR + " /opt/solr/collection1/conf" + " -" + CONFNAME + " myconf");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LINKCONFIG + " -" + COLLECTION + " collection1" + " -" + CONFNAME + " myconf");
- System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr/data.txt 'config data'");
+ System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + MAKEPATH + " /apache/solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + PUT + " /solr.conf 'conf data'");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/cloud/ZkController.java Tue Aug 13 06:33:29 2013
@@ -130,7 +130,7 @@ public final class ZkController {
protected volatile Overseer overseer;
- private String leaderVoteWait;
+ private int leaderVoteWait;
private boolean genericCoreNodeNames;
@@ -141,7 +141,7 @@ public final class ZkController {
private UpdateShardHandler updateShardHandler;
public ZkController(final CoreContainer cc, String zkServerAddress, int zkClientTimeout, int zkClientConnectTimeout, String localHost, String locaHostPort,
- String localHostContext, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
+ String localHostContext, int leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout, final CurrentCoreDescriptorProvider registerOnReconnect) throws InterruptedException,
TimeoutException, IOException {
if (cc == null) throw new IllegalArgumentException("CoreContainer cannot be null.");
this.cc = cc;
@@ -242,7 +242,7 @@ public final class ZkController {
init(registerOnReconnect);
}
- public String getLeaderVoteWait() {
+ public int getLeaderVoteWait() {
return leaderVoteWait;
}
@@ -769,7 +769,7 @@ public final class ZkController {
// in this case, we want to wait for the leader as long as the leader might
// wait for a vote, at least - but also long enough that a large cluster has
// time to get its act together
- String leaderUrl = getLeader(cloudDesc, Integer.parseInt(leaderVoteWait) + 600000);
+ String leaderUrl = getLeader(cloudDesc, leaderVoteWait + 600000);
String ourUrl = ZkCoreNodeProps.getCoreUrl(baseUrl, coreName);
log.info("We are " + ourUrl + " and leader is " + leaderUrl);
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/Config.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/Config.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/Config.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/Config.java Tue Aug 13 06:33:29 2013
@@ -233,14 +233,13 @@ public class Config {
}
public Node getNode(String path, Document doc, boolean errIfMissing) {
- XPath xpath = xpathFactory.newXPath();
- Node nd = null;
- String xstr = normalize(path);
+ XPath xpath = xpathFactory.newXPath();
+ String xstr = normalize(path);
try {
- nd = (Node)xpath.evaluate(xstr, doc, XPathConstants.NODE);
-
- if (nd==null) {
+ NodeList nodes = (NodeList)xpath.evaluate(xstr, doc,
+ XPathConstants.NODESET);
+ if (nodes==null || 0 == nodes.getLength() ) {
if (errIfMissing) {
throw new RuntimeException(name + " missing "+path);
} else {
@@ -248,7 +247,11 @@ public class Config {
return null;
}
}
-
+ if ( 1 < nodes.getLength() ) {
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR,
+ name + " contains more than one value for config path: " + path);
+ }
+ Node nd = nodes.item(0);
log.trace(name + ":" + path + "=" + nd);
return nd;
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolr.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolr.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolr.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolr.java Tue Aug 13 06:33:29 2013
@@ -21,6 +21,7 @@ import com.google.common.base.Charsets;
import com.google.common.io.ByteStreams;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.SolrException;
+import org.apache.solr.logging.LogWatcherConfig;
import org.apache.solr.util.DOMUtil;
import org.apache.solr.util.PropertiesUtil;
import org.slf4j.Logger;
@@ -100,20 +101,106 @@ public abstract class ConfigSolr {
public abstract CoresLocator getCoresLocator();
-
public PluginInfo getShardHandlerFactoryPluginInfo() {
Node node = config.getNode(getShardHandlerFactoryConfigPath(), false);
return (node == null) ? null : new PluginInfo(node, "shardHandlerFactory", false, true);
}
- public Node getUnsubsititutedShardHandlerFactoryPluginNode() {
- return config.getUnsubstitutedNode(getShardHandlerFactoryConfigPath(), false);
+ protected abstract String getShardHandlerFactoryConfigPath();
+
+ public String getZkHost() {
+ String sysZkHost = System.getProperty("zkHost");
+ if (sysZkHost != null)
+ return sysZkHost;
+ return get(CfgProp.SOLR_ZKHOST, null);
}
- protected abstract String getShardHandlerFactoryConfigPath();
+ public int getZkClientTimeout() {
+ String sysProp = System.getProperty("zkClientTimeout");
+ if (sysProp != null)
+ return Integer.parseInt(sysProp);
+ return getInt(CfgProp.SOLR_ZKCLIENTTIMEOUT, DEFAULT_ZK_CLIENT_TIMEOUT);
+ }
+
+ private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 15000;
+ private static final int DEFAULT_LEADER_VOTE_WAIT = 180000; // 3 minutes
+ private static final int DEFAULT_CORE_LOAD_THREADS = 3;
+
+ protected static final String DEFAULT_CORE_ADMIN_PATH = "/admin/cores";
+
+ public String getZkHostPort() {
+ return get(CfgProp.SOLR_HOSTPORT, null);
+ }
+
+ public String getZkHostContext() {
+ return get(CfgProp.SOLR_HOSTCONTEXT, null);
+ }
+
+ public String getHost() {
+ return get(CfgProp.SOLR_HOST, null);
+ }
+
+ public int getLeaderVoteWait() {
+ return getInt(CfgProp.SOLR_LEADERVOTEWAIT, DEFAULT_LEADER_VOTE_WAIT);
+ }
+
+ public boolean getGenericCoreNodeNames() {
+ return getBool(CfgProp.SOLR_GENERICCORENODENAMES, false);
+ }
+
+ public int getDistributedConnectionTimeout() {
+ return getInt(CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, 0);
+ }
+
+ public int getDistributedSocketTimeout() {
+ return getInt(CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, 0);
+ }
+
+ public int getCoreLoadThreadCount() {
+ return getInt(ConfigSolr.CfgProp.SOLR_CORELOADTHREADS, DEFAULT_CORE_LOAD_THREADS);
+ }
+
+ public String getSharedLibDirectory() {
+ return get(ConfigSolr.CfgProp.SOLR_SHAREDLIB , null);
+ }
+
+ public String getDefaultCoreName() {
+ return get(CfgProp.SOLR_CORES_DEFAULT_CORE_NAME, null);
+ }
+
+ public abstract boolean isPersistent();
+
+ public String getAdminPath() {
+ return get(CfgProp.SOLR_ADMINPATH, DEFAULT_CORE_ADMIN_PATH);
+ }
+
+ public String getCoreAdminHandlerClass() {
+ return get(CfgProp.SOLR_ADMINHANDLER, "org.apache.solr.handler.admin.CoreAdminHandler");
+ }
+
+ public boolean hasSchemaCache() {
+ return getBool(ConfigSolr.CfgProp.SOLR_SHARESCHEMA, false);
+ }
+
+ public String getManagementPath() {
+ return get(CfgProp.SOLR_MANAGEMENTPATH, null);
+ }
+
+ public LogWatcherConfig getLogWatcherConfig() {
+ return new LogWatcherConfig(
+ getBool(CfgProp.SOLR_LOGGING_ENABLED, false),
+ get(CfgProp.SOLR_LOGGING_CLASS, null),
+ get(CfgProp.SOLR_LOGGING_WATCHER_THRESHOLD, null),
+ getInt(CfgProp.SOLR_LOGGING_WATCHER_SIZE, 50)
+ );
+ }
+
+ public int getTransientCacheSize() {
+ return getInt(CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE);
+ }
// Ugly for now, but we'll at least be able to centralize all of the differences between 4x and 5x.
- public static enum CfgProp {
+ protected static enum CfgProp {
SOLR_ADMINHANDLER,
SOLR_CORELOADTHREADS,
SOLR_COREROOTDIRECTORY,
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXml.java Tue Aug 13 06:33:29 2013
@@ -126,11 +126,26 @@ public class ConfigSolrXml extends Confi
}
@Override
+ public String getDefaultCoreName() {
+ return "collection1";
+ }
+
+ @Override
+ public boolean isPersistent() {
+ return true;
+ }
+
+ @Override
protected String getShardHandlerFactoryConfigPath() {
return "solr/shardHandlerFactory";
}
@Override
+ public String getAdminPath() {
+ return DEFAULT_CORE_ADMIN_PATH;
+ }
+
+ @Override
public CoresLocator getCoresLocator() {
return coresLocator;
}
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ConfigSolrXmlOld.java Tue Aug 13 06:33:29 2013
@@ -110,9 +110,15 @@ public class ConfigSolrXmlOld extends Co
}
}
+ @Override
public boolean isPersistent() {
return config.getBool("solr/@persistent", false);
}
+
+ @Override
+ public String getDefaultCoreName() {
+ return get(CfgProp.SOLR_CORES_DEFAULT_CORE_NAME, DEFAULT_DEFAULT_CORE_NAME);
+ }
private void fillPropMap() {
@@ -268,15 +274,17 @@ public class ConfigSolrXmlOld extends Co
return new Properties();
}
+ public static final String DEFAULT_DEFAULT_CORE_NAME = "collection1";
+
public static final String DEF_SOLR_XML = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n"
+ "<solr persistent=\"false\">\n"
+ " <cores adminPath=\"/admin/cores\" defaultCoreName=\""
- + CoreContainer.DEFAULT_DEFAULT_CORE_NAME
+ + DEFAULT_DEFAULT_CORE_NAME
+ "\""
+ " host=\"${host:}\" hostPort=\"${hostPort:}\" hostContext=\"${hostContext:}\" zkClientTimeout=\"${zkClientTimeout:15000}\""
+ ">\n"
+ " <core name=\""
- + CoreContainer.DEFAULT_DEFAULT_CORE_NAME
+ + DEFAULT_DEFAULT_CORE_NAME
+ "\" shard=\"${shard:}\" collection=\"${collection:collection1}\" instanceDir=\"collection1\" />\n"
+ " </cores>\n" + "</solr>";
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/CoreContainer.java Tue Aug 13 06:33:29 2013
@@ -55,10 +55,9 @@ import java.util.concurrent.CompletionSe
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
import static com.google.common.base.Preconditions.checkNotNull;
@@ -67,55 +66,34 @@ import static com.google.common.base.Pre
*
* @since solr 1.3
*/
-public class CoreContainer
-{
- private static final String LEADER_VOTE_WAIT = "180000"; // 3 minutes
- private static final int CORE_LOAD_THREADS = 3;
-
- private static final int DEFAULT_ZK_CLIENT_TIMEOUT = 15000;
- public static final String DEFAULT_DEFAULT_CORE_NAME = "collection1";
- private static final boolean DEFAULT_SHARE_SCHEMA = false;
-
- protected static Logger log = LoggerFactory.getLogger(CoreContainer.class);
+public class CoreContainer {
+ protected static final Logger log = LoggerFactory.getLogger(CoreContainer.class);
private final SolrCores solrCores = new SolrCores(this);
protected final Map<String,Exception> coreInitFailures =
Collections.synchronizedMap(new LinkedHashMap<String,Exception>());
-
- protected boolean persistent = false;
- protected String adminPath = null;
- protected volatile String managementPath = null;
protected CoreAdminHandler coreAdminHandler = null;
protected CollectionsHandler collectionsHandler = null;
- protected String libDir = null;
+ private InfoHandler infoHandler;
protected Properties containerProperties;
+
protected Map<String ,IndexSchema> indexSchemaCache;
- protected String adminHandler;
protected boolean shareSchema;
- protected Integer zkClientTimeout;
- protected String defaultCoreName = null;
- protected int distribUpdateConnTimeout = 0;
- protected int distribUpdateSoTimeout = 0;
protected ZkContainer zkSys = new ZkContainer();
-
private ShardHandlerFactory shardHandlerFactory;
+
protected LogWatcher logging = null;
- private String zkHost;
- private int transientCacheSize = Integer.MAX_VALUE;
- private int coreLoadThreads;
private CloserThread backgroundCloser = null;
-
protected final ConfigSolr cfg;
protected final SolrResourceLoader loader;
- protected final String solrHome;
- private InfoHandler infoHandler;
+ protected final String solrHome;
protected final CoresLocator coresLocator;
@@ -202,10 +180,8 @@ public class CoreContainer
log.info("Loading cores into CoreContainer [instanceDir={}]", loader.getInstanceDir());
- ThreadPoolExecutor coreLoadExecutor = null;
-
// add the sharedLib to the shared resource loader before initializing cfg based plugins
- libDir = cfg.get(ConfigSolr.CfgProp.SOLR_SHAREDLIB , null);
+ String libDir = cfg.getSharedLibDirectory();
if (libDir != null) {
File f = FileUtils.resolvePath(new File(solrHome), libDir);
log.info("loading shared library: " + f.getAbsolutePath());
@@ -215,79 +191,32 @@ public class CoreContainer
shardHandlerFactory = ShardHandlerFactory.newInstance(cfg.getShardHandlerFactoryPluginInfo(), loader);
- solrCores.allocateLazyCores(cfg, loader);
-
- logging = JulWatcher.newRegisteredLogWatcher(cfg, loader);
+ solrCores.allocateLazyCores(cfg.getTransientCacheSize(), loader);
- if (cfg instanceof ConfigSolrXmlOld) { //TODO: Remove for 5.0
- String dcoreName = cfg.get(ConfigSolr.CfgProp.SOLR_CORES_DEFAULT_CORE_NAME, null);
- if (dcoreName != null && !dcoreName.isEmpty()) {
- defaultCoreName = dcoreName;
- }
- persistent = cfg.getBool(ConfigSolr.CfgProp.SOLR_PERSISTENT, false);
- adminPath = cfg.get(ConfigSolr.CfgProp.SOLR_ADMINPATH, "/admin/cores");
- } else {
- adminPath = "/admin/cores";
- defaultCoreName = DEFAULT_DEFAULT_CORE_NAME;
- }
- zkHost = cfg.get(ConfigSolr.CfgProp.SOLR_ZKHOST, null);
- coreLoadThreads = cfg.getInt(ConfigSolr.CfgProp.SOLR_CORELOADTHREADS, CORE_LOAD_THREADS);
-
-
- shareSchema = cfg.getBool(ConfigSolr.CfgProp.SOLR_SHARESCHEMA, DEFAULT_SHARE_SCHEMA);
- zkClientTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_ZKCLIENTTIMEOUT, DEFAULT_ZK_CLIENT_TIMEOUT);
-
- distribUpdateConnTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_DISTRIBUPDATECONNTIMEOUT, 0);
- distribUpdateSoTimeout = cfg.getInt(ConfigSolr.CfgProp.SOLR_DISTRIBUPDATESOTIMEOUT, 0);
+ logging = JulWatcher.newRegisteredLogWatcher(cfg.getLogWatcherConfig(), loader);
- // Note: initZooKeeper will apply hardcoded default if cloud mode
- String hostPort = cfg.get(ConfigSolr.CfgProp.SOLR_HOSTPORT, null);
- // Note: initZooKeeper will apply hardcoded default if cloud mode
- String hostContext = cfg.get(ConfigSolr.CfgProp.SOLR_HOSTCONTEXT, null);
-
- String host = cfg.get(ConfigSolr.CfgProp.SOLR_HOST, null);
-
- String leaderVoteWait = cfg.get(ConfigSolr.CfgProp.SOLR_LEADERVOTEWAIT, LEADER_VOTE_WAIT);
-
- adminHandler = cfg.get(ConfigSolr.CfgProp.SOLR_ADMINHANDLER, null);
- managementPath = cfg.get(ConfigSolr.CfgProp.SOLR_MANAGEMENTPATH, null);
-
- transientCacheSize = cfg.getInt(ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE);
-
- boolean genericCoreNodeNames = cfg.getBool(ConfigSolr.CfgProp.SOLR_GENERICCORENODENAMES, false);
+ shareSchema = cfg.hasSchemaCache();
if (shareSchema) {
indexSchemaCache = new ConcurrentHashMap<String,IndexSchema>();
}
- zkClientTimeout = Integer.parseInt(System.getProperty("zkClientTimeout",
- Integer.toString(zkClientTimeout)));
- zkSys.initZooKeeper(this, solrHome, zkHost, zkClientTimeout, hostPort, hostContext, host, leaderVoteWait, genericCoreNodeNames, distribUpdateConnTimeout, distribUpdateSoTimeout);
-
- if (isZooKeeperAware() && coreLoadThreads <= 1) {
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "SolrCloud requires a value of at least 2 in solr.xml for coreLoadThreads");
- }
-
- if (adminPath != null) {
- if (adminHandler == null) {
- coreAdminHandler = new CoreAdminHandler(this);
- } else {
- coreAdminHandler = this.createMultiCoreHandler(adminHandler);
- }
- }
-
+ zkSys.initZooKeeper(this, solrHome, cfg);
+
collectionsHandler = new CollectionsHandler(this);
infoHandler = new InfoHandler(this);
+ coreAdminHandler = createMultiCoreHandler(cfg.getCoreAdminHandlerClass());
+
containerProperties = cfg.getSolrProperties("solr");
// setup executor to load cores in parallel
- coreLoadExecutor = new ThreadPoolExecutor(coreLoadThreads, coreLoadThreads, 1,
- TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(),
+ ExecutorService coreLoadExecutor = Executors.newFixedThreadPool(cfg.getCoreLoadThreadCount(),
new DefaultSolrThreadFactory("coreLoadExecutor"));
+
try {
CompletionService<SolrCore> completionService = new ExecutorCompletionService<SolrCore>(
coreLoadExecutor);
+
Set<Future<SolrCore>> pending = new HashSet<Future<SolrCore>>();
List<CoreDescriptor> cds = coresLocator.discover(this);
@@ -341,7 +270,7 @@ public class CoreContainer
SolrException.log(log, null, ex);
}
}
-
+
while (pending != null && pending.size() > 0) {
try {
@@ -358,7 +287,7 @@ public class CoreContainer
} catch (ExecutionException e) {
SolrException.log(SolrCore.log, "Error loading core", e);
}
-
+
} catch (InterruptedException e) {
throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
"interrupted while loading core", e);
@@ -759,7 +688,7 @@ public class CoreContainer
//5.0 remove all checkDefaults?
private String checkDefault(String name) {
- return (null == name || name.isEmpty()) ? defaultCoreName : name;
+ return (null == name || name.isEmpty()) ? getDefaultCoreName() : name;
}
/**
@@ -918,46 +847,34 @@ public class CoreContainer
* the default core name, or null if there is no default core name
*/
public String getDefaultCoreName() {
- return defaultCoreName;
+ return cfg.getDefaultCoreName();
}
// all of the following properties aren't synchronized
// but this should be OK since they normally won't be changed rapidly
@Deprecated
public boolean isPersistent() {
- return persistent;
- }
-
- @Deprecated
- public void setPersistent(boolean persistent) {
- this.persistent = persistent;
+ return cfg.isPersistent();
}
public String getAdminPath() {
- return adminPath;
+ return cfg.getAdminPath();
}
- public String getManagementPath() {
- return managementPath;
- }
-
/**
- * Sets the alternate path for multicore handling:
+ * Gets the alternate path for multicore handling:
* This is used in case there is a registered unnamed core (aka name is "") to
* declare an alternate way of accessing named cores.
* This can also be used in a pseudo single-core environment so admins can prepare
* a new version before swapping.
*/
- public void setManagementPath(String path) {
- this.managementPath = path;
+ public String getManagementPath() {
+ return cfg.getManagementPath();
}
-
+
public LogWatcher getLogging() {
return logging;
}
- public void setLogging(LogWatcher v) {
- logging = v;
- }
/**
* Determines whether the core is already loaded or not but does NOT load the core
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java Tue Aug 13 06:33:29 2013
@@ -27,12 +27,8 @@ import org.apache.solr.util.plugin.Named
* Factory used to build a new IndexReader instance.
*/
public abstract class IndexReaderFactory implements NamedListInitializedPlugin {
- protected int termInfosIndexDivisor = 1;//IndexReader.DEFAULT_TERMS_INDEX_DIVISOR; Set this once Lucene makes this public.
/**
- * Potentially initializes {@link #termInfosIndexDivisor}. Overriding classes should call super.init() in order
- * to make sure termInfosIndexDivisor is set.
- * <p>
- * <code>init</code> will be called just once, immediately after creation.
+ * init will be called just once, immediately after creation.
* <p>
* The args are user-level initialization parameters that may be specified
* when declaring an indexReaderFactory in solrconfig.xml
@@ -40,18 +36,10 @@ public abstract class IndexReaderFactory
*/
@Override
public void init(NamedList args) {
- Integer v = (Integer)args.get("setTermIndexDivisor");
- if (v != null) {
- termInfosIndexDivisor = v.intValue();
- }
- }
-
- /**
- *
- * @return The setting of {@link #termInfosIndexDivisor}
- */
- public int getTermInfosIndexDivisor() {
- return termInfosIndexDivisor;
+ Object v = args.get("setTermIndexDivisor");
+ if (v != null) {
+ throw new IllegalArgumentException("Illegal parameter 'setTermIndexDivisor'");
+ }
}
/**
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/JmxMonitoredMap.java Tue Aug 13 06:33:29 2013
@@ -190,6 +190,11 @@ public class JmxMonitoredMap<K, V> exten
return ObjectName.getInstance(jmxRootName, map);
}
+ /** For test verification */
+ public MBeanServer getServer() {
+ return server;
+ }
+
/**
* DynamicMBean is used to dynamically expose all SolrInfoMBean
* getStatistics() NameList keys as String getters.
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrConfig.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrConfig.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrConfig.java Tue Aug 13 06:33:29 2013
@@ -124,8 +124,8 @@ public class SolrConfig extends Config {
// Old indexDefaults and mainIndex sections are deprecated and fails fast for luceneMatchVersion=>LUCENE_40.
// For older solrconfig.xml's we allow the old sections, but never mixed with the new <indexConfig>
- boolean hasDeprecatedIndexConfig = get("indexDefaults/text()", null) != null || get("mainIndex/text()", null) != null;
- boolean hasNewIndexConfig = get("indexConfig/text()", null) != null;
+ boolean hasDeprecatedIndexConfig = (getNode("indexDefaults", false) != null) || (getNode("mainIndex", false) != null);
+ boolean hasNewIndexConfig = getNode("indexConfig", false) != null;
if(hasDeprecatedIndexConfig){
if(luceneMatchVersion.onOrAfter(Version.LUCENE_40)) {
throw new SolrException(ErrorCode.FORBIDDEN, "<indexDefaults> and <mainIndex> configuration sections are discontinued. Use <indexConfig> instead.");
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrCores.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrCores.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrCores.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/SolrCores.java Tue Aug 13 06:33:29 2013
@@ -65,14 +65,13 @@ class SolrCores {
// Trivial helper method for load, note it implements LRU on transient cores. Also note, if
// there is no setting for max size, nothing is done and all cores go in the regular "cores" list
- protected void allocateLazyCores(final ConfigSolr cfg, final SolrResourceLoader loader) {
- final int transientCacheSize = cfg.getInt(ConfigSolr.CfgProp.SOLR_TRANSIENTCACHESIZE, Integer.MAX_VALUE);
- if (transientCacheSize != Integer.MAX_VALUE) {
- CoreContainer.log.info("Allocating transient cache for {} transient cores", transientCacheSize);
- transientCores = new LinkedHashMap<String, SolrCore>(transientCacheSize, 0.75f, true) {
+ protected void allocateLazyCores(final int cacheSize, final SolrResourceLoader loader) {
+ if (cacheSize != Integer.MAX_VALUE) {
+ CoreContainer.log.info("Allocating transient cache for {} transient cores", cacheSize);
+ transientCores = new LinkedHashMap<String, SolrCore>(cacheSize, 0.75f, true) {
@Override
protected boolean removeEldestEntry(Map.Entry<String, SolrCore> eldest) {
- if (size() > transientCacheSize) {
+ if (size() > cacheSize) {
synchronized (modifyLock) {
SolrCore coreToClose = eldest.getValue();
logger.info("Closing transient core [{}]", coreToClose.getName());
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java Tue Aug 13 06:33:29 2013
@@ -31,6 +31,6 @@ public class StandardIndexReaderFactory
@Override
public DirectoryReader newReader(Directory indexDir, SolrCore core) throws IOException {
- return DirectoryReader.open(indexDir, termInfosIndexDivisor);
+ return DirectoryReader.open(indexDir);
}
}
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ZkContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ZkContainer.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ZkContainer.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/core/ZkContainer.java Tue Aug 13 06:33:29 2013
@@ -17,14 +17,6 @@ package org.apache.solr.core;
* limitations under the License.
*/
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.TimeoutException;
-
import org.apache.solr.cloud.CurrentCoreDescriptorProvider;
import org.apache.solr.cloud.SolrZkServer;
import org.apache.solr.cloud.ZkController;
@@ -40,6 +32,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.TimeoutException;
+
public class ZkContainer {
protected static Logger log = LoggerFactory.getLogger(ZkContainer.class);
@@ -49,53 +49,31 @@ public class ZkContainer {
private String hostPort;
private String hostContext;
private String host;
- private String leaderVoteWait;
+ private int leaderVoteWait;
private Boolean genericCoreNodeNames;
private int distribUpdateConnTimeout;
-
- public SolrZkServer getZkServer() {
- return zkServer;
- }
-
- public int getZkClientTimeout() {
- return zkClientTimeout;
- }
-
- public String getHostPort() {
- return hostPort;
- }
-
- public String getHostContext() {
- return hostContext;
- }
-
- public String getHost() {
- return host;
- }
-
- public String getLeaderVoteWait() {
- return leaderVoteWait;
- }
+ private int distribUpdateSoTimeout;
- public boolean getGenericCoreNodeNames() {
- return genericCoreNodeNames;
+ public ZkContainer() {
+
}
- public int getDistribUpdateConnTimeout() {
- return distribUpdateConnTimeout;
- }
+ public void initZooKeeper(final CoreContainer cc, String solrHome, ConfigSolr config) {
- public int getDistribUpdateSoTimeout() {
- return distribUpdateSoTimeout;
- }
+ if (config.getCoreLoadThreadCount() <= 1) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "SolrCloud requires a value of at least 2 in solr.xml for coreLoadThreads");
+ }
- private int distribUpdateSoTimeout;
-
- public ZkContainer() {
-
+ initZooKeeper(cc, solrHome,
+ config.getZkHost(), config.getZkClientTimeout(), config.getZkHostPort(), config.getZkHostContext(),
+ config.getHost(), config.getLeaderVoteWait(), config.getGenericCoreNodeNames(),
+ config.getDistributedConnectionTimeout(), config.getDistributedSocketTimeout());
}
- public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort, String hostContext, String host, String leaderVoteWait, boolean genericCoreNodeNames, int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
+ public void initZooKeeper(final CoreContainer cc, String solrHome, String zkHost, int zkClientTimeout, String hostPort,
+ String hostContext, String host, int leaderVoteWait, boolean genericCoreNodeNames,
+ int distribUpdateConnTimeout, int distribUpdateSoTimeout) {
ZkController zkController = null;
// if zkHost sys property is not set, we are not using ZooKeeper
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Tue Aug 13 06:33:29 2013
@@ -17,10 +17,6 @@ package org.apache.solr.handler.admin;
* limitations under the License.
*/
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
@@ -32,6 +28,7 @@ import org.apache.solr.cloud.OverseerCol
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -49,6 +46,22 @@ import org.apache.zookeeper.KeeperExcept
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.COLL_CONF;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.CREATESHARD;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.CREATE_NODE_SET;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.ROUTER;
+import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
+import static org.apache.solr.common.cloud.DocRouter.ROUTE_FIELD;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
public class CollectionsHandler extends RequestHandlerBase {
protected static Logger log = LoggerFactory.getLogger(CollectionsHandler.class);
@@ -139,6 +152,9 @@ public class CollectionsHandler extends
case DELETESHARD: {
this.handleDeleteShardAction(req, rsp);
break;
+ }case CREATESHARD: {
+ this.handleCreateShard(req, rsp);
+ break;
}
default: {
@@ -260,13 +276,7 @@ public class CollectionsHandler extends
private void handleCreateAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws InterruptedException, KeeperException {
log.info("Creating Collection : " + req.getParamString());
- Integer numReplicas = req.getParams().getInt(OverseerCollectionProcessor.REPLICATION_FACTOR, 1);
String name = req.getParams().required().get("name");
- String configName = req.getParams().get("collection.configName");
- String numShards = req.getParams().get(OverseerCollectionProcessor.NUM_SLICES);
- String maxShardsPerNode = req.getParams().get(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE);
- String createNodeSetStr = req.getParams().get(OverseerCollectionProcessor.CREATE_NODE_SET);
-
if (name == null) {
log.error("Collection name is required to create a new collection");
throw new SolrException(ErrorCode.BAD_REQUEST,
@@ -276,19 +286,45 @@ public class CollectionsHandler extends
Map<String,Object> props = new HashMap<String,Object>();
props.put(Overseer.QUEUE_OPERATION,
OverseerCollectionProcessor.CREATECOLLECTION);
- props.put(OverseerCollectionProcessor.REPLICATION_FACTOR, numReplicas.toString());
- props.put("name", name);
- if (configName != null) {
- props.put("collection.configName", configName);
- }
- props.put(OverseerCollectionProcessor.NUM_SLICES, numShards);
- props.put(OverseerCollectionProcessor.MAX_SHARDS_PER_NODE, maxShardsPerNode);
- props.put(OverseerCollectionProcessor.CREATE_NODE_SET, createNodeSetStr);
-
- ZkNodeProps m = new ZkNodeProps(props);
+ copyIfNotNull(req.getParams(),props,
+ "name",
+ REPLICATION_FACTOR,
+ COLL_CONF,
+ NUM_SLICES,
+ MAX_SHARDS_PER_NODE,
+ CREATE_NODE_SET ,
+ ROUTER,
+ SHARDS_PROP,
+ ROUTE_FIELD);
+
+
+ ZkNodeProps m = new ZkNodeProps(props);
handleResponse(OverseerCollectionProcessor.CREATECOLLECTION, m, rsp);
}
+
+ private void handleCreateShard(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
+ log.info("Create shard: " + req.getParamString());
+ req.getParams().required().check(COLLECTION_PROP, SHARD_ID_PROP);
+ ClusterState clusterState = coreContainer.getZkController().getClusterState();
+ if(!ImplicitDocRouter.NAME.equals( clusterState.getCollection(req.getParams().get(COLLECTION_PROP)).getStr(ROUTER)))
+ throw new SolrException(ErrorCode.BAD_REQUEST, "shards can be added only to 'implicit' collections" );
+
+ Map<String, Object> map = OverseerCollectionProcessor.asMap(QUEUE_OPERATION, CREATESHARD);
+ copyIfNotNull(req.getParams(),map,COLLECTION_PROP, SHARD_ID_PROP, REPLICATION_FACTOR);
+ ZkNodeProps m = new ZkNodeProps(map);
+ handleResponse(CREATESHARD, m, rsp);
+ }
+
+ private static void copyIfNotNull(SolrParams params, Map<String, Object> props, String... keys) {
+ if(keys !=null){
+ for (String key : keys) {
+ String v = params.get(key);
+ if(v != null) props.put(key,v);
+ }
+ }
+
+ }
private void handleDeleteShardAction(SolrQueryRequest req,
SolrQueryResponse rsp) throws InterruptedException, KeeperException {
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java Tue Aug 13 06:33:29 2013
@@ -18,6 +18,7 @@
package org.apache.solr.handler.admin;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.DirectoryReader;
@@ -295,17 +296,17 @@ public class CoreAdminHandler extends Re
}
- protected void handleMergeAction(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ protected void handleMergeAction(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
SolrParams params = req.getParams();
String cname = params.required().get(CoreAdminParams.CORE);
SolrCore core = coreContainer.getCore(cname);
SolrQueryRequest wrappedReq = null;
- SolrCore[] sourceCores = null;
- RefCounted<SolrIndexSearcher>[] searchers = null;
+ List<SolrCore> sourceCores = Lists.newArrayList();
+ List<RefCounted<SolrIndexSearcher>> searchers = Lists.newArrayList();
// stores readers created from indexDir param values
- DirectoryReader[] readersToBeClosed = null;
- Directory[] dirsToBeReleased = null;
+ List<DirectoryReader> readersToBeClosed = Lists.newArrayList();
+ List<Directory> dirsToBeReleased = Lists.newArrayList();
if (core != null) {
try {
String[] dirNames = params.getParams(CoreAdminParams.INDEX_DIR);
@@ -315,38 +316,34 @@ public class CoreAdminHandler extends Re
throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
"At least one indexDir or srcCore must be specified");
- sourceCores = new SolrCore[sources.length];
for (int i = 0; i < sources.length; i++) {
String source = sources[i];
SolrCore srcCore = coreContainer.getCore(source);
if (srcCore == null)
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Core: " + source + " does not exist");
- sourceCores[i] = srcCore;
+ sourceCores.add(srcCore);
}
} else {
- readersToBeClosed = new DirectoryReader[dirNames.length];
- dirsToBeReleased = new Directory[dirNames.length];
DirectoryFactory dirFactory = core.getDirectoryFactory();
for (int i = 0; i < dirNames.length; i++) {
Directory dir = dirFactory.get(dirNames[i], DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
- dirsToBeReleased[i] = dir;
+ dirsToBeReleased.add(dir);
// TODO: why doesn't this use the IR factory? what is going on here?
- readersToBeClosed[i] = DirectoryReader.open(dir);
+ readersToBeClosed.add(DirectoryReader.open(dir));
}
}
- DirectoryReader[] readers = null;
- if (readersToBeClosed != null) {
+ List<DirectoryReader> readers = null;
+ if (readersToBeClosed.size() > 0) {
readers = readersToBeClosed;
} else {
- readers = new DirectoryReader[sourceCores.length];
- searchers = new RefCounted[sourceCores.length];
- for (int i = 0; i < sourceCores.length; i++) {
- SolrCore solrCore = sourceCores[i];
+ readers = Lists.newArrayList();
+ for (SolrCore solrCore: sourceCores) {
// record the searchers so that we can decref
- searchers[i] = solrCore.getSearcher();
- readers[i] = searchers[i].get().getIndexReader();
+ RefCounted<SolrIndexSearcher> searcher = solrCore.getSearcher();
+ searchers.add(searcher);
+ readers.add(searcher.get().getIndexReader());
}
}
@@ -356,23 +353,21 @@ public class CoreAdminHandler extends Re
UpdateRequestProcessor processor =
processorChain.createProcessor(wrappedReq, rsp);
processor.processMergeIndexes(new MergeIndexesCommand(readers, req));
+ } catch (Exception e) {
+ // log and rethrow so that if the finally fails we don't lose the original problem
+ log.error("ERROR executing merge:", e);
+ throw e;
} finally {
- if (searchers != null) {
- for (RefCounted<SolrIndexSearcher> searcher : searchers) {
- if (searcher != null) searcher.decref();
- }
+ for (RefCounted<SolrIndexSearcher> searcher : searchers) {
+ if (searcher != null) searcher.decref();
}
- if (sourceCores != null) {
- for (SolrCore solrCore : sourceCores) {
- if (solrCore != null) solrCore.close();
- }
+ for (SolrCore solrCore : sourceCores) {
+ if (solrCore != null) solrCore.close();
}
- if (readersToBeClosed != null) IOUtils.closeWhileHandlingException(readersToBeClosed);
- if (dirsToBeReleased != null) {
- for (Directory dir : dirsToBeReleased) {
- DirectoryFactory dirFactory = core.getDirectoryFactory();
- dirFactory.release(dir);
- }
+ IOUtils.closeWhileHandlingException(readersToBeClosed);
+ for (Directory dir : dirsToBeReleased) {
+ DirectoryFactory dirFactory = core.getDirectoryFactory();
+ dirFactory.release(dir);
}
if (wrappedReq != null) wrappedReq.close();
core.close();
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Tue Aug 13 06:33:29 2013
@@ -576,7 +576,7 @@ public class LukeRequestHandler extends
throws IOException {
SolrParams params = req.getParams();
- int numTerms = params.getInt( NUMTERMS, DEFAULT_COUNT );
+ final int numTerms = params.getInt( NUMTERMS, DEFAULT_COUNT );
TopTermQueue tiq = new TopTermQueue(numTerms + 1); // Something to collect the top N terms in.
@@ -600,7 +600,7 @@ public class LukeRequestHandler extends
int freq = termsEnum.docFreq(); // This calculation seems odd, but it gives the same results as it used to.
int slot = 32 - Integer.numberOfLeadingZeros(Math.max(0, freq - 1));
buckets[slot] = buckets[slot] + 1;
- if (freq > tiq.minFreq) {
+ if (numTerms > 0 && freq > tiq.minFreq) {
UnicodeUtil.UTF8toUTF16(text, spare);
String t = spare.toString();
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/HttpShardHandler.java Tue Aug 13 06:33:29 2013
@@ -16,18 +16,6 @@ package org.apache.solr.handler.componen
* limitations under the License.
*/
-import java.net.ConnectException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
import org.apache.http.client.HttpClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
@@ -44,7 +32,6 @@ import org.apache.solr.common.cloud.DocC
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
-import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
@@ -55,6 +42,18 @@ import org.apache.solr.common.util.StrUt
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.request.SolrQueryRequest;
+import java.net.ConnectException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
public class HttpShardHandler extends ShardHandler {
private HttpShardHandlerFactory httpShardHandlerFactory;
@@ -277,7 +276,8 @@ public class HttpShardHandler extends Sh
// we weren't provided with an explicit list of slices to query via "shards", so use the cluster state
clusterState = zkController.getClusterState();
- String shardKeys = params.get(ShardParams.SHARD_KEYS);
+ String shardKeys = params.get(ShardParams._ROUTE_);
+ if(shardKeys == null) shardKeys = params.get(ShardParams.SHARD_KEYS);//eprecated
// This will be the complete list of slices we need to query for this request.
slices = new HashMap<String,Slice>();
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Tue Aug 13 06:33:29 2013
@@ -562,7 +562,7 @@ public class QueryElevationComponent ext
for (String id : elevations.ids) {
term.copyChars(id);
- if (seen.contains(id) == false && termsEnum.seekExact(term, false)) {
+ if (seen.contains(id) == false && termsEnum.seekExact(term)) {
docsEnum = termsEnum.docs(liveDocs, docsEnum, DocsEnum.FLAG_NONE);
if (docsEnum != null) {
int docId = docsEnum.nextDoc();
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java Tue Aug 13 06:33:29 2013
@@ -163,7 +163,7 @@ public class TermsComponent extends Sear
BytesRef term = null;
if (lowerBytes != null) {
- if (termsEnum.seekCeil(lowerBytes, true) == TermsEnum.SeekStatus.END) {
+ if (termsEnum.seekCeil(lowerBytes) == TermsEnum.SeekStatus.END) {
termsEnum = null;
} else {
term = termsEnum.term();
Modified: lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java?rev=1513364&r1=1513363&r2=1513364&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java (original)
+++ lucene/dev/branches/lucene3069/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java Tue Aug 13 06:33:29 2013
@@ -25,8 +25,6 @@ import org.apache.solr.common.params.Sol
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.ContentStream;
-import org.apache.solr.schema.IndexSchema;
-import org.apache.solr.schema.SchemaField;
import org.apache.solr.update.*;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.internal.csv.CSVStrategy;
@@ -63,15 +61,13 @@ abstract class CSVLoaderBase extends Con
public static Logger log = LoggerFactory.getLogger(CSVLoaderBase.class);
- final IndexSchema schema;
final SolrParams params;
final CSVStrategy strategy;
final UpdateRequestProcessor processor;
// hashmap to save any literal fields and their values
- HashMap <SchemaField, String> literals;
+ HashMap <String, String> literals;
String[] fieldnames;
- SchemaField[] fields;
CSVLoaderBase.FieldAdder[] adders;
String rowId = null;// if not null, add a special field by the name given with the line number/row id as the value
@@ -92,7 +88,7 @@ abstract class CSVLoaderBase extends Con
private class FieldAdder {
void add(SolrInputDocument doc, int line, int column, String val) {
if (val.length() > 0) {
- doc.addField(fields[column].getName(),val,1.0f);
+ doc.addField(fieldnames[column],val,1.0f);
}
}
}
@@ -101,7 +97,7 @@ abstract class CSVLoaderBase extends Con
private class FieldAdderEmpty extends CSVLoaderBase.FieldAdder {
@Override
void add(SolrInputDocument doc, int line, int column, String val) {
- doc.addField(fields[column].getName(),val,1.0f);
+ doc.addField(fieldnames[column],val,1.0f);
}
}
@@ -168,8 +164,7 @@ abstract class CSVLoaderBase extends Con
CSVLoaderBase(SolrQueryRequest req, UpdateRequestProcessor processor) {
this.processor = processor;
this.params = req.getParams();
- schema = req.getSchema();
- this.literals = new HashMap<SchemaField, String>();
+ this.literals = new HashMap<String, String>();
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite=params.getBool(OVERWRITE,true);
@@ -243,7 +238,6 @@ abstract class CSVLoaderBase extends Con
// from a POST, one could cache all of this setup info based on the params.
// The link from FieldAdder to this would need to be severed for that to happen.
- fields = new SchemaField[fieldnames.length];
adders = new CSVLoaderBase.FieldAdder[fieldnames.length];
String skipStr = params.get(SKIP);
List<String> skipFields = skipStr==null ? null : StrUtils.splitSmart(skipStr,',');
@@ -251,12 +245,11 @@ abstract class CSVLoaderBase extends Con
CSVLoaderBase.FieldAdder adder = new CSVLoaderBase.FieldAdder();
CSVLoaderBase.FieldAdder adderKeepEmpty = new CSVLoaderBase.FieldAdderEmpty();
- for (int i=0; i<fields.length; i++) {
+ for (int i=0; i<fieldnames.length; i++) {
String fname = fieldnames[i];
// to skip a field, leave the entries in fields and addrs null
if (fname.length()==0 || (skipFields!=null && skipFields.contains(fname))) continue;
- fields[i] = schema.getField(fname);
boolean keepEmpty = params.getFieldBool(fname,EMPTY,false);
adders[i] = keepEmpty ? adderKeepEmpty : adder;
@@ -297,11 +290,7 @@ abstract class CSVLoaderBase extends Con
if (!pname.startsWith(LITERALS_PREFIX)) continue;
String name = pname.substring(LITERALS_PREFIX.length());
- //TODO: need to look at this in light of schemaless
- SchemaField sf = schema.getFieldOrNull(name);
- if(sf == null)
- throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid field name for literal:'"+ name +"'");
- literals.put(sf, params.get(pname));
+ literals.put(name, params.get(pname));
}
}
@@ -368,8 +357,8 @@ abstract class CSVLoaderBase extends Con
}
if (vals==null) break;
- if (vals.length != fields.length) {
- input_err("expected "+fields.length+" values but got "+vals.length, vals, line);
+ if (vals.length != fieldnames.length) {
+ input_err("expected "+fieldnames.length+" values but got "+vals.length, vals, line);
}
addDoc(line,vals);
@@ -389,16 +378,15 @@ abstract class CSVLoaderBase extends Con
// the line number is passed for error reporting in MT mode as well as for optional rowId.
// first, create the lucene document
for (int i=0; i<vals.length; i++) {
- if (fields[i]==null) continue; // ignore this field
+ if (adders[i]==null) continue; // skip this field
String val = vals[i];
adders[i].add(doc, line, i, val);
}
// add any literals
- for (SchemaField sf : literals.keySet()) {
- String fn = sf.getName();
- String val = literals.get(sf);
- doc.addField(fn, val);
+ for (String fname : literals.keySet()) {
+ String val = literals.get(fname);
+ doc.addField(fname, val);
}
if (rowId != null){
doc.addField(rowId, line + rowIdOffset);