You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/05/22 13:38:49 UTC
svn commit: r1596817 [5/5] - in /lucene/dev/branches/lucene5675: ./
dev-tools/ dev-tools/idea/lucene/spatial/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/scripts/ lucene/
lucene/codecs/ lucene/codecs/src/java/org/apache/lucene/codecs/memory/...
Modified: lucene/dev/branches/lucene5675/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java Thu May 22 11:38:47 2014
@@ -20,12 +20,11 @@ package org.apache.solr.update.processor
import static org.apache.solr.update.processor.DistributingUpdateProcessorFactory.DISTRIB_UPDATE_PARAM;
import java.io.IOException;
-import java.net.ConnectException;
-import java.net.SocketException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -33,14 +32,12 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.http.NoHttpResponseException;
-import org.apache.http.conn.ConnectTimeoutException;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.solr.client.solrj.impl.HttpSolrServer;
-import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestRecovery;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.DistributedQueue;
import org.apache.solr.cloud.LeaderInitiatedRecoveryThread;
@@ -62,7 +59,6 @@ import org.apache.solr.common.cloud.ZkCo
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.cloud.ZooKeeperException;
-import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
@@ -71,7 +67,6 @@ import org.apache.solr.common.util.Hash;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.RealTimeGetComponent;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
@@ -128,7 +123,103 @@ public class DistributedUpdateProcessor
}
}
}
+
+ /**
+ * Keeps track of the replication factor achieved for a distributed update request
+ * originated in this distributed update processor.
+ */
+ public static class RequestReplicationTracker {
+ int minRf;
+ // if a leader is driving the update request, then this will be non-null
+ // however a replica may also be driving the update request (forwards to leaders)
+ // in which case we leave this as null so we only count the rf back from the leaders
+ String onLeaderShardId;
+ // track number of nodes we sent requests to and how many resulted in errors
+ // there may be multiple requests per node when processing a batch
+ Map<String,AtomicInteger> nodeErrorTracker;
+ // if not using DirectUpdates, a leader may end up forwarding to other
+ // leaders, so we need to keep the achieved rf for each of those too
+ Map<String,Integer> otherLeaderRf;
+
+ private RequestReplicationTracker(String shardId, int minRf) {
+ this.minRf = minRf;
+ this.onLeaderShardId = shardId;
+ this.nodeErrorTracker = new HashMap<>(5);
+ this.otherLeaderRf = new HashMap<>();
+ }
+
+ // gives the replication factor that was achieved for this request
+ public int getAchievedRf() {
+ // look across all shards to find the minimum achieved replication
+ // factor ... unless the client is using direct updates from CloudSolrServer
+ // there may be multiple shards at play here
+ int achievedRf = 1;
+ if (onLeaderShardId != null) {
+ synchronized (nodeErrorTracker) {
+ for (AtomicInteger nodeErrors : nodeErrorTracker.values()) {
+ if (nodeErrors.get() == 0)
+ ++achievedRf;
+ }
+ }
+ } else {
+ // the node driving this updateRequest is not a leader and so
+ // it only forwards to other leaders, so its local result doesn't count
+ achievedRf = Integer.MAX_VALUE;
+ }
+
+ // min achieved may come from a request to another leader
+ synchronized (otherLeaderRf) {
+ for (Integer otherRf : otherLeaderRf.values()) {
+ if (otherRf < achievedRf)
+ achievedRf = otherRf;
+ }
+ }
+
+ return (achievedRf == Integer.MAX_VALUE) ? 1 : achievedRf;
+ }
+
+ public void trackRequestResult(Node node, boolean success, Integer rf) {
+ String shardId = node.getShardId();
+ if (log.isDebugEnabled())
+ log.debug("trackRequestResult("+node+"): success? "+success+" rf="+rf+
+ ", shardId="+shardId+" onLeaderShardId="+onLeaderShardId);
+
+ if (onLeaderShardId == null || !onLeaderShardId.equals(shardId)) {
+ // result from another leader that we forwarded to
+ synchronized (otherLeaderRf) {
+ otherLeaderRf.put(shardId, rf != null ? rf : new Integer(1));
+ }
+ return;
+ }
+
+ if (onLeaderShardId != null) {
+ // track result for this leader
+ String nodeUrl = node.getUrl();
+ AtomicInteger nodeErrors = null;
+ // potentially many results flooding into this method from multiple nodes concurrently
+ synchronized (nodeErrorTracker) {
+ nodeErrors = nodeErrorTracker.get(nodeUrl);
+ if (nodeErrors == null) {
+ nodeErrors = new AtomicInteger(0);
+ nodeErrorTracker.put(nodeUrl, nodeErrors);
+ }
+ }
+
+ if (!success)
+ nodeErrors.incrementAndGet();
+ }
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder("RequestReplicationTracker");
+ sb.append(": onLeaderShardId=").append(String.valueOf(onLeaderShardId));
+ sb.append(", minRf=").append(minRf);
+ sb.append(", achievedRf=").append(getAchievedRf());
+ return sb.toString();
+ }
+ }
+
public static final String COMMIT_END_POINT = "commit_end_point";
public static final String LOG_REPLAY = "log_replay";
@@ -168,6 +259,9 @@ public class DistributedUpdateProcessor
private UpdateCommand updateCommand; // the current command this processor is working on.
+ //used for keeping track of replicas that have processed an add/update from the leader
+ private RequestReplicationTracker replicationTracker = null;
+
public DistributedUpdateProcessor(SolrQueryRequest req,
SolrQueryResponse rsp, UpdateRequestProcessor next) {
super(next);
@@ -559,7 +653,41 @@ public class DistributedUpdateProcessor
} else {
isLeader = getNonZkLeaderAssumption(req);
}
-
+
+ // check if client has requested minimum replication factor information
+ int minRf = -1; // disabled by default
+ if (replicationTracker != null) {
+ minRf = replicationTracker.minRf; // for subsequent requests in the same batch
+ } else {
+ SolrParams rp = cmd.getReq().getParams();
+ String distribUpdate = rp.get(DISTRIB_UPDATE_PARAM);
+ // somewhat tricky logic here: we only activate the replication tracker if we're on
+ // a leader or this is the top-level request processor
+ if (distribUpdate == null || distribUpdate.equals(DistribPhase.TOLEADER.toString())) {
+ String minRepFact = rp.get(UpdateRequest.MIN_REPFACT);
+ if (minRepFact != null) {
+ try {
+ minRf = Integer.parseInt(minRepFact);
+ } catch (NumberFormatException nfe) {
+ minRf = -1;
+ }
+
+ if (minRf <= 0)
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid value "+minRepFact+" for "+UpdateRequest.MIN_REPFACT+
+ "; must be >0 and less than or equal to the collection replication factor.");
+ }
+
+ if (minRf > 1) {
+ String myShardId = forwardToLeader ? null : cloudDesc.getShardId();
+ replicationTracker = new RequestReplicationTracker(myShardId, minRf);
+ }
+ }
+ }
+
+ // TODO: if minRf > 1 and we know the leader is the only active replica, we could fail
+ // the request right here but for now I think it is better to just return the status
+ // to the client that the minRf wasn't reached and let them handle it
+
boolean dropCmd = false;
if (!forwardToLeader) {
dropCmd = versionAdd(cmd);
@@ -593,15 +721,15 @@ public class DistributedUpdateProcessor
params.set(DISTRIB_FROM_COLLECTION, req.getCore().getCoreDescriptor().getCloudDescriptor().getCollectionName());
params.set(DISTRIB_FROM_SHARD, req.getCore().getCoreDescriptor().getCloudDescriptor().getShardId());
+
for (Node nodesByRoutingRule : nodesByRoutingRules) {
cmdDistrib.distribAdd(cmd, Collections.singletonList(nodesByRoutingRule), params, true);
}
}
}
-
+
ModifiableSolrParams params = null;
if (nodes != null) {
-
params = new ModifiableSolrParams(filterParams(req.getParams()));
params.set(DISTRIB_UPDATE_PARAM,
(isLeader || isSubShardLeader ?
@@ -609,7 +737,11 @@ public class DistributedUpdateProcessor
DistribPhase.TOLEADER.toString()));
params.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
zkController.getBaseUrl(), req.getCore().getName()));
- cmdDistrib.distribAdd(cmd, nodes, params);
+
+ if (replicationTracker != null && minRf > 1)
+ params.set(UpdateRequest.MIN_REPFACT, String.valueOf(minRf));
+
+ cmdDistrib.distribAdd(cmd, nodes, params, false, replicationTracker);
}
// TODO: what to do when no idField?
@@ -632,9 +764,9 @@ public class DistributedUpdateProcessor
// TODO: optionally fail if n replicas are not reached...
private void doFinish() {
// TODO: if not a forward and replication req is not specified, we could
- // send in a background thread
-
- cmdDistrib.finish();
+ // send in a background thread
+
+ cmdDistrib.finish();
List<Error> errors = cmdDistrib.getErrors();
// TODO - we may need to tell about more than one error...
@@ -724,6 +856,12 @@ public class DistributedUpdateProcessor
ExecutorService executor = coreContainer.getUpdateShardHandler().getUpdateExecutor();
executor.execute(lirThread);
}
+
+ if (replicationTracker != null) {
+ rsp.getResponseHeader().add(UpdateRequest.REPFACT, replicationTracker.getAchievedRf());
+ rsp.getResponseHeader().add(UpdateRequest.MIN_REPFACT, replicationTracker.minRf);
+ replicationTracker = null;
+ }
}
@@ -1052,7 +1190,7 @@ public class DistributedUpdateProcessor
if (zkEnabled && isLeader && !isSubShardLeader) {
DocCollection coll = zkController.getClusterState().getCollection(collection);
- List<Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), null, null);
+ List<Node> subShardLeaders = getSubShardLeaders(coll, cloudDesc.getShardId(), cmd.getId(), null);
// the list<node> will actually have only one element for an add request
if (subShardLeaders != null && !subShardLeaders.isEmpty()) {
ModifiableSolrParams params = new ModifiableSolrParams(filterParams(req.getParams()));
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java Thu May 22 11:38:47 2014
@@ -26,7 +26,6 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.JSONTestUtil;
@@ -59,10 +58,11 @@ public class HttpPartitionTest extends A
// To prevent the test assertions firing too fast before cluster state
// recognizes (and propagates) partitions
- private static final long sleepMsBeforeHealPartition = 1000L;
+ private static final long sleepMsBeforeHealPartition = 2000L;
+
+ private static final int maxWaitSecsToSeeAllActive = 30;
private Map<URI,SocketProxy> proxies = new HashMap<URI,SocketProxy>();
- private AtomicInteger portCounter = new AtomicInteger(0);
public HttpPartitionTest() {
super();
@@ -157,7 +157,7 @@ public class HttpPartitionTest extends A
sendDoc(1);
Replica notLeader =
- ensureAllReplicasAreActive(testCollectionName, 2, 10).get(0);
+ ensureAllReplicasAreActive(testCollectionName, 2, maxWaitSecsToSeeAllActive).get(0);
// ok, now introduce a network partition between the leader and the replica
SocketProxy proxy = getProxyForReplica(notLeader);
@@ -179,7 +179,7 @@ public class HttpPartitionTest extends A
proxy.reopen();
List<Replica> notLeaders =
- ensureAllReplicasAreActive(testCollectionName, 2, 20); // shouldn't take 20 secs but just to be safe
+ ensureAllReplicasAreActive(testCollectionName, 2, maxWaitSecsToSeeAllActive);
sendDoc(3);
@@ -211,7 +211,7 @@ public class HttpPartitionTest extends A
proxy.reopen();
}
- notLeaders = ensureAllReplicasAreActive(testCollectionName, 2, 20);
+ notLeaders = ensureAllReplicasAreActive(testCollectionName, 2, maxWaitSecsToSeeAllActive);
// verify all docs received
assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, numDocs + 3);
@@ -226,10 +226,10 @@ public class HttpPartitionTest extends A
sendDoc(1);
List<Replica> notLeaders =
- ensureAllReplicasAreActive(testCollectionName, 3, 10);
+ ensureAllReplicasAreActive(testCollectionName, 3, maxWaitSecsToSeeAllActive);
assertTrue("Expected 2 replicas for collection " + testCollectionName
+ " but found " + notLeaders.size() + "; clusterState: "
- + cloudClient.getZkStateReader().getClusterState(),
+ + printClusterStateInfo(),
notLeaders.size() == 2);
sendDoc(1);
@@ -256,7 +256,7 @@ public class HttpPartitionTest extends A
proxy1.reopen();
// sent 4 docs in so far, verify they are on the leader and replica
- notLeaders = ensureAllReplicasAreActive(testCollectionName, 3, 20);
+ notLeaders = ensureAllReplicasAreActive(testCollectionName, 3, maxWaitSecsToSeeAllActive);
sendDoc(4);
@@ -274,10 +274,10 @@ public class HttpPartitionTest extends A
sendDoc(1);
List<Replica> notLeaders =
- ensureAllReplicasAreActive(testCollectionName, 3, 10);
+ ensureAllReplicasAreActive(testCollectionName, 3, maxWaitSecsToSeeAllActive);
assertTrue("Expected 2 replicas for collection " + testCollectionName
+ " but found " + notLeaders.size() + "; clusterState: "
- + cloudClient.getZkStateReader().getClusterState(),
+ + printClusterStateInfo(),
notLeaders.size() == 2);
sendDoc(1);
@@ -305,7 +305,7 @@ public class HttpPartitionTest extends A
proxy1.reopen();
// sent 4 docs in so far, verify they are on the leader and replica
- notLeaders = ensureAllReplicasAreActive(testCollectionName, 3, 20);
+ notLeaders = ensureAllReplicasAreActive(testCollectionName, 3, maxWaitSecsToSeeAllActive);
sendDoc(4);
@@ -314,11 +314,12 @@ public class HttpPartitionTest extends A
Replica leader =
cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, "shard1");
String leaderNode = leader.getNodeName();
- assertNotNull("Could not find leader for shard1 of "+testCollectionName, leader);
+ assertNotNull("Could not find leader for shard1 of "+
+ testCollectionName+"; clusterState: "+printClusterStateInfo(), leader);
JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(leader));
// since maxShardsPerNode is 1, we're safe to kill the leader
- notLeaders = ensureAllReplicasAreActive(testCollectionName, 3, 20);
+ notLeaders = ensureAllReplicasAreActive(testCollectionName, 3, maxWaitSecsToSeeAllActive);
proxy0 = getProxyForReplica(notLeaders.get(0));
proxy0.close();
@@ -343,16 +344,17 @@ public class HttpPartitionTest extends A
log.warn("No SocketProxy found for old leader node "+leaderNode);
}
- Thread.sleep(sleepMsBeforeHealPartition);
+ Thread.sleep(10000); // give chance for new leader to be elected.
Replica newLeader =
- cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, "shard1", 30000);
+ cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, "shard1", 60000);
- assertNotNull("No new leader was elected after 30 seconds", newLeader);
+ assertNotNull("No new leader was elected after 60 seconds; clusterState: "+
+ printClusterStateInfo(),newLeader);
assertTrue("Expected node "+shouldNotBeNewLeaderNode+
" to NOT be the new leader b/c it was out-of-sync with the old leader! ClusterState: "+
- cloudClient.getZkStateReader().getClusterState(),
+ printClusterStateInfo(),
!shouldNotBeNewLeaderNode.equals(newLeader.getNodeName()));
proxy0.reopen();
@@ -362,12 +364,19 @@ public class HttpPartitionTest extends A
cloudClient.getZkStateReader().updateClusterState(true);
List<Replica> activeReps = getActiveOrRecoveringReplicas(testCollectionName, "shard1");
- assertTrue("Expected 2 of 3 replicas to be active but only found "+activeReps.size()+"; "+activeReps, activeReps.size() == 2);
+ assertTrue("Expected 2 of 3 replicas to be active but only found "+
+ activeReps.size()+"; "+activeReps+"; clusterState: "+printClusterStateInfo(),
+ activeReps.size() == 2);
sendDoc(6);
assertDocsExistInAllReplicas(activeReps, testCollectionName, 1, 6);
- }
+ }
+
+ protected String printClusterStateInfo() throws Exception {
+ cloudClient.getZkStateReader().updateClusterState(true);
+ return String.valueOf(cloudClient.getZkStateReader().getClusterState());
+ }
protected List<Replica> getActiveOrRecoveringReplicas(String testCollectionName, String shardId) throws Exception {
Map<String,Replica> activeReplicas = new HashMap<String,Replica>();
@@ -492,10 +501,12 @@ public class HttpPartitionTest extends A
} // end while
if (!allReplicasUp)
- fail("Didn't see all replicas come up within " + maxWaitMs + " ms! ClusterState: " + cs);
+ fail("Didn't see all replicas come up within " + maxWaitMs +
+ " ms! ClusterState: " + printClusterStateInfo());
if (notLeaders.isEmpty())
- fail("Didn't isolate any replicas that are not the leader! ClusterState: " + cs);
+ fail("Didn't isolate any replicas that are not the leader! ClusterState: " +
+ printClusterStateInfo());
long diffMs = (System.currentTimeMillis() - startMs);
log.info("Took " + diffMs + " ms to see all replicas become active.");
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java Thu May 22 11:38:47 2014
@@ -31,12 +31,13 @@ import org.apache.solr.common.params.Cor
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.handler.component.ShardHandler;
+import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
import org.apache.zookeeper.CreateMode;
import org.easymock.Capture;
+import org.easymock.EasyMock;
import org.easymock.IAnswer;
-import org.eclipse.jetty.util.BlockingArrayQueue;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -44,14 +45,17 @@ import org.junit.BeforeClass;
import org.junit.Test;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Queue;
import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
import static org.easymock.EasyMock.anyBoolean;
import static org.easymock.EasyMock.anyObject;
@@ -74,36 +78,32 @@ public class OverseerCollectionProcessor
private static DistributedMap runningMapMock;
private static DistributedMap completedMapMock;
private static DistributedMap failureMapMock;
+ private static ShardHandlerFactory shardHandlerFactoryMock;
private static ShardHandler shardHandlerMock;
private static ZkStateReader zkStateReaderMock;
private static ClusterState clusterStateMock;
private static SolrZkClient solrZkClientMock;
private final Map zkMap = new HashMap();
private final Set collectionsSet = new HashSet();
+ private SolrResponse lastProcessMessageResult;
+
private OverseerCollectionProcessorToBeTested underTest;
private Thread thread;
- private Queue<QueueEvent> queue = new BlockingArrayQueue<>();
+ private Queue<QueueEvent> queue = new ArrayBlockingQueue<>(10);
private class OverseerCollectionProcessorToBeTested extends
OverseerCollectionProcessor {
- private SolrResponse lastProcessMessageResult;
-
+
public OverseerCollectionProcessorToBeTested(ZkStateReader zkStateReader,
- String myId, ShardHandler shardHandler, String adminPath,
+ String myId, ShardHandlerFactory shardHandlerFactory,
+ String adminPath,
DistributedQueue workQueue, DistributedMap runningMap,
DistributedMap completedMap,
DistributedMap failureMap) {
- super(zkStateReader, myId, shardHandler, adminPath, new Overseer.Stats(), workQueue, runningMap, completedMap, failureMap);
- }
-
- @Override
- protected SolrResponse processMessage(ZkNodeProps message, String operation) {
- lastProcessMessageResult = super.processMessage(message, operation);
- log.info("1 : "+System.currentTimeMillis());
- return lastProcessMessageResult;
+ super(zkStateReader, myId, shardHandlerFactory, adminPath, new Overseer.Stats(), workQueue, runningMap, completedMap, failureMap);
}
@Override
@@ -119,10 +119,12 @@ public class OverseerCollectionProcessor
runningMapMock = createMock(DistributedMap.class);
completedMapMock = createMock(DistributedMap.class);
failureMapMock = createMock(DistributedMap.class);
+ shardHandlerFactoryMock = createMock(ShardHandlerFactory.class);
shardHandlerMock = createMock(ShardHandler.class);
zkStateReaderMock = createMock(ZkStateReader.class);
clusterStateMock = createMock(ClusterState.class);
solrZkClientMock = createMock(SolrZkClient.class);
+
}
@AfterClass
@@ -131,6 +133,7 @@ public class OverseerCollectionProcessor
runningMapMock = null;
completedMapMock = null;
failureMapMock = null;
+ shardHandlerFactoryMock = null;
shardHandlerMock = null;
zkStateReaderMock = null;
clusterStateMock = null;
@@ -145,12 +148,13 @@ public class OverseerCollectionProcessor
reset(runningMapMock);
reset(completedMapMock);
reset(failureMapMock);
+ reset(shardHandlerFactoryMock);
reset(shardHandlerMock);
reset(zkStateReaderMock);
reset(clusterStateMock);
reset(solrZkClientMock);
underTest = new OverseerCollectionProcessorToBeTested(zkStateReaderMock,
- "1234", shardHandlerMock, ADMIN_PATH, workQueueMock, runningMapMock,
+ "1234", shardHandlerFactoryMock, ADMIN_PATH, workQueueMock, runningMapMock,
completedMapMock, failureMapMock);
zkMap.clear();
collectionsSet.clear();
@@ -163,6 +167,44 @@ public class OverseerCollectionProcessor
}
protected Set<String> commonMocks(int liveNodesCount) throws Exception {
+
+ shardHandlerFactoryMock.getShardHandler();
+ expectLastCall().andAnswer(new IAnswer<ShardHandler>() {
+ @Override
+ public ShardHandler answer() throws Throwable {
+ log.info("SHARDHANDLER");
+ return shardHandlerMock;
+ }
+ }).anyTimes();
+ workQueueMock.peekTopN(EasyMock.anyInt(), anyObject(Set.class), EasyMock.anyLong());
+ expectLastCall().andAnswer(new IAnswer<List>() {
+ @Override
+ public List answer() throws Throwable {
+ Object result;
+ int count = 0;
+ while ((result = queue.peek()) == null) {
+ Thread.sleep(1000);
+ count++;
+ if (count > 1) return null;
+ }
+
+ return Arrays.asList(result);
+ }
+ }).anyTimes();
+
+ workQueueMock.getTailId();
+ expectLastCall().andAnswer(new IAnswer<Object>() {
+ @Override
+ public Object answer() throws Throwable {
+ Object result = null;
+ Iterator iter = queue.iterator();
+ while(iter.hasNext()) {
+ result = iter.next();
+ }
+ return result==null ? null : ((QueueEvent)result).getId();
+ }
+ }).anyTimes();
+
workQueueMock.peek(true);
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
@@ -191,7 +233,7 @@ public class OverseerCollectionProcessor
return queue.poll();
}
}).anyTimes();
-
+
zkStateReaderMock.getClusterState();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
@@ -207,8 +249,8 @@ public class OverseerCollectionProcessor
return solrZkClientMock;
}
}).anyTimes();
-
-
+
+
clusterStateMock.getCollections();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
@@ -239,7 +281,7 @@ public class OverseerCollectionProcessor
return new HashMap();
}
});
-
+
solrZkClientMock.getZkClientTimeout();
expectLastCall().andAnswer(new IAnswer<Object>() {
@Override
@@ -378,7 +420,12 @@ public class OverseerCollectionProcessor
OverseerCollectionProcessor.MAX_SHARDS_PER_NODE,
maxShardsPerNode.toString());
}
- QueueEvent qe = new QueueEvent("id", ZkStateReader.toJSON(props), null);
+ QueueEvent qe = new QueueEvent("id", ZkStateReader.toJSON(props), null){
+ @Override
+ public void setBytes(byte[] bytes) {
+ lastProcessMessageResult = SolrResponse.deserialize( bytes);
+ }
+ };
queue.add(qe);
}
@@ -542,20 +589,23 @@ public class OverseerCollectionProcessor
replay(solrZkClientMock);
replay(zkStateReaderMock);
replay(clusterStateMock);
+ replay(shardHandlerFactoryMock);
replay(shardHandlerMock);
- log.info("clusterstate " +clusterStateMock.hashCode());
+
+ log.info("clusterstate " + clusterStateMock.hashCode());
startComponentUnderTest();
- issueCreateJob(numberOfSlices, replicationFactor, maxShardsPerNode, (createNodeListOption != CreateNodeListOptions.SEND_NULL)?createNodeList:null, (createNodeListOption != CreateNodeListOptions.DONT_SEND));
+ issueCreateJob(numberOfSlices, replicationFactor, maxShardsPerNode, (createNodeListOption != CreateNodeListOptions.SEND_NULL) ? createNodeList : null, (createNodeListOption != CreateNodeListOptions.DONT_SEND));
waitForEmptyQueue(10000);
if (collectionExceptedToBeCreated) {
- assertNotNull(underTest.lastProcessMessageResult.getResponse().toString(), underTest.lastProcessMessageResult);
+ assertNotNull(lastProcessMessageResult.getResponse().toString(), lastProcessMessageResult);
}
+ verify(shardHandlerFactoryMock);
verify(shardHandlerMock);
-
+
if (collectionExceptedToBeCreated) {
verifySubmitCaptures(submitCaptures, numberOfSlices, replicationFactor,
createNodeList);
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/OverseerStatusTest.java Thu May 22 11:38:47 2014
@@ -123,22 +123,4 @@ public class OverseerStatusTest extends
waitForThingsToLevelOut(15);
}
-
- private NamedList<Object> invokeCollectionApi(String... args) throws SolrServerException, IOException {
- ModifiableSolrParams params = new ModifiableSolrParams();
- SolrRequest request = new QueryRequest(params);
- for (int i = 0; i < args.length - 1; i+=2) {
- params.add(args[i], args[i+1]);
- }
- request.setPath("/admin/collections");
-
- String baseUrl = ((HttpSolrServer) shardToJetty.get(SHARD1).get(0).client.solrClient)
- .getBaseURL();
- baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
-
- HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
- baseServer.setConnectionTimeout(15000);
- baseServer.setSoTimeout(60000 * 5);
- return baseServer.request(request);
- }
}
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java Thu May 22 11:38:47 2014
@@ -38,7 +38,6 @@ import org.apache.solr.common.cloud.ZkNo
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.update.DirectUpdateHandler2;
import org.junit.After;
import org.junit.Before;
@@ -52,14 +51,12 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
-import org.apache.lucene.util.LuceneTestCase.BadApple;
import org.apache.lucene.util.LuceneTestCase.Slow;
import static org.apache.solr.cloud.OverseerCollectionProcessor.MAX_SHARDS_PER_NODE;
import static org.apache.solr.cloud.OverseerCollectionProcessor.NUM_SLICES;
import static org.apache.solr.cloud.OverseerCollectionProcessor.REPLICATION_FACTOR;
@Slow
-@BadApple(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5309")
public class ShardSplitTest extends BasicDistributedZkTest {
public static final String SHARD1_0 = SHARD1 + "_0";
@@ -235,7 +232,7 @@ public class ShardSplitTest extends Basi
}
}
- waitForRecoveriesToFinish(false);
+ waitForRecoveriesToFinish(true);
checkDocCountsAndShardStates(docCounts, numReplicas);
}
@@ -466,8 +463,8 @@ public class ShardSplitTest extends Basi
logDebugHelp(docCounts, response, shard10Count, response2, shard11Count);
- assertEquals("Wrong doc count on shard1_0", docCounts[0], shard10Count);
- assertEquals("Wrong doc count on shard1_1", docCounts[1], shard11Count);
+ assertEquals("Wrong doc count on shard1_0. See SOLR-5309", docCounts[0], shard10Count);
+ assertEquals("Wrong doc count on shard1_1. See SOLR-5309", docCounts[1], shard11Count);
}
protected void checkSubShardConsistency(String shard) throws SolrServerException {
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java Thu May 22 11:38:47 2014
@@ -79,9 +79,8 @@ public class TestMiniSolrCloudCluster ex
@BeforeClass
public static void startup() throws Exception {
- String testHome = SolrTestCaseJ4.TEST_HOME();
- miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, new File(testHome, "solr-no-core.xml"),
- null, null);
+ File solrXml = new File(SolrTestCaseJ4.TEST_HOME(), "solr-no-core.xml");
+ miniCluster = new MiniSolrCloudCluster(NUM_SERVERS, null, createTempDir(), solrXml, null, null);
}
@AfterClass
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/MockShardHandlerFactory.java Thu May 22 11:38:47 2014
@@ -57,6 +57,11 @@ public class MockShardHandlerFactory ext
@Override
public void cancelAll() {}
+
+ @Override
+ public ShardHandlerFactory getShardHandlerFactory() {
+ return MockShardHandlerFactory.this;
+ }
};
}
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/TestConfigSets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/TestConfigSets.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/TestConfigSets.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/core/TestConfigSets.java Thu May 22 11:38:47 2014
@@ -56,17 +56,19 @@ public class TestConfigSets extends Solr
@Test
public void testConfigSetServiceFindsConfigSets() {
CoreContainer container = null;
+ SolrCore core1 = null;
try {
container = setupContainer(getFile("solr/configsets").getAbsolutePath());
- String testDirectory = container.getResourceLoader().getInstanceDir();
+ String testDirectory = new File(container.getResourceLoader().getInstanceDir()).getAbsolutePath();
- SolrCore core1 = container.create("core1", testDirectory + "/core1", "configSet", "configset-2");
+ core1 = container.create("core1", testDirectory + "core1", "configSet", "configset-2");
assertThat(core1.getCoreDescriptor().getName(), is("core1"));
- assertThat(core1.getDataDir(), is(testDirectory + "/core1" + File.separator + "data" + File.separator));
- core1.close();
-
+ assertThat(core1.getDataDir(), is(testDirectory + "core1" + File.separator + "data" + File.separator));
}
finally {
+ if (core1 != null) {
+ core1.close();
+ }
if (container != null)
container.shutdown();
}
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Thu May 22 11:38:47 2014
@@ -114,12 +114,12 @@ public class TestReplicationHandler exte
// System.setProperty("solr.directoryFactory", "solr.StandardDirectoryFactory");
// For manual testing only
// useFactory(null); // force an FS factory.
- master = new SolrInstance("master", null);
+ master = new SolrInstance(createTempDir("solr-instance"), "master", null);
master.setUp();
masterJetty = createJetty(master);
masterClient = createNewSolrServer(masterJetty.getLocalPort());
- slave = new SolrInstance("slave", masterJetty.getLocalPort());
+ slave = new SolrInstance(createTempDir("solr-instance"), "slave", masterJetty.getLocalPort());
slave.setUp();
slaveJetty = createJetty(slave);
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
@@ -323,7 +323,7 @@ public class TestReplicationHandler exte
JettySolrRunner repeaterJetty = null;
SolrServer repeaterClient = null;
try {
- repeater = new SolrInstance("repeater", masterJetty.getLocalPort());
+ repeater = new SolrInstance(createTempDir("solr-instance"), "repeater", masterJetty.getLocalPort());
repeater.setUp();
repeaterJetty = createJetty(repeater);
repeaterClient = createNewSolrServer(repeaterJetty.getLocalPort());
@@ -911,7 +911,7 @@ public class TestReplicationHandler exte
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
try {
- repeater = new SolrInstance("repeater", null);
+ repeater = new SolrInstance(createTempDir("solr-instance"), "repeater", null);
repeater.setUp();
repeater.copyConfigFile(CONF_DIR + "solrconfig-repeater.xml",
"solrconfig.xml");
@@ -1647,13 +1647,15 @@ public class TestReplicationHandler exte
private File dataDir;
/**
- * @param name used to pick new solr home dir, as well as which
+ * @param homeDir Base directory to build solr configuration and index in
+ * @param name used to pick which
* "solrconfig-${name}.xml" file gets copied
* to solrconfig.xml in new conf dir.
* @param testPort if not null, used as a replacement for
* TEST_PORT in the cloned config files.
*/
- public SolrInstance(String name, Integer testPort) {
+ public SolrInstance(File homeDir, String name, Integer testPort) {
+ this.homeDir = homeDir;
this.name = name;
this.testPort = testPort;
}
@@ -1687,11 +1689,6 @@ public class TestReplicationHandler exte
System.setProperty("solr.test.sys.prop1", "propone");
System.setProperty("solr.test.sys.prop2", "proptwo");
- File home = new File(dataDir,
- getClass().getName() + "-" +
- System.currentTimeMillis());
-
- homeDir = new File(home, name);
dataDir = new File(homeDir + "/collection1", "data");
confDir = new File(homeDir + "/collection1", "conf");
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/request/TestFaceting.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/request/TestFaceting.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/request/TestFaceting.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/request/TestFaceting.java Thu May 22 11:38:47 2014
@@ -23,6 +23,7 @@ import java.util.Locale;
import java.util.Random;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
@@ -955,9 +956,27 @@ public class TestFaceting extends SolrTe
SortedDocValues singleton1 = DocValues.unwrapSingleton(dv1);
SortedDocValues singleton2 = DocValues.unwrapSingleton(dv2);
if (singleton1 == null || singleton2 == null) {
- assertSame(dv1, dv2);
+ // actually a multi-valued field
+ if (dv1 instanceof MultiDocValues.MultiSortedSetDocValues) {
+ // if we produced more than one segment, ensure the core ordinal map is the same object
+ assertTrue(dv2 instanceof MultiDocValues.MultiSortedSetDocValues);
+ assertSame(((MultiDocValues.MultiSortedSetDocValues) dv1).mapping,
+ ((MultiDocValues.MultiSortedSetDocValues) dv2).mapping);
+ } else {
+ // otherwise, same atomic instance
+ assertSame(dv1, dv2);
+ }
} else {
- assertSame(singleton1, singleton2);
+ // just wrapping a field that is actually single-valued
+ if (singleton1 instanceof MultiDocValues.MultiSortedDocValues) {
+ // if we produced more than one segment, ensure the core ordinal map is the same object
+ assertTrue(singleton2 instanceof MultiDocValues.MultiSortedDocValues);
+ assertSame(((MultiDocValues.MultiSortedDocValues) singleton1).mapping,
+ ((MultiDocValues.MultiSortedDocValues) singleton2).mapping);
+ } else {
+ // otherwise, same atomic instance
+ assertSame(singleton1, singleton2);
+ }
}
}
}
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/rest/TestManagedResourceStorage.java Thu May 22 11:38:47 2014
@@ -16,6 +16,7 @@ package org.apache.solr.rest;
* limitations under the License.
*/
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -66,12 +67,12 @@ public class TestManagedResourceStorage
* Runs persisted managed resource creation and update tests on JSON storage.
*/
@Test
- public void testFileBasedJsonStorage() throws Exception {
- SolrResourceLoader loader = new SolrResourceLoader("./");
- // Solr unit tests can only write to their working directory due to
- // a custom Java Security Manager installed in the test environment
+ public void testFileBasedJsonStorage() throws Exception {
+ File instanceDir = createTempDir("json-storage");
+ SolrResourceLoader loader = new SolrResourceLoader(instanceDir.getAbsolutePath());
NamedList<String> initArgs = new NamedList<>();
- initArgs.add(ManagedResourceStorage.STORAGE_DIR_INIT_ARG, "./managed");
+ String managedDir = instanceDir.getAbsolutePath() + File.separator + "managed";
+ initArgs.add(ManagedResourceStorage.STORAGE_DIR_INIT_ARG, managedDir);
FileStorageIO fileStorageIO = new FileStorageIO();
fileStorageIO.configure(loader, initArgs);
doStorageTests(loader, fileStorageIO);
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestAnalyzeInfixSuggestions.java Thu May 22 11:38:47 2014
@@ -1,10 +1,7 @@
package org.apache.solr.spelling.suggest;
-import java.io.File;
-
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.SpellingParams;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
/*
@@ -33,16 +30,6 @@ public class TestAnalyzeInfixSuggestions
assertQ(req("qt", URI_DEFAULT, "q", "", SpellingParams.SPELLCHECK_BUILD, "true"));
}
- @AfterClass
- public static void afterClass() throws Exception {
- File indexPathDir = new File("analyzingInfixSuggesterIndexDir");
- File indexPathDirTmp = new File("analyzingInfixSuggesterIndexDir.tmp");
- if (indexPathDir.exists())
- recurseDelete(indexPathDir);
- if (indexPathDirTmp.exists())
- recurseDelete(indexPathDirTmp);
- }
-
public void testSingle() throws Exception {
assertQ(req("qt", URI_DEFAULT, "q", "japan", SpellingParams.SPELLCHECK_COUNT, "1"),
Modified: lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java (original)
+++ lucene/dev/branches/lucene5675/solr/core/src/test/org/apache/solr/spelling/suggest/TestBlendedInfixSuggestions.java Thu May 22 11:38:47 2014
@@ -17,10 +17,7 @@ package org.apache.solr.spelling.suggest
* limitations under the License.
*/
-import java.io.File;
-
import org.apache.solr.SolrTestCaseJ4;
-import org.junit.AfterClass;
import org.junit.BeforeClass;
public class TestBlendedInfixSuggestions extends SolrTestCaseJ4 {
@@ -31,17 +28,7 @@ public class TestBlendedInfixSuggestions
initCore("solrconfig-phrasesuggest.xml","schema-phrasesuggest.xml");
assertQ(req("qt", URI, "q", "", SuggesterParams.SUGGEST_BUILD_ALL, "true"));
}
-
- @AfterClass
- public static void afterClass() throws Exception {
- File indexPathDir = new File("blendedInfixSuggesterIndexDir");
- File indexPathDirTmp = new File("blendedInfixSuggesterIndexDir.tmp");
- if (indexPathDir.exists())
- recurseDelete(indexPathDir);
- if (indexPathDirTmp.exists())
- recurseDelete(indexPathDirTmp);
- }
-
+
public void testLinearBlenderType() {
assertQ(req("qt", URI, "q", "the", SuggesterParams.SUGGEST_COUNT, "10", SuggesterParams.SUGGEST_DICT, "blended_infix_suggest_linear"),
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/int[@name='numFound'][.='3']",
@@ -55,7 +42,6 @@ public class TestBlendedInfixSuggestions
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/long[@name='weight'][.='7']",
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_linear']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']"
);
-
}
public void testReciprocalBlenderType() {
@@ -97,5 +83,4 @@ public class TestBlendedInfixSuggestions
"//lst[@name='suggest']/lst[@name='blended_infix_suggest_reciprocal']/lst[@name='the']/arr[@name='suggestions']/lst[3]/str[@name='payload'][.='star']"
);
}
-
}
Modified: lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original)
+++ lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Thu May 22 11:38:47 2014
@@ -436,19 +436,33 @@ public class CloudSolrServer extends Sol
public RouteResponse condenseResponse(NamedList response, long timeMillis) {
RouteResponse condensed = new RouteResponse();
int status = 0;
+ Integer rf = null;
+ Integer minRf = null;
for(int i=0; i<response.size(); i++) {
NamedList shardResponse = (NamedList)response.getVal(i);
- NamedList header = (NamedList)shardResponse.get("responseHeader");
+ NamedList header = (NamedList)shardResponse.get("responseHeader");
Integer shardStatus = (Integer)header.get("status");
int s = shardStatus.intValue();
if(s > 0) {
status = s;
}
+ Object rfObj = header.get(UpdateRequest.REPFACT);
+ if (rfObj != null && rfObj instanceof Integer) {
+ Integer routeRf = (Integer)rfObj;
+ if (rf == null || routeRf < rf)
+ rf = routeRf;
+ }
+ minRf = (Integer)header.get(UpdateRequest.MIN_REPFACT);
}
NamedList cheader = new NamedList();
cheader.add("status", status);
cheader.add("QTime", timeMillis);
+ if (rf != null)
+ cheader.add(UpdateRequest.REPFACT, rf);
+ if (minRf != null)
+ cheader.add(UpdateRequest.MIN_REPFACT, minRf);
+
condensed.add("responseHeader", cheader);
return condensed;
}
@@ -692,4 +706,73 @@ public class CloudSolrServer extends Sol
return updatesToLeaders;
}
+ /**
+ * Useful for determining the minimum achieved replication factor across
+ * all shards involved in processing an update request, typically useful
+ * for gauging the replication factor of a batch.
+ */
+ @SuppressWarnings("rawtypes")
+ public int getMinAchievedReplicationFactor(String collection, NamedList resp) {
+ // it's probably already on the top-level header set by condense
+ NamedList header = (NamedList)resp.get("responseHeader");
+ Integer achRf = (Integer)header.get(UpdateRequest.REPFACT);
+ if (achRf != null)
+ return achRf.intValue();
+
+ // not on the top-level header, walk the shard route tree
+ Map<String,Integer> shardRf = getShardReplicationFactor(collection, resp);
+ for (Integer rf : shardRf.values()) {
+ if (achRf == null || rf < achRf) {
+ achRf = rf;
+ }
+ }
+ return (achRf != null) ? achRf.intValue() : -1;
+ }
+
+ /**
+ * Walks the NamedList response after performing an update request looking for
+ * the replication factor that was achieved in each shard involved in the request.
+ * For single doc updates, there will be only one shard in the return value.
+ */
+ @SuppressWarnings("rawtypes")
+ public Map<String,Integer> getShardReplicationFactor(String collection, NamedList resp) {
+ connect();
+
+ Map<String,Integer> results = new HashMap<String,Integer>();
+ if (resp instanceof CloudSolrServer.RouteResponse) {
+ NamedList routes = ((CloudSolrServer.RouteResponse)resp).getRouteResponses();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ Map<String,String> leaders = new HashMap<String,String>();
+ for (Slice slice : clusterState.getActiveSlices(collection)) {
+ Replica leader = slice.getLeader();
+ if (leader != null) {
+ ZkCoreNodeProps zkProps = new ZkCoreNodeProps(leader);
+ String leaderUrl = zkProps.getBaseUrl() + "/" + zkProps.getCoreName();
+ leaders.put(leaderUrl, slice.getName());
+ String altLeaderUrl = zkProps.getBaseUrl() + "/" + collection;
+ leaders.put(altLeaderUrl, slice.getName());
+ }
+ }
+
+ Iterator<Map.Entry<String,Object>> routeIter = routes.iterator();
+ while (routeIter.hasNext()) {
+ Map.Entry<String,Object> next = routeIter.next();
+ String host = next.getKey();
+ NamedList hostResp = (NamedList)next.getValue();
+ Integer rf = (Integer)((NamedList)hostResp.get("responseHeader")).get(UpdateRequest.REPFACT);
+ if (rf != null) {
+ String shard = leaders.get(host);
+ if (shard == null) {
+ if (host.endsWith("/"))
+ shard = leaders.get(host.substring(0,host.length()-1));
+ if (shard == null) {
+ shard = host;
+ }
+ }
+ results.put(shard, rf);
+ }
+ }
+ }
+ return results;
+ }
}
Modified: lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java (original)
+++ lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ConcurrentUpdateSolrServer.java Thu May 22 11:38:47 2014
@@ -239,6 +239,8 @@ public class ConcurrentUpdateSolrServer
msg.append("\n\n");
msg.append("request: ").append(method.getURI());
handleError(new SolrException(ErrorCode.getErrorCode(statusCode), msg.toString()));
+ } else {
+ onSuccess(response);
}
} finally {
try {
@@ -405,6 +407,13 @@ public class ConcurrentUpdateSolrServer
public void handleError(Throwable ex) {
log.error("error", ex);
}
+
+ /**
+ * Intended to be used as an extension point for doing post processing after a request completes.
+ */
+ public void onSuccess(HttpResponse resp) {
+ // no-op by design, override to add functionality
+ }
@Override
public void shutdown() {
Modified: lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java (original)
+++ lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/client/solrj/request/UpdateRequest.java Thu May 22 11:38:47 2014
@@ -46,7 +46,8 @@ import org.apache.solr.common.util.XML;
* @since solr 1.3
*/
public class UpdateRequest extends AbstractUpdateRequest {
-
+ public static final String REPFACT = "rf";
+ public static final String MIN_REPFACT = "min_rf";
public static final String VER = "ver";
public static final String OVERWRITE = "ow";
public static final String COMMIT_WITHIN = "cw";
Modified: lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/lucene5675/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Thu May 22 11:38:47 2014
@@ -90,7 +90,7 @@ public class ZkStateReader {
public static final String LEGACY_CLOUD = "legacyCloud";
public static final String URL_SCHEME = "urlScheme";
-
+
private volatile ClusterState clusterState;
private static final long SOLRCLOUD_UPDATE_DELAY = Long.parseLong(System.getProperty("solrcloud.update.delay", "5000"));
Modified: lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java (original)
+++ lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/TestLBHttpSolrServer.java Thu May 22 11:38:47 2014
@@ -92,7 +92,7 @@ public class TestLBHttpSolrServer extend
httpClient = HttpClientUtil.createClient(null);
HttpClientUtil.setConnectionTimeout(httpClient, 1000);
for (int i = 0; i < solr.length; i++) {
- solr[i] = new SolrInstance("solr/collection1" + i, 0);
+ solr[i] = new SolrInstance("solr/collection1" + i, createTempDir("instance-" + i), 0);
solr[i].setUp();
solr[i].startJetty();
addDocs(solr[i]);
@@ -250,9 +250,13 @@ public class TestLBHttpSolrServer extend
int port;
JettySolrRunner jetty;
- public SolrInstance(String name, int port) {
+ public SolrInstance(String name, File homeDir, int port) {
this.name = name;
+ this.homeDir = homeDir;
this.port = port;
+
+ dataDir = new File(homeDir + "/collection1", "data");
+ confDir = new File(homeDir + "/collection1", "conf");
}
public String getHomeDir() {
@@ -285,12 +289,6 @@ public class TestLBHttpSolrServer extend
public void setUp() throws Exception {
- File home = new File(dataDir,
- getClass().getName() + "-" + System.currentTimeMillis());
- homeDir = new File(home, name);
- dataDir = new File(homeDir + "/collection1", "data");
- confDir = new File(homeDir + "/collection1", "conf");
-
homeDir.mkdirs();
dataDir.mkdirs();
confDir.mkdirs();
@@ -301,7 +299,6 @@ public class TestLBHttpSolrServer extend
FileUtils.copyFile(SolrTestCaseJ4.getFile(getSolrConfigFile()), f);
f = new File(confDir, "schema.xml");
FileUtils.copyFile(SolrTestCaseJ4.getFile(getSchemaFile()), f);
-
}
public void tearDown() throws Exception {
Modified: lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java (original)
+++ lucene/dev/branches/lucene5675/solr/solrj/src/test/org/apache/solr/client/solrj/request/SolrPingTest.java Thu May 22 11:38:47 2014
@@ -19,6 +19,7 @@ package org.apache.solr.client.solrj.req
import junit.framework.Assert;
+import org.apache.commons.io.FileUtils;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.common.SolrException;
@@ -27,6 +28,8 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.File;
+
/**
* Test SolrPing in Solrj
*/
@@ -34,7 +37,9 @@ public class SolrPingTest extends SolrJe
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig.xml", "schema.xml", "solrj/solr", "collection1");
+ File testHome = createTempDir();
+ FileUtils.copyDirectory(getFile("solrj/solr"), testHome);
+ initCore("solrconfig.xml", "schema.xml", testHome.getAbsolutePath(), "collection1");
}
@Before
Modified: lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java (original)
+++ lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java Thu May 22 11:38:47 2014
@@ -68,6 +68,7 @@ import org.apache.solr.common.cloud.ZkSt
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
@@ -1719,6 +1720,25 @@ public abstract class AbstractFullDistri
}
}
+ protected NamedList<Object> invokeCollectionApi(String... args) throws SolrServerException, IOException {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ SolrRequest request = new QueryRequest(params);
+ for (int i = 0; i < args.length - 1; i+=2) {
+ params.add(args[i], args[i+1]);
+ }
+ request.setPath("/admin/collections");
+
+ String baseUrl = ((HttpSolrServer) shardToJetty.get(SHARD1).get(0).client.solrClient)
+ .getBaseURL();
+ baseUrl = baseUrl.substring(0, baseUrl.length() - "collection1".length());
+
+ HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
+ baseServer.setConnectionTimeout(15000);
+ baseServer.setSoTimeout(60000 * 5);
+ NamedList r = baseServer.request(request);
+ baseServer.shutdown();
+ return r;
+ }
protected void createCollection(String collName,
CloudSolrServer client,
Modified: lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java?rev=1596817&r1=1596816&r2=1596817&view=diff
==============================================================================
--- lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java (original)
+++ lucene/dev/branches/lucene5675/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java Thu May 22 11:38:47 2014
@@ -48,14 +48,15 @@ public class MiniSolrCloudCluster {
* "Mini" SolrCloud cluster to be used for testing
* @param numServers number of Solr servers to start
* @param hostContext context path of Solr servers used by Jetty
+ * @param baseDir base directory that the mini cluster should be run from
* @param solrXml solr.xml file to be uploaded to ZooKeeper
* @param extraServlets Extra servlets to be started by Jetty
* @param extraRequestFilters extra filters to be started by Jetty
*/
- public MiniSolrCloudCluster(int numServers, String hostContext, File solrXml,
+ public MiniSolrCloudCluster(int numServers, String hostContext, File baseDir, File solrXml,
SortedMap<ServletHolder, String> extraServlets,
SortedMap<Class, String> extraRequestFilters) throws Exception {
- testDir = Files.createTempDir();
+ testDir = baseDir;
String zkDir = testDir.getAbsolutePath() + File.separator
+ "zookeeper/server1/data";