You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/08/09 12:21:07 UTC
svn commit: r1371142 [31/32] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/maven/lucene/
dev-tools/maven/lucene/analysis/common/
dev-tools/maven/lucene/analysis/icu/ dev-tools/maven/lucene/analysis/ku...
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java Thu Aug 9 10:20:53 2012
@@ -22,6 +22,9 @@ import java.util.List;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.update.DirectUpdateHandler2;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -29,13 +32,13 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
@Ignore("SOLR-3126")
-public class ChaosMonkeySafeLeaderTest extends FullSolrCloudTest {
+public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase {
private static final int BASE_RUN_LENGTH = 120000;
@BeforeClass
public static void beforeSuperClass() {
-
+
}
@AfterClass
@@ -77,9 +80,10 @@ public class ChaosMonkeySafeLeaderTest e
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
- // we cannot do delete by query
- // as it's not supported for recovery
- //del("*:*");
+ // randomly turn on 5 seconds 'soft' commit
+ randomlyEnableAutoSoftCommit();
+
+ del("*:*");
List<StopableIndexingThread> threads = new ArrayList<StopableIndexingThread>();
int threadCount = 2;
@@ -116,6 +120,24 @@ public class ChaosMonkeySafeLeaderTest e
if (VERBOSE) System.out.println("control docs:" + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + "\n\n");
}
+
+ private void randomlyEnableAutoSoftCommit() {
+ if (r.nextBoolean()) {
+ log.info("Turning on auto soft commit");
+ for (CloudJettyRunner jetty : shardToJetty.get("shard1")) {
+ SolrCore core = ((SolrDispatchFilter) jetty.jetty.getDispatchFilter()
+ .getFilter()).getCores().getCore("collection1");
+ try {
+ ((DirectUpdateHandler2) core.getUpdateHandler()).getCommitTracker()
+ .setTimeUpperBound(5000);
+ } finally {
+ core.close();
+ }
+ }
+ } else {
+ log.info("Not turning on auto soft commit");
+ }
+ }
// skip the randoms - they can deadlock...
protected void indexr(Object... fields) throws Exception {
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/FullSolrCloudDistribCmdsTest.java Thu Aug 9 10:20:53 2012
@@ -23,8 +23,8 @@ import org.apache.lucene.util.LuceneTest
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
@@ -37,14 +37,13 @@ import org.apache.solr.common.params.Mod
import org.apache.solr.update.VersionInfo;
import org.apache.solr.update.processor.DistributedUpdateProcessor;
import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
import org.junit.BeforeClass;
/**
* Super basic testing, no shard restarting or anything.
*/
@Slow
-public class FullSolrCloudDistribCmdsTest extends FullSolrCloudTest {
+public class FullSolrCloudDistribCmdsTest extends AbstractFullDistribZkTestBase {
@BeforeClass
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java Thu Aug 9 10:20:53 2012
@@ -36,7 +36,7 @@ import javax.xml.parsers.ParserConfigura
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.cloud.CloudState;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -153,7 +153,7 @@ public class OverseerTest extends SolrTe
}
private String getShardId(final String coreName) {
- Map<String,Slice> slices = zkStateReader.getCloudState().getSlices(
+ Map<String,Slice> slices = zkStateReader.getClusterState().getSlices(
collection);
if (slices != null) {
for (Slice slice : slices.values()) {
@@ -340,8 +340,8 @@ public class OverseerTest extends SolrTe
// make sure all cores have been assigned a id in cloudstate
for (int i = 0; i < 40; i++) {
- reader.updateCloudState(true);
- CloudState state = reader.getCloudState();
+ reader.updateClusterState(true);
+ ClusterState state = reader.getClusterState();
Map<String,Slice> slices = state.getSlices("collection1");
int count = 0;
for (String name : slices.keySet()) {
@@ -419,8 +419,8 @@ public class OverseerTest extends SolrTe
private void waitForCollections(ZkStateReader stateReader, String... collections) throws InterruptedException, KeeperException {
int maxIterations = 100;
while (0 < maxIterations--) {
- stateReader.updateCloudState(true);
- final CloudState state = stateReader.getCloudState();
+ stateReader.updateClusterState(true);
+ final ClusterState state = stateReader.getClusterState();
Set<String> availableCollections = state.getCollections();
int availableCount = 0;
for(String requiredCollection: collections) {
@@ -431,7 +431,7 @@ public class OverseerTest extends SolrTe
Thread.sleep(50);
}
}
- log.warn("Timeout waiting for collections: " + Arrays.asList(collections) + " state:" + stateReader.getCloudState());
+ log.warn("Timeout waiting for collections: " + Arrays.asList(collections) + " state:" + stateReader.getClusterState());
}
@Test
@@ -472,8 +472,8 @@ public class OverseerTest extends SolrTe
waitForCollections(reader, "collection1");
- assertEquals(reader.getCloudState().toString(), ZkStateReader.RECOVERING,
- reader.getCloudState().getSlice("collection1", "shard1").getShards()
+ assertEquals(reader.getClusterState().toString(), ZkStateReader.RECOVERING,
+ reader.getClusterState().getSlice("collection1", "shard1").getShards()
.get("node1_core1").get(ZkStateReader.STATE_PROP));
//publish node state (active)
@@ -503,7 +503,7 @@ public class OverseerTest extends SolrTe
int maxIterations = 100;
String coreState = null;
while(maxIterations-->0) {
- Slice slice = reader.getCloudState().getSlice("collection1", "shard1");
+ Slice slice = reader.getClusterState().getSlice("collection1", "shard1");
if(slice!=null) {
coreState = slice.getShards().get("node1_core1").get(ZkStateReader.STATE_PROP);
if(coreState.equals(expectedState)) {
@@ -512,14 +512,14 @@ public class OverseerTest extends SolrTe
}
Thread.sleep(50);
}
- fail("Illegal state, was:" + coreState + " expected:" + expectedState + "cloudState:" + reader.getCloudState());
+ fail("Illegal state, was:" + coreState + " expected:" + expectedState + "clusterState:" + reader.getClusterState());
}
private void verifyShardLeader(ZkStateReader reader, String collection, String shard, String expectedCore) throws InterruptedException, KeeperException {
int maxIterations = 100;
while(maxIterations-->0) {
- reader.updateCloudState(true); // poll state
- ZkNodeProps props = reader.getCloudState().getLeader(collection, shard);
+ reader.updateClusterState(true); // poll state
+ ZkNodeProps props = reader.getClusterState().getLeader(collection, shard);
if(props!=null) {
if(expectedCore.equals(props.get(ZkStateReader.CORE_NAME_PROP))) {
return;
@@ -528,7 +528,7 @@ public class OverseerTest extends SolrTe
Thread.sleep(100);
}
- assertEquals("Unexpected shard leader coll:" + collection + " shard:" + shard, expectedCore, (reader.getCloudState().getLeader(collection, shard)!=null)?reader.getCloudState().getLeader(collection, shard).get(ZkStateReader.CORE_NAME_PROP):null);
+ assertEquals("Unexpected shard leader coll:" + collection + " shard:" + shard, expectedCore, (reader.getClusterState().getLeader(collection, shard)!=null)?reader.getClusterState().getLeader(collection, shard).get(ZkStateReader.CORE_NAME_PROP):null);
}
@Test
@@ -562,35 +562,35 @@ public class OverseerTest extends SolrTe
waitForCollections(reader, "collection1");
verifyStatus(reader, ZkStateReader.RECOVERING);
- int version = getCloudStateVersion(controllerClient);
+ int version = getClusterStateVersion(controllerClient);
mockController.publishState("core1", ZkStateReader.ACTIVE, 1);
- while(version == getCloudStateVersion(controllerClient));
+ while(version == getClusterStateVersion(controllerClient));
verifyStatus(reader, ZkStateReader.ACTIVE);
- version = getCloudStateVersion(controllerClient);
+ version = getClusterStateVersion(controllerClient);
overseerClient.close();
Thread.sleep(1000); //wait for overseer to get killed
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
- version = getCloudStateVersion(controllerClient);
+ version = getClusterStateVersion(controllerClient);
overseerClient = electNewOverseer(server.getZkAddress());
- while(version == getCloudStateVersion(controllerClient));
+ while(version == getClusterStateVersion(controllerClient));
verifyStatus(reader, ZkStateReader.RECOVERING);
- assertEquals("Live nodes count does not match", 1, reader.getCloudState()
+ assertEquals("Live nodes count does not match", 1, reader.getClusterState()
.getLiveNodes().size());
- assertEquals("Shard count does not match", 1, reader.getCloudState()
+ assertEquals("Shard count does not match", 1, reader.getClusterState()
.getSlice("collection1", "shard1").getShards().size());
- version = getCloudStateVersion(controllerClient);
+ version = getClusterStateVersion(controllerClient);
mockController.publishState("core1", null,1);
- while(version == getCloudStateVersion(controllerClient));
+ while(version == getClusterStateVersion(controllerClient));
Thread.sleep(500);
- assertFalse("collection1 should be gone after publishing the null state", reader.getCloudState().getCollections().contains("collection1"));
+ assertFalse("collection1 should be gone after publishing the null state", reader.getClusterState().getCollections().contains("collection1"));
} finally {
close(mockController);
@@ -740,15 +740,15 @@ public class OverseerTest extends SolrTe
mockController.close();
- int version = getCloudStateVersion(controllerClient);
+ int version = getClusterStateVersion(controllerClient);
mockController = new MockZKController(server.getZkAddress(), "node1", "collection1");
mockController.publishState("core1", ZkStateReader.RECOVERING, 1);
- while (version == getCloudStateVersion(controllerClient));
+ while (version == getClusterStateVersion(controllerClient));
- reader.updateCloudState(true);
- CloudState state = reader.getCloudState();
+ reader.updateClusterState(true);
+ ClusterState state = reader.getClusterState();
int numFound = 0;
for (Map<String,Slice> collection : state.getCollectionStates().values()) {
@@ -758,7 +758,7 @@ public class OverseerTest extends SolrTe
}
}
}
- assertEquals("Shard was found in more than 1 times in CloudState", 1,
+ assertEquals("Shard was found more than once in ClusterState", 1,
numFound);
} finally {
close(overseerClient);
@@ -800,7 +800,7 @@ public class OverseerTest extends SolrTe
waitForCollections(reader, "collection1");
- assertEquals("Slicecount does not match", 12, reader.getCloudState().getSlices("collection1").size());
+ assertEquals("Slicecount does not match", 12, reader.getClusterState().getSlices("collection1").size());
} finally {
close(overseerClient);
@@ -872,12 +872,12 @@ public class OverseerTest extends SolrTe
queue.offer(ZkStateReader.toJSON(m));
for(int i=0;i<100;i++) {
- Slice s = reader.getCloudState().getSlice("collection1", "s1");
+ Slice s = reader.getClusterState().getSlice("collection1", "s1");
if(s!=null && s.getShards().size()==3) break;
Thread.sleep(100);
}
- assertNotNull(reader.getCloudState().getSlice("collection1", "s1"));
- assertEquals(3, reader.getCloudState().getSlice("collection1", "s1").getShards().size());
+ assertNotNull(reader.getClusterState().getSlice("collection1", "s1"));
+ assertEquals(3, reader.getClusterState().getSlice("collection1", "s1").getShards().size());
} finally {
close(overseerClient);
close(zkClient);
@@ -898,18 +898,22 @@ public class OverseerTest extends SolrTe
}
}
- private int getCloudStateVersion(SolrZkClient controllerClient)
+ private int getClusterStateVersion(SolrZkClient controllerClient)
throws KeeperException, InterruptedException {
return controllerClient.exists(ZkStateReader.CLUSTER_STATE, null, false).getVersion();
}
private SolrZkClient electNewOverseer(String address) throws InterruptedException,
- TimeoutException, IOException, KeeperException, ParserConfigurationException, SAXException {
- SolrZkClient zkClient = new SolrZkClient(address, TIMEOUT);
+ TimeoutException, IOException,
+ KeeperException, ParserConfigurationException, SAXException {
+ SolrZkClient zkClient = new SolrZkClient(address, TIMEOUT);
ZkStateReader reader = new ZkStateReader(zkClient);
LeaderElector overseerElector = new LeaderElector(zkClient);
- ElectionContext ec = new OverseerElectionContext(new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", address.replaceAll("/", "_"), reader);
+ // TODO: close Overseer
+ Overseer overseer = new Overseer(
+ new HttpShardHandlerFactory().getShardHandler(), "/admin/cores", reader);
+ ElectionContext ec = new OverseerElectionContext(zkClient, overseer, address.replaceAll("/", "_"));
overseerElector.setup(ec);
overseerElector.joinElection(ec);
return zkClient;
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java Thu Aug 9 10:20:53 2012
@@ -26,12 +26,11 @@ import org.apache.solr.client.solrj.embe
import org.apache.solr.common.SolrInputDocument;
import org.junit.AfterClass;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Slow
-public class RecoveryZkTest extends FullSolrCloudTest {
+public class RecoveryZkTest extends AbstractFullDistribZkTestBase {
//private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain";
private static Logger log = LoggerFactory.getLogger(RecoveryZkTest.class);
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java Thu Aug 9 10:20:53 2012
@@ -44,7 +44,7 @@ import org.junit.BeforeClass;
* elected.
*/
@Slow
-public class SyncSliceTest extends FullSolrCloudTest {
+public class SyncSliceTest extends AbstractFullDistribZkTestBase {
@BeforeClass
public static void beforeSuperClass() {
@@ -76,7 +76,7 @@ public class SyncSliceTest extends FullS
public SyncSliceTest() {
super();
sliceCount = 1;
- shardCount = TEST_NIGHTLY ? 7 : 3;
+ shardCount = TEST_NIGHTLY ? 7 : 4;
}
@Override
@@ -86,7 +86,7 @@ public class SyncSliceTest extends FullS
handle.put("QTime", SKIPVAL);
handle.put("timestamp", SKIPVAL);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
del("*:*");
List<String> skipServers = new ArrayList<String>();
@@ -129,7 +129,7 @@ public class SyncSliceTest extends FullS
HttpSolrServer baseServer = new HttpSolrServer(baseUrl);
baseServer.request(request);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
checkShardConsistency(false, true);
@@ -157,9 +157,9 @@ public class SyncSliceTest extends FullS
// we are careful to make sure the downed node is no longer in the state,
// because on some systems (especially freebsd w/ blackhole enabled), trying
// to talk to a downed node causes grief
- waitToSeeDownInCloudState(leaderJetty, jetties);
+ waitToSeeDownInClusterState(leaderJetty, jetties);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
checkShardConsistency(false, true);
@@ -180,18 +180,18 @@ public class SyncSliceTest extends FullS
// give a moment to be sure it has started recovering
Thread.sleep(2000);
- waitForThingsToLevelOut();
+ waitForThingsToLevelOut(15);
waitForRecoveriesToFinish(false);
skipServers = getRandomOtherJetty(leaderJetty, null);
-
+ skipServers.addAll( getRandomOtherJetty(leaderJetty, null));
// skip list should be
//System.out.println("leader:" + leaderJetty.url);
//System.out.println("skip list:" + skipServers);
- // we are skipping the leader and one node
- assertEquals(1, skipServers.size());
+ // we are skipping one nodes
+ assertEquals(2, skipServers.size());
// more docs than can peer sync
for (int i = 0; i < 300; i++) {
@@ -217,19 +217,13 @@ public class SyncSliceTest extends FullS
// kill the current leader
chaosMonkey.killJetty(leaderJetty);
- waitToSeeDownInCloudState(leaderJetty, jetties);
+ waitToSeeDownInClusterState(leaderJetty, jetties);
Thread.sleep(4000);
waitForRecoveriesToFinish(false);
-
-
- // TODO: for now, we just check consistency -
- // there will be 305 or 5 docs depending on who
- // becomes the leader - eventually we want that to
- // always be the 305
- //checkShardConsistency(true, true);
- checkShardConsistency(false, true);
+
+ checkShardConsistency(true, true);
}
@@ -255,7 +249,7 @@ public class SyncSliceTest extends FullS
return skipServers;
}
- private void waitToSeeDownInCloudState(CloudJettyRunner leaderJetty,
+ private void waitToSeeDownInClusterState(CloudJettyRunner leaderJetty,
Set<CloudJettyRunner> jetties) throws InterruptedException {
for (CloudJettyRunner cjetty : jetties) {
@@ -265,34 +259,6 @@ public class SyncSliceTest extends FullS
}
waitToSeeNotLive(cloudClient.getZkStateReader(), leaderJetty);
}
-
- private void waitForThingsToLevelOut() throws Exception {
- int cnt = 0;
- boolean retry = false;
- do {
- waitForRecoveriesToFinish(false);
-
- commit();
-
- updateMappingsFromZk(jettys, clients);
-
- Set<String> theShards = shardToJetty.keySet();
- String failMessage = null;
- for (String shard : theShards) {
- failMessage = checkShardConsistency(shard, false);
- }
-
- if (failMessage != null) {
- retry = true;
- } else {
- retry = false;
- }
-
- cnt++;
- if (cnt > 10) break;
- Thread.sleep(2000);
- } while (retry);
- }
protected void indexDoc(List<String> skipServers, Object... fields) throws IOException,
SolrServerException {
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java Thu Aug 9 10:20:53 2012
@@ -202,7 +202,7 @@ public class ZkControllerTest extends So
assertNotNull(reader.getLeaderUrl("collection1", "shard1", 15000));
- assertEquals("Shard(s) missing from cloudstate", 2, zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size());
+ assertEquals("Shard(s) missing from cloudstate", 2, zkController.getZkStateReader().getClusterState().getSlice("collection1", "shard1").getShards().size());
// unregister current leader
final ZkNodeProps shard1LeaderProps = reader.getLeaderProps(
@@ -224,10 +224,10 @@ public class ZkControllerTest extends So
reader.getLeaderUrl("collection1", "shard1", 15000));
for(int i=0;i<30;i++) {
- if(zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size()==1) break;
+ if(zkController.getZkStateReader().getClusterState().getSlice("collection1", "shard1").getShards().size()==1) break;
Thread.sleep(500);
}
- assertEquals("shard was not unregistered", 1, zkController.getZkStateReader().getCloudState().getSlice("collection1", "shard1").getShards().size());
+ assertEquals("shard was not unregistered", 1, zkController.getZkStateReader().getClusterState().getSlice("collection1", "shard1").getShards().size());
} finally {
System.clearProperty("solrcloud.skip.autorecovery");
System.clearProperty(ZkStateReader.NUM_SHARDS_PROP);
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/core/CoreContainerCoreInitFailuresTest.java Thu Aug 9 10:20:53 2012
@@ -123,6 +123,11 @@ public class CoreContainerCoreInitFailur
}
public void testFlowBadFromStart() throws Exception {
+ // TODO: even if we close all solr cores in the container, there is still a leaked dir?
+ // maybe from one that didnt load right?
+
+ // TODO: make SolrCore closeable since its has close()
+ System.setProperty("solr.directoryFactory", "org.apache.solr.core.SimpleFSDirectoryFactory");
// reused state
Map<String,Exception> failures = null;
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/DummyCustomParamSpellChecker.java Thu Aug 9 10:20:53 2012
@@ -8,8 +8,10 @@ import org.apache.solr.spelling.Spelling
import org.apache.solr.spelling.SpellingResult;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -49,12 +51,20 @@ public class DummyCustomParamSpellChecke
SpellingResult result = new SpellingResult();
//just spit back out the results
+
+ // sort the keys to make ordering predictable
Iterator<String> iterator = options.customParams.getParameterNamesIterator();
+ List<String> lst = new ArrayList<String>();
+ while (iterator.hasNext()) {
+ lst.add(iterator.next());
+ }
+ Collections.sort(lst);
+
int i = 0;
- while (iterator.hasNext()){
- String name = iterator.next();
+ for (String name : lst) {
String value = options.customParams.get(name);
- result.add(new Token(name, i++, i++), Collections.singletonList(value));
+ result.add(new Token(name, i, i+1), Collections.singletonList(value));
+ i += 2;
}
return result;
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/SpellCheckComponentTest.java Thu Aug 9 10:20:53 2012
@@ -20,6 +20,7 @@ package org.apache.solr.handler.componen
import java.io.File;
import java.util.*;
+import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.params.CommonParams;
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/handler/component/TermVectorComponentTest.java Thu Aug 9 10:20:53 2012
@@ -121,7 +121,7 @@ public class TermVectorComponentTest ext
@Test
public void testBasics() throws Exception {
assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true", TermVectorParams.TF, "true")
- ,"/termVectors=={'doc-0':{'uniqueKey':'0'," +
+ ,"/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_posofftv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
@@ -136,7 +136,7 @@ public class TermVectorComponentTest ext
"tv.fl", "test_basictv,test_offtv",
TermVectorComponent.COMPONENT_NAME, "true",
TermVectorParams.TF, "true")
- ,"/termVectors=={'doc-0':{'uniqueKey':'0'," +
+ ,"/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}}," +
" 'uniqueKeyFieldName':'id'}"
@@ -150,7 +150,7 @@ public class TermVectorComponentTest ext
"tv.fl","test_offtv",
TermVectorComponent.COMPONENT_NAME, "true",
TermVectorParams.TF, "true")
- ,"/termVectors=={'doc-0':{'uniqueKey':'0'," +
+ ,"/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}}," +
" 'uniqueKeyFieldName':'id'}"
@@ -162,7 +162,7 @@ public class TermVectorComponentTest ext
"fl", "*,score",
TermVectorComponent.COMPONENT_NAME, "true",
TermVectorParams.TF, "true")
- ,"/termVectors=={'doc-0':{'uniqueKey':'0'," +
+ ,"/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_offtv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_posofftv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
@@ -176,7 +176,7 @@ public class TermVectorComponentTest ext
"fl", "score,test_basictv,[docid],test_postv,val:sum(3,4)",
TermVectorComponent.COMPONENT_NAME, "true",
TermVectorParams.TF, "true")
- ,"/termVectors=={'doc-0':{'uniqueKey':'0'," +
+ ,"/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}," +
" 'uniqueKeyFieldName':'id'}"
@@ -189,7 +189,7 @@ public class TermVectorComponentTest ext
"fl", "[docid],test_postv,val:sum(3,4)",
TermVectorComponent.COMPONENT_NAME, "true",
TermVectorParams.TF, "true")
- ,"/termVectors=={'doc-0':{'uniqueKey':'0'," +
+ ,"/termVectors=={'0':{'uniqueKey':'0'," +
" 'test_basictv':{'anoth':{'tf':1},'titl':{'tf':2}}," +
" 'test_postv':{'anoth':{'tf':1},'titl':{'tf':2}}}," +
" 'uniqueKeyFieldName':'id'}"
@@ -201,12 +201,12 @@ public class TermVectorComponentTest ext
public void testOptions() throws Exception {
assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
, TermVectorParams.TF, "true", TermVectorParams.DF, "true", TermVectorParams.OFFSETS, "true", TermVectorParams.POSITIONS, "true", TermVectorParams.TF_IDF, "true")
- ,"/termVectors/doc-0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':1}, 'df':2, 'tf-idf':0.5}"
+ ,"/termVectors/0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':1}, 'df':2, 'tf-idf':0.5}"
);
assertJQ(req("json.nl","map", "qt",tv, "q", "id:0", TermVectorComponent.COMPONENT_NAME, "true"
, TermVectorParams.ALL, "true")
- ,"/termVectors/doc-0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':1}, 'df':2, 'tf-idf':0.5}"
+ ,"/termVectors/0/test_posofftv/anoth=={'tf':1, 'offsets':{'start':20, 'end':27}, 'positions':{'position':1}, 'df':2, 'tf-idf':0.5}"
);
// test each combination at random
@@ -217,7 +217,7 @@ public class TermVectorComponentTest ext
{ TermVectorParams.POSITIONS, "'positions':{'position':1}" },
{ TermVectorParams.DF, "'df':2" },
{ TermVectorParams.TF_IDF, "'tf-idf':0.5" } };
- StringBuilder expected = new StringBuilder("/termVectors/doc-0/test_posofftv/anoth=={");
+ StringBuilder expected = new StringBuilder("/termVectors/0/test_posofftv/anoth=={");
boolean first = true;
for (int i = 0; i < options.length; i++) {
final boolean use = random().nextBoolean();
@@ -248,59 +248,13 @@ public class TermVectorComponentTest ext
,"f.test_basictv." + TermVectorParams.TF, "false"
,"f.test_basictv." + TermVectorParams.TF_IDF, "false"
)
- ,"/termVectors/doc-0/test_basictv=={'anoth':{},'titl':{}}"
- ,"/termVectors/doc-0/test_postv/anoth=={'tf':1, 'positions':{'position':1}, 'df':2, 'tf-idf':0.5}"
- ,"/termVectors/doc-0/test_offtv/anoth=={'tf':1, 'df':2, 'tf-idf':0.5}"
+ ,"/termVectors/0/test_basictv=={'anoth':{},'titl':{}}"
+ ,"/termVectors/0/test_postv/anoth=={'tf':1, 'positions':{'position':1}, 'df':2, 'tf-idf':0.5}"
+ ,"/termVectors/0/test_offtv/anoth=={'tf':1, 'df':2, 'tf-idf':0.5}"
,"/termVectors/warnings=={ 'noTermVectors':['test_notv'], 'noPositions':['test_basictv', 'test_offtv'], 'noOffsets':['test_basictv', 'test_postv']}"
);
}
-
- // TODO: this test is really fragile since it pokes around in solr's guts and makes many assumptions.
- // it should be rewritten to use the real distributed interface
- @Test
- public void testDistributed() throws Exception {
- SolrCore core = h.getCore();
- TermVectorComponent tvComp = (TermVectorComponent) core.getSearchComponent("tvComponent");
- assertTrue("tvComp is null and it shouldn't be", tvComp != null);
- ModifiableSolrParams params = new ModifiableSolrParams();
- params.add(CommonParams.Q, "id:0");
- params.add(CommonParams.QT, "tvrh");
- params.add(TermVectorParams.TF, "true");
- params.add(TermVectorParams.DF, "true");
- params.add(TermVectorParams.OFFSETS, "true");
- params.add(TermVectorParams.POSITIONS, "true");
- params.add(TermVectorComponent.COMPONENT_NAME, "true");
-
- ResponseBuilder rb = new ResponseBuilder(new LocalSolrQueryRequest(core, params), new SolrQueryResponse(), (List)Arrays.asList(tvComp));
- rb.stage = ResponseBuilder.STAGE_GET_FIELDS;
- rb.shards = new String[]{"localhost:0", "localhost:1", "localhost:2", "localhost:3"};//we don't actually call these, since we are going to invoke distributedProcess directly
- rb.resultIds = new HashMap<Object, ShardDoc>();
-
- rb.outgoing = new ArrayList<ShardRequest>();
- //one doc per shard, but make sure there are enough docs to go around
- for (int i = 0; i < rb.shards.length; i++){
- ShardDoc doc = new ShardDoc();
- doc.id = i; //must be a valid doc that was indexed.
- doc.score = 1 - (i / (float)rb.shards.length);
- doc.positionInResponse = i;
- doc.shard = rb.shards[i];
- doc.orderInShard = 0;
- rb.resultIds.put(doc.id, doc);
- }
-
- int result = tvComp.distributedProcess(rb);
- assertTrue(result + " does not equal: " + ResponseBuilder.STAGE_DONE, result == ResponseBuilder.STAGE_DONE);
- //one outgoing per shard
- assertTrue("rb.outgoing Size: " + rb.outgoing.size() + " is not: " + rb.shards.length, rb.outgoing.size() == rb.shards.length);
- for (ShardRequest request : rb.outgoing) {
- ModifiableSolrParams solrParams = request.params;
- log.info("Shard: " + Arrays.asList(request.shards) + " Params: " + solrParams);
- }
-
- rb.req.close();
- }
-
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/schema/BadIndexSchemaTest.java Thu Aug 9 10:20:53 2012
@@ -49,6 +49,7 @@ public class BadIndexSchemaTest extends
public void testSevereErrorsForUnexpectedAnalyzer() throws Exception {
doTest("bad-schema-nontext-analyzer.xml", "StrField (bad_type)");
+ doTest("bad-schema-analyzer-class-and-nested.xml", "bad_type");
}
public void testBadExternalFileField() throws Exception {
@@ -61,6 +62,8 @@ public class BadIndexSchemaTest extends
"can not be the dest of a copyField");
doTest("bad-schema-uniquekey-uses-default.xml",
"can not be configured with a default value");
+ doTest("bad-schema-uniquekey-multivalued.xml",
+ "can not be configured to be multivalued");
}
public void testPerFieldtypeSimButNoSchemaSimFactory() throws Exception {
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/QueryEqualityTest.java Thu Aug 9 10:20:53 2012
@@ -648,6 +648,11 @@ public class QueryEqualityTest extends S
"foo_i");
}
+ public void testTestFuncs() throws Exception {
+ assertFuncEquals("sleep(1,5)", "sleep(1,5)");
+ assertFuncEquals("threadid()", "threadid()");
+ }
+
/**
* this test does not assert anything itself, it simply toggles a static
* boolean informing an @AfterClass method to assert that every default
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRTGBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRTGBase.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRTGBase.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRTGBase.java Thu Aug 9 10:20:53 2012
@@ -134,7 +134,7 @@ public class TestRTGBase extends SolrTes
if (!termsEnum.seekExact(termBytes, false)) {
return -1;
}
- DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(r), null, false);
+ DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(r), null, 0);
int id = docs.nextDoc();
if (id != DocIdSetIterator.NO_MORE_DOCS) {
int next = docs.nextDoc();
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRecovery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRecovery.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRecovery.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/search/TestRecovery.java Thu Aug 9 10:20:53 2012
@@ -559,7 +559,22 @@ public class TestRecovery extends SolrTe
assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);
+ ulog.bufferUpdates();
+ // simulate receiving no updates
+ ulog.applyBufferedUpdates();
+ updateJ(jsonAdd(sdoc("id","Q7", "_version_","117")), params(DISTRIB_UPDATE_PARAM,FROM_LEADER)); // do another add to make sure flags are back to normal
+ req.close();
+ h.close();
+ createCore();
+
+ req = req();
+ uhandler = req.getCore().getUpdateHandler();
+ ulog = uhandler.getUpdateLog();
+
+ assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0); // check flags on Q7
+
+ logReplayFinish.acquire();
assertEquals(UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
} finally {
DirectUpdateHandler2.commitOnClose = true;
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/servlet/SolrRequestParserTest.java Thu Aug 9 10:20:53 2012
@@ -21,6 +21,8 @@ import static org.easymock.EasyMock.crea
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
+import java.net.HttpURLConnection;
+import java.net.SocketTimeoutException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
@@ -114,10 +116,13 @@ public class SolrRequestParserTest exten
String url = "http://www.apache.org/dist/lucene/solr/";
byte[] bytes = null;
try {
- URLConnection connection = new URL(url).openConnection();
+ URL u = new URL(url);
+ HttpURLConnection connection = (HttpURLConnection)u.openConnection();
connection.setConnectTimeout(5000);
connection.setReadTimeout(5000);
connection.connect();
+ int code = connection.getResponseCode();
+ assumeTrue("wrong response code from server: " + code, 200 == code);
bytes = IOUtils.toByteArray( connection.getInputStream());
}
catch( Exception ex ) {
@@ -134,8 +139,13 @@ public class SolrRequestParserTest exten
List<ContentStream> streams = new ArrayList<ContentStream>();
SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
assertEquals( 1, streams.size() );
- assertArrayEquals( bytes, IOUtils.toByteArray( streams.get(0).getStream() ) );
- req.close();
+ try {
+ assertArrayEquals( bytes, IOUtils.toByteArray( streams.get(0).getStream() ) );
+ } catch (SocketTimeoutException ex) {
+ assumeNoException("Problems retrieving from " + url + " to run the test.", ex);
+ } finally {
+ req.close();
+ }
}
@Test
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/SolrCmdDistributorTest.java Thu Aug 9 10:20:53 2012
@@ -24,13 +24,16 @@ import java.util.List;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.update.SolrCmdDistributor.Node;
import org.apache.solr.update.SolrCmdDistributor.Response;
import org.apache.solr.update.SolrCmdDistributor.StdNode;
@@ -40,7 +43,7 @@ public class SolrCmdDistributorTest exte
public SolrCmdDistributorTest() {
fixShardCount = true;
- shardCount = 1;
+ shardCount = 4;
stress = 0;
}
@@ -80,7 +83,7 @@ public class SolrCmdDistributorTest exte
@Override
public void doTest() throws Exception {
- //del("*:*");
+ del("*:*");
SolrCmdDistributor cmdDistrib = new SolrCmdDistributor(8);
@@ -159,6 +162,7 @@ public class SolrCmdDistributorTest exte
assertEquals(response.errors.toString(), 0, response.errors.size());
+
results = controlClient.query(new SolrQuery("*:*")).getResults();
numFound = results.getNumFound();
assertEquals(results.toString(), 2, numFound);
@@ -166,5 +170,48 @@ public class SolrCmdDistributorTest exte
numFound = client.query(new SolrQuery("*:*")).getResults()
.getNumFound();
assertEquals(results.toString(), 2, numFound);
+
+ // debug stuff
+ for (SolrServer c : clients) {
+ c.optimize();
+ // distrib optimize is not working right yet, so call it on each client
+ //System.out.println(clients.get(0).request(new LukeRequest()));
+ }
+
+ int id = 5;
+
+ cmdDistrib = new SolrCmdDistributor(8);
+
+ nodes.clear();
+ int cnt = atLeast(200);
+ for (int i = 0; i < cnt; i++) {
+ nodes.clear();
+ for (SolrServer c : clients) {
+ if (random().nextBoolean()) {
+ continue;
+ }
+ HttpSolrServer httpClient = (HttpSolrServer) c;
+ nodeProps = new ZkNodeProps(ZkStateReader.BASE_URL_PROP,
+ httpClient.getBaseURL(), ZkStateReader.CORE_NAME_PROP, "");
+ nodes.add(new StdNode(new ZkCoreNodeProps(nodeProps)));
+
+ }
+
+ cmd.solrDoc = sdoc("id", id++);
+ cmdDistrib.distribAdd(cmd, nodes, params);
+ }
+
+ cmdDistrib.finish();
+
+ cmdDistrib.distribCommit(ccmd, nodes, params);
+
+ for (SolrServer c : clients) {
+ NamedList<Object> resp = c.request(new LukeRequest());
+ System.out.println(resp);
+ assertEquals("SOLR-3428: We only did adds - there should be no deletes",
+ ((NamedList<Object>) resp.get("index")).get("numDocs"),
+ ((NamedList<Object>) resp.get("index")).get("maxDoc"));
+ }
+
}
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/ScriptEngineTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/ScriptEngineTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/ScriptEngineTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/ScriptEngineTest.java Thu Aug 9 10:20:53 2012
@@ -25,6 +25,9 @@ import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.StringReader;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+
/**
* Sanity tests basic functionality of {@link ScriptEngineManager} and
* {@link ScriptEngine} w/o excercising any Lucene specific code.
@@ -33,6 +36,12 @@ public class ScriptEngineTest extends Lu
private ScriptEngineManager manager;
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ Assume.assumeNotNull((new ScriptEngineManager()).getEngineByExtension("js"));
+ Assume.assumeNotNull((new ScriptEngineManager()).getEngineByName("JavaScript"));
+ }
+
@Override
public void setUp() throws Exception {
super.setUp();
@@ -83,13 +92,17 @@ public class ScriptEngineTest extends Lu
assertEquals(3, result.intValue());
}
-// public void testJRuby() throws ScriptException, NoSuchMethodException { // Simply adding jruby.jar to Solr's lib/ directory gets this test passing
-// ScriptEngine engine = manager.getEngineByName("jruby");
-// assertNotNull(engine);
-// engine.eval("def add(a,b); a + b; end");
-// Long result = (Long) ((Invocable)engine).invokeFunction("add", 1, 2);
-// assertNotNull(result);
-// assertEquals(3, result.intValue());
-// }
+ public void testJRuby() throws ScriptException, NoSuchMethodException {
+ // Simply adding jruby.jar to Solr's lib/ directory gets this test passing
+ ScriptEngine engine = manager.getEngineByName("jruby");
+
+ Assume.assumeNotNull(engine);
+
+ assertNotNull(engine);
+ engine.eval("def add(a,b); a + b; end");
+ Long result = (Long) ((Invocable)engine).invokeFunction("add", 1, 2);
+ assertNotNull(result);
+ assertEquals(3, result.intValue());
+ }
}
Modified: lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactoryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactoryTest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactoryTest.java (original)
+++ lucene/dev/branches/lucene3312/solr/core/src/test/org/apache/solr/update/processor/StatelessScriptUpdateProcessorFactoryTest.java Thu Aug 9 10:20:53 2012
@@ -45,9 +45,8 @@ public class StatelessScriptUpdateProces
@BeforeClass
public static void beforeClass() throws Exception {
- initCore("solrconfig-script-updateprocessor.xml", "schema12.xml");
-
Assume.assumeNotNull((new ScriptEngineManager()).getEngineByExtension("js"));
+ initCore("solrconfig-script-updateprocessor.xml", "schema12.xml");
}
/**
Modified: lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/db/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/db/conf/schema.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/db/conf/schema.xml (original)
+++ lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/db/conf/schema.xml Thu Aug 9 10:20:53 2012
@@ -259,7 +259,7 @@
best performance.
-->
- <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<field name="sku" type="textTight" indexed="true" stored="true" omitNorms="true"/>
<field name="name" type="text" indexed="true" stored="true"/>
<field name="nameSort" type="string" indexed="true" stored="false"/>
Modified: lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/rss/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/rss/conf/schema.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/rss/conf/schema.xml (original)
+++ lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/rss/conf/schema.xml Thu Aug 9 10:20:53 2012
@@ -291,7 +291,7 @@
<field name="subject" type="text" indexed="true" stored="true" />
<field name="title" type="text" indexed="true" stored="true" />
- <field name="link" type="string" indexed="true" stored="true" />
+ <field name="link" type="string" indexed="true" stored="true" multiValued="false" />
<field name="description" type="html" indexed="true" stored="true" />
<field name="creator" type="string" indexed="false" stored="true" />
<field name="item-subject" type="string" indexed="true" stored="false" />
Modified: lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/solr/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/solr/conf/schema.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/solr/conf/schema.xml (original)
+++ lucene/dev/branches/lucene3312/solr/example/example-DIH/solr/solr/conf/schema.xml Thu Aug 9 10:20:53 2012
@@ -259,7 +259,7 @@
best performance.
-->
- <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<field name="sku" type="textTight" indexed="true" stored="true" omitNorms="true"/>
<field name="name" type="text" indexed="true" stored="true"/>
<field name="nameSort" type="string" indexed="true" stored="false"/>
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/schema.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/schema.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/schema.xml (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/schema.xml Thu Aug 9 10:20:53 2012
@@ -91,7 +91,7 @@
trailing underscores (e.g. _version_) are reserved.
-->
- <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
<field name="sku" type="text_en_splitting_tight" indexed="true" stored="true" omitNorms="true"/>
<field name="name" type="text_general" indexed="true" stored="true"/>
<field name="manu" type="text_general" indexed="true" stored="true" omitNorms="true"/>
@@ -109,7 +109,10 @@
<!-- Common metadata fields, named specifically to match up with
SolrCell metadata when parsing rich documents such as Word, PDF.
Some fields are multiValued only because Tika currently may return
- multiple values for them.
+ multiple values for them. Some metadata is parsed from the documents,
+ but there are some which come from the client context:
+ "content_type": From the HTTP headers of incoming stream
+ "resourcename": From SolrCell request param resource.name
-->
<field name="title" type="text_general" indexed="true" stored="true" multiValued="true"/>
<field name="subject" type="text_general" indexed="true" stored="true"/>
@@ -118,10 +121,18 @@
<field name="author" type="text_general" indexed="true" stored="true"/>
<field name="keywords" type="text_general" indexed="true" stored="true"/>
<field name="category" type="text_general" indexed="true" stored="true"/>
+ <field name="resourcename" type="text_general" indexed="true" stored="true"/>
+ <field name="url" type="text_general" indexed="true" stored="true"/>
<field name="content_type" type="string" indexed="true" stored="true" multiValued="true"/>
<field name="last_modified" type="date" indexed="true" stored="true"/>
<field name="links" type="string" indexed="true" stored="true" multiValued="true"/>
+ <!-- Main body of document extracted by SolrCell.
+ NOTE: This field is not indexed by default, since it is also copied to "text"
+ using copyField below. This is to save space. Use this field for returning and
+ highlighting document content. Use the "text" field to search the content. -->
+ <field name="content" type="text_general" indexed="false" stored="true" multiValued="true"/>
+
<!-- catchall field, containing all other searchable text fields (implemented
via copyField further on in this schema -->
@@ -137,7 +148,6 @@
<field name="payloads" type="payloads" indexed="true" stored="true"/>
-
<field name="_version_" type="long" indexed="true" stored="true"/>
<!-- Uncommenting the following will create a "timestamp" field using
@@ -233,6 +243,19 @@
<!-- Copy the price into a currency enabled field (default USD) -->
<copyField source="price" dest="price_c"/>
+
+ <!-- Text fields from SolrCell to search by default in our catch-all field -->
+ <copyField source="title" dest="text"/>
+ <copyField source="author" dest="text"/>
+ <copyField source="description" dest="text"/>
+ <copyField source="keywords" dest="text"/>
+ <copyField source="content" dest="text"/>
+ <copyField source="content_type" dest="text"/>
+ <copyField source="resourcename" dest="text"/>
+ <copyField source="url" dest="text"/>
+
+ <!-- Create a string version of author for faceting -->
+ <copyField source="author" dest="author_s"/>
<!-- Above, multiple source fields are copied to the [text] field.
Another way to map multiple source fields to the same
@@ -607,12 +630,30 @@
</analyzer>
</fieldType>
- <fieldType name="text_path" class="solr.TextField" positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.PathHierarchyTokenizerFactory"/>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at index time, so
+ queries for paths match documents at that path, or in descendent paths
+ -->
+ <fieldType name="descendent_path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ </fieldType>
+ <!--
+ Example of using PathHierarchyTokenizerFactory at query time, so
+ queries for paths match documents at that path, or in ancestor paths
+ -->
+ <fieldType name="ancestor_path" class="solr.TextField">
+ <analyzer type="index">
+ <tokenizer class="solr.KeywordTokenizerFactory" />
+ </analyzer>
+ <analyzer type="query">
+ <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
</analyzer>
</fieldType>
-
<!-- since fields of this type are by default not stored or indexed,
any data added to them will be ignored outright. -->
@@ -1041,6 +1082,7 @@
<filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>
</analyzer>
</fieldType>
+
</types>
<!-- Similarity is the scoring routine for each document vs. a query.
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/solrconfig.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/solrconfig.xml (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/solrconfig.xml Thu Aug 9 10:20:53 2012
@@ -821,6 +821,8 @@
<requestHandler name="/get" class="solr.RealTimeGetHandler">
<lst name="defaults">
<str name="omitHeader">true</str>
+ <str name="wt">json</str>
+ <str name="indent">true</str>
</lst>
</requestHandler>
@@ -848,6 +850,7 @@
<str name="defType">edismax</str>
<str name="qf">
text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
</str>
<str name="df">text</str>
<str name="mm">100%</str>
@@ -857,14 +860,17 @@
<str name="mlt.qf">
text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4
+ title^10.0 description^5.0 keywords^5.0 author^2.0 resourcename^1.0
</str>
- <str name="mlt.fl">text,features,name,sku,id,manu,cat</str>
+ <str name="mlt.fl">text,features,name,sku,id,manu,cat,title,description,keywords,author,resourcename</str>
<int name="mlt.count">3</int>
<!-- Faceting defaults -->
<str name="facet">on</str>
<str name="facet.field">cat</str>
<str name="facet.field">manu_exact</str>
+ <str name="facet.field">content_type</str>
+ <str name="facet.field">author_s</str>
<str name="facet.query">ipod</str>
<str name="facet.query">GB</str>
<str name="facet.mincount">1</str>
@@ -887,9 +893,18 @@
<!-- Highlighting defaults -->
<str name="hl">on</str>
- <str name="hl.fl">text features name</str>
+ <str name="hl.fl">content features title name</str>
+ <str name="hl.encoder">html</str>
+ <str name="hl.simple.pre"><b></str>
+ <str name="hl.simple.post"></b></str>
+ <str name="f.title.hl.fragsize">0</str>
+ <str name="f.title.hl.alternateField">title</str>
<str name="f.name.hl.fragsize">0</str>
<str name="f.name.hl.alternateField">name</str>
+ <str name="f.content.hl.snippets">3</str>
+ <str name="f.content.hl.fragsize">200</str>
+ <str name="f.content.hl.alternateField">content</str>
+ <str name="f.content.hl.maxAlternateFieldLength">750</str>
<!-- Spell checking defaults -->
<str name="spellcheck">on</str>
@@ -950,9 +965,6 @@
startup="lazy"
class="solr.extraction.ExtractingRequestHandler" >
<lst name="defaults">
- <!-- All the main content goes into "text"... if you need to return
- the extracted text or do highlighting, use a stored field. -->
- <str name="fmap.content">text</str>
<str name="lowernames">true</str>
<str name="uprefix">ignored_</str>
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/update-script.js
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/update-script.js?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/update-script.js (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/update-script.js Thu Aug 9 10:20:53 2012
@@ -10,7 +10,7 @@
function processAdd(cmd) {
doc = cmd.solrDoc; // org.apache.solr.common.SolrInputDocument
- id =doc.getFieldValue("id");
+ id = doc.getFieldValue("id");
logger.info("update-script#processAdd: id=" + id);
// Set a field value:
@@ -27,7 +27,7 @@ function processAdd(cmd) {
// field_names = doc.getFieldNames().toArray();
// for(i=0; i < field_names.length; i++) {
// field_name = field_names[i];
-// if (/attr_.*/.test(field_name)) { doc.addField("field_name_ss", field_names[i]); }
+// if (/attr_.*/.test(field_name)) { doc.addField("attribute_ss", field_names[i]); }
// }
}
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/VM_global_library.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/VM_global_library.vm?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/VM_global_library.vm (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/VM_global_library.vm Thu Aug 9 10:20:53 2012
@@ -133,10 +133,14 @@
#macro(field $f)
#if($response.response.highlighting.get($docId).get($f).get(0))
- $!response.response.highlighting.get($docId).get($f).get(0)
+ #set($pad = "")
+ #foreach($v in $response.response.highlighting.get($docId).get($f))
+$pad$v##
+ #set($pad = " ... ")
+ #end
#else
#foreach($v in $doc.getFieldValues($f))
- $v
+$v##
#end
#end
#end
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_fields.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_fields.vm?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_fields.vm (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_fields.vm Thu Aug 9 10:20:53 2012
@@ -1,6 +1,8 @@
#if($response.facetFields)
<h2 #annTitle("Facets generated by adding &facet.field= to the request")>Field Facets</h2>
#foreach($field in $response.facetFields)
+ ## Hide facets without value
+ #if($field.values.size() > 0)
<span class="facet-field">$field.name</span>
<ul>
@@ -8,5 +10,6 @@
<li><a href="#url_for_facet_filter($field.name, $facet.name)">$facet.name</a> ($facet.count)</li>
#end
</ul>
+ #end
#end
#end
\ No newline at end of file
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_ranges.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_ranges.vm?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_ranges.vm (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/facet_ranges.vm Thu Aug 9 10:20:53 2012
@@ -1,5 +1,7 @@
<h2 #annTitle("Facets generated by adding &facet.range= to the request")>Range Facets</h2>
#foreach ($field in $response.response.facet_counts.facet_ranges)
+ ## Hide facets without value
+ #if($field.value.counts.size() > 0)
#set($name = $field.key)
#set($display = $name)
#set($f = $field.value.counts)
@@ -9,4 +11,5 @@
#set($before = $field.value.before)
#set($after = $field.value.after)
#display_facet_range($f, $display, $name, $start, $end, $gap, $before, $after)
+ #end
#end
\ No newline at end of file
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hit.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hit.vm?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hit.vm (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hit.vm Thu Aug 9 10:20:53 2012
@@ -1,5 +1,11 @@
#set($docId = $doc.getFieldValue('id'))
<div class="result-document">
- #parse("doc.vm")
+#if($doc.getFieldValue('name'))
+ #parse("product-doc.vm")
+#elseif($doc.getFieldValue('compName_s'))
+ #parse("join-doc.vm")
+#else
+ #parse("richtext-doc.vm")
+#end
</div>
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hitGrouped.vm
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hitGrouped.vm?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hitGrouped.vm (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/hitGrouped.vm Thu Aug 9 10:20:53 2012
@@ -6,7 +6,13 @@
<div class="group-doclist" #annTitle("Contains the top scoring documents in the group")>
#foreach ($doc in $group.doclist)
#set($docId = $doc.getFieldValue('id'))
- #parse("doc.vm")
+ #if($doc.getFieldValue('name'))
+ #parse("product-doc.vm")
+ #elseif($doc.getFieldValue('compName_s'))
+ #parse("join-doc.vm")
+ #else
+ #parse("richtext-doc.vm")
+ #end
#end
</div>
#end</div>
Modified: lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/main.css
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/main.css?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/main.css (original)
+++ lucene/dev/branches/lucene3312/solr/example/solr/collection1/conf/velocity/main.css Thu Aug 9 10:20:53 2012
@@ -167,6 +167,10 @@ a {
width:60%;
}
+.result-body{
+ background: #ddd;
+}
+
.mlt{
}
Modified: lucene/dev/branches/lucene3312/solr/licenses/xercesImpl-2.9.1.jar.sha1
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/licenses/xercesImpl-2.9.1.jar.sha1?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/licenses/xercesImpl-2.9.1.jar.sha1 (original)
+++ lucene/dev/branches/lucene3312/solr/licenses/xercesImpl-2.9.1.jar.sha1 Thu Aug 9 10:20:53 2012
@@ -1 +1 @@
-7bc7e49ddfe4fb5f193ed37ecc96c12292c8ceb6
\ No newline at end of file
+7bc7e49ddfe4fb5f193ed37ecc96c12292c8ceb6
Modified: lucene/dev/branches/lucene3312/solr/solrj/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/solrj/ivy.xml?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/solrj/ivy.xml (original)
+++ lucene/dev/branches/lucene3312/solr/solrj/ivy.xml Thu Aug 9 10:20:53 2012
@@ -20,7 +20,7 @@
<info organisation="org.apache.solr" module="solrj"/>
<dependencies>
- <dependency org="org.apache.zookeeper" name="zookeeper" rev="3.3.5" transitive="false"/>
+ <dependency org="org.apache.zookeeper" name="zookeeper" rev="3.3.6" transitive="false"/>
<dependency org="org.slf4j" name="log4j-over-slf4j" rev="1.6.4" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpcore" rev="4.1.4" transitive="false"/>
<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.1.3" transitive="false"/>
Modified: lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java (original)
+++ lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrServer.java Thu Aug 9 10:20:53 2012
@@ -32,9 +32,10 @@ import org.apache.http.client.HttpClient
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.request.IsUpdateRequest;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.CloudState;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkNodeProps;
@@ -61,6 +62,18 @@ public class CloudSolrServer extends Sol
private LBHttpSolrServer lbServer;
private HttpClient myClient;
Random rand = new Random();
+
+ // since the state shouldn't change often, should be very cheap reads
+ private volatile List<String> urlList;
+
+ private volatile List<String> leaderUrlList;
+ private volatile List<String> replicasList;
+
+ private volatile int lastClusterStateHashCode;
+
+ private final boolean updatesToLeaders;
+
+
/**
* @param zkHost The client endpoint of the zookeeper quorum containing the cloud state,
* in the form HOST:PORT.
@@ -69,6 +82,7 @@ public class CloudSolrServer extends Sol
this.zkHost = zkHost;
this.myClient = HttpClientUtil.createClient(null);
this.lbServer = new LBHttpSolrServer(myClient);
+ this.updatesToLeaders = true;
}
/**
@@ -79,6 +93,19 @@ public class CloudSolrServer extends Sol
public CloudSolrServer(String zkHost, LBHttpSolrServer lbServer) {
this.zkHost = zkHost;
this.lbServer = lbServer;
+ this.updatesToLeaders = true;
+ }
+
+ /**
+ * @param zkHost The client endpoint of the zookeeper quorum containing the cloud state,
+ * in the form HOST:PORT.
+ * @param lbServer LBHttpSolrServer instance for requests.
+ * @param updatesToLeaders sends updates only to leaders - defaults to true
+ */
+ public CloudSolrServer(String zkHost, LBHttpSolrServer lbServer, boolean updatesToLeaders) {
+ this.zkHost = zkHost;
+ this.lbServer = lbServer;
+ this.updatesToLeaders = updatesToLeaders;
}
public ZkStateReader getZkStateReader() {
@@ -139,7 +166,14 @@ public class CloudSolrServer extends Sol
// TODO: if you can hash here, you could favor the shard leader
- CloudState cloudState = zkStateReader.getCloudState();
+ ClusterState clusterState = zkStateReader.getClusterState();
+ boolean sendToLeaders = false;
+ List<String> replicas = null;
+
+ if (request instanceof IsUpdateRequest && updatesToLeaders) {
+ sendToLeaders = true;
+ replicas = new ArrayList<String>();
+ }
SolrParams reqParams = request.getParams();
if (reqParams == null) {
@@ -154,42 +188,70 @@ public class CloudSolrServer extends Sol
// Extract each comma separated collection name and store in a List.
List<String> collectionList = StrUtils.splitSmart(collection, ",", true);
+ // TODO: not a big deal because of the caching, but we could avoid looking at every shard
+ // when getting leaders if we tweaked some things
+
// Retrieve slices from the cloud state and, for each collection specified,
// add it to the Map of slices.
Map<String,Slice> slices = new HashMap<String,Slice>();
for (int i = 0; i < collectionList.size(); i++) {
String coll= collectionList.get(i);
- ClientUtils.appendMap(coll, slices, cloudState.getSlices(coll));
+ ClientUtils.appendMap(coll, slices, clusterState.getSlices(coll));
}
- Set<String> liveNodes = cloudState.getLiveNodes();
+ Set<String> liveNodes = clusterState.getLiveNodes();
- // IDEA: have versions on various things... like a global cloudState version
- // or shardAddressVersion (which only changes when the shards change)
- // to allow caching.
-
- // build a map of unique nodes
- // TODO: allow filtering by group, role, etc
- Map<String,ZkNodeProps> nodes = new HashMap<String,ZkNodeProps>();
- List<String> urlList = new ArrayList<String>();
- for (Slice slice : slices.values()) {
- for (ZkNodeProps nodeProps : slice.getShards().values()) {
- ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
- String node = coreNodeProps.getNodeName();
- if (!liveNodes.contains(coreNodeProps.getNodeName())
- || !coreNodeProps.getState().equals(
- ZkStateReader.ACTIVE)) continue;
- if (nodes.put(node, nodeProps) == null) {
- String url = coreNodeProps.getCoreUrl();
- urlList.add(url);
+ if (sendToLeaders && leaderUrlList == null || !sendToLeaders && urlList == null || clusterState.hashCode() != this.lastClusterStateHashCode) {
+
+ // build a map of unique nodes
+ // TODO: allow filtering by group, role, etc
+ Map<String,ZkNodeProps> nodes = new HashMap<String,ZkNodeProps>();
+ List<String> urlList = new ArrayList<String>();
+ for (Slice slice : slices.values()) {
+ for (ZkNodeProps nodeProps : slice.getShards().values()) {
+ ZkCoreNodeProps coreNodeProps = new ZkCoreNodeProps(nodeProps);
+ String node = coreNodeProps.getNodeName();
+ if (!liveNodes.contains(coreNodeProps.getNodeName())
+ || !coreNodeProps.getState().equals(ZkStateReader.ACTIVE)) continue;
+ if (nodes.put(node, nodeProps) == null) {
+ if (!sendToLeaders || (sendToLeaders && coreNodeProps.isLeader())) {
+ String url = coreNodeProps.getCoreUrl();
+ urlList.add(url);
+ } else if (sendToLeaders) {
+ String url = coreNodeProps.getCoreUrl();
+ replicas.add(url);
+ }
+ }
}
}
+ if (sendToLeaders) {
+ this.leaderUrlList = urlList;
+ this.replicasList = replicas;
+ } else {
+ this.urlList = urlList;
+ }
+ this.lastClusterStateHashCode = clusterState.hashCode();
+ }
+
+ List<String> theUrlList;
+ if (sendToLeaders) {
+ theUrlList = new ArrayList<String>(leaderUrlList.size());
+ theUrlList.addAll(leaderUrlList);
+ } else {
+ theUrlList = new ArrayList<String>(urlList.size());
+ theUrlList.addAll(urlList);
}
+ Collections.shuffle(theUrlList, rand);
+ if (replicas != null) {
+ ArrayList<String> theReplicas = new ArrayList<String>(replicasList.size());
+ theReplicas.addAll(replicasList);
+ Collections.shuffle(theReplicas, rand);
- Collections.shuffle(urlList, rand);
- //System.out.println("########################## MAKING REQUEST TO " + urlList);
+ theUrlList.addAll(theReplicas);
+ }
+ //System.out.println("########################## MAKING REQUEST TO " + theUrlList);
- LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, urlList);
+ LBHttpSolrServer.Req req = new LBHttpSolrServer.Req(request, theUrlList);
LBHttpSolrServer.Rsp rsp = lbServer.request(req);
return rsp.getResponse();
}
@@ -211,4 +273,16 @@ public class CloudSolrServer extends Sol
public LBHttpSolrServer getLbServer() {
return lbServer;
}
+
+ List<String> getUrlList() {
+ return urlList;
+ }
+
+ List<String> getLeaderUrlList() {
+ return leaderUrlList;
+ }
+
+ List<String> getReplicasList() {
+ return replicasList;
+ }
}
Modified: lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java (original)
+++ lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/AbstractUpdateRequest.java Thu Aug 9 10:20:53 2012
@@ -30,7 +30,7 @@ import java.io.IOException;
*
*
**/
-public abstract class AbstractUpdateRequest extends SolrRequest {
+public abstract class AbstractUpdateRequest extends SolrRequest implements IsUpdateRequest {
protected ModifiableSolrParams params;
protected int commitWithin = -1;
Modified: lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java (original)
+++ lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/client/solrj/request/DirectXmlRequest.java Thu Aug 9 10:20:53 2012
@@ -34,7 +34,7 @@ import org.apache.solr.common.util.Conte
*
* @since solr 1.3
*/
-public class DirectXmlRequest extends SolrRequest
+public class DirectXmlRequest extends SolrRequest implements IsUpdateRequest
{
final String xml;
private SolrParams params;
Modified: lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java (original)
+++ lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ConnectionManager.java Thu Aug 9 10:20:53 2012
@@ -47,6 +47,8 @@ class ConnectionManager implements Watch
private OnReconnect onReconnect;
+ private volatile boolean isClosed = false;
+
public ConnectionManager(String name, SolrZkClient client, String zkServerAddress, int zkClientTimeout, ZkClientConnectionStrategy strat, OnReconnect onConnect) {
this.name = name;
this.client = client;
@@ -68,6 +70,8 @@ class ConnectionManager implements Watch
log.info("Watcher " + this + " name:" + name + " got event " + event
+ " path:" + event.getPath() + " type:" + event.getType());
}
+
+ checkClosed();
state = event.getState();
if (state == KeeperState.SyncConnected) {
@@ -81,11 +85,18 @@ class ConnectionManager implements Watch
connectionStrategy.reconnect(zkServerAddress, zkClientTimeout, this,
new ZkClientConnectionStrategy.ZkUpdate() {
@Override
- public void update(SolrZooKeeper keeper)
- throws InterruptedException, TimeoutException {
+ public void update(SolrZooKeeper keeper) throws TimeoutException {
synchronized (connectionStrategy) {
- waitForConnected(SolrZkClient.DEFAULT_CLIENT_CONNECT_TIMEOUT);
- client.updateKeeper(keeper);
+ checkClosed();
+ try {
+ waitForConnected(SolrZkClient.DEFAULT_CLIENT_CONNECT_TIMEOUT);
+ checkClosed();
+ client.updateKeeper(keeper);
+ } catch (InterruptedException e) {
+ // we must have been asked to stop
+ throw new RuntimeException("Giving up on connecting - we were interrupted");
+ }
+ checkClosed();
if (onReconnect != null) {
onReconnect.command();
}
@@ -95,6 +106,7 @@ class ConnectionManager implements Watch
}
}
+
});
} catch (Exception e) {
SolrException.log(log, "", e);
@@ -109,7 +121,13 @@ class ConnectionManager implements Watch
}
public synchronized boolean isConnected() {
- return connected;
+ return !isClosed && connected;
+ }
+
+ // we use a volatile rather than sync
+ // to avoid deadlock on shutdown
+ public void close() {
+ this.isClosed = true;
}
public synchronized KeeperState state() {
@@ -122,12 +140,20 @@ class ConnectionManager implements Watch
long left = waitForConnection;
while (!connected && left > 0) {
wait(left);
+ checkClosed();
left = expire - System.currentTimeMillis();
}
if (!connected) {
throw new TimeoutException("Could not connect to ZooKeeper " + zkServerAddress + " within " + waitForConnection + " ms");
}
}
+
+ private synchronized void checkClosed() {
+ if (isClosed) {
+ log.info("Not acting because I am closed");
+ return;
+ }
+ }
public synchronized void waitForDisconnected(long timeout)
throws InterruptedException, TimeoutException {
Modified: lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java (original)
+++ lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/SolrZkClient.java Thu Aug 9 10:20:53 2012
@@ -654,7 +654,11 @@ public class SolrZkClient {
public void close() throws InterruptedException {
if (isClosed) return; // it's okay if we over close - same as solrcore
isClosed = true;
- keeper.close();
+ try {
+ keeper.close();
+ } finally {
+ connManager.close();
+ }
numCloses.incrementAndGet();
}
Modified: lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java (original)
+++ lucene/dev/branches/lucene3312/solr/solrj/src/java/org/apache/solr/common/cloud/ZkCoreNodeProps.java Thu Aug 9 10:20:53 2012
@@ -74,5 +74,9 @@ public class ZkCoreNodeProps {
return nodeProps;
}
+ public boolean isLeader() {
+ return nodeProps.containsKey(ZkStateReader.LEADER_PROP);
+ }
+
}