You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2010/08/02 23:33:20 UTC
svn commit: r981690 -
/lucene/dev/trunk/solr/src/test/org/apache/solr/handler/TestReplicationHandler.java
Author: yonik
Date: Mon Aug 2 21:33:19 2010
New Revision: 981690
URL: http://svn.apache.org/viewvc?rev=981690&view=rev
Log:
SOLR-1469: fix TestReplicationHandler failures
Modified:
lucene/dev/trunk/solr/src/test/org/apache/solr/handler/TestReplicationHandler.java
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=981690&r1=981689&r2=981690&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/handler/TestReplicationHandler.java Mon Aug 2 21:33:19 2010
@@ -64,6 +64,10 @@ public class TestReplicationHandler exte
static String context = "/solr";
+ // number of docs to index... decremented for each test case to tell if we accidentally reuse
+ // index from previous test method
+ static int nDocs = 500;
+
@BeforeClass
public static void beforeClass() throws Exception {
master = new SolrInstance("master", null);
@@ -77,17 +81,15 @@ public class TestReplicationHandler exte
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
}
- @Before
- public void setUp() throws Exception {
- super.setUp();
-
+ public void clearIndexWithReplication() throws Exception {
NamedList res = query("*:*", masterClient);
SolrDocumentList docs = (SolrDocumentList)res.get("response");
if (docs.getNumFound() != 0) {
masterClient.deleteByQuery("*:*");
masterClient.commit();
// wait for replication to sync
- rQuery(0, "*:*", slaveClient);
+ res = rQuery(0, "*:*", slaveClient);
+ assertEquals(0, ((SolrDocumentList) res.get("response")).getNumFound());
}
}
@@ -160,8 +162,8 @@ public class TestReplicationHandler exte
@Test
public void testReplicateAfterWrite2Slave() throws Exception {
- //add 50 docs to master
- int nDocs = 50;
+ clearIndexWithReplication();
+ nDocs--;
for (int i = 0; i < nDocs; i++) {
index(masterClient, "id", i, "name", "name = " + i);
}
@@ -225,21 +227,22 @@ public class TestReplicationHandler exte
@Test
public void testIndexAndConfigReplication() throws Exception {
+ clearIndexWithReplication();
- //add 500 docs to master
- for (int i = 0; i < 500; i++)
+ nDocs--;
+ for (int i = 0; i < nDocs; i++)
index(masterClient, "id", i, "name", "name = " + i);
masterClient.commit();
- NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
+ NamedList masterQueryRsp = rQuery(nDocs, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
- assertEquals(500, masterQueryResult.getNumFound());
+ assertEquals(nDocs, masterQueryResult.getNumFound());
//get docs from slave and check if number is equal to master
- NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
+ NamedList slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
- assertEquals(500, slaveQueryResult.getNumFound());
+ assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
@@ -278,102 +281,34 @@ public class TestReplicationHandler exte
}
@Test
- public void testIndexAndConfigAliasReplication() throws Exception {
-
- //add 500 docs to master
- for (int i = 0; i < 500; i++)
- index(masterClient, "id", i, "name", "name = " + i);
-
- masterClient.commit();
-
- NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
- SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
- assertEquals(500, masterQueryResult.getNumFound());
-
- //get docs from slave and check if number is equal to master
- NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
- SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
-
- assertEquals(500, slaveQueryResult.getNumFound());
-
- //compare results
- String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
- assertEquals(null, cmp);
-
- //start config files replication test
- //clear master index
- masterClient.deleteByQuery("*:*");
- masterClient.commit();
-
- //change solrconfig on master
- copyFile(new File(CONF_DIR + "solrconfig-master1.xml"), new File(master.getConfDir(), "solrconfig.xml"));
-
- //change schema on master
- copyFile(new File(CONF_DIR + "schema-replication2.xml"), new File(master.getConfDir(), "schema.xml"));
-
- //keep a copy of the new schema
- copyFile(new File(CONF_DIR + "schema-replication2.xml"), new File(master.getConfDir(), "schema-replication2.xml"));
-
- masterJetty.stop();
-
- masterJetty = createJetty(master);
- masterClient = createNewSolrServer(masterJetty.getLocalPort());
-
- copyFile(new File(SLAVE_CONFIG), new File(slave.getConfDir(), "solrconfig.xml"), masterJetty.getLocalPort());
-
- slaveJetty.stop();
- slaveJetty = createJetty(slave);
- slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
-
- //add a doc with new field and commit on master to trigger snappull from slave.
- index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);
- masterClient.commit();
-
- NamedList masterQueryRsp2 = rQuery(1, "*:*", masterClient);
- SolrDocumentList masterQueryResult2 = (SolrDocumentList) masterQueryRsp2.get("response");
- assertEquals(1, masterQueryResult2.getNumFound());
-
- NamedList slaveQueryRsp2 = rQuery(1, "*:*", slaveClient);
- SolrDocumentList slaveQueryResult2 = (SolrDocumentList) slaveQueryRsp2.get("response");
- assertEquals(1, slaveQueryResult2.getNumFound());
-
- index(slaveClient, "id", "2000", "name", "name = " + 2001, "newname", "newname = " + 2001);
- slaveClient.commit();
-
- slaveQueryRsp = rQuery(1, "*:*", slaveClient);
- SolrDocument d = ((SolrDocumentList) slaveQueryRsp.get("response")).get(0);
- assertEquals("newname = 2001", (String) d.getFieldValue("newname"));
-
- }
-
- @Test
public void testStopPoll() throws Exception {
+ clearIndexWithReplication();
+
// Test:
// setup master/slave.
// stop polling on slave, add a doc to master and verify slave hasn't picked it.
-
- //add 500 docs to master
- for (int i = 0; i < 500; i++)
+ nDocs--;
+ for (int i = 0; i < nDocs; i++)
index(masterClient, "id", i, "name", "name = " + i);
masterClient.commit();
- NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
+ NamedList masterQueryRsp = rQuery(nDocs, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
- assertEquals(500, masterQueryResult.getNumFound());
+ assertEquals(nDocs, masterQueryResult.getNumFound());
//get docs from slave and check if number is equal to master
- NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
+ NamedList slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
- assertEquals(500, slaveQueryResult.getNumFound());
+ assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
// start stop polling test
- String masterUrl = "http://localhost:" + slaveJetty.getLocalPort() + "/solr/replication?command=disablepoll";
- URL url = new URL(masterUrl);
+ String slaveURL = "http://localhost:" + slaveJetty.getLocalPort() + "/solr/replication?command=disablepoll";
+ URL url = new URL(slaveURL);
InputStream stream = url.openStream();
try {
stream.close();
@@ -384,39 +319,54 @@ public class TestReplicationHandler exte
masterClient.commit();
//get docs from master and check if number is equal to master
- masterQueryRsp = rQuery(501, "*:*", masterClient);
+ masterQueryRsp = rQuery(nDocs+1, "*:*", masterClient);
masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
- assertEquals(501, masterQueryResult.getNumFound());
+ assertEquals(nDocs+1, masterQueryResult.getNumFound());
// NOTE: this test is wierd, we want to verify it DOESNT replicate...
// for now, add a sleep for this.., but the logic is wierd.
Thread.sleep(3000);
//get docs from slave and check if number is not equal to master; polling is disabled
- slaveQueryRsp = rQuery(500, "*:*", slaveClient);
+ slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
- assertEquals(500, slaveQueryResult.getNumFound());
+ assertEquals(nDocs, slaveQueryResult.getNumFound());
+ // re-enable replication
+ slaveURL = "http://localhost:" + slaveJetty.getLocalPort() + "/solr/replication?command=enablepoll";
+ url = new URL(slaveURL);
+ stream = url.openStream();
+ try {
+ stream.close();
+ } catch (IOException e) {
+ //e.printStackTrace();
+ }
+
+ slaveQueryRsp = rQuery(nDocs+1, "*:*", slaveClient);
+ slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
+ assertEquals(nDocs+1, slaveQueryResult.getNumFound());
}
+
@Test
public void testSnapPullWithMasterUrl() throws Exception {
//change solrconfig on slave
//this has no entry for pollinginterval
- copyFile(new File(CONF_DIR + "solrconfig-slave1.xml"), new File(slave.getConfDir(), "solrconfig.xml"));
+ copyFile(new File(CONF_DIR + "solrconfig-slave1.xml"), new File(slave.getConfDir(), "solrconfig.xml"), masterJetty.getLocalPort());
slaveJetty.stop();
slaveJetty = createJetty(slave);
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
- //add 500 docs to master
- for (int i = 0; i < 500; i++)
+ masterClient.deleteByQuery("*:*");
+ nDocs--;
+ for (int i = 0; i < nDocs; i++)
index(masterClient, "id", i, "name", "name = " + i);
masterClient.commit();
- NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
+ NamedList masterQueryRsp = rQuery(nDocs, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
- assertEquals(500, masterQueryResult.getNumFound());
+ assertEquals(nDocs, masterQueryResult.getNumFound());
// snappull
String masterUrl = "http://localhost:" + slaveJetty.getLocalPort() + "/solr/replication?command=fetchindex&masterUrl=";
@@ -430,28 +380,37 @@ public class TestReplicationHandler exte
}
//get docs from slave and check if number is equal to master
- NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
+ NamedList slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
- assertEquals(500, slaveQueryResult.getNumFound());
+ assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
+
+ // NOTE: at this point, the slave is not polling any more
+ // restore it.
+ copyFile(new File(CONF_DIR + "solrconfig-slave.xml"), new File(slave.getConfDir(), "solrconfig.xml"), masterJetty.getLocalPort());
+ slaveJetty.stop();
+ slaveJetty = createJetty(slave);
+ slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
}
+
@Test
public void testReplicateAfterStartup() throws Exception {
//stop slave
slaveJetty.stop();
- //add 500 docs to master
- for (int i = 0; i < 500; i++)
+ nDocs--;
+ masterClient.deleteByQuery("*:*");
+ for (int i = 0; i < nDocs; i++)
index(masterClient, "id", i, "name", "name = " + i);
masterClient.commit();
- NamedList masterQueryRsp = rQuery(500, "*:*", masterClient);
+ NamedList masterQueryRsp = rQuery(nDocs, "*:*", masterClient);
SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
- assertEquals(500, masterQueryResult.getNumFound());
+ assertEquals(nDocs, masterQueryResult.getNumFound());
//change solrconfig having 'replicateAfter startup' option on master
copyFile(new File(CONF_DIR + "solrconfig-master2.xml"),
@@ -469,14 +428,94 @@ public class TestReplicationHandler exte
slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
//get docs from slave and check if number is equal to master
- NamedList slaveQueryRsp = rQuery(500, "*:*", slaveClient);
+ NamedList slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
- assertEquals(500, slaveQueryResult.getNumFound());
+ assertEquals(nDocs, slaveQueryResult.getNumFound());
//compare results
String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
assertEquals(null, cmp);
+ // NOTE: the master only replicates after startup now!
+ // revert that change.
+ copyFile(new File(CONF_DIR + "solrconfig-master.xml"), new File(master.getConfDir(), "solrconfig.xml"));
+ masterJetty.stop();
+ masterJetty = createJetty(master);
+ masterClient = createNewSolrServer(masterJetty.getLocalPort());
+ copyFile(new File(SLAVE_CONFIG), new File(slave.getConfDir(), "solrconfig.xml"), masterJetty.getLocalPort());
+ //start slave
+ slaveJetty = createJetty(slave);
+ slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
+ }
+
+
+ @Test
+ public void testIndexAndConfigAliasReplication() throws Exception {
+ clearIndexWithReplication();
+
+ nDocs--;
+ for (int i = 0; i < nDocs; i++)
+ index(masterClient, "id", i, "name", "name = " + i);
+
+ masterClient.commit();
+
+ NamedList masterQueryRsp = rQuery(nDocs, "*:*", masterClient);
+ SolrDocumentList masterQueryResult = (SolrDocumentList) masterQueryRsp.get("response");
+ assertEquals(nDocs, masterQueryResult.getNumFound());
+
+ //get docs from slave and check if number is equal to master
+ NamedList slaveQueryRsp = rQuery(nDocs, "*:*", slaveClient);
+ SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
+
+ assertEquals(nDocs, slaveQueryResult.getNumFound());
+
+ //compare results
+ String cmp = TestDistributedSearch.compare(masterQueryResult, slaveQueryResult, 0, null);
+ assertEquals(null, cmp);
+
+ //start config files replication test
+ //clear master index
+ masterClient.deleteByQuery("*:*");
+ masterClient.commit();
+
+ //change solrconfig on master
+ copyFile(new File(CONF_DIR + "solrconfig-master1.xml"), new File(master.getConfDir(), "solrconfig.xml"));
+
+ //change schema on master
+ copyFile(new File(CONF_DIR + "schema-replication2.xml"), new File(master.getConfDir(), "schema.xml"));
+
+ //keep a copy of the new schema
+ copyFile(new File(CONF_DIR + "schema-replication2.xml"), new File(master.getConfDir(), "schema-replication2.xml"));
+
+ masterJetty.stop();
+
+ masterJetty = createJetty(master);
+ masterClient = createNewSolrServer(masterJetty.getLocalPort());
+
+ copyFile(new File(SLAVE_CONFIG), new File(slave.getConfDir(), "solrconfig.xml"), masterJetty.getLocalPort());
+
+ slaveJetty.stop();
+ slaveJetty = createJetty(slave);
+ slaveClient = createNewSolrServer(slaveJetty.getLocalPort());
+
+ //add a doc with new field and commit on master to trigger snappull from slave.
+ index(masterClient, "id", "2000", "name", "name = " + 2000, "newname", "newname = " + 2000);
+ masterClient.commit();
+
+ NamedList masterQueryRsp2 = rQuery(1, "*:*", masterClient);
+ SolrDocumentList masterQueryResult2 = (SolrDocumentList) masterQueryRsp2.get("response");
+ assertEquals(1, masterQueryResult2.getNumFound());
+
+ NamedList slaveQueryRsp2 = rQuery(1, "*:*", slaveClient);
+ SolrDocumentList slaveQueryResult2 = (SolrDocumentList) slaveQueryRsp2.get("response");
+ assertEquals(1, slaveQueryResult2.getNumFound());
+
+ index(slaveClient, "id", "2000", "name", "name = " + 2001, "newname", "newname = " + 2001);
+ slaveClient.commit();
+
+ slaveQueryRsp = rQuery(1, "*:*", slaveClient);
+ SolrDocument d = ((SolrDocumentList) slaveQueryRsp.get("response")).get(0);
+ assertEquals("newname = 2001", (String) d.getFieldValue("newname"));
}
@@ -491,8 +530,9 @@ public class TestReplicationHandler exte
masterClient = createNewSolrServer(masterJetty.getLocalPort());
- //add 500 docs to master
- for (int i = 0; i < 500; i++)
+ nDocs--;
+ masterClient.deleteByQuery("*:*");
+ for (int i = 0; i < nDocs; i++)
index(masterClient, "id", i, "name", "name = " + i);
masterClient.commit();
@@ -579,7 +619,7 @@ public class TestReplicationHandler exte
IndexSearcher searcher = new IndexSearcher(dir, true);
TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
- assertEquals(500, hits.totalHits);
+ assertEquals(nDocs, hits.totalHits);
searcher.close();
dir.close();
}