You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2014/06/04 08:11:24 UTC
svn commit: r1599943 - in
/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler:
TestReplicationHandler.java TestReplicationHandlerBackup.java
Author: dweiss
Date: Wed Jun 4 06:11:24 2014
New Revision: 1599943
URL: http://svn.apache.org/r1599943
Log:
SOLR-6119: refactored doTestBackup into a separate class.
Added:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java (with props)
Modified:
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java?rev=1599943&r1=1599942&r2=1599943&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java Wed Jun 4 06:11:24 2014
@@ -20,7 +20,6 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
-import java.io.FilenameFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -33,21 +32,10 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
-import java.util.List;
import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.commons.io.IOUtils;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.MatchAllDocsQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.SimpleFSDirectory;
-import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util.TestUtil;
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
@@ -72,7 +60,6 @@ import org.apache.solr.core.CoreContaine
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.StandardDirectoryFactory;
import org.apache.solr.servlet.SolrDispatchFilter;
-import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.FileUtils;
import org.junit.After;
import org.junit.Before;
@@ -1308,261 +1295,7 @@ public class TestReplicationHandler exte
checkForSingleIndex(masterJetty);
checkForSingleIndex(slaveJetty);
}
-
-
- @Test
- public void doTestBackup() throws Exception {
- String configFile = "solrconfig-master1.xml";
- boolean addNumberToKeepInRequest = true;
- String backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM;
- if(random().nextBoolean()) {
- configFile = "solrconfig-master1-keepOneBackup.xml";
- addNumberToKeepInRequest = false;
- backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_INIT_PARAM;
- }
-
- masterJetty.stop();
- master.copyConfigFile(CONF_DIR + configFile,
- "solrconfig.xml");
-
- masterJetty = createJetty(master);
- masterClient.shutdown();
- masterClient = createNewSolrServer(masterJetty.getLocalPort());
-
- nDocs--;
- masterClient.deleteByQuery("*:*");
- for (int i = 0; i < nDocs; i++)
- index(masterClient, "id", i, "name", "name = " + i);
-
- masterClient.commit();
-
- class BackupThread extends Thread {
- volatile String fail = null;
- final boolean addNumberToKeepInRequest;
- String backupKeepParamName;
- String backupName;
- String cmd;
- BackupThread(boolean addNumberToKeepInRequest, String backupKeepParamName, String command) {
- this.addNumberToKeepInRequest = addNumberToKeepInRequest;
- this.backupKeepParamName = backupKeepParamName;
- this.cmd = command;
- }
- BackupThread(String backupName, String command) {
- this.backupName = backupName;
- addNumberToKeepInRequest = false;
- this.cmd = command;
- }
- @Override
- public void run() {
- String masterUrl = null;
- if(backupName != null) {
- masterUrl = buildUrl(masterJetty.getLocalPort()) + "/replication?command=" + cmd +
- "&name=" + backupName;
- } else {
- masterUrl = buildUrl(masterJetty.getLocalPort()) + "/replication?command=" + cmd +
- (addNumberToKeepInRequest ? "&" + backupKeepParamName + "=1" : "");
- }
-
- URL url;
- InputStream stream = null;
- try {
- url = new URL(masterUrl);
- stream = url.openStream();
- stream.close();
- } catch (Exception e) {
- fail = e.getMessage();
- } finally {
- IOUtils.closeQuietly(stream);
- }
-
- };
- };
-
- class CheckDeleteBackupStatus {
- String response = null;
- boolean success = false;
- String fail = null;
-
- public void fetchStatus() {
- String masterUrl = buildUrl(masterJetty.getLocalPort()) + "/replication?command=" + ReplicationHandler.CMD_DETAILS;
- URL url;
- InputStream stream = null;
- try {
- url = new URL(masterUrl);
- stream = url.openStream();
- response = IOUtils.toString(stream, "UTF-8");
- if(response.contains("<str name=\"status\">success</str>")) {
- success = true;
- }
- stream.close();
- } catch (Exception e) {
- fail = e.getMessage();
- } finally {
- IOUtils.closeQuietly(stream);
- }
- };
- }
-
- class CheckBackupStatus {
- String fail = null;
- String response = null;
- boolean success = false;
- String backupTimestamp = null;
- final String lastBackupTimestamp;
- final Pattern p = Pattern.compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
-
- CheckBackupStatus(String lastBackupTimestamp) {
- this.lastBackupTimestamp = lastBackupTimestamp;
- }
-
- public void fetchStatus() {
- String masterUrl = buildUrl(masterJetty.getLocalPort()) + "/replication?command=" + ReplicationHandler.CMD_DETAILS;
- URL url;
- InputStream stream = null;
- try {
- url = new URL(masterUrl);
- stream = url.openStream();
- response = IOUtils.toString(stream, "UTF-8");
- if(response.contains("<str name=\"status\">success</str>")) {
- Matcher m = p.matcher(response);
- if(!m.find()) {
- fail("could not find the completed timestamp in response.");
- }
- backupTimestamp = m.group(1);
- if(!backupTimestamp.equals(lastBackupTimestamp)) {
- success = true;
- }
- }
- stream.close();
- } catch (Exception e) {
- fail = e.getMessage();
- } finally {
- IOUtils.closeQuietly(stream);
- }
-
- };
- };
-
- File[] snapDir = new File[2];
- boolean namedBackup = random().nextBoolean();
- try {
- String firstBackupTimestamp = null;
-
- String[] backupNames = null;
- if (namedBackup) {
- backupNames = new String[2];
- }
- for (int i = 0; i < 2; i++) {
- BackupThread backupThread;
- final String backupName = TestUtil.randomSimpleString(random(), 1, 20);
- if (!namedBackup) {
- backupThread = new BackupThread(addNumberToKeepInRequest, backupKeepParamName, ReplicationHandler.CMD_BACKUP);
- } else {
- backupThread = new BackupThread(backupName, ReplicationHandler.CMD_BACKUP);
- backupNames[i] = backupName;
- }
- backupThread.start();
-
- File dataDir = new File(master.getDataDir());
-
- int waitCnt = 0;
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus(firstBackupTimestamp);
- while (true) {
- checkBackupStatus.fetchStatus();
- if (checkBackupStatus.fail != null) {
- fail(checkBackupStatus.fail);
- }
- if (checkBackupStatus.success) {
- if (i == 0) {
- firstBackupTimestamp = checkBackupStatus.backupTimestamp;
- Thread.sleep(1000); //ensure the next backup will have a different timestamp.
- }
- break;
- }
- Thread.sleep(200);
- if (waitCnt == 20) {
- fail("Backup success not detected:" + checkBackupStatus.response);
- }
- waitCnt++;
- }
-
- if (backupThread.fail != null) {
- fail(backupThread.fail);
- }
- File[] files = null;
- if (!namedBackup) {
- files = dataDir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- if (name.startsWith("snapshot")) {
- return true;
- }
- return false;
- }
- });
- } else {
- files = dataDir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- if (name.equals("snapshot." + backupName)) {
- return true;
- }
- return false;
- }
- });
- }
- assertEquals(1, files.length);
- snapDir[i] = files[0];
- Directory dir = new SimpleFSDirectory(snapDir[i].getAbsoluteFile());
- IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
- TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
- assertEquals(nDocs, hits.totalHits);
- reader.close();
- dir.close();
-
- }
-
- if (!namedBackup && snapDir[0].exists()) {
- fail("The first backup should have been cleaned up because " + backupKeepParamName + " was set to 1.");
- }
-
- //Test Deletion of named backup
- if(namedBackup) {
- for (int i = 0; i < 2; i++) {
- BackupThread deleteBackupThread = new BackupThread(backupNames[i], ReplicationHandler.CMD_DELETE_BACKUP);
- deleteBackupThread.start();
- int waitCnt = 0;
- CheckDeleteBackupStatus checkDeleteBackupStatus = new CheckDeleteBackupStatus();
- while (true) {
- checkDeleteBackupStatus.fetchStatus();
- if (checkDeleteBackupStatus.fail != null) {
- fail(checkDeleteBackupStatus.fail);
- }
- if (checkDeleteBackupStatus.success) {
- break;
- }
- Thread.sleep(200);
- if (waitCnt == 20) {
- fail("Delete Backup success not detected:" + checkDeleteBackupStatus.response);
- }
- waitCnt++;
- }
-
- if (deleteBackupThread.fail != null) {
- fail(deleteBackupThread.fail);
- }
- }
- }
-
- } finally {
- if(!namedBackup) {
- TestUtil.rm(snapDir);
- }
-
- }
- }
-
+
/**
* character copy of file using UTF-8. If port is non-null, will be substituted any time "TEST_PORT" is found.
*/
@@ -1640,7 +1373,7 @@ public class TestReplicationHandler exte
return buildUrl(port, context);
}
- private static class SolrInstance {
+ static class SolrInstance {
private String name;
private Integer testPort;
Added: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java?rev=1599943&view=auto
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java (added)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java Wed Jun 4 06:11:24 2014
@@ -0,0 +1,363 @@
+package org.apache.solr.handler;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.io.IOUtils;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.SimpleFSDirectory;
+import org.apache.lucene.util.TestUtil;
+import org.apache.solr.SolrJettyTestBase;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.impl.HttpSolrServer;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.util.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@SolrTestCaseJ4.SuppressSSL // Currently unknown why SSL does not work with this test
+public class TestReplicationHandlerBackup extends SolrJettyTestBase {
+
+ JettySolrRunner masterJetty;
+ TestReplicationHandler.SolrInstance master = null;
+ SolrServer masterClient;
+
+ private static final String CONF_DIR = "solr"
+ + File.separator + "collection1" + File.separator + "conf"
+ + File.separator;
+
+ private static String context = "/solr";
+
+ boolean addNumberToKeepInRequest = true;
+ String backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM;
+
+ private static JettySolrRunner createJetty(TestReplicationHandler.SolrInstance instance) throws Exception {
+ System.setProperty("solr.data.dir", instance.getDataDir());
+ FileUtils.copyFile(new File(SolrTestCaseJ4.TEST_HOME(), "solr.xml"), new File(instance.getHomeDir(), "solr.xml"));
+ JettySolrRunner jetty = new JettySolrRunner(instance.getHomeDir(), "/solr", 0);
+
+ jetty.start();
+ return jetty;
+ }
+
+ private static SolrServer createNewSolrServer(int port) {
+ try {
+ // setup the server...
+ HttpSolrServer s = new HttpSolrServer(buildUrl(port, context));
+ s.setConnectionTimeout(15000);
+ s.setSoTimeout(60000);
+ s.setDefaultMaxConnectionsPerHost(100);
+ s.setMaxTotalConnections(100);
+ return s;
+ }
+ catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ String configFile = "solrconfig-master1.xml";
+
+ if(random().nextBoolean()) {
+ configFile = "solrconfig-master1-keepOneBackup.xml";
+ addNumberToKeepInRequest = false;
+ backupKeepParamName = ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_INIT_PARAM;
+ }
+ master = new TestReplicationHandler.SolrInstance(createTempDir("solr-instance"), "master", null);
+ master.setUp();
+ master.copyConfigFile(CONF_DIR + configFile, "solrconfig.xml");
+
+ masterJetty = createJetty(master);
+ masterClient = createNewSolrServer(masterJetty.getLocalPort());
+ }
+
+ @Override
+ @After
+ public void tearDown() throws Exception {
+ super.tearDown();
+ masterJetty.stop();
+ master.tearDown();
+ masterJetty = null;
+ master = null;
+ masterClient.shutdown();
+ masterClient = null;
+ }
+
+
+ @Test
+ public void doTestBackup() throws Exception {
+
+ int nDocs = TestUtil.nextInt(random(), 1, 100);
+ masterClient.deleteByQuery("*:*");
+ for (int i = 0; i < nDocs; i++) {
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.addField("id", i);
+ doc.addField("name", "name = " + i);
+ masterClient.add(doc);
+ }
+
+ masterClient.commit();
+
+ File[] snapDir = new File[2];
+ boolean namedBackup = random().nextBoolean();
+ try {
+ String firstBackupTimestamp = null;
+
+ String[] backupNames = null;
+ if (namedBackup) {
+ backupNames = new String[2];
+ }
+ for (int i = 0; i < 2; i++) {
+ BackupThread backupThread;
+ final String backupName = TestUtil.randomSimpleString(random(), 1, 20);
+ if (!namedBackup) {
+ backupThread = new BackupThread(addNumberToKeepInRequest, backupKeepParamName, ReplicationHandler.CMD_BACKUP);
+ } else {
+ backupThread = new BackupThread(backupName, ReplicationHandler.CMD_BACKUP);
+ backupNames[i] = backupName;
+ }
+ backupThread.start();
+
+ File dataDir = new File(master.getDataDir());
+
+ int waitCnt = 0;
+ CheckBackupStatus checkBackupStatus = new CheckBackupStatus(firstBackupTimestamp);
+ while (true) {
+ checkBackupStatus.fetchStatus();
+ if (checkBackupStatus.fail != null) {
+ fail(checkBackupStatus.fail);
+ }
+ if (checkBackupStatus.success) {
+ if (i == 0) {
+ firstBackupTimestamp = checkBackupStatus.backupTimestamp;
+ Thread.sleep(1000); //ensure the next backup will have a different timestamp.
+ }
+ break;
+ }
+ Thread.sleep(200);
+ if (waitCnt == 20) {
+ fail("Backup success not detected:" + checkBackupStatus.response);
+ }
+ waitCnt++;
+ }
+
+ if (backupThread.fail != null) {
+ fail(backupThread.fail);
+ }
+ File[] files = null;
+ if (!namedBackup) {
+ files = dataDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name.startsWith("snapshot")) {
+ return true;
+ }
+ return false;
+ }
+ });
+ } else {
+ files = dataDir.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name.equals("snapshot." + backupName)) {
+ return true;
+ }
+ return false;
+ }
+ });
+ }
+ assertEquals(1, files.length);
+ snapDir[i] = files[0];
+ Directory dir = new SimpleFSDirectory(snapDir[i].getAbsoluteFile());
+ IndexReader reader = DirectoryReader.open(dir);
+ IndexSearcher searcher = new IndexSearcher(reader);
+ TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
+ assertEquals(nDocs, hits.totalHits);
+ reader.close();
+ dir.close();
+
+ }
+
+ if (!namedBackup && snapDir[0].exists()) {
+ fail("The first backup should have been cleaned up because " + backupKeepParamName + " was set to 1.");
+ }
+
+ //Test Deletion of named backup
+ if(namedBackup) {
+ testDeleteNamedBackup(backupNames);
+ }
+
+ } finally {
+ if(!namedBackup) {
+ TestUtil.rm(snapDir);
+ }
+
+ }
+ }
+
+ private void testDeleteNamedBackup(String backupNames[]) throws InterruptedException {
+ for (int i = 0; i < 2; i++) {
+ BackupThread deleteBackupThread = new BackupThread(backupNames[i], ReplicationHandler.CMD_DELETE_BACKUP);
+ deleteBackupThread.start();
+ int waitCnt = 0;
+ CheckDeleteBackupStatus checkDeleteBackupStatus = new CheckDeleteBackupStatus();
+ while (true) {
+ checkDeleteBackupStatus.fetchStatus();
+ if (checkDeleteBackupStatus.fail != null) {
+ fail(checkDeleteBackupStatus.fail);
+ }
+ if (checkDeleteBackupStatus.success) {
+ break;
+ }
+ Thread.sleep(200);
+ if (waitCnt == 20) {
+ fail("Delete Backup success not detected:" + checkDeleteBackupStatus.response);
+ }
+ waitCnt++;
+ }
+
+ if (deleteBackupThread.fail != null) {
+ fail(deleteBackupThread.fail);
+ }
+ }
+ }
+
+ private class CheckBackupStatus {
+ String fail = null;
+ String response = null;
+ boolean success = false;
+ String backupTimestamp = null;
+ final String lastBackupTimestamp;
+ final Pattern p = Pattern.compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
+
+ CheckBackupStatus(String lastBackupTimestamp) {
+ this.lastBackupTimestamp = lastBackupTimestamp;
+ }
+
+ public void fetchStatus() {
+ String masterUrl = buildUrl(masterJetty.getLocalPort(), "/solr") + "/replication?command=" + ReplicationHandler.CMD_DETAILS;
+ URL url;
+ InputStream stream = null;
+ try {
+ url = new URL(masterUrl);
+ stream = url.openStream();
+ response = IOUtils.toString(stream, "UTF-8");
+ if(response.contains("<str name=\"status\">success</str>")) {
+ Matcher m = p.matcher(response);
+ if(!m.find()) {
+ fail("could not find the completed timestamp in response.");
+ }
+ backupTimestamp = m.group(1);
+ if(!backupTimestamp.equals(lastBackupTimestamp)) {
+ success = true;
+ }
+ }
+ stream.close();
+ } catch (Exception e) {
+ fail = e.getMessage();
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+
+ };
+ }
+
+ private class BackupThread extends Thread {
+ volatile String fail = null;
+ final boolean addNumberToKeepInRequest;
+ String backupKeepParamName;
+ String backupName;
+ String cmd;
+ BackupThread(boolean addNumberToKeepInRequest, String backupKeepParamName, String command) {
+ this.addNumberToKeepInRequest = addNumberToKeepInRequest;
+ this.backupKeepParamName = backupKeepParamName;
+ this.cmd = command;
+ }
+ BackupThread(String backupName, String command) {
+ this.backupName = backupName;
+ addNumberToKeepInRequest = false;
+ this.cmd = command;
+ }
+ @Override
+ public void run() {
+ String masterUrl = null;
+ if(backupName != null) {
+ masterUrl = buildUrl(masterJetty.getLocalPort(), context) + "/replication?command=" + cmd +
+ "&name=" + backupName;
+ } else {
+ masterUrl = buildUrl(masterJetty.getLocalPort(), context) + "/replication?command=" + cmd +
+ (addNumberToKeepInRequest ? "&" + backupKeepParamName + "=1" : "");
+ }
+
+ URL url;
+ InputStream stream = null;
+ try {
+ url = new URL(masterUrl);
+ stream = url.openStream();
+ stream.close();
+ } catch (Exception e) {
+ fail = e.getMessage();
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+
+ };
+ }
+
+ private class CheckDeleteBackupStatus {
+ String response = null;
+ boolean success = false;
+ String fail = null;
+
+ public void fetchStatus() {
+ String masterUrl = buildUrl(masterJetty.getLocalPort(), context) + "/replication?command=" + ReplicationHandler.CMD_DETAILS;
+ URL url;
+ InputStream stream = null;
+ try {
+ url = new URL(masterUrl);
+ stream = url.openStream();
+ response = IOUtils.toString(stream, "UTF-8");
+ if(response.contains("<str name=\"status\">success</str>")) {
+ success = true;
+ }
+ stream.close();
+ } catch (Exception e) {
+ fail = e.getMessage();
+ } finally {
+ IOUtils.closeQuietly(stream);
+ }
+ };
+ }
+}