You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ho...@apache.org on 2019/11/15 17:32:59 UTC
[lucene-solr] branch branch_8x updated: SOLR-13909:
ReplicationHandler testing: Replace the completely broken CheckBackupStatus
with a new BackupStatusChecker helper class
This is an automated email from the ASF dual-hosted git repository.
hossman pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new 95e7916 SOLR-13909: ReplicationHandler testing: Replace the completely broken CheckBackupStatus with a new BackupStatusChecker helper class
95e7916 is described below
commit 95e791622b97cd9e7ea59e748d287c3e7f17c76e
Author: Chris Hostetter <ho...@apache.org>
AuthorDate: Fri Nov 15 10:17:26 2019 -0700
SOLR-13909: ReplicationHandler testing: Replace the completely broken CheckBackupStatus with a new BackupStatusChecker helper class
(cherry picked from commit 805305c41046da3385d2146de290e52c556b373c)
---
.../java/org/apache/solr/handler/SnapShooter.java | 14 +-
.../apache/solr/cloud/BasicDistributedZk2Test.java | 31 +--
.../apache/solr/handler/BackupStatusChecker.java | 294 +++++++++++++++++++++
.../org/apache/solr/handler/CheckBackupStatus.java | 69 -----
.../solr/handler/TestHdfsBackupRestoreCore.java | 8 +-
.../solr/handler/TestReplicationHandler.java | 53 ++--
.../solr/handler/TestReplicationHandlerBackup.java | 154 ++++-------
.../org/apache/solr/handler/TestRestoreCore.java | 40 +--
.../solr/handler/TestStressThreadBackup.java | 42 +--
9 files changed, 414 insertions(+), 291 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
index 9bb88b5..1187af3 100644
--- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
+++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
@@ -207,13 +207,13 @@ public class SnapShooter {
public void createSnapAsync(final int numberToKeep, Consumer<NamedList> result) throws IOException {
//TODO should use Solr's ExecutorUtil
new Thread(() -> {
+ NamedList snapShootDetails;
try {
- result.accept(createSnapshot());
+ snapShootDetails = createSnapshot();
} catch (Exception e) {
log.error("Exception while creating snapshot", e);
- NamedList snapShootDetails = new NamedList<>();
+ snapShootDetails = new NamedList<>();
snapShootDetails.add("exception", e.getMessage());
- result.accept(snapShootDetails);
}
if (snapshotName == null) {
try {
@@ -222,6 +222,7 @@ public class SnapShooter {
log.warn("Unable to delete old snapshots ", e);
}
}
+ if (null != snapShootDetails) result.accept(snapShootDetails);
}).start();
}
@@ -260,8 +261,8 @@ public class SnapShooter {
details.add("status", "success");
details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString()
details.add("snapshotName", snapshotName);
- log.info("Done creating backup snapshot: " + (snapshotName == null ? "<not named>" : snapshotName) +
- " at " + baseSnapDirPath);
+ details.add("directoryName", directoryName);
+ log.info("Done creating backup snapshot: {} into {}", (snapshotName == null ? "<not named>" : snapshotName), snapshotDirPath);
success = true;
return details;
} finally {
@@ -302,7 +303,8 @@ public class SnapShooter {
log.info("Deleting snapshot: " + snapshotName);
NamedList<Object> details = new NamedList<>();
-
+ details.add("snapshotName", snapshotName);
+
try {
URI path = baseSnapDirPath.resolve("snapshot." + snapshotName);
backupRepo.deleteDirectory(path);
diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
index aa93202..73487d7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
+++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java
@@ -16,14 +16,8 @@
*/
package org.apache.solr.cloud;
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import com.google.common.collect.Lists;
import org.apache.lucene.mockfile.FilterPath;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
@@ -41,7 +35,7 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.handler.CheckBackupStatus;
+import org.apache.solr.handler.BackupStatusChecker;
import org.apache.solr.handler.ReplicationHandler;
import org.junit.Test;
@@ -404,9 +398,11 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
// try a backup command
try(final HttpSolrClient client = getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) {
+ final String backupName = "the_backup";
ModifiableSolrParams params = new ModifiableSolrParams();
params.set("qt", ReplicationHandler.PATH);
params.set("command", "backup");
+ params.set("name", backupName);
Path location = createTempDir();
location = FilterPath.unwrap(location).toRealPath();
params.set("location", location.toString());
@@ -414,25 +410,16 @@ public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase {
QueryRequest request = new QueryRequest(params);
client.request(request, DEFAULT_TEST_COLLECTION_NAME);
- checkForBackupSuccess(client, location);
- client.close();
- }
-
- }
- private void checkForBackupSuccess(HttpSolrClient client, Path location) throws InterruptedException, IOException {
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus(client, DEFAULT_TEST_COLLECTION_NAME);
- while (!checkBackupStatus.success) {
- checkBackupStatus.fetchStatus();
- Thread.sleep(1000);
- }
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(location, "snapshot*")) {
- ArrayList<Path> files = Lists.newArrayList(stream.iterator());
- assertEquals(Arrays.asList(files).toString(), 1, files.size());
+ final BackupStatusChecker backupStatus
+ = new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication");
+ final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30);
+ assertTrue("Backup dir does not exist: " + backupDirName,
+ Files.exists(location.resolve(backupDirName)));
}
}
-
+
private void addNewReplica() throws Exception {
waitForRecoveriesToFinish(false);
diff --git a/solr/core/src/test/org/apache/solr/handler/BackupStatusChecker.java b/solr/core/src/test/org/apache/solr/handler/BackupStatusChecker.java
new file mode 100644
index 0000000..3a461b7
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/BackupStatusChecker.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler;
+
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.response.SimpleSolrResponse;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.TimeSource;
+import org.apache.solr.util.TimeOut;
+
+import static org.apache.solr.SolrTestCaseJ4.params;
+
+import static org.apache.lucene.util.LuceneTestCase.assertNotNull;
+import static org.apache.lucene.util.LuceneTestCase.assertNull;
+import static org.apache.lucene.util.LuceneTestCase.assertTrue;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Helper class for validating when the replication handler has finished a backup.
+ *
+ */
+public final class BackupStatusChecker {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ final SolrClient client;
+ final String path;
+
+ /**
+ * @param client the client to use in all requests, will not be closed
+ * @param path the path to use for accessing the /replication handle when using the client
+ */
+ public BackupStatusChecker(final SolrClient client, String path) {
+ this.client = client;
+ this.path = path;
+ }
+
+ /**
+ * Defaults to a path of <code>/replication</code>
+ * (ie: assumes client is configured with a core specific solr URL).
+ *
+ * @see #BackupStatusChecker(SolrClient,String)
+ */
+ public BackupStatusChecker(final SolrClient client) {
+ this(client, "/replication");
+ }
+
+ /**
+ * Convinience wrapper
+ * @see #waitForBackupSuccess(String,TimeOut)
+ */
+ public String waitForBackupSuccess(final String backupName, final int timeLimitInSeconds) throws Exception {
+ return waitForBackupSuccess(backupName,
+ new TimeOut(timeLimitInSeconds, TimeUnit.SECONDS, TimeSource.NANO_TIME));
+ }
+
+ /**
+ * Polls the replication handler's status until the it reports that the specified backupName is
+ * completed as a <code>"success"</code> (in which case the method returns the directoryName of the backup)
+ * or either <code>"exception"</code> is reported or the <code>timeOut</code> expires
+ * (in either case an assertion is thrown)
+ *
+ * <p>
+ * <b>NOTE:</b> this method is <em>NOT</em> suitable/safe to use in a test where multiple backups are
+ * being taken/deleted concurrently, because the replication handler API provides no reliable way to check
+ * the results of a specific backup before the results of another backup may overwrite them internally.
+ * </p>
+ *
+ * @param backupName to look for
+ * @param timeOut limiting how long we wait
+ * @return the (new) directoryName of the specified backup
+ * @see #checkBackupSuccess(String)
+ */
+ public String waitForBackupSuccess(final String backupName, final TimeOut timeOut) throws Exception {
+ assertNotNull("backupName must not be null", backupName);
+ while (!timeOut.hasTimedOut()) {
+ final String newDirName = checkBackupSuccess(backupName);
+ if (null != newDirName) {
+ return newDirName;
+ }
+ timeOut.sleep(50);
+ }
+
+ // total TimeOut elapsed, so one last check or fail whole test.
+ final String newDirName = checkBackupSuccess(backupName);
+ assertNotNull(backupName + " did not succeed before the TimeOut elapsed",
+ newDirName);
+ return newDirName;
+ }
+
+ /**
+ * Convinience wrapper
+ * @see #waitForDifferentBackupDir(String,TimeOut)
+ */
+ public String waitForDifferentBackupDir(final String directoryName,
+ final int timeLimitInSeconds) throws Exception {
+
+ return waitForDifferentBackupDir(directoryName,
+ new TimeOut(timeLimitInSeconds, TimeUnit.SECONDS, TimeSource.NANO_TIME));
+ }
+
+ /**
+ * Polls the replication handler's status until the it reports that <em>any</em> backup has
+ * completed as a <code>"success"</code> with a different <code>"directoryName"</code> then the
+ * one specified (in which case the method returns the new directoryName) or either an
+ * <code>"exception"</code> is reported or the <code>timeOut</code> expires
+ * (in either case an assertion is thrown)
+ *
+ * <p>
+ * <b>NOTE:</b> this method is <em>NOT</em> suitable/safe to use in a test where multiple backups are
+ * being taken/deleted concurrently, because the replication handler API provides no reliable way to determine
+ * if the the most recently reported status to the a particular backup request.
+ * </p>
+ *
+ * @param directoryName to compare to, may be null
+ * @param timeOut limiting how long we wait
+ * @return the (new) directoryName of the latests successful backup
+ * @see #checkBackupSuccess()
+ */
+ public String waitForDifferentBackupDir(final String directoryName, final TimeOut timeOut) throws Exception {
+ while (!timeOut.hasTimedOut()) {
+ final String newDirName = checkBackupSuccess();
+ if (null != newDirName && ! newDirName.equals(directoryName)) {
+ return newDirName;
+ }
+ timeOut.sleep(50);
+ }
+
+ // total TimeOut elapsed, so one last check or fail whole test...
+ final String newDirName = checkBackupSuccess();
+ assertTrue("No successful backup with different directoryName then "
+ + directoryName + " before TimeOut elapsed",
+ (null != newDirName && ! newDirName.equals(directoryName)));
+ return newDirName;
+ }
+
+ /**
+ * Does a single check of the replication handler's status to determine if the mostrecently
+ * completed backup was a success.
+ * Throws a test assertion failure if any <code>"exception"</code> message is ever encountered
+ * (The Replication Handler API does not make it possible to know <em>which</em> backup
+ * this exception was related to)
+ *
+ * <p>
+ * <b>NOTE:</b> this method is <em>NOT</em> suitable/safe to use in a test where multiple backups are
+ * being taken/deleted concurrently, because the replication handler API provides no reliable way to determine
+ * if the the most recently reported status to the a particular backup request.
+ * </p>
+ *
+ * @returns the "directoryName" of the backup if the response indicates that a is completed successfully, otherwise null
+ */
+ public String checkBackupSuccess() throws Exception {
+ return _checkBackupSuccess(null);
+ }
+
+ /**
+ * Does a single check of the replication handler's status to determine if the specified name matches
+ * the most recently completed backup, and if that backup was a success.
+ * Throws a test assertion failure if any <code>"exception"</code> message is ever encountered
+ * (The Replication Handler API does not make it possible to know <em>which</em> backup
+ * this exception was related to)
+ *
+ * @returns the "directoryName" of the backup if the response indicates that the specified backupName is completed successfully, otherwise null
+ * @see #waitForBackupSuccess(String,TimeOut)
+ */
+ public String checkBackupSuccess(final String backupName) throws Exception {
+ assertNotNull("backupName must not be null", backupName);
+ return _checkBackupSuccess(backupName);
+ }
+
+ /**
+ * Helper method that works with either named or unnamemed backups
+ */
+ private String _checkBackupSuccess(final String backupName) throws Exception {
+ final String label = (null == backupName ? "latest backup" : backupName);
+ final SimpleSolrResponse rsp = new GenericSolrRequest(GenericSolrRequest.METHOD.GET, path,
+ params("command", "details")).process(client);
+ final NamedList data = rsp.getResponse();
+ log.info("Checking Status of {}: {}", label, data);
+ final NamedList<String> backupData = (NamedList<String>) data.findRecursive("details","backup");
+ if (null == backupData) {
+ // no backup has finished yet
+ return null;
+ }
+
+ final Object exception = backupData.get("exception");
+ assertNull("Backup failure: " + label, exception);
+
+ if ("success".equals(backupData.get("status"))
+ && (null == backupName || backupName.equals(backupData.get("snapshotName"))) ) {
+ assert null != backupData.get("directoryName");
+ return backupData.get("directoryName");
+ }
+ return null;
+ }
+
+
+
+ /**
+ * Convinience wrapper
+ * @see #waitForBackupDeletionSuccess(String,TimeOut)
+ */
+ public void waitForBackupDeletionSuccess(final String backupName, final int timeLimitInSeconds) throws Exception {
+ waitForBackupDeletionSuccess(backupName,
+ new TimeOut(timeLimitInSeconds, TimeUnit.SECONDS, TimeSource.NANO_TIME));
+ }
+
+ /**
+ * Polls the replication handler's status until the it reports that the specified backupName is
+ * deleted or either <code>"Unable to delete"</code> status is reported or the <code>timeOut</code> expires
+ * (in either case an assertion is thrown)
+ *
+ * <p>
+ * <b>NOTE:</b> this method is <em>NOT</em> suitable/safe to use in a test where multiple backups are
+ * being taken/deleted concurrently, because the replication handler API provides no reliable way to check
+ * the results of a specific backup before the results of another backup may overwrite them internally.
+ * </p>
+ *
+ * @param backupName to look for in status
+ * @param timeOut limiting how long we wait
+ * @see #checkBackupSuccess(String)
+ */
+ public void waitForBackupDeletionSuccess(final String backupName, final TimeOut timeOut) throws Exception {
+ assertNotNull("backumpName must not be null", backupName);
+ while (!timeOut.hasTimedOut()) {
+ if (checkBackupDeletionSuccess(backupName)) {
+ return;
+ }
+ timeOut.sleep(50);
+ }
+
+ // total TimeOut elapsed, so one last check or fail whole test.
+ assertTrue(backupName + " was not reported as deleted before the TimeOut elapsed",
+ checkBackupDeletionSuccess(backupName));
+ }
+
+ /**
+ * Does a single check of the replication handler's status to determine if the specified name matches
+ * the most recently deleted backup, and if deleting that backup was a success.
+ * Throws a test assertion failure if the status is about this backupName but the starts message
+ * with <code>"Unable to delete"</code>
+ *
+ * @returns true if the replication status info indicates the backup was deleted, false otherwise
+ * @see #waitForBackupDeletionSuccess(String,TimeOut)
+ */
+ public boolean checkBackupDeletionSuccess(final String backupName) throws Exception {
+ assertNotNull("backumpName must not be null", backupName);
+ final SimpleSolrResponse rsp = new GenericSolrRequest(GenericSolrRequest.METHOD.GET, path,
+ params("command", "details")).process(client);
+ final NamedList data = rsp.getResponse();
+ log.info("Checking Deletion Status of {}: {}", backupName, data);
+ final NamedList<String> backupData = (NamedList<String>) data.findRecursive("details","backup");
+ if (null == backupData
+ || null == backupData.get("status")
+ || ! backupName.equals(backupData.get("snapshotName")) ) {
+ // either no backup activity at all,
+ // or most recent activity isn't something we can infer anything from,
+ // or is not about the backup we care about...
+ return false;
+ }
+
+ final Object status = backupData.get("status");
+ if (status.toString().startsWith("Unable to delete")) {
+ // we already know backupData is about our backup
+ assertNull("Backup Deleting failure: " + backupName, status);
+ }
+
+ if ("success".equals(status) && null != backupData.get("snapshotDeletedAt")) {
+ return true; // backup done
+ }
+
+ // if we're still here then this status is about our backup, but doesn't seem to be a deletion
+ return false;
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java b/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java
deleted file mode 100644
index bd9b1eb..0000000
--- a/solr/core/src/test/org/apache/solr/handler/CheckBackupStatus.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.handler;
-
-import java.io.IOException;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.BasicResponseHandler;
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-
-public class CheckBackupStatus extends SolrTestCaseJ4 {
- String response = null;
- public boolean success = false;
- String backupTimestamp = null;
- final String lastBackupTimestamp;
- final Pattern p = Pattern.compile("<str name=\"snapshotCompletedAt\">(.*?)</str>");
- final Pattern pException = Pattern.compile("<str name=\"snapShootException\">(.*?)</str>");
- final HttpSolrClient client;
- final String coreName;
-
- public CheckBackupStatus(final HttpSolrClient client, String coreName, String lastBackupTimestamp) {
- this.client = client;
- this.lastBackupTimestamp = lastBackupTimestamp;
- this.coreName = coreName;
- }
-
- public CheckBackupStatus(final HttpSolrClient client, String coreName) {
- this(client, coreName, null);
- }
-
- public void fetchStatus() throws IOException {
- String masterUrl = client.getBaseURL() + "/" + coreName + ReplicationHandler.PATH + "?wt=xml&command=" + ReplicationHandler.CMD_DETAILS;
- response = client.getHttpClient().execute(new HttpGet(masterUrl), new BasicResponseHandler());
- if(pException.matcher(response).find()) {
- fail("Failed to create backup");
- }
- if(response.contains("<str name=\"status\">success</str>")) {
- Matcher m = p.matcher(response);
- if(!m.find()) {
- fail("could not find the completed timestamp in response.");
- }
- if (lastBackupTimestamp != null) {
- backupTimestamp = m.group(1);
- if (backupTimestamp.equals(lastBackupTimestamp)) {
- success = true;
- }
- } else {
- success = true;
- }
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
index f51c4ac..fc7a58e 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestHdfsBackupRestoreCore.java
@@ -187,11 +187,9 @@ public class TestHdfsBackupRestoreCore extends SolrCloudTestCase {
if (testViaReplicationHandler) {
log.info("Running Backup via replication handler");
BackupRestoreUtils.runReplicationHandlerCommand(baseUrl, coreName, ReplicationHandler.CMD_BACKUP, "hdfs", backupName);
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus(masterClient, coreName, null);
- while (!checkBackupStatus.success) {
- checkBackupStatus.fetchStatus();
- Thread.sleep(1000);
- }
+ final BackupStatusChecker backupStatus
+ = new BackupStatusChecker(masterClient, "/" + coreName + "/replication");
+ backupStatus.waitForBackupSuccess(backupName, 30);
} else {
log.info("Running Backup via core admin api");
Map<String,String> params = new HashMap<>();
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
index 62fb67a..6f99201 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandler.java
@@ -40,11 +40,15 @@ import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.SimpleFSDirectory;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.TestUtil;
+
import org.apache.solr.BaseDistributedSearchTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
@@ -1579,52 +1583,51 @@ public class TestReplicationHandler extends SolrTestCaseJ4 {
@Test
public void testEmptyBackups() throws Exception {
final File backupDir = createTempDir().toFile();
- final CheckBackupStatus backupStatus = new CheckBackupStatus(masterClient, /* Silly API */ ".");
+ final BackupStatusChecker backupStatus = new BackupStatusChecker(masterClient);
{ // initial request w/o any committed docs
+ final String backupName = "empty_backup1";
final GenericSolrRequest req = new GenericSolrRequest
(SolrRequest.METHOD.GET, "/replication",
params("command", "backup",
"location", backupDir.getAbsolutePath(),
- "name", "empty_backup1"));
+ "name", backupName));
final TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
final SimpleSolrResponse rsp = req.process(masterClient);
-
- while (!timeout.hasTimedOut()) {
- backupStatus.fetchStatus();
- if (backupStatus.success) {
- break;
- }
- timeout.sleep(50);
- }
- assertTrue(backupStatus.success);
-
- assertTrue("snapshot.empty_backup1 doesn't exist in expected location",
- new File(backupDir, "snapshot.empty_backup1").exists());
+
+ final String dirName = backupStatus.waitForBackupSuccess(backupName, timeout);
+ assertEquals("Did not get expected dir name for backup, did API change?",
+ "snapshot.empty_backup1", dirName);
+ assertTrue(dirName + " doesn't exist in expected location for backup " + backupName,
+ new File(backupDir, dirName).exists());
}
index(masterClient, "id", "1", "name", "foo");
{ // second backup w/uncommited doc
+ final String backupName = "empty_backup2";
final GenericSolrRequest req = new GenericSolrRequest
(SolrRequest.METHOD.GET, "/replication",
params("command", "backup",
"location", backupDir.getAbsolutePath(),
- "name", "empty_backup2"));
+ "name", backupName));
final TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
final SimpleSolrResponse rsp = req.process(masterClient);
- while (!timeout.hasTimedOut()) {
- backupStatus.fetchStatus();
- if (backupStatus.success) {
- break;
- }
- timeout.sleep(50);
+ final String dirName = backupStatus.waitForBackupSuccess(backupName, timeout);
+ assertEquals("Did not get expected dir name for backup, did API change?",
+ "snapshot.empty_backup2", dirName);
+ assertTrue(dirName + " doesn't exist in expected location for backup " + backupName,
+ new File(backupDir, dirName).exists());
+ }
+
+ // confirm backups really are empty
+ for (int i = 1; i <=2; i++) {
+ final String name = "snapshot.empty_backup"+i;
+ try (Directory dir = new SimpleFSDirectory(new File(backupDir, name).toPath());
+ IndexReader reader = DirectoryReader.open(dir)) {
+ assertEquals(name + " is not empty", 0, reader.numDocs());
}
- assertTrue(backupStatus.success);
-
- assertTrue("snapshot.empty_backup2 doesn't exist in expected location",
- new File(backupDir, "snapshot.empty_backup2").exists());
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
index e57241d..68bf77c 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestReplicationHandlerBackup.java
@@ -25,10 +25,10 @@ import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.Properties;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.DirectoryReader;
@@ -128,24 +128,24 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {
@Test
public void testBackupOnCommit() throws Exception {
+ final BackupStatusChecker backupStatus
+ = new BackupStatusChecker(masterClient, "/" + DEFAULT_TEST_CORENAME + "/replication");
+
+ final String lastBackupDir = backupStatus.checkBackupSuccess();
+ // sanity check no backups yet
+ assertNull("Already have a successful backup",
+ lastBackupDir);
+
//Index
int nDocs = BackupRestoreUtils.indexDocs(masterClient, DEFAULT_TEST_COLLECTION_NAME, docsSeed);
-
- //Confirm if completed
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, DEFAULT_TEST_CORENAME);
- while (!checkBackupStatus.success) {
- checkBackupStatus.fetchStatus();
- Thread.sleep(1000);
- }
-
+
+ final String newBackupDir = backupStatus.waitForDifferentBackupDir(lastBackupDir, 30);
//Validate
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot*")) {
- Path snapDir = stream.iterator().next();
- verify(snapDir, nDocs);
- }
+ verify(Paths.get(master.getDataDir(), newBackupDir), nDocs);
}
private void verify(Path backup, int nDocs) throws IOException {
+ log.info("Verifying ndocs={} in {}", nDocs, backup);
try (Directory dir = new SimpleFSDirectory(backup);
IndexReader reader = DirectoryReader.open(dir)) {
IndexSearcher searcher = new IndexSearcher(reader);
@@ -157,62 +157,43 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {
@Test
public void doTestBackup() throws Exception {
+ final BackupStatusChecker backupStatus
+ = new BackupStatusChecker(masterClient, "/" + DEFAULT_TEST_CORENAME + "/replication");
- int nDocs = BackupRestoreUtils.indexDocs(masterClient, DEFAULT_TEST_COLLECTION_NAME, docsSeed);
+ String lastBackupDir = backupStatus.checkBackupSuccess();
+ assertNull("Already have a successful backup",
+ lastBackupDir);
- //Confirm if completed
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, DEFAULT_TEST_CORENAME);
- while (!checkBackupStatus.success) {
- checkBackupStatus.fetchStatus();
- Thread.sleep(1000);
- }
-
- Path[] snapDir = new Path[5]; //One extra for the backup on commit
+ final Path[] snapDir = new Path[5]; //One extra for the backup on commit
//First snapshot location
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot*")) {
- snapDir[0] = stream.iterator().next();
- }
+
+ int nDocs = BackupRestoreUtils.indexDocs(masterClient, DEFAULT_TEST_COLLECTION_NAME, docsSeed);
- boolean namedBackup = random().nextBoolean();
- String firstBackupTimestamp = null;
+ lastBackupDir = backupStatus.waitForDifferentBackupDir(lastBackupDir, 30);
+ snapDir[0] = Paths.get(master.getDataDir(), lastBackupDir);
+
+ final boolean namedBackup = random().nextBoolean();
String[] backupNames = null;
if (namedBackup) {
backupNames = new String[4];
}
for (int i = 0; i < 4; i++) {
- final String backupName = TestUtil.randomSimpleString(random(), 1, 20);
+ final String backupName = TestUtil.randomSimpleString(random(), 1, 20) + "_" + i;
if (!namedBackup) {
if (addNumberToKeepInRequest) {
runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, "&" + backupKeepParamName + "=2");
} else {
runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, "");
}
+ lastBackupDir = backupStatus.waitForDifferentBackupDir(lastBackupDir, 30);
} else {
- runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, "&name=" + backupName);
+ runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, "&name=" + backupName);
+ lastBackupDir = backupStatus.waitForBackupSuccess(backupName, 30);
backupNames[i] = backupName;
}
-
- checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, DEFAULT_TEST_CORENAME, firstBackupTimestamp);
- while (!checkBackupStatus.success) {
- checkBackupStatus.fetchStatus();
- Thread.sleep(1000);
- }
- if (i == 0) {
- firstBackupTimestamp = checkBackupStatus.backupTimestamp;
- }
-
- if (!namedBackup) {
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot*")) {
- snapDir[i+1] = stream.iterator().next();
- }
- } else {
- try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot." + backupName)) {
- snapDir[i+1] = stream.iterator().next();
- }
- }
+ snapDir[i+1] = Paths.get(master.getDataDir(), lastBackupDir);
verify(snapDir[i+1], nDocs);
-
}
@@ -222,24 +203,26 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {
} else {
//5 backups got created. 4 explicitly and one because a commit was called.
// Only the last two should still exist.
- int count =0;
+ final List<String> remainingBackups = new ArrayList<>();
+
try (DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(master.getDataDir()), "snapshot*")) {
Iterator<Path> iter = stream.iterator();
while (iter.hasNext()) {
- iter.next();
- count ++;
+ remainingBackups.add(iter.next().getFileName().toString());
}
}
- //There will be 2 backups, otherwise 1
+ // Depending on the use of backupKeepParamName there should either be 2 or 1 backups remaining
if (backupKeepParamName.equals(ReplicationHandler.NUMBER_BACKUPS_TO_KEEP_REQUEST_PARAM)) {
- assertEquals(2, count);
+ assertEquals(remainingBackups.toString(),
+ 2, remainingBackups.size());
if (Files.exists(snapDir[0]) || Files.exists(snapDir[1]) || Files.exists(snapDir[2])) {
fail("Backup should have been cleaned up because " + backupKeepParamName + " was set to 2.");
}
} else {
- assertEquals(1, count);
+ assertEquals(remainingBackups.toString(),
+ 1, remainingBackups.size());
if (Files.exists(snapDir[0]) || Files.exists(snapDir[1]) || Files.exists(snapDir[2])
|| Files.exists(snapDir[3])) {
@@ -250,23 +233,19 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {
}
}
- private void testDeleteNamedBackup(String backupNames[]) throws InterruptedException, IOException {
- String lastTimestamp = null;
+ private void testDeleteNamedBackup(String backupNames[]) throws Exception {
+ final BackupStatusChecker backupStatus
+ = new BackupStatusChecker(masterClient, "/" + DEFAULT_TEST_CORENAME + "/replication");
for (int i = 0; i < 2; i++) {
+ final Path p = Paths.get(master.getDataDir(), "snapshot." + backupNames[i]);
+ assertTrue("WTF: Backup doesn't exist: " + p.toString(),
+ Files.exists(p));
runBackupCommand(masterJetty, ReplicationHandler.CMD_DELETE_BACKUP, "&name=" +backupNames[i]);
- CheckDeleteBackupStatus checkDeleteBackupStatus = new CheckDeleteBackupStatus(backupNames[i], lastTimestamp);
- while (true) {
- boolean success = checkDeleteBackupStatus.fetchStatus();
- if (success) {
- lastTimestamp = checkDeleteBackupStatus.lastTimestamp;
- if (i == 0) {
- Thread.sleep(1000); //make the timestamp change
- }
- break;
- }
- Thread.sleep(200);
- }
+ backupStatus.waitForBackupDeletionSuccess(backupNames[i], 30);
+ assertFalse("backup still exists after deletion: " + p.toString(),
+ Files.exists(p));
}
+
}
public static void runBackupCommand(JettySolrRunner masterJetty, String cmd, String params) throws IOException {
@@ -282,39 +261,4 @@ public class TestReplicationHandlerBackup extends SolrJettyTestBase {
}
}
- private class CheckDeleteBackupStatus {
- String response = null;
- private String backupName;
- final Pattern p = Pattern.compile("<str name=\"snapshotDeletedAt\">(.*?)</str>");
- String lastTimestamp;
-
- private CheckDeleteBackupStatus(String backupName, String lastTimestamp) {
- this.backupName = backupName;
- this.lastTimestamp = lastTimestamp;
- }
-
- public boolean fetchStatus() throws IOException {
- String masterUrl = buildUrl(masterJetty.getLocalPort(), context) + "/" + DEFAULT_TEST_CORENAME + ReplicationHandler.PATH + "?wt=xml&command=" + ReplicationHandler.CMD_DETAILS;
- URL url;
- InputStream stream = null;
- try {
- url = new URL(masterUrl);
- stream = url.openStream();
- response = IOUtils.toString(stream, "UTF-8");
- if(response.contains("<str name=\"status\">success</str>")) {
- Matcher m = p.matcher(response);
- if(m.find() && (lastTimestamp == null || !lastTimestamp.equals(m.group(1)))) {
- lastTimestamp = m.group(1);
- return true;
- }
- } else if(response.contains("<str name=\"status\">Unable to delete snapshot: " + backupName + "</str>" )) {
- return false;
- }
- stream.close();
- } finally {
- IOUtils.closeQuietly(stream);
- }
- return false;
- }
- }
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java
index ab12439..8b04acc 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestRestoreCore.java
@@ -114,7 +114,10 @@ public class TestRestoreCore extends SolrJettyTestBase {
int nDocs = usually() ? BackupRestoreUtils.indexDocs(masterClient, "collection1", docsSeed) : 0;
- String snapshotName;
+ final BackupStatusChecker backupStatus
+ = new BackupStatusChecker(masterClient, "/" + DEFAULT_TEST_CORENAME + "/replication");
+ final String oldBackupDir = backupStatus.checkBackupSuccess();
+ String snapshotName = null;
String location;
String params = "";
String baseUrl = masterJetty.getBaseUrl().toString();
@@ -133,10 +136,10 @@ public class TestRestoreCore extends SolrJettyTestBase {
TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, params);
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, DEFAULT_TEST_CORENAME, null);
- while (!checkBackupStatus.success) {
- checkBackupStatus.fetchStatus();
- Thread.sleep(1000);
+ if (null == snapshotName) {
+ backupStatus.waitForDifferentBackupDir(oldBackupDir, 30);
+ } else {
+ backupStatus.waitForBackupSuccess(snapshotName, 30);
}
int numRestoreTests = nDocs > 0 ? TestUtil.nextInt(random(), 1, 5) : 1;
@@ -189,15 +192,14 @@ public class TestRestoreCore extends SolrJettyTestBase {
TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_BACKUP, params);
- CheckBackupStatus checkBackupStatus = new CheckBackupStatus((HttpSolrClient) masterClient, DEFAULT_TEST_CORENAME, null);
- while (!checkBackupStatus.success) {
- checkBackupStatus.fetchStatus();
- Thread.sleep(1000);
- }
+ final BackupStatusChecker backupStatus
+ = new BackupStatusChecker(masterClient, "/" + DEFAULT_TEST_CORENAME + "/replication");
+ final String backupDirName = backupStatus.waitForBackupSuccess(snapshotName, 30);
//Remove the segments_n file so that the backup index is corrupted.
//Restore should fail and it should automatically rollback to the original index.
- Path restoreIndexPath = Paths.get(location).resolve("snapshot." + snapshotName);
+ final Path restoreIndexPath = Paths.get(location, backupDirName);
+ assertTrue("Does not exist: " + restoreIndexPath, Files.exists(restoreIndexPath));
try (DirectoryStream<Path> stream = Files.newDirectoryStream(restoreIndexPath, IndexFileNames.SEGMENTS + "*")) {
Path segmentFileName = stream.iterator().next();
Files.delete(segmentFileName);
@@ -205,14 +207,14 @@ public class TestRestoreCore extends SolrJettyTestBase {
TestReplicationHandlerBackup.runBackupCommand(masterJetty, ReplicationHandler.CMD_RESTORE, params);
- try {
- while (!fetchRestoreStatus(baseUrl, DEFAULT_TEST_CORENAME)) {
- Thread.sleep(1000);
- }
- fail("Should have thrown an error because restore could not have been successful");
- } catch (AssertionError e) {
- //supposed to happen
- }
+ expectThrows(AssertionError.class, () -> {
+ for (int i = 0; i < 10; i++) {
+ // this will throw an assertion once we get what we expect
+ fetchRestoreStatus(baseUrl, DEFAULT_TEST_CORENAME);
+ Thread.sleep(50);
+ }
+ // if we never got an assertion let expectThrows complain
+ });
BackupRestoreUtils.verifyDocs(nDocs, masterClient, DEFAULT_TEST_CORENAME);
diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
index d4b6035..ad28b79 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
@@ -42,7 +42,6 @@ import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.SimpleSolrResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.Replica;
@@ -51,7 +50,6 @@ import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.util.TimeOut;
@@ -112,6 +110,7 @@ public class TestStressThreadBackup extends SolrCloudTestCase {
// Create a custom BackupAPIImpl which uses ReplicatoinHandler for the backups
// but still defaults to CoreAdmin for making named snapshots (since that's what's documented)
testSnapshotsAndBackupsDuringConcurrentCommitsAndOptimizes(new BackupAPIImpl() {
+ final BackupStatusChecker backupStatus = new BackupStatusChecker(coreClient);
/** no solrj API for ReplicationHandler */
private GenericSolrRequest makeReplicationReq(SolrParams p) {
return new GenericSolrRequest(GenericSolrRequest.METHOD.GET, "/replication", p);
@@ -130,44 +129,7 @@ public class TestStressThreadBackup extends SolrCloudTestCase {
p.add(CoreAdminParams.COMMIT_NAME, snapName);
}
makeReplicationReq(p).process(coreClient);
-
- // "/replication" handler is all async, need to poll untill we see *this*
- // backupName report success
- while (!timeout.hasTimedOut()) {
- if (checkBackupSuccess(backupName)) {
- return;
- }
- timeout.sleep(50);
- }
-
- // total TimeOut elapsed, so one last check or fail whole test.
- assertTrue(backupName + " never succeeded after waiting excessive amount of time",
- checkBackupSuccess(backupName));
- }
-
- /**
- * Returns true if the replication handler's 'details' command indicates that
- * the most recently (succcessfully) completed backup has the specified name.
- * "fails" the test if 'details' ever indicates there was a backup exception.
- */
- private boolean checkBackupSuccess(final String backupName) throws Exception {
- final SimpleSolrResponse rsp = makeReplicationReq(params("command", "details")).process(coreClient);
- final NamedList data = rsp.getResponse();
- log.info("Checking Status of {}: {}", backupName, data);
- final NamedList<String> backupData = (NamedList<String>) data.findRecursive("details","backup");
- if (null == backupData) {
- // no backup has finished yet
- return false;
- }
-
- final Object exception = backupData.get("exception");
- assertNull("Backup failure", exception);
-
- if (backupName.equals(backupData.get("snapshotName"))
- && "success".equals(backupData.get("status"))) {
- return true;
- }
- return false;
+ backupStatus.waitForBackupSuccess(backupName, timeout);
}
});