You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ge...@apache.org on 2021/02/10 20:39:58 UTC
[lucene-solr] branch master updated: SOLR-13608: Fix
incremental-backup test failures
This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new aea2946 SOLR-13608: Fix incremental-backup test failures
aea2946 is described below
commit aea29464f120feb6f5c3c9e3e7e6c5374770cccd
Author: Jason Gerlowski <ja...@lucidworks.com>
AuthorDate: Tue Feb 9 15:45:27 2021 -0500
SOLR-13608: Fix incremental-backup test failures
The initial incremental-backup commit introduced several test failures
on Windows test runs that I neglected to catch before committing. Most
of these failures were the result of bad 'location' path handling in the
test logic itself, though there were a few tweaks made to Solr code
itself to better handle Windows paths as well.
---
.../solr/cloud/api/collections/BackupCmd.java | 53 +++++++--------
.../cloud/api/collections/DeleteBackupCmd.java | 14 ++--
.../OverseerCollectionMessageHandler.java | 51 ++++++---------
.../repository/LocalFileSystemRepository.java | 32 +++------
.../apache/solr/handler/admin/BackupCoreOp.java | 2 +-
.../solr/handler/admin/CollectionsHandler.java | 2 -
.../LocalFSCloudIncrementalBackupTest.java | 13 ++--
.../solr/handler/TestIncrementalCoreBackup.java | 76 ++++++++++++----------
.../solr/handler/TestStressIncrementalBackup.java | 22 ++++---
.../solr/handler/TestStressThreadBackup.java | 5 +-
10 files changed, 124 insertions(+), 146 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
index 9de6b6e..afa1368 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
@@ -16,19 +16,6 @@
*/
package org.apache.solr.cloud.api.collections;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
-import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
-import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
-import static org.apache.solr.common.params.CommonParams.NAME;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Optional;
-
import org.apache.solr.cloud.api.collections.OverseerCollectionMessageHandler.ShardRequestTracker;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
@@ -44,6 +31,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.backup.BackupFilePaths;
import org.apache.solr.core.backup.BackupManager;
import org.apache.solr.core.backup.BackupProperties;
import org.apache.solr.core.backup.ShardBackupId;
@@ -52,11 +40,23 @@ import org.apache.solr.core.snapshots.CollectionSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.CoreSnapshotMetaData;
import org.apache.solr.core.snapshots.CollectionSnapshotMetaData.SnapshotStatus;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
-import org.apache.solr.core.backup.BackupFilePaths;
import org.apache.solr.handler.component.ShardHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
+import static org.apache.solr.common.params.CollectionAdminParams.FOLLOW_ALIASES;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+import static org.apache.solr.common.params.CommonParams.NAME;
+
public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -90,30 +90,26 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
try (BackupRepository repository = cc.newBackupRepository(repo)) {
// Backup location
- if (log.isDebugEnabled()) {
- log.debug("Backup location received by overseer is: {}", message.getStr(CoreAdminParams.BACKUP_LOCATION));
- }
URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
- log.debug("Resolved backup URI in overseer is: {}", location);
- final URI backupPath = createAndValidateBackupPath(repository, incremental, location, backupName, collectionName);
+ final URI backupUri = createAndValidateBackupPath(repository, incremental, location, backupName, collectionName);
BackupManager backupMgr = (incremental) ?
- BackupManager.forIncrementalBackup(repository, ocmh.zkStateReader, backupPath) :
- BackupManager.forBackup(repository, ocmh.zkStateReader, backupPath);
+ BackupManager.forIncrementalBackup(repository, ocmh.zkStateReader, backupUri) :
+ BackupManager.forBackup(repository, ocmh.zkStateReader, backupUri);
String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
switch (strategy) {
case CollectionAdminParams.COPY_FILES_STRATEGY: {
if (incremental) {
try {
- incrementalCopyIndexFiles(backupPath, collectionName, message, results, backupProperties, backupMgr);
+ incrementalCopyIndexFiles(backupUri, collectionName, message, results, backupProperties, backupMgr);
} catch (SolrException e) {
log.error("Error happened during incremental backup for collection:{}", collectionName, e);
- ocmh.cleanBackup(repository, backupPath, backupMgr.getBackupId());
+ ocmh.cleanBackup(repository, backupUri, backupMgr.getBackupId());
throw e;
}
} else {
- copyIndexFiles(backupPath, collectionName, message, results);
+ copyIndexFiles(backupUri, collectionName, message, results);
}
break;
}
@@ -143,14 +139,13 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
int maxNumBackup = message.getInt(CoreAdminParams.MAX_NUM_BACKUP_POINTS, -1);
if (incremental && maxNumBackup != -1) {
- ocmh.deleteBackup(repository, backupPath, maxNumBackup, results);
+ ocmh.deleteBackup(repository, backupUri, maxNumBackup, results);
}
}
}
private URI createAndValidateBackupPath(BackupRepository repository, boolean incremental, URI location, String backupName, String collection) throws IOException{
final URI backupNamePath = repository.resolve(location, backupName);
- log.debug("Location {} and backup-name {} combine to make path {}", location, backupName, backupNamePath);
if ( (!incremental) && repository.exists(backupNamePath)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupNamePath);
@@ -209,7 +204,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
return r.get();
}
- private void incrementalCopyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request,
+ private void incrementalCopyIndexFiles(URI backupUri, String collectionName, ZkNodeProps request,
NamedList<Object> results, BackupProperties backupProperties,
BackupManager backupManager) throws IOException {
String backupName = request.getStr(NAME);
@@ -218,7 +213,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler();
log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
- backupPath);
+ backupUri);
Optional<BackupProperties> previousProps = backupManager.tryReadBackupProperties();
final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
@@ -232,7 +227,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
}
String coreName = replica.getStr(CORE_NAME_PROP);
- ModifiableSolrParams params = coreBackupParams(backupPath, repoName, slice, coreName, true /* incremental backup */);
+ ModifiableSolrParams params = coreBackupParams(backupUri, repoName, slice, coreName, true /* incremental backup */);
params.set(CoreAdminParams.BACKUP_INCREMENTAL, true);
previousProps.flatMap(bp -> bp.getShardBackupIdFor(slice.getName()))
.ifPresent(prevBackupPoint -> params.set(CoreAdminParams.PREV_SHARD_BACKUP_ID, prevBackupPoint.getIdAsString()));
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java
index 7709751..0a48f1d 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteBackupCmd.java
@@ -23,14 +23,14 @@ import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.backup.BackupId;
import org.apache.solr.core.backup.AggregateBackupStats;
+import org.apache.solr.core.backup.BackupFilePaths;
+import org.apache.solr.core.backup.BackupId;
import org.apache.solr.core.backup.BackupManager;
import org.apache.solr.core.backup.BackupProperties;
import org.apache.solr.core.backup.ShardBackupId;
import org.apache.solr.core.backup.ShardBackupMetadata;
import org.apache.solr.core.backup.repository.BackupRepository;
-import org.apache.solr.core.backup.BackupFilePaths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,10 +117,10 @@ public class DeleteBackupCmd implements OverseerCollectionMessageHandler.Cmd {
deleteBackupIds(backupPath, repository, new HashSet<>(backupIdDeletes), results);
}
- void deleteBackupIds(URI backupPath, BackupRepository repository,
+ void deleteBackupIds(URI backupUri, BackupRepository repository,
Set<BackupId> backupIdsDeletes,
@SuppressWarnings({"rawtypes"}) NamedList results) throws IOException {
- BackupFilePaths incBackupFiles = new BackupFilePaths(repository, backupPath);
+ BackupFilePaths incBackupFiles = new BackupFilePaths(repository, backupUri);
URI shardBackupMetadataDir = incBackupFiles.getShardBackupMetadataDir();
Set<String> referencedIndexFiles = new HashSet<>();
@@ -167,15 +167,15 @@ public class DeleteBackupCmd implements OverseerCollectionMessageHandler.Cmd {
repository.delete(incBackupFiles.getIndexDir(), unusedFiles, true);
try {
for (BackupId backupId : backupIdsDeletes) {
- repository.deleteDirectory(repository.resolve(backupPath, BackupFilePaths.getZkStateDir(backupId)));
+ repository.deleteDirectory(repository.resolve(backupUri, BackupFilePaths.getZkStateDir(backupId)));
}
} catch (FileNotFoundException e) {
//ignore this
}
//add details to result before deleting backupPropFiles
- addResult(backupPath, repository, backupIdsDeletes, backupIdToCollectionBackupPoint, results);
- repository.delete(backupPath, backupIdsDeletes.stream().map(id -> BackupFilePaths.getBackupPropsName(id)).collect(Collectors.toList()), true);
+ addResult(backupUri, repository, backupIdsDeletes, backupIdToCollectionBackupPoint, results);
+ repository.delete(backupUri, backupIdsDeletes.stream().map(id -> BackupFilePaths.getBackupPropsName(id)).collect(Collectors.toList()), true);
}
@SuppressWarnings("unchecked")
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
index b7f9c6d..53a02fa 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/OverseerCollectionMessageHandler.java
@@ -16,24 +16,6 @@
*/
package org.apache.solr.cloud.api.collections;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.SynchronousQueue;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang3.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
@@ -95,16 +77,25 @@ import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.PROPERTY_VALUE_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.REPLICA_PROP;
-import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.solr.common.cloud.ZkStateReader.*;
import static org.apache.solr.common.params.CollectionAdminParams.COLLECTION;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
@@ -630,9 +621,9 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler,
}
@SuppressWarnings({"rawtypes"})
- void cleanBackup(BackupRepository repository, URI backupPath, BackupId backupId) throws Exception {
+ void cleanBackup(BackupRepository repository, URI backupUri, BackupId backupId) throws Exception {
((DeleteBackupCmd)commandMap.get(DELETEBACKUP))
- .deleteBackupIds(backupPath, repository, Collections.singleton(backupId), new NamedList());
+ .deleteBackupIds(backupUri, repository, Collections.singleton(backupId), new NamedList());
}
void deleteBackup(BackupRepository repository, URI backupPath,
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
index 1fcdec7..e8cfef5 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
@@ -17,9 +17,18 @@
package org.apache.solr.core.backup.repository;
+import com.google.common.base.Preconditions;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.NIOFSDirectory;
+import org.apache.lucene.store.NoLockFactory;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.DirectoryFactory;
+
import java.io.IOException;
import java.io.OutputStream;
-import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.FileVisitResult;
@@ -33,26 +42,12 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collection;
import java.util.Objects;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.NIOFSDirectory;
-import org.apache.lucene.store.NoLockFactory;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.core.DirectoryFactory;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
* A concrete implementation of {@linkplain BackupRepository} interface supporting backup/restore of Solr indexes to a
* local file-system. (Note - This can even be used for a shared file-system if it is exposed via a local file-system
* interface e.g. NFS).
*/
public class LocalFileSystemRepository implements BackupRepository {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@SuppressWarnings("rawtypes")
private NamedList config = null;
@@ -71,7 +66,6 @@ public class LocalFileSystemRepository implements BackupRepository {
@Override
public URI createURI(String location) {
Objects.requireNonNull(location);
- log.debug("Creating URI from location: {}", location);
URI result = null;
try {
@@ -83,14 +77,12 @@ public class LocalFileSystemRepository implements BackupRepository {
result = Paths.get(location).toUri();
}
- log.debug("Created URI is: {}", result);
return result;
}
@Override
public URI resolve(URI baseUri, String... pathComponents) {
Preconditions.checkArgument(pathComponents.length > 0);
- log.debug("Resolving URI from base: {}", baseUri);
Path result = Paths.get(baseUri);
for (int i = 0; i < pathComponents.length; i++) {
@@ -103,9 +95,6 @@ public class LocalFileSystemRepository implements BackupRepository {
}
- if (log.isDebugEnabled()) {
- log.debug("Resolved URI is: {}", result.toUri());
- }
return result.toUri();
}
@@ -136,7 +125,6 @@ public class LocalFileSystemRepository implements BackupRepository {
@Override
public boolean exists(URI path) throws IOException {
- log.debug("Checking whether URI exists: {}", path);
return Files.exists(Paths.get(path));
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
index c2c7806..5543f04 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
@@ -60,7 +60,7 @@ class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {
if (incremental) {
if ("file".equals(locationUri.getScheme())) {
- core.getCoreContainer().assertPathAllowed(Paths.get(location));
+ core.getCoreContainer().assertPathAllowed(Paths.get(locationUri));
}
final ShardBackupId prevShardBackupId = prevShardBackupIdStr != null ? ShardBackupId.from(prevShardBackupIdStr) : null;
BackupFilePaths incBackupFiles = new BackupFilePaths(repository, locationUri);
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 60f56d1..54aaf28 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -1028,12 +1028,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
+ " parameter or as a default repository property or as a cluster property.");
}
}
- log.debug("Provided backup location is: {}", location);
boolean incremental = req.getParams().getBool(CoreAdminParams.BACKUP_INCREMENTAL, true);
// Check if the specified location is valid for this repository.
final URI uri = repository.createURI(location);
- log.debug("Provided backup location uri is: {}", uri);
try {
if (!repository.exists(uri)) {
throw new SolrException(ErrorCode.SERVER_ERROR, "specified location " + uri + " does not exist.");
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java
index b247ae6..b552c6c 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java
@@ -20,8 +20,9 @@ package org.apache.solr.cloud.api.collections;
import org.junit.BeforeClass;
public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackupTest {
- public static final String SOLR_XML = "<solr>\n" +
+ private static final String SOLR_XML = "<solr>\n" +
"\n" +
+ " <str name=\"allowPaths\">ALLOWPATHS_TEMPLATE_VAL</str>\n" +
" <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
" <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
" <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +
@@ -57,17 +58,17 @@ public class LocalFSCloudIncrementalBackupTest extends AbstractIncrementalBackup
@BeforeClass
public static void setupClass() throws Exception {
- configureCluster(NUM_SHARDS)// nodes
- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
- .withSolrXml(SOLR_XML)
- .configure();
-
boolean whitespacesInPath = random().nextBoolean();
if (whitespacesInPath) {
backupLocation = createTempDir("my backup").toAbsolutePath().toString();
} else {
backupLocation = createTempDir("mybackup").toAbsolutePath().toString();
}
+
+ configureCluster(NUM_SHARDS)// nodes
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+ .withSolrXml(SOLR_XML.replace("ALLOWPATHS_TEMPLATE_VAL", backupLocation))
+ .configure();
}
@Override
diff --git a/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java b/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java
index d8301da..5035827 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestIncrementalCoreBackup.java
@@ -27,18 +27,16 @@ import org.apache.solr.core.backup.ShardBackupMetadata;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.util.LogLevel;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
import java.io.IOException;
import java.net.URI;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
-@LogLevel("org.apache.solr.cloud.api.collections.BackupCmd=DEBUG;org.apache.solr.handler.admin.BackupCoreOp=DEBUG;org.apache.solr.core.backup.repository.LocalFileSystemRepository=DEBUG")
public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
@Before // unique core per test
public void coreInit() throws Exception {
@@ -64,21 +62,22 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
assertQ(req("q", "id:2"), "//result[@numFound='0']");
//call backup
- final URI location = createAndBootstrapLocationForBackup();
+ final Path locationPath = createBackupLocation();
+ final URI locationUri = bootstrapBackupLocation(locationPath);
final ShardBackupId shardBackupId = new ShardBackupId("shard1", BackupId.zero());
final CoreContainer cores = h.getCoreContainer();
- cores.getAllowPaths().add(Paths.get(location));
+ cores.getAllowPaths().add(Paths.get(locationUri));
try (final CoreAdminHandler admin = new CoreAdminHandler(cores)) {
SolrQueryResponse resp = new SolrQueryResponse();
admin.handleRequestBody
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, shardBackupId.getIdAsString())
, resp);
assertNull("Backup should have succeeded", resp.getException());
- simpleBackupCheck(location, shardBackupId);
+ simpleBackupCheck(locationUri, shardBackupId);
}
}
@@ -102,7 +101,8 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
final CoreContainer cores = h.getCoreContainer();
final CoreAdminHandler admin = new CoreAdminHandler(cores);
- final URI location = createAndBootstrapLocationForBackup();
+ final Path locationPath = createBackupLocation();
+ final URI locationUri = bootstrapBackupLocation(locationPath);
final ShardBackupId firstShardBackup = new ShardBackupId("shard1", BackupId.zero());
{ // first a backup before we've ever done *anything*...
@@ -110,11 +110,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
admin.handleRequestBody
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, firstShardBackup.getIdAsString()),
resp);
assertNull("Backup should have succeeded", resp.getException());
- simpleBackupCheck(location, firstShardBackup, initialEmptyIndexSegmentFileName);
+ simpleBackupCheck(locationUri, firstShardBackup, initialEmptyIndexSegmentFileName);
}
{ // Empty (named) snapshot..
@@ -135,11 +135,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
admin.handleRequestBody
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, secondShardBackupId.getIdAsString()),
resp);
assertNull("Backup should have succeeded", resp.getException());
- simpleBackupCheck(location, secondShardBackupId, initialEmptyIndexSegmentFileName);
+ simpleBackupCheck(locationUri, secondShardBackupId, initialEmptyIndexSegmentFileName);
}
{ // Second empty (named) snapshot..
@@ -156,7 +156,7 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
assertU(commit());
for (ShardBackupId shardBackupId: Arrays.asList(firstShardBackup, secondShardBackupId)) {
- simpleBackupCheck(location, shardBackupId, initialEmptyIndexSegmentFileName);
+ simpleBackupCheck(locationUri, shardBackupId, initialEmptyIndexSegmentFileName);
}
// Make backups from each of the snapshots and check they are still empty as well...
@@ -167,11 +167,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"commitName", "empty_snapshotA",
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, thirdShardBackup.getIdAsString()),
resp);
assertNull("Backup from snapshot empty_snapshotA should have succeeded", resp.getException());
- simpleBackupCheck(location, thirdShardBackup, initialEmptyIndexSegmentFileName);
+ simpleBackupCheck(locationUri, thirdShardBackup, initialEmptyIndexSegmentFileName);
}
{
final ShardBackupId fourthShardBackup = new ShardBackupId("shard1", new BackupId(3));
@@ -180,11 +180,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"commitName", "empty_snapshotB",
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, fourthShardBackup.getIdAsString()),
resp);
assertNull("Backup from snapshot empty_snapshotB should have succeeded", resp.getException());
- simpleBackupCheck(location, fourthShardBackup, initialEmptyIndexSegmentFileName);
+ simpleBackupCheck(locationUri, fourthShardBackup, initialEmptyIndexSegmentFileName);
}
admin.close();
}
@@ -211,7 +211,8 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
final CoreContainer cores = h.getCoreContainer();
final CoreAdminHandler admin = new CoreAdminHandler(cores);
- final URI location = createAndBootstrapLocationForBackup();
+ final Path locationPath = createBackupLocation();
+ final URI locationUri = bootstrapBackupLocation(locationPath);
final ShardBackupId firstShardBackupId = new ShardBackupId("shard1", BackupId.zero());
{ // take an initial 'backup1a' containing our 1 document
@@ -220,11 +221,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"name", "backup1a",
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, firstShardBackupId.getIdAsString()),
resp);
assertNull("Backup should have succeeded", resp.getException());
- simpleBackupCheck(location, firstShardBackupId, oneDocSegmentFile);
+ simpleBackupCheck(locationUri, firstShardBackupId, oneDocSegmentFile);
}
{ // and an initial "snapshot1a' that should eventually match
@@ -253,11 +254,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
admin.handleRequestBody
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, secondShardBackupId.getIdAsString()),
resp);
assertNull("Backup should have succeeded", resp.getException());
- simpleBackupCheck(location, secondShardBackupId, oneDocSegmentFile);
+ simpleBackupCheck(locationUri, secondShardBackupId, oneDocSegmentFile);
}
{ // and a second "snapshot1b' should also still be identical
@@ -272,8 +273,8 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
// Hard Committing the 2nd doc now should not affect the existing backups or snapshots...
assertU(commit());
- simpleBackupCheck(location, firstShardBackupId, oneDocSegmentFile); // backup1a
- simpleBackupCheck(location, secondShardBackupId, oneDocSegmentFile); // backup1b
+ simpleBackupCheck(locationUri, firstShardBackupId, oneDocSegmentFile); // backup1a
+ simpleBackupCheck(locationUri, secondShardBackupId, oneDocSegmentFile); // backup1b
final ShardBackupId thirdShardBackupId = new ShardBackupId("shard1", new BackupId(2));
{ // But we should be able to confirm both docs appear in a new backup (not based on a previous snapshot)
@@ -281,13 +282,13 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
admin.handleRequestBody
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, thirdShardBackupId.getIdAsString()),
resp);
assertNull("Backup should have succeeded", resp.getException());
// TODO This doesn't actually check that backup has both docs! Can we do better than this without doing a full restore?
// Maybe validate the new segments_X file at least to show that it's picked up the latest commit?
- simpleBackupCheck(location, thirdShardBackupId);
+ simpleBackupCheck(locationUri, thirdShardBackupId);
}
// if we go back and create backups from our earlier snapshots they should still only
@@ -300,11 +301,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"commitName", "snapshot1a",
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, fourthShardBackupId.getIdAsString()),
resp);
assertNull("Backup of snapshot1a should have succeeded", resp.getException());
- simpleBackupCheck(location, fourthShardBackupId, oneDocSegmentFile);
+ simpleBackupCheck(locationUri, fourthShardBackupId, oneDocSegmentFile);
}
final ShardBackupId fifthShardBackupId = new ShardBackupId("shard1", new BackupId(4));
{
@@ -313,11 +314,11 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString(),
"core", DEFAULT_TEST_COLLECTION_NAME,
"commitName", "snapshot1b",
- "location", location.getPath(),
+ "location", locationPath.toString(),
CoreAdminParams.SHARD_BACKUP_ID, fifthShardBackupId.getIdAsString()),
resp);
assertNull("Backup of snapshot1b should have succeeded", resp.getException());
- simpleBackupCheck(location, fifthShardBackupId, oneDocSegmentFile);
+ simpleBackupCheck(locationUri, fifthShardBackupId, oneDocSegmentFile);
}
admin.close();
@@ -355,15 +356,18 @@ public class TestIncrementalCoreBackup extends SolrTestCaseJ4 {
}
}
- private URI createAndBootstrapLocationForBackup() throws IOException {
- final File locationFile = createTempDir().toFile();
- final String location = locationFile.getAbsolutePath();
+ private Path createBackupLocation() {
+ return createTempDir().toAbsolutePath();
+ }
- h.getCoreContainer().getAllowPaths().add(locationFile.toPath());
+ private URI bootstrapBackupLocation(Path locationPath) throws IOException {
+ final String locationPathStr = locationPath.toString();
+ h.getCoreContainer().getAllowPaths().add(locationPath);
try (BackupRepository backupRepo = h.getCoreContainer().newBackupRepository(null)) {
- final BackupFilePaths backupFilePaths = new BackupFilePaths(backupRepo, backupRepo.createURI(location));
+ final URI locationUri = backupRepo.createURI(locationPathStr);
+ final BackupFilePaths backupFilePaths = new BackupFilePaths(backupRepo, locationUri);
backupFilePaths.createIncrementalBackupFolders();
- return backupRepo.createURI(location);
+ return locationUri;
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java
index 871d349..0e507a4 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestStressIncrementalBackup.java
@@ -17,11 +17,6 @@
package org.apache.solr.handler;
-import java.io.File;
-import java.lang.invoke.MethodHandles;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -32,26 +27,31 @@ import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.util.LogLevel;
import org.junit.After;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.lang.invoke.MethodHandles;
+import java.nio.file.Path;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+
import static org.apache.solr.handler.TestStressThreadBackup.makeDoc;
//@LuceneTestCase.Nightly
@LuceneTestCase.SuppressCodecs({"SimpleText"})
-@LogLevel("org.apache.solr.handler.SnapShooter=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG")
public class TestStressIncrementalBackup extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private File backupDir;
+ private Path backupPath;
private SolrClient adminClient;
private SolrClient coreClient;
+
@Before
public void beforeTest() throws Exception {
- backupDir = createTempDir(getTestClass().getSimpleName() + "_backups").toFile();
+ backupPath = createTempDir(getTestClass().getSimpleName() + "_backups");
+ System.setProperty("solr.allowPaths", backupPath.toString());
// NOTE: we don't actually care about using SolrCloud, but we want to use SolrClient and I can't
// bring myself to deal with the nonsense that is SolrJettyTestBase.
@@ -87,6 +87,8 @@ public class TestStressIncrementalBackup extends SolrCloudTestCase {
if (null != coreClient) {
coreClient.close();
}
+
+ System.clearProperty("solr.allowPaths");
}
public void testCoreAdminHandler() throws Exception {
@@ -158,7 +160,7 @@ public class TestStressIncrementalBackup extends SolrCloudTestCase {
public void makeBackup() throws Exception {
CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(DEFAULT_TEST_COLLECTION_NAME, "stressBackup")
- .setLocation(backupDir.getAbsolutePath())
+ .setLocation(backupPath.toString())
.setIncremental(true)
.setMaxNumberBackupPoints(5);
if (random().nextBoolean()) {
diff --git a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
index 8c7128b..f8f33a3 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestStressThreadBackup.java
@@ -37,7 +37,6 @@ import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.util.LogLevel;
import org.apache.solr.util.TimeOut;
import org.junit.After;
import org.junit.AfterClass;
@@ -63,7 +62,6 @@ import java.util.regex.Pattern;
@Nightly
@SuppressCodecs({"SimpleText"})
-@LogLevel("org.apache.solr.handler.SnapShooter=DEBUG;org.apache.solr.core.IndexDeletionPolicyWrapper=DEBUG")
public class TestStressThreadBackup extends SolrCloudTestCase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@@ -369,7 +367,8 @@ public class TestStressThreadBackup extends SolrCloudTestCase {
public void makeBackup(final String backupName, final String snapName) throws Exception {
ModifiableSolrParams p = params(CoreAdminParams.CORE, coreName,
CoreAdminParams.NAME, backupName,
- CoreAdminParams.BACKUP_LOCATION, backupDir.getAbsolutePath());
+ CoreAdminParams.BACKUP_LOCATION, backupDir.getAbsolutePath(),
+ CoreAdminParams.BACKUP_INCREMENTAL, "false");
if (null != snapName) {
p.add(CoreAdminParams.COMMIT_NAME, snapName);
}