You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2019/07/05 03:32:39 UTC

[lucene-solr] branch jira/SOLR-13608 created (now 38a9afe)

This is an automated email from the ASF dual-hosted git repository.

datcm pushed a change to branch jira/SOLR-13608
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git.


      at 38a9afe  SOLR-13608: Incremental backup for Solr

This branch includes the following new commits:

     new 38a9afe  SOLR-13608: Incremental backup for Solr

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[lucene-solr] 01/01: SOLR-13608: Incremental backup for Solr

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

datcm pushed a commit to branch jira/SOLR-13608
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git

commit 38a9afe0ce65a5f9970da9ec4953a399b2ea5101
Author: Cao Manh Dat <da...@apache.org>
AuthorDate: Fri Jul 5 10:32:26 2019 +0700

    SOLR-13608: Incremental backup for Solr
---
 .../solr/cloud/api/collections/BackupCmd.java      |  41 ++--
 .../solr/cloud/api/collections/RestoreCmd.java     |  10 +-
 .../org/apache/solr/core/backup/BackupManager.java | 132 +++++++---
 .../apache/solr/core/backup/FileGeneration.java    |  99 ++++++++
 .../core/backup/repository/BackupRepository.java   |  77 ++++++
 .../backup/repository/BackupRepositoryFactory.java |   5 +
 .../backup/repository/HdfsBackupRepository.java    |  16 ++
 .../repository/LocalFileSystemRepository.java      |  26 ++
 .../java/org/apache/solr/handler/SnapShooter.java  |  78 +++++-
 .../apache/solr/handler/admin/BackupCoreOp.java    |   3 +-
 .../solr/handler/admin/CollectionsHandler.java     |   6 +-
 .../AbstractCloudBackupRestoreTestCase.java        | 271 +++++++++++++++++++--
 .../collections/TestHdfsCloudBackupRestore.java    |  14 +-
 .../collections/TestLocalFSCloudBackupRestore.java |  41 +++-
 .../solr/core/backup/FileGenerationTest.java       |  49 ++++
 .../solrj/request/CollectionAdminRequest.java      |   7 +
 .../apache/solr/common/params/CoreAdminParams.java |   5 +
 .../apache/solr/core/TrackingBackupRepository.java | 145 +++++++++++
 18 files changed, 916 insertions(+), 109 deletions(-)

diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
index ceec4e2..ceac8af 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/BackupCmd.java
@@ -77,29 +77,32 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
     }
     String backupName = message.getStr(NAME);
     String repo = message.getStr(CoreAdminParams.BACKUP_REPOSITORY);
+    boolean incremental = message.getBool(CoreAdminParams.BACKUP_INCREMENTAL, false);
 
     Instant startTime = Instant.now();
 
     CoreContainer cc = ocmh.overseer.getCoreContainer();
     BackupRepository repository = cc.newBackupRepository(Optional.ofNullable(repo));
-    BackupManager backupMgr = new BackupManager(repository, ocmh.zkStateReader);
 
     // Backup location
     URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
     URI backupPath = repository.resolve(location, backupName);
 
+    BackupManager backupMgr = BackupManager.forBackup(repository, ocmh.zkStateReader, location, backupName);
+
     //Validating if the directory already exists.
     if (repository.exists(backupPath)) {
-      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath);
+      if (!incremental)
+        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "The backup directory already exists: " + backupPath);
+    } else {
+      // Create a directory to store backup details.
+      repository.createDirectory(backupPath);
     }
 
-    // Create a directory to store backup details.
-    repository.createDirectory(backupPath);
-
     String strategy = message.getStr(CollectionAdminParams.INDEX_BACKUP_STRATEGY, CollectionAdminParams.COPY_FILES_STRATEGY);
     switch (strategy) {
       case CollectionAdminParams.COPY_FILES_STRATEGY: {
-        copyIndexFiles(backupPath, collectionName, message, results);
+        copyIndexFiles(backupPath, collectionName, message, results, incremental);
         break;
       }
       case CollectionAdminParams.NO_INDEX_BACKUP_STRATEGY: {
@@ -111,15 +114,16 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
 
     //Download the configs
     String configName = ocmh.zkStateReader.readConfigName(collectionName);
-    backupMgr.downloadConfigDir(location, backupName, configName);
+    backupMgr.downloadConfigDir(configName);
 
     //Save the collection's state. Can be part of the monolithic clusterstate.json or a individual state.json
     //Since we don't want to distinguish we extract the state and back it up as a separate json
     DocCollection collectionState = ocmh.zkStateReader.getClusterState().getCollection(collectionName);
-    backupMgr.writeCollectionState(location, backupName, collectionName, collectionState);
+    backupMgr.writeCollectionState(collectionName, collectionState);
+    backupMgr.downloadCollectionProperties(collectionName);
 
+    // We should write the properties last, the exist of file guarantee that the backup success
     Properties properties = new Properties();
-
     properties.put(BackupManager.BACKUP_NAME_PROP, backupName);
     properties.put(BackupManager.COLLECTION_NAME_PROP, collectionName);
     properties.put(BackupManager.COLLECTION_ALIAS_PROP, extCollectionName);
@@ -130,9 +134,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
     //if they are not the same then we can throw an error or have an 'overwriteConfig' flag
     //TODO save numDocs for the shardLeader. We can use it to sanity check the restore.
 
-    backupMgr.writeBackupProperties(location, backupName, properties);
-
-    backupMgr.downloadCollectionProperties(location, backupName, collectionName);
+    backupMgr.writeBackupProperties(properties);
 
     log.info("Completed backing up ZK data for backupName={}", backupName);
   }
@@ -142,7 +144,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
     // If that is not possible, we choose any other replica for the given shard.
     Collection<CoreSnapshotMetaData> snapshots = snapshotMeta.getReplicaSnapshotsForShard(slice.getName());
 
-    Optional<CoreSnapshotMetaData> leaderCore = snapshots.stream().filter(x -> x.isLeader()).findFirst();
+    Optional<CoreSnapshotMetaData> leaderCore = snapshots.stream().filter(CoreSnapshotMetaData::isLeader).findFirst();
     if (leaderCore.isPresent()) {
       log.info("Replica {} was the leader when snapshot {} was created.", leaderCore.get().getCoreName(), snapshotMeta.getName());
       Replica r = slice.getReplica(leaderCore.get().getCoreName());
@@ -155,7 +157,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
                                .filter(x -> x.getState() != State.DOWN && snapshotMeta.isSnapshotExists(slice.getName(), x))
                                .findFirst();
 
-    if (!r.isPresent()) {
+    if (r.isEmpty()) {
       throw new SolrException(ErrorCode.SERVER_ERROR,
           "Unable to find any live replica with a snapshot named " + snapshotMeta.getName() + " for shard " + slice.getName());
     }
@@ -163,7 +165,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
     return r.get();
   }
 
-  private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, NamedList results) throws Exception {
+  private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, NamedList results, boolean incremental) throws Exception {
     String backupName = request.getStr(NAME);
     String asyncId = request.getStr(ASYNC);
     String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
@@ -174,7 +176,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
     if (commitName != null) {
       SolrZkClient zkClient = ocmh.zkStateReader.getZkClient();
       snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
-      if (!snapshotMeta.isPresent()) {
+      if (snapshotMeta.isEmpty()) {
         throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
             + " does not exist for collection " + collectionName);
       }
@@ -194,7 +196,7 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
 
     final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
     for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
-      Replica replica = null;
+      Replica replica;
 
       if (snapshotMeta.isPresent()) {
         if (!shardsToConsider.contains(slice.getName())) {
@@ -219,9 +221,8 @@ public class BackupCmd implements OverseerCollectionMessageHandler.Cmd {
       params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
       params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
       params.set(CORE_NAME_PROP, coreName);
-      if (snapshotMeta.isPresent()) {
-        params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
-      }
+      params.set(CoreAdminParams.BACKUP_INCREMENTAL, incremental);
+      snapshotMeta.ifPresent(collectionSnapshotMetaData -> params.set(CoreAdminParams.COMMIT_NAME, collectionSnapshotMetaData.getName()));
 
       shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);
       log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
index 20dfc6d..de62ed9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/RestoreCmd.java
@@ -103,12 +103,12 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
     URI location = repository.createURI(message.getStr(CoreAdminParams.BACKUP_LOCATION));
     URI backupPath = repository.resolve(location, backupName);
     ZkStateReader zkStateReader = ocmh.zkStateReader;
-    BackupManager backupMgr = new BackupManager(repository, zkStateReader);
+    BackupManager backupMgr = BackupManager.forRestore(repository, zkStateReader, location, backupName);
 
-    Properties properties = backupMgr.readBackupProperties(location, backupName);
+    Properties properties = backupMgr.readBackupProperties();
     String backupCollection = properties.getProperty(BackupManager.COLLECTION_NAME_PROP);
     String backupCollectionAlias = properties.getProperty(BackupManager.COLLECTION_ALIAS_PROP);
-    DocCollection backupCollectionState = backupMgr.readCollectionState(location, backupName, backupCollection);
+    DocCollection backupCollectionState = backupMgr.readCollectionState(backupCollection);
 
     // Get the Solr nodes to restore a collection.
     final List<String> nodeList = Assign.getLiveOrLiveAndCreateNodeSetList(
@@ -149,7 +149,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
       //TODO add overwrite option?
     } else {
       log.info("Uploading config {}", restoreConfigName);
-      backupMgr.uploadConfigDir(location, backupName, configName, restoreConfigName);
+      backupMgr.uploadConfigDir(configName, restoreConfigName);
     }
 
     log.info("Starting restore into collection={} with backup_name={} at location={}", restoreCollectionName, backupName,
@@ -209,7 +209,7 @@ public class RestoreCmd implements OverseerCollectionMessageHandler.Cmd {
     }
 
     // Restore collection properties
-    backupMgr.uploadCollectionProperties(location, backupName, restoreCollectionName);
+    backupMgr.uploadCollectionProperties(restoreCollectionName);
 
     DocCollection restoreCollection = zkStateReader.getClusterState().getCollection(restoreCollectionName);
 
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index b15bbfe..0820d1b 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -28,6 +28,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Properties;
 
 import com.google.common.base.Preconditions;
@@ -70,9 +71,61 @@ public class BackupManager {
   protected final ZkStateReader zkStateReader;
   protected final BackupRepository repository;
 
-  public BackupManager(BackupRepository repository, ZkStateReader zkStateReader) {
+  // Multiple generations of a backup can coexist under a backupId
+  protected FileGeneration propertiesFile;
+  // In case of generation > 0, all zk related files are stored under this subFolder
+  // If generation = 0, value of subFolder will be ignored to maintain backward compatibility
+  protected final String subFolder;
+  protected final URI backupPath;
+
+  private BackupManager(BackupRepository repository,
+                       ZkStateReader zkStateReader,
+                       URI backupPath,
+                       FileGeneration propertiesFile) {
     this.repository = Objects.requireNonNull(repository);
     this.zkStateReader = Objects.requireNonNull(zkStateReader);
+    this.backupPath = backupPath;
+    this.propertiesFile = propertiesFile;
+    this.subFolder = "gen-" + propertiesFile.gen;
+  }
+
+  public static BackupManager forBackup(BackupRepository repository,
+                                        ZkStateReader stateReader,
+                                        URI backupLoc,
+                                        String backupId) throws IOException{
+    Objects.requireNonNull(repository);
+    Objects.requireNonNull(stateReader);
+
+    URI backupPath = repository.resolve(backupLoc, backupId);
+
+    Optional<FileGeneration> opFileGen = FileGeneration.findMostRecent(BACKUP_PROPS_FILE,
+        repository.listAllOrEmpty(backupPath));
+    if (opFileGen.isPresent()) {
+      return new BackupManager(repository, stateReader, backupPath, opFileGen.get().nextGen());
+    } else {
+      return new BackupManager(repository, stateReader, backupPath, FileGeneration.zeroGen(BACKUP_PROPS_FILE));
+    }
+  }
+
+  public static BackupManager forRestore(BackupRepository repository,
+                                         ZkStateReader stateReader,
+                                         URI backupLoc,
+                                         String backupId) throws IOException {
+    Objects.requireNonNull(repository);
+    Objects.requireNonNull(stateReader);
+
+    URI backupPath = repository.resolve(backupLoc, backupId);
+    if (!repository.exists(backupPath)) {
+      throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't restore since doesn't exist: " + backupPath);
+    }
+
+    Optional<FileGeneration> opFileGen = FileGeneration.findMostRecent(BACKUP_PROPS_FILE,
+        repository.listAll(repository.resolve(backupLoc, backupId)));
+    if (opFileGen.isPresent()) {
+      return new BackupManager(repository, stateReader, backupPath, opFileGen.get());
+    } else {
+      throw new IllegalStateException("No " + BACKUP_PROPS_FILE + " was found, the backup does not exist or not complete");
+    }
   }
 
   /**
@@ -85,24 +138,13 @@ public class BackupManager {
   /**
    * This method returns the configuration parameters for the specified backup.
    *
-   * @param backupLoc The base path used to store the backup data.
-   * @param backupId  The unique name for the backup whose configuration params are required.
    * @return the configuration parameters for the specified backup.
    * @throws IOException In case of errors.
    */
-  public Properties readBackupProperties(URI backupLoc, String backupId) throws IOException {
-    Objects.requireNonNull(backupLoc);
-    Objects.requireNonNull(backupId);
-
-    // Backup location
-    URI backupPath = repository.resolve(backupLoc, backupId);
-    if (!repository.exists(backupPath)) {
-      throw new SolrException(ErrorCode.SERVER_ERROR, "Couldn't restore since doesn't exist: " + backupPath);
-    }
-
+  public Properties readBackupProperties() throws IOException {
     Properties props = new Properties();
     try (Reader is = new InputStreamReader(new PropertiesInputStream(
-        repository.openInput(backupPath, BACKUP_PROPS_FILE, IOContext.DEFAULT)), StandardCharsets.UTF_8)) {
+        repository.openInput(backupPath, propertiesFile.getFileName(), IOContext.DEFAULT)), StandardCharsets.UTF_8)) {
       props.load(is);
       return props;
     }
@@ -111,13 +153,11 @@ public class BackupManager {
   /**
    * This method stores the backup properties at the specified location in the repository.
    *
-   * @param backupLoc  The base path used to store the backup data.
-   * @param backupId  The unique name for the backup whose configuration params are required.
    * @param props The backup properties
    * @throws IOException in case of I/O error
    */
-  public void writeBackupProperties(URI backupLoc, String backupId, Properties props) throws IOException {
-    URI dest = repository.resolve(backupLoc, backupId, BACKUP_PROPS_FILE);
+  public void writeBackupProperties(Properties props) throws IOException {
+    URI dest = repository.resolve(backupPath, propertiesFile.getFileName());
     try (Writer propsWriter = new OutputStreamWriter(repository.createOutput(dest), StandardCharsets.UTF_8)) {
       props.store(propsWriter, "Backup properties file");
     }
@@ -126,16 +166,14 @@ public class BackupManager {
   /**
    * This method reads the meta-data information for the backed-up collection.
    *
-   * @param backupLoc The base path used to store the backup data.
-   * @param backupId The unique name for the backup.
    * @param collectionName The name of the collection whose meta-data is to be returned.
    * @return the meta-data information for the backed-up collection.
    * @throws IOException in case of errors.
    */
-  public DocCollection readCollectionState(URI backupLoc, String backupId, String collectionName) throws IOException {
+  public DocCollection readCollectionState(String collectionName) throws IOException {
     Objects.requireNonNull(collectionName);
 
-    URI zkStateDir = repository.resolve(backupLoc, backupId, ZK_STATE_DIR);
+    URI zkStateDir = getUriUnderSubFolder(ZK_STATE_DIR);
     try (IndexInput is = repository.openInput(zkStateDir, COLLECTION_PROPS_FILE, IOContext.DEFAULT)) {
       byte[] arr = new byte[(int) is.length()]; // probably ok since the json file should be small.
       is.readBytes(arr, 0, (int) is.length());
@@ -144,18 +182,32 @@ public class BackupManager {
     }
   }
 
+  private URI getUriUnderSubFolder(String... files) {
+    if (propertiesFile.gen == 0) {
+      if (files.length == 0) {
+        return backupPath;
+      }
+
+      return repository.resolve(backupPath, files);
+    } else {
+      URI subFolderURI = repository.resolve(backupPath, subFolder);
+      if (files.length == 0) {
+        return subFolderURI;
+      }
+      return repository.resolve(subFolderURI, files);
+    }
+  }
+
   /**
    * This method writes the collection meta-data to the specified location in the repository.
    *
-   * @param backupLoc The base path used to store the backup data.
-   * @param backupId  The unique name for the backup.
    * @param collectionName The name of the collection whose meta-data is being stored.
    * @param collectionState The collection meta-data to be stored.
    * @throws IOException in case of I/O errors.
    */
-  public void writeCollectionState(URI backupLoc, String backupId, String collectionName,
+  public void writeCollectionState(String collectionName,
                                    DocCollection collectionState) throws IOException {
-    URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, COLLECTION_PROPS_FILE);
+    URI dest = getUriUnderSubFolder(ZK_STATE_DIR, COLLECTION_PROPS_FILE);
     try (OutputStream collectionStateOs = repository.createOutput(dest)) {
       collectionStateOs.write(Utils.toJSON(Collections.singletonMap(collectionName, collectionState)));
     }
@@ -164,15 +216,13 @@ public class BackupManager {
   /**
    * This method uploads the Solr configuration files to the desired location in Zookeeper.
    *
-   * @param backupLoc  The base path used to store the backup data.
-   * @param backupId  The unique name for the backup.
    * @param sourceConfigName The name of the config to be copied
    * @param targetConfigName  The name of the config to be created.
    * @throws IOException in case of I/O errors.
    */
-  public void uploadConfigDir(URI backupLoc, String backupId, String sourceConfigName, String targetConfigName)
+  public void uploadConfigDir(String sourceConfigName, String targetConfigName)
       throws IOException {
-    URI source = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, sourceConfigName);
+    URI source = getUriUnderSubFolder(ZK_STATE_DIR, CONFIG_STATE_DIR, sourceConfigName);
     String zkPath = ZkConfigManager.CONFIGS_ZKNODE + "/" + targetConfigName;
     uploadToZk(zkStateReader.getZkClient(), source, zkPath);
   }
@@ -180,22 +230,24 @@ public class BackupManager {
   /**
    * This method stores the contents of a specified Solr config at the specified location in repository.
    *
-   * @param backupLoc  The base path used to store the backup data.
-   * @param backupId  The unique name for the backup.
    * @param configName The name of the config to be saved.
    * @throws IOException in case of I/O errors.
    */
-  public void downloadConfigDir(URI backupLoc, String backupId, String configName) throws IOException {
-    URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
-    repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR));
-    repository.createDirectory(repository.resolve(backupLoc, backupId, ZK_STATE_DIR, CONFIG_STATE_DIR));
+  public void downloadConfigDir(String configName) throws IOException {
+    URI generationFolder = getUriUnderSubFolder();
+    if (!repository.exists(generationFolder))
+      repository.createDirectory(generationFolder);
+
+    URI dest = repository.resolve(generationFolder, ZK_STATE_DIR, CONFIG_STATE_DIR, configName);
+    repository.createDirectory(repository.resolve(generationFolder, ZK_STATE_DIR));
+    repository.createDirectory(repository.resolve(generationFolder, ZK_STATE_DIR, CONFIG_STATE_DIR));
     repository.createDirectory(dest);
 
     downloadFromZK(zkStateReader.getZkClient(), ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, dest);
   }
 
-  public void uploadCollectionProperties(URI backupLoc, String backupId, String collectionName) throws IOException {
-    URI sourceDir = repository.resolve(backupLoc, backupId, ZK_STATE_DIR);
+  public void uploadCollectionProperties(String collectionName) throws IOException {
+    URI sourceDir = getUriUnderSubFolder(ZK_STATE_DIR);
     URI source = repository.resolve(sourceDir, ZkStateReader.COLLECTION_PROPS_ZKNODE);
     if (!repository.exists(source)) {
       // No collection properties to restore
@@ -213,8 +265,8 @@ public class BackupManager {
     }
   }
 
-  public void downloadCollectionProperties(URI backupLoc, String backupId, String collectionName) throws IOException {
-    URI dest = repository.resolve(backupLoc, backupId, ZK_STATE_DIR, ZkStateReader.COLLECTION_PROPS_ZKNODE);
+  public void downloadCollectionProperties(String collectionName) throws IOException {
+    URI dest = getUriUnderSubFolder(ZK_STATE_DIR, ZkStateReader.COLLECTION_PROPS_ZKNODE);
     String zkPath = ZkStateReader.COLLECTIONS_ZKNODE + '/' + collectionName + '/' + ZkStateReader.COLLECTION_PROPS_ZKNODE;
 
 
diff --git a/solr/core/src/java/org/apache/solr/core/backup/FileGeneration.java b/solr/core/src/java/org/apache/solr/core/backup/FileGeneration.java
new file mode 100644
index 0000000..763e9c7
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/core/backup/FileGeneration.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Optional;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Finding and represent file with different generation.
+ * i.e: backup-10.properties
+ */
+class FileGeneration {
+  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+  private final String prefix;
+  private final String suffix;
+  public final long gen;
+
+  private FileGeneration(String prefix, String suffix, long gen) {
+    this.prefix = prefix;
+    this.suffix = suffix;
+    this.gen = gen;
+  }
+
+  FileGeneration nextGen() {
+    return new FileGeneration(prefix, suffix, gen+1);
+  }
+
+  static FileGeneration zeroGen(String zeroGenFile) {
+    int index = zeroGenFile.indexOf('.');
+    if (index == -1) {
+      return new FileGeneration(zeroGenFile, "", 0);
+    }
+    return new FileGeneration(zeroGenFile.substring(0, index), zeroGenFile.substring(index), 0);
+  }
+
+  static Optional<FileGeneration> findMostRecent(String zeroGenFile, String[] listFiles) {
+    if (listFiles == null || listFiles.length == 0) {
+      return Optional.empty();
+    }
+
+    int index = zeroGenFile.indexOf('.');
+    if (index == -1) {
+      return findMostRecent(zeroGenFile, "", listFiles);
+    }
+    return findMostRecent(zeroGenFile.substring(0, index), zeroGenFile.substring(index), listFiles);
+  }
+
+  private static Optional<FileGeneration> findMostRecent(String prefix, String suffix, String[] listFiles) {
+    long gen = -1;
+    for (String file : listFiles) {
+      if (file.startsWith(prefix) && file.endsWith(suffix)) {
+        if (prefix.length() + suffix.length() == file.length()) {
+          gen = Math.max(gen, 0);
+        } else {
+          if (file.charAt(prefix.length()) != '-') {
+            log.info("This file {} does not match with the file generation", file);
+            continue;
+          }
+
+          try {
+            gen = Math.max(gen, Long.parseLong(file.substring(prefix.length() + 1, file.length() - suffix.length())));
+          } catch (NumberFormatException e) {
+            log.info("This file {} does not match with the file generation", file);
+          }
+        }
+      }
+    }
+
+    if (gen == -1)
+      return Optional.empty();
+
+    return Optional.of(new FileGeneration(prefix, suffix, gen));
+  }
+
+  public String getFileName() {
+    if (gen == 0) {
+      return prefix + suffix;
+    }
+    else return String.format("%s-%d%s", prefix, gen, suffix);
+  }
+}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java
index 875be18..e7e47a3 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepository.java
@@ -20,8 +20,12 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URI;
+import java.util.Collection;
+import java.util.Objects;
 import java.util.Optional;
 
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -181,4 +185,77 @@ public interface BackupRepository extends NamedListInitializedPlugin, Closeable
    *           in case of errors.
    */
   void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException;
+
+  /**
+   * Delete {@code files} at {@code path}
+   * @since 8.2.0
+   */
+  default void delete(URI path, Collection<String> files) throws IOException {
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Get checksum of {@code fileName} at {@code path}
+   * This method only be called on Lucene index files
+   * @since 8.2.0
+   */
+  default Checksum checksum(URI path, String fileName) throws IOException {
+    throw new UnsupportedOperationException();
+  }
+
+  /**
+   * Get checksum of {@code fileName} at {@code dir}.
+   * This method only be called on Lucene index files
+   * @since 8.2.0
+   */
+  default Checksum checksum(Directory dir, String fileName) throws IOException {
+    try (IndexInput in = dir.openChecksumInput(fileName, IOContext.READONCE)) {
+      final long length = in.length();
+      if (length < CodecUtil.footerLength()) {
+        throw new CorruptIndexException("File: " + fileName + " is corrupted, its length must be >= " +
+            CodecUtil.footerLength() + " but was: " + in.length(), in);
+      }
+
+      return new BackupRepository.Checksum(fileName, String.valueOf(CodecUtil.retrieveChecksum(in)), in.length());
+    }
+  }
+
+  /**
+   * List all files or directories directly under {@code path}.
+   * @return an empty array in case of IOException
+   */
+  default String[] listAllOrEmpty(URI path) {
+    try {
+      return this.listAll(path);
+    } catch (IOException e) {
+      return new String[0];
+    }
+  }
+
+  class Checksum {
+    public final String fileName;
+    public final String checksum;
+    public final long size;
+
+    Checksum(String fileName, String checksum, long size) {
+      this.fileName = fileName;
+      this.checksum = checksum;
+      this.size = size;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) return true;
+      if (o == null || getClass() != o.getClass()) return false;
+      Checksum checksum = (Checksum) o;
+      return size == checksum.size &&
+          Objects.equals(fileName, checksum.fileName) &&
+          Objects.equals(this.checksum, checksum.checksum);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(fileName, checksum, size);
+    }
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
index 9e02b21..bfc1e51 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
@@ -72,6 +72,11 @@ public class BackupRepositoryFactory {
         "Could not find a backup repository with name " + name);
 
     BackupRepository result = loader.newInstance(repo.className, BackupRepository.class);
+    if ("trackingBackupRepo".equals(name) && repo.initArgs.get("factory") == null) {
+      repo.initArgs.add("factory", this);
+      repo.initArgs.add("loader", loader);
+    }
+
     result.init(repo.initArgs);
     return result;
   }
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
index 6c0b04c..be46478 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/HdfsBackupRepository.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Collection;
 import java.util.Objects;
 
 import com.google.common.base.Preconditions;
@@ -196,4 +197,19 @@ public class HdfsBackupRepository implements BackupRepository {
       dest.copyFrom(dir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
     }
   }
+
+  @Override
+  public void delete(URI path, Collection<String> files) throws IOException {
+    for (String file : files) {
+      fileSystem.delete(new Path(new Path(path), file), true);
+    }
+  }
+
+  @Override
+  public Checksum checksum(URI repo, String fileName) throws IOException {
+    try (HdfsDirectory dir = new HdfsDirectory(new Path(repo), NoLockFactory.INSTANCE,
+        hdfsConfig, copyBufferSize)) {
+      return checksum(dir, fileName);
+    }
+  }
 }
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
index 01810f6..8173701 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/LocalFileSystemRepository.java
@@ -27,6 +27,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Collection;
 import java.util.Objects;
 
 import org.apache.lucene.store.Directory;
@@ -129,6 +130,12 @@ public class LocalFileSystemRepository implements BackupRepository {
 
   @Override
   public String[] listAll(URI dirPath) throws IOException {
+    // It is better to check the existence of the directory first since
+    // creating a FSDirectory will create a corresponds folder if the directory does not exist
+    if (!exists(dirPath)) {
+      return new String[0];
+    }
+
     try (FSDirectory dir = new SimpleFSDirectory(Paths.get(dirPath), NoLockFactory.INSTANCE)) {
       return dir.listAll();
     }
@@ -154,5 +161,24 @@ public class LocalFileSystemRepository implements BackupRepository {
   }
 
   @Override
+  public Checksum checksum(URI repo, String fileName) throws IOException {
+    try (FSDirectory dir = new SimpleFSDirectory(Paths.get(repo), NoLockFactory.INSTANCE)) {
+      return checksum(dir, fileName);
+    }
+  }
+
+  @Override
+  public void delete(URI path, Collection<String> files) throws IOException {
+    if (files.isEmpty())
+      return;
+
+    try (FSDirectory dir = new SimpleFSDirectory(Paths.get(path), NoLockFactory.INSTANCE)) {
+      for (String file : files) {
+        dir.deleteFile(file);
+      }
+    }
+  }
+
+  @Override
   public void close() throws IOException {}
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
index 2c3c691..0192a29 100644
--- a/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
+++ b/solr/core/src/java/org/apache/solr/handler/SnapShooter.java
@@ -22,15 +22,19 @@ import java.net.URI;
 import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Date;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Consumer;
 
+import org.apache.lucene.index.CorruptIndexException;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.store.Directory;
 import org.apache.solr.common.SolrException;
@@ -62,8 +66,10 @@ public class SnapShooter {
   private URI snapshotDirPath = null;
   private BackupRepository backupRepo = null;
   private String commitName; // can be null
+  private final boolean incremental;
 
   @Deprecated
+  // Deprecated since 8.2.0
   public SnapShooter(SolrCore core, String location, String snapshotName) {
     String snapDirStr = null;
     // Note - This logic is only applicable to the usecase where a shared file-system is exposed via
@@ -74,10 +80,16 @@ public class SnapShooter {
     } else {
       snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
     }
+    this.incremental = false;
     initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null);
   }
 
   public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName) {
+    this(backupRepo, core, location, snapshotName, commitName, false);
+  }
+
+  public SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName, boolean incremental) {
+    this.incremental = incremental;
     initialize(backupRepo, core, location, snapshotName, commitName);
   }
 
@@ -142,7 +154,7 @@ public class SnapShooter {
           " Directory does not exist: " + snapshotDirPath);
     }
 
-    if (backupRepo.exists(snapshotDirPath)) {
+    if (!incremental && backupRepo.exists(snapshotDirPath)) {
       throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
           "Snapshot directory already exists: " + snapshotDirPath);
     }
@@ -234,8 +246,12 @@ public class SnapShooter {
       Collection<String> files = indexCommit.getFileNames();
       Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
       try {
-        for(String fileName : files) {
-          backupRepo.copyFileFrom(dir, fileName, snapshotDirPath);
+        if (incremental) {
+          incrementalCopy(indexCommit, files, dir);
+        } else {
+          for(String fileName : files) {
+            backupRepo.copyFileFrom(dir, fileName, snapshotDirPath);
+          }
         }
       } finally {
         solrCore.getDirectoryFactory().release(dir);
@@ -260,6 +276,54 @@ public class SnapShooter {
     }
   }
 
+  private void incrementalCopy(IndexCommit indexCommit, Collection<String> indexFiles, Directory dir) throws IOException {
+    Set<String> existedFiles = new HashSet<>(Arrays.asList(backupRepo.listAllOrEmpty(snapshotDirPath)));
+
+    // Files in destination with same name as files in indexCommit but with different checksum or length should be deleted first
+    List<String> corruptedFiles = new ArrayList<>();
+    List<String> filesNeedCopyOver = new ArrayList<>();
+
+    for(String fileName : indexFiles) {
+      if (existedFiles.contains(fileName)) {
+        BackupRepository.Checksum originalFileCS = backupRepo.checksum(dir, fileName);
+        try {
+          BackupRepository.Checksum existedFileCS = backupRepo.checksum(snapshotDirPath, fileName);
+          if (Objects.equals(originalFileCS, existedFileCS)) {
+            continue;
+          }
+        } catch (CorruptIndexException e) {
+          log.info("Found a corrupted file in backup repository {}", fileName);
+        }
+
+        corruptedFiles.add(fileName);
+      }
+
+      filesNeedCopyOver.add(fileName);
+    }
+
+    backupRepo.delete(snapshotDirPath, corruptedFiles);
+
+    boolean copySegmentsFile = false;
+    for (String fileName : filesNeedCopyOver) {
+      if (fileName.equals(indexCommit.getSegmentsFileName())) {
+        copySegmentsFile = true;
+        continue;
+      }
+
+      backupRepo.copyFileFrom(dir, fileName, snapshotDirPath);
+    }
+
+    if (copySegmentsFile) {
+      // copy segments_N last, in case of failures on copy new files, the backup still work
+      backupRepo.copyFileFrom(dir, indexCommit.getSegmentsFileName(), snapshotDirPath);
+    }
+
+    // finally delete unused files
+    //TODO keeping multiple indexCommit
+    existedFiles.removeAll(indexFiles);
+    backupRepo.delete(snapshotDirPath, existedFiles);
+  }
+
   private void deleteOldBackups(int numberToKeep) throws IOException {
     String[] paths = backupRepo.listAll(baseSnapDirPath);
     List<OldBackupDirectory> dirs = new ArrayList<>();
@@ -303,6 +367,14 @@ public class SnapShooter {
     replicationHandler.snapShootDetails = details;
   }
 
+  private static String[] listAllOrEmpty(BackupRepository repo, URI dir) {
+    try {
+      return repo.listAll(dir);
+    } catch (IOException e) {
+      return new String[0];
+    }
+  }
+
   public static final String DATE_FMT = "yyyyMMddHHmmssSSS";
 
 }
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
index 503eed0..3e61c4f 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/BackupCoreOp.java
@@ -37,6 +37,7 @@ class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {
 
     String cname = params.required().get(CoreAdminParams.CORE);
     String name = params.required().get(NAME);
+    boolean incremental = params.getBool(CoreAdminParams.BACKUP_INCREMENTAL, false);
 
     String repoName = params.get(CoreAdminParams.BACKUP_REPOSITORY);
     BackupRepository repository = it.handler.coreContainer.newBackupRepository(Optional.ofNullable(repoName));
@@ -53,7 +54,7 @@ class BackupCoreOp implements CoreAdminHandler.CoreAdminOp {
 
     URI locationUri = repository.createURI(location);
     try (SolrCore core = it.handler.coreContainer.getCore(cname)) {
-      SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName);
+      SnapShooter snapShooter = new SnapShooter(repository, core, locationUri, name, commitName, incremental);
       // validateCreateSnapshot will create parent dirs instead of throw; that choice is dubious.
       //  But we want to throw. One reason is that
       //  this dir really should, in fact must, already exist here if triggered via a collection backup on a shared
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index d292976..a2d4cc2 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -1085,6 +1085,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
         }
       }
 
+      boolean incremental = req.getParams().getBool(CoreAdminParams.BACKUP_INCREMENTAL, false);
       // Check if the specified location is valid for this repository.
       URI uri = repository.createURI(location);
       try {
@@ -1099,10 +1100,11 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
       if (!CollectionAdminParams.INDEX_BACKUP_STRATEGIES.contains(strategy)) {
         throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown index backup strategy " + strategy);
       }
-
-      Map<String, Object> params = copy(req.getParams(), null, NAME, COLLECTION_PROP, FOLLOW_ALIASES, CoreAdminParams.COMMIT_NAME);
+      
+      Map<String, Object> params = copy(req.getParams(), null, NAME, COLLECTION_PROP, FOLLOW_ALIASES, CoreAdminParams.COMMIT_NAME, CoreAdminParams.BACKUP_REPOSITORY);
       params.put(CoreAdminParams.BACKUP_LOCATION, location);
       params.put(CollectionAdminParams.INDEX_BACKUP_STRATEGY, strategy);
+      params.put(CoreAdminParams.BACKUP_INCREMENTAL, incremental);
       return params;
     }),
     RESTORE_OP(RESTORE, (req, rsp, h) -> {
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
index 9a41afe..b3dd3b1 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java
@@ -16,19 +16,27 @@
  */
 package org.apache.solr.cloud.api.collections;
 
-import static org.hamcrest.CoreMatchers.hasItem;
-import static org.hamcrest.CoreMatchers.not;
-
 import java.io.IOException;
+import java.io.OutputStream;
 import java.lang.invoke.MethodHandles;
+import java.net.URI;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.Random;
+import java.util.Set;
 import java.util.TreeMap;
+import java.util.stream.Collectors;
 
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.TestUtil;
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
@@ -36,6 +44,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.HttpSolrClient;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest;
 import org.apache.solr.client.solrj.request.CollectionAdminRequest.ClusterProp;
+import org.apache.solr.client.solrj.request.UpdateRequest;
 import org.apache.solr.client.solrj.response.RequestStatusState;
 import org.apache.solr.cloud.AbstractDistribZkTestBase;
 import org.apache.solr.cloud.SolrCloudTestCase;
@@ -44,14 +53,23 @@ import org.apache.solr.common.SolrException.ErrorCode;
 import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.cloud.DocCollection;
 import org.apache.solr.common.cloud.ImplicitDocRouter;
+import org.apache.solr.common.cloud.Replica;
 import org.apache.solr.common.cloud.Slice;
 import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.core.DirectoryFactory;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.TrackingBackupRepository;
+import org.apache.solr.core.backup.repository.BackupRepository;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.solr.core.TrackingBackupRepository.copiedFiles;
+import static org.hamcrest.CoreMatchers.hasItem;
+import static org.hamcrest.CoreMatchers.not;
+
 /**
  * This class implements the logic required to test Solr cloud backup/restore capability.
  */
@@ -80,11 +98,6 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
   public abstract String getCollectionNamePrefix();
 
   /**
-   * @return The name of the backup repository to use.
-   */
-  public abstract String getBackupRepoName();
-
-  /**
    * @return The absolute path for the backup location.
    *         Could return null.
    */
@@ -104,9 +117,7 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
     setTestSuffix("testok");
     boolean isImplicit = random().nextBoolean();
     boolean doSplitShardOperation = !isImplicit && random().nextBoolean();
-    replFactor = TestUtil.nextInt(random(), 1, 2);
-    numTlogReplicas = TestUtil.nextInt(random(), 0, 1);
-    numPullReplicas = TestUtil.nextInt(random(), 0, 1);
+    randomizeReplicaTypes();
     int backupReplFactor = replFactor + numPullReplicas + numTlogReplicas;
 
     CollectionAdminRequest.Create create = isImplicit ?
@@ -162,11 +173,86 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
   }
 
   @Test
+  @Slow
+  public void testBackupIncremental() throws Exception {
+    TrackingBackupRepository.clear();
+
+    setTestSuffix("testbackupinc");
+    randomizeReplicaTypes();
+    CloudSolrClient solrClient = cluster.getSolrClient();
+
+    CollectionAdminRequest
+        .createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor, numTlogReplicas, numPullReplicas)
+        .setMaxShardsPerNode(-1)
+        .process(solrClient);
+
+    indexDocs(getCollectionName(), false);
+
+    String backupName = BACKUPNAME_PREFIX + testSuffix;
+    try (BackupRepository repository = cluster.getJettySolrRunner(0).getCoreContainer()
+        .newBackupRepository(Optional.of(getBackupRepoName()))) {
+      String backupLocation = repository.getBackupLocation(getBackupLocation());
+      IncrementalBackupVerifier verifier = new IncrementalBackupVerifier(repository, backupLocation, backupName);
+      backupRestoreCorruptBackupFilesThenCheck(solrClient, verifier, backupLocation, backupName);
+
+      indexDocs(getCollectionName(), false);
+      backupRestoreCorruptBackupFilesThenCheck(solrClient, verifier, backupLocation, backupName);
+
+      for (int i = 0; i < 15; i++) {
+        indexDocs(getCollectionName(), 5,false);
+      }
+
+      backupRestoreCorruptBackupFilesThenCheck(solrClient, verifier, backupLocation, backupName);
+
+      new UpdateRequest()
+          .deleteByQuery("*:*")
+          .commit(cluster.getSolrClient(), getCollectionName());
+
+      backupRestoreCorruptBackupFilesThenCheck(solrClient, verifier, backupLocation, backupName);
+
+      indexDocs(getCollectionName(), false);
+      backupRestoreCorruptBackupFilesThenCheck(solrClient, verifier, backupLocation, backupName);
+    }
+  }
+
+  private void backupRestoreCorruptBackupFilesThenCheck(CloudSolrClient solrClient,
+                                                        IncrementalBackupVerifier verifier,
+                                                        String backupLocation,
+                                                        String backupName) throws Exception {
+    verifier.incrementalBackupThenVerify();
+
+    if( random().nextBoolean() )
+      simpleRestoreAndCheckDocCount(solrClient, backupLocation, backupName);
+
+    if ( random().nextBoolean() )
+      verifier.corruptBackupFiles();
+  }
+
+  private void simpleRestoreAndCheckDocCount(CloudSolrClient solrClient, String backupLocation, String backupName) throws Exception{
+    Map<String, Integer> origShardToDocCount = getShardToDocCountMap(solrClient, getCollectionState(getCollectionName()));
+
+    String restoreCollectionName = getCollectionName() + "_restored";
+    CollectionAdminRequest.restoreCollection(restoreCollectionName, backupName)
+        .setLocation(backupLocation).setRepositoryName(getBackupRepoName()).process(solrClient);
+
+    AbstractDistribZkTestBase.waitForRecoveriesToFinish(
+        restoreCollectionName, cluster.getSolrClient().getZkStateReader(), log.isDebugEnabled(), true, 30);
+
+    // check num docs are the same
+    assertEquals(origShardToDocCount, getShardToDocCountMap(solrClient, getCollectionState(restoreCollectionName)));
+
+    // this methods may get invoked multiple times, collection must be cleanup
+    CollectionAdminRequest.deleteCollection(restoreCollectionName).process(solrClient);
+  }
+
+  protected String getBackupRepoName() {
+    return "trackingBackupRepo";
+  }
+
+  @Test
   public void testRestoreFailure() throws Exception {
     setTestSuffix("testfailure");
-    replFactor = TestUtil.nextInt(random(), 1, 2);
-    numTlogReplicas = TestUtil.nextInt(random(), 0, 1);
-    numPullReplicas = TestUtil.nextInt(random(), 0, 1);
+    randomizeReplicaTypes();
 
     CollectionAdminRequest.Create create =
         CollectionAdminRequest.createCollection(getCollectionName(), "conf1", NUM_SHARDS, replFactor, numTlogReplicas, numPullReplicas);
@@ -213,6 +299,12 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
     }
   }
 
+  private void randomizeReplicaTypes() {
+    replFactor = TestUtil.nextInt(random(), 1, 2);
+    numTlogReplicas = TestUtil.nextInt(random(), 0, 1);
+    numPullReplicas = TestUtil.nextInt(random(), 0, 1);
+  }
+
   /**
    * This test validates the backup of collection configuration using
    *  {@linkplain CollectionAdminParams#NO_INDEX_BACKUP_STRATEGY}.
@@ -266,18 +358,13 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
     return cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices().size();
   }
 
-  private int indexDocs(String collectionName, boolean useUUID) throws Exception {
-    Random random = new Random(docsSeed);// use a constant seed for the whole test run so that we can easily re-index.
-    int numDocs = random.nextInt(100);
-    if (numDocs == 0) {
-      log.info("Indexing ZERO test docs");
-      return 0;
-    }
+  private void indexDocs(String collectionName, int numDocs, boolean useUUID) throws Exception {
+    Random random = new Random(docsSeed);
 
     List<SolrInputDocument> docs = new ArrayList<>(numDocs);
     for (int i=0; i<numDocs; i++) {
       SolrInputDocument doc = new SolrInputDocument();
-      doc.addField("id", ((useUUID == true) ? java.util.UUID.randomUUID().toString() : i));
+      doc.addField("id", (useUUID ? java.util.UUID.randomUUID().toString() : i));
       doc.addField("shard_s", "shard" + (1 + random.nextInt(NUM_SHARDS))); // for implicit router
       docs.add(doc);
     }
@@ -287,7 +374,17 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
     client.commit(collectionName);
 
     log.info("Indexed {} docs to collection: {}", numDocs, collectionName);
+  }
 
+  private int indexDocs(String collectionName, boolean useUUID) throws Exception {
+    Random random = new Random(docsSeed);// use a constant seed for the whole test run so that we can easily re-index.
+    int numDocs = random.nextInt(100);
+    if (numDocs == 0) {
+      log.info("Indexing ZERO test docs");
+      return 0;
+    }
+
+    indexDocs(collectionName, numDocs, useUUID);
     return numDocs;
   }
 
@@ -373,7 +470,7 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
     props.setProperty("customKey", "customVal");
     restore.setProperties(props);
 
-    if (sameConfig==false) {
+    if (!sameConfig) {
       restore.setConfigName("customConfigName");
     }
     if (random().nextBoolean()) {
@@ -451,4 +548,132 @@ public abstract class AbstractCloudBackupRestoreTestCase extends SolrCloudTestCa
     }
     return shardToDocCount;
   }
+
+  private class IncrementalBackupVerifier {
+    private BackupRepository repository;
+    private String backupLocation;
+    private String backupName;
+    private Map<String, Collection<String>> lastShardCommitToBackupFiles = new HashMap<>();
+    private List<URI> corruptedFiles = new ArrayList<>();
+
+    IncrementalBackupVerifier(BackupRepository repository, String backupLocation, String backupName) {
+      this.repository = repository;
+      this.backupLocation = backupLocation;
+      this.backupName = backupName;
+    }
+
+    private void corruptBackupFiles() throws Exception {
+      for(Slice slice : getCollectionState(getCollectionName()).getSlices()) {
+        Replica leader = slice.getLeader();
+
+        try (SolrCore solrCore = cluster.getReplicaJetty(leader).getCoreContainer().getCore(leader.getCoreName())) {
+          Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
+          try {
+            URI snapshotDirPath = repository.resolve(repository.createURI(backupLocation), backupName, "snapshot." + slice.getName());
+
+            String[] listFiles = repository.listAll(snapshotDirPath);
+            for (String file : listFiles) {
+              if (random().nextBoolean())
+                continue;
+
+              repository.delete(snapshotDirPath, Collections.singleton(file));
+              try (OutputStream os = repository.createOutput(repository.resolve(snapshotDirPath, file))) {
+                byte[] corruptedData = new byte[random().nextInt(30) + 1];
+                random().nextBytes(corruptedData);
+                os.write(corruptedData);
+              }
+              corruptedFiles.add(repository.resolve(snapshotDirPath, file));
+            }
+          } finally {
+            solrCore.getDirectoryFactory().release(dir);
+          }
+        }
+      }
+    }
+
+    private void backupThenWait() throws SolrServerException, IOException {
+      CollectionAdminRequest.Backup backup = CollectionAdminRequest.backupCollection(getCollectionName(), backupName)
+          .setLocation(backupLocation).setIncremental(true).setRepositoryName(getBackupRepoName());
+      assertEquals(0, backup.process(cluster.getSolrClient()).getStatus());
+    }
+
+    void incrementalBackupThenVerify() throws IOException, SolrServerException {
+      int numCopiedFiles = copiedFiles().size();
+      backupThenWait();
+      List<URI> newFilesCopiedOver = copiedFiles().subList(numCopiedFiles, copiedFiles().size());
+      verify(newFilesCopiedOver, corruptedFiles);
+
+      // these corrupted files are no longer valid for the next check since they are healed by the last backup
+      corruptedFiles.clear();
+    }
+
+    public void verify(List<URI> newFilesCopiedOver, List<URI> corruptedFiles) throws IOException {
+      //TODO Datcm verify each backup will reupload config files
+
+      // verify indexes file
+      for(Slice slice : getCollectionState(getCollectionName()).getSlices()) {
+        Replica leader = slice.getLeader();
+
+        try (SolrCore solrCore = cluster.getReplicaJetty(leader).getCoreContainer().getCore(leader.getCoreName())) {
+          Directory dir = solrCore.getDirectoryFactory().get(solrCore.getIndexDir(), DirectoryFactory.DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
+          try {
+
+            URI snapshotDirPath = repository.resolve(repository.createURI(backupLocation), backupName, "snapshot." + slice.getName());
+            IndexCommit lastCommit = solrCore.getDeletionPolicy().getLatestCommit();
+            Collection<URI> newFilesInIndex = newFilesComparedToLastBackup(slice.getName(), lastCommit)
+                .stream()
+                .map(f -> repository.resolve(snapshotDirPath, f))
+                .collect(Collectors.toList());
+
+            String[] listFiles = repository.listAll(snapshotDirPath);
+
+            lastCommit.getFileNames().forEach(
+                f -> {
+                  URI remoteURI = repository.resolve(snapshotDirPath, f);
+                  if (newFilesInIndex.contains(remoteURI) || corruptedFiles.contains(remoteURI)) {
+                    assertTrue(newFilesCopiedOver.contains(remoteURI));
+                  } else {
+                    assertFalse(newFilesCopiedOver.contains(remoteURI));
+                  }
+                }
+            );
+
+            assertEquals("Incremental backup stored more files than needed stored: " + Arrays.toString(listFiles) +
+                " commit-files: " + lastCommit.getFileNames(),lastCommit.getFileNames().size(), listFiles.length);
+
+            Set<String> allChecksums = new HashSet<>();
+            // assert checksum are same
+            Arrays.stream(listFiles)
+                .forEach(f -> {
+                  try {
+                    BackupRepository.Checksum localChecksum = repository.checksum(dir, f);
+                    BackupRepository.Checksum remoteChecksum = repository.checksum(snapshotDirPath, f);
+                    allChecksums.add(localChecksum.checksum);
+                    allChecksums.add(remoteChecksum.checksum);
+                    assertEquals(repository.checksum(dir, f), repository.checksum(snapshotDirPath, f));
+                  } catch (IOException e) {
+                    throw new IllegalStateException(e);
+                  }
+                });
+
+            if (listFiles.length > 7) {
+              assertNotEquals("All files have same checksum, this likely be a bug in checksum implementation",1, allChecksums.size());
+            }
+          } finally {
+            solrCore.getDirectoryFactory().release(dir);
+          }
+        }
+      }
+    }
+
+    private Collection<String> newFilesComparedToLastBackup(String shardName, IndexCommit currentCommit) throws IOException {
+      Collection<String> oldFiles = lastShardCommitToBackupFiles.put(shardName, currentCommit.getFileNames());
+      if (oldFiles == null)
+        oldFiles = new ArrayList<>();
+
+      List<String> newFiles = new ArrayList<>(currentCommit.getFileNames());
+      newFiles.removeAll(oldFiles);
+      return newFiles;
+    }
+  }
 }
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
index 2e7ea10..388d2ef 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestHdfsCloudBackupRestore.java
@@ -88,6 +88,9 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
       "  </solrcloud>\n" +
       "  \n" +
       "  <backup>\n" +
+      "    <repository name=\"trackingBackupRepo\" default=\"true\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n" +
+      "      <str name=\"delegateRepoName\">hdfs</str>\n" +
+      "    </repository>\n" +
       "    <repository  name=\"hdfs\" class=\"org.apache.solr.core.backup.repository.HdfsBackupRepository\"> \n" +
       "      <str name=\"location\">${solr.hdfs.default.backup.path}</str>\n" +
       "      <str name=\"solr.hdfs.home\">${solr.hdfs.home:}</str>\n" +
@@ -162,11 +165,6 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
   }
 
   @Override
-  public String getBackupRepoName() {
-    return "hdfs";
-  }
-
-  @Override
   public String getBackupLocation() {
     return null;
   }
@@ -186,17 +184,17 @@ public class TestHdfsCloudBackupRestore extends AbstractCloudBackupRestoreTestCa
 
     HdfsBackupRepository repo = new HdfsBackupRepository();
     repo.init(new NamedList<>(params));
-    BackupManager mgr = new BackupManager(repo, solrClient.getZkStateReader());
 
     URI baseLoc = repo.createURI("/backup");
 
-    Properties props = mgr.readBackupProperties(baseLoc, backupName);
+    BackupManager mgr = BackupManager.forRestore(repo, solrClient.getZkStateReader(), baseLoc, backupName);
+    Properties props = mgr.readBackupProperties();
     assertNotNull(props);
     assertEquals(collectionName, props.getProperty(COLLECTION_NAME_PROP));
     assertEquals(backupName, props.getProperty(BACKUP_NAME_PROP));
     assertEquals(configName, props.getProperty(COLL_CONF));
 
-    DocCollection collectionState = mgr.readCollectionState(baseLoc, backupName, collectionName);
+    DocCollection collectionState = mgr.readCollectionState(collectionName);
     assertNotNull(collectionState);
     assertEquals(collectionName, collectionState.getName());
 
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java
index 2b6abf1..5c4228b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestLocalFSCloudBackupRestore.java
@@ -17,7 +17,6 @@
 package org.apache.solr.cloud.api.collections;
 
 import org.apache.hadoop.fs.Path;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.apache.solr.common.cloud.ZkConfigManager;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -27,8 +26,40 @@ import org.junit.Test;
  * Solr backup/restore still requires a "shared" file-system. Its just that in this case such file-system would be
  * exposed via local file-system API.
  */
-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-12866")
 public class TestLocalFSCloudBackupRestore extends AbstractCloudBackupRestoreTestCase {
+  public static final String SOLR_XML = "<solr>\n" +
+      "\n" +
+      "  <str name=\"shareSchema\">${shareSchema:false}</str>\n" +
+      "  <str name=\"configSetBaseDir\">${configSetBaseDir:configsets}</str>\n" +
+      "  <str name=\"coreRootDirectory\">${coreRootDirectory:.}</str>\n" +
+      "\n" +
+      "  <shardHandlerFactory name=\"shardHandlerFactory\" class=\"HttpShardHandlerFactory\">\n" +
+      "    <str name=\"urlScheme\">${urlScheme:}</str>\n" +
+      "    <int name=\"socketTimeout\">${socketTimeout:90000}</int>\n" +
+      "    <int name=\"connTimeout\">${connTimeout:15000}</int>\n" +
+      "  </shardHandlerFactory>\n" +
+      "\n" +
+      "  <solrcloud>\n" +
+      "    <str name=\"host\">127.0.0.1</str>\n" +
+      "    <int name=\"hostPort\">${hostPort:8983}</int>\n" +
+      "    <str name=\"hostContext\">${hostContext:solr}</str>\n" +
+      "    <int name=\"zkClientTimeout\">${solr.zkclienttimeout:30000}</int>\n" +
+      "    <bool name=\"genericCoreNodeNames\">${genericCoreNodeNames:true}</bool>\n" +
+      "    <int name=\"leaderVoteWait\">10000</int>\n" +
+      "    <int name=\"distribUpdateConnTimeout\">${distribUpdateConnTimeout:45000}</int>\n" +
+      "    <int name=\"distribUpdateSoTimeout\">${distribUpdateSoTimeout:340000}</int>\n" +
+      "  </solrcloud>\n" +
+      "  \n" +
+      "  <backup>\n" +
+      "    <repository name=\"trackingBackupRepo\" class=\"org.apache.solr.core.TrackingBackupRepository\"> \n" +
+      "      <str name=\"delegateRepoName\">localfs</str>\n" +
+      "    </repository>\n" +
+      "    <repository name=\"localfs\" class=\"org.apache.solr.core.backup.repository.LocalFileSystemRepository\"> \n" +
+      "    </repository>\n" +
+      "  </backup>\n" +
+      "  \n" +
+      "</solr>\n";
+
   private static String backupLocation;
 
   @BeforeClass
@@ -36,6 +67,7 @@ public class TestLocalFSCloudBackupRestore extends AbstractCloudBackupRestoreTes
     configureCluster(NUM_SHARDS)// nodes
         .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
         .addConfig("confFaulty", TEST_PATH().resolve("configsets").resolve("cloud-minimal").resolve("conf"))
+        .withSolrXml(SOLR_XML)
         .configure();
     cluster.getZkClient().delete(ZkConfigManager.CONFIGS_ZKNODE + Path.SEPARATOR + "confFaulty" + Path.SEPARATOR + "solrconfig.xml", -1, true);
 
@@ -53,11 +85,6 @@ public class TestLocalFSCloudBackupRestore extends AbstractCloudBackupRestoreTes
   }
 
   @Override
-  public String getBackupRepoName() {
-    return null;
-  }
-
-  @Override
   public String getBackupLocation() {
     return backupLocation;
   }
diff --git a/solr/core/src/test/org/apache/solr/core/backup/FileGenerationTest.java b/solr/core/src/test/org/apache/solr/core/backup/FileGenerationTest.java
new file mode 100644
index 0000000..0401f85
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/core/backup/FileGenerationTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core.backup;
+
+import java.util.Optional;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class FileGenerationTest {
+
+  @Test
+  public void test() {
+    FileGeneration fileGen = FileGeneration.zeroGen("backup.properties");
+    assertEquals("backup.properties", fileGen.getFileName());
+    fileGen = fileGen.nextGen();
+    assertEquals("backup-1.properties", fileGen.getFileName());
+    fileGen = fileGen.nextGen();
+    assertEquals("backup-2.properties", fileGen.getFileName());
+
+    fileGen = FileGeneration.findMostRecent("backup.properties",
+        new String[] {"aaa", "baa.properties", "backup.properties",
+            "backup-1.properties", "backup-2.properties", "backup-neqewq.properties", "backup999.properties"}).get();
+    assertEquals("backup-2.properties", fileGen.getFileName());
+    fileGen = fileGen.nextGen();
+    assertEquals("backup-3.properties", fileGen.getFileName());
+
+    Optional<FileGeneration> op = FileGeneration.findMostRecent("backup.properties",
+        new String[0]);
+    assertTrue(op.isEmpty());
+  }
+}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 7f3cbd4..c461b37 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -1008,6 +1008,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
     protected String location;
     protected Optional<String> commitName = Optional.empty();
     protected Optional<String> indexBackupStrategy = Optional.empty();
+    protected boolean incremental = false;
 
     public Backup(String collection, String name) {
       super(CollectionAction.BACKUP, collection);
@@ -1051,6 +1052,11 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
       return this;
     }
 
+    public Backup setIncremental(boolean incremental) {
+      this.incremental = incremental;
+      return this;
+    }
+
     @Override
     public SolrParams getParams() {
       ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
@@ -1066,6 +1072,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
       if (indexBackupStrategy.isPresent()) {
         params.set(CollectionAdminParams.INDEX_BACKUP_STRATEGY, indexBackupStrategy.get());
       }
+      params.set(CoreAdminParams.BACKUP_INCREMENTAL, incremental);
       return params;
     }
 
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
index 9103450..736a406 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CoreAdminParams.java
@@ -121,6 +121,11 @@ public abstract class CoreAdminParams
   public static final String BACKUP_LOCATION = "location";
 
   /**
+   * A parameter to specify whether incremental backup is used
+   */
+  public static final String BACKUP_INCREMENTAL = "incremental";
+
+  /**
    * A parameter to specify the name of the commit to be stored during the backup operation.
    */
   public static final String COMMIT_NAME = "commitName";
diff --git a/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java b/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java
new file mode 100644
index 0000000..376baee
--- /dev/null
+++ b/solr/test-framework/src/java/org/apache/solr/core/TrackingBackupRepository.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.core;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.core.backup.repository.BackupRepositoryFactory;
+
+public class TrackingBackupRepository implements BackupRepository {
+  private static final List<URI> COPIED_FILES = Collections.synchronizedList(new ArrayList<>());
+
+  private BackupRepository delegate;
+
+  @Override
+  public <T> T getConfigProperty(String name) {
+    return delegate.getConfigProperty(name);
+  }
+
+  @Override
+  public URI createURI(String path) {
+    return delegate.createURI(path);
+  }
+
+  @Override
+  public URI resolve(URI baseUri, String... pathComponents) {
+    return delegate.resolve(baseUri, pathComponents);
+  }
+
+  @Override
+  public boolean exists(URI path) throws IOException {
+    return delegate.exists(path);
+  }
+
+  @Override
+  public PathType getPathType(URI path) throws IOException {
+    return delegate.getPathType(path);
+  }
+
+  @Override
+  public String[] listAll(URI path) throws IOException {
+    return delegate.listAll(path);
+  }
+
+  @Override
+  public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException {
+    return delegate.openInput(dirPath, fileName, ctx);
+  }
+
+  @Override
+  public OutputStream createOutput(URI path) throws IOException {
+    return delegate.createOutput(path);
+  }
+
+  @Override
+  public void createDirectory(URI path) throws IOException {
+    delegate.createDirectory(path);
+  }
+
+  @Override
+  public void deleteDirectory(URI path) throws IOException {
+    delegate.deleteDirectory(path);
+  }
+
+  @Override
+  public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException {
+    COPIED_FILES.add(delegate.resolve(dest, fileName));
+    delegate.copyFileFrom(sourceDir, fileName, dest);
+  }
+
+  @Override
+  public void copyFileTo(URI sourceRepo, String fileName, Directory dest) throws IOException {
+    delegate.copyFileTo(sourceRepo, fileName, dest);
+  }
+
+  @Override
+  public void close() throws IOException {
+    delegate.close();
+  }
+
+
+  @Override
+  public void delete(URI path, Collection<String> files) throws IOException {
+    delegate.delete(path, files);
+  }
+
+  @Override
+  public Checksum checksum(URI path, String fileName) throws IOException {
+    return delegate.checksum(path, fileName);
+  }
+
+  @Override
+  public Checksum checksum(Directory dir, String fileName) throws IOException {
+    return delegate.checksum(dir, fileName);
+  }
+
+  @Override
+  public void init(NamedList args) {
+    BackupRepositoryFactory factory = (BackupRepositoryFactory) args.get("factory");
+    SolrResourceLoader loader = (SolrResourceLoader) args.get("loader");
+    String repoName = (String) args.get("delegateRepoName");
+
+    this.delegate = factory.newInstance(loader, repoName);
+  }
+
+  /**
+   * @return list of files were copied by using {@link #copyFileFrom(Directory, String, URI)}
+   */
+  public static List<URI> copiedFiles() {
+    return new ArrayList<>(COPIED_FILES);
+  }
+
+  /**
+   * Clear all tracking data
+   */
+  public static void clear() {
+    COPIED_FILES.clear();
+  }
+
+}