You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2023/03/02 19:05:41 UTC
[ozone] branch master updated: HDDS-7905. [Snapshot] Snapdiff should only return modifications done to the bucket. (#4273)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1f8795b4fd HDDS-7905. [Snapshot] Snapdiff should only return modifications done to the bucket. (#4273)
1f8795b4fd is described below
commit 1f8795b4fd9108865f8b1f52bbe3de48ba5f324a
Author: Sadanand Shenoy <sa...@gmail.com>
AuthorDate: Fri Mar 3 00:35:35 2023 +0530
HDDS-7905. [Snapshot] Snapdiff should only return modifications done to the bucket. (#4273)
---
hadoop-hdds/common/pom.xml | 4 ++
.../db/managed/ManagedBlockBasedTableConfig.java | 0
.../hdds/utils/db/managed/ManagedBloomFilter.java | 0
.../hdds/utils/db/managed/ManagedCheckpoint.java | 0
.../db/managed/ManagedColumnFamilyOptions.java | 0
.../db/managed/ManagedCompactRangeOptions.java | 0
.../hdds/utils/db/managed/ManagedDBOptions.java | 0
.../hdds/utils/db/managed/ManagedEnvOptions.java | 0
.../hdds/utils/db/managed/ManagedFlushOptions.java | 0
.../managed/ManagedIngestExternalFileOptions.java | 0
.../hdds/utils/db/managed/ManagedLRUCache.java | 0
.../hdds/utils/db/managed/ManagedObject.java | 0
.../hdds/utils/db/managed/ManagedOptions.java | 0
.../hdds/utils/db/managed/ManagedReadOptions.java | 0
.../hdds/utils/db/managed/ManagedRocksDB.java | 0
.../utils/db/managed/ManagedRocksIterator.java | 0
.../db/managed/ManagedRocksObjectMetrics.java | 0
.../utils/db/managed/ManagedRocksObjectUtils.java | 0
.../hadoop/hdds/utils/db/managed/ManagedSlice.java | 0
.../utils/db/managed/ManagedSstFileReader.java} | 16 +++--
.../db/managed/ManagedSstFileReaderIterator.java} | 21 +++---
.../utils/db/managed/ManagedSstFileWriter.java | 0
.../hdds/utils/db/managed/ManagedStatistics.java | 0
.../db/managed/ManagedTransactionLogIterator.java | 0
.../hdds/utils/db/managed/ManagedWriteBatch.java | 0
.../hdds/utils/db/managed/ManagedWriteOptions.java | 0
.../hadoop/hdds/utils/db/managed/package-info.java | 0
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 5 ++
.../common/src/main/resources/ozone-default.xml | 10 +++
hadoop-hdds/framework/pom.xml | 4 --
.../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 44 ++++---------
.../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 67 +++++++++++++++++--
.../rocksdiff/TestRocksDBCheckpointDiffer.java | 3 +-
.../hadoop/ozone/freon/TestOMSnapshotDAG.java | 2 +-
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 53 ++++++++++++---
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 3 +-
.../ozone/om/snapshot/SnapshotDiffManager.java | 77 +++++++++++++++++-----
37 files changed, 224 insertions(+), 85 deletions(-)
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index d63515b3b7..d1e22ab79a 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -242,6 +242,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>spotbugs-annotations</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.rocksdb</groupId>
+ <artifactId>rocksdbjni</artifactId>
+ </dependency>
</dependencies>
<build>
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBlockBasedTableConfig.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedBloomFilter.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCheckpoint.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCheckpoint.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCheckpoint.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCheckpoint.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedColumnFamilyOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedColumnFamilyOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedColumnFamilyOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedColumnFamilyOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCompactRangeOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCompactRangeOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCompactRangeOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedCompactRangeOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedDBOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedEnvOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedEnvOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedEnvOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedEnvOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedFlushOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedFlushOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedFlushOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedFlushOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedIngestExternalFileOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedIngestExternalFileOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedIngestExternalFileOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedIngestExternalFileOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedLRUCache.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedLRUCache.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedLRUCache.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedLRUCache.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedObject.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedReadOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedReadOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedReadOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedReadOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksDB.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksIterator.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksIterator.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksIterator.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectUtils.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSlice.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSlice.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSlice.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSlice.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReader.java
similarity index 75%
copy from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
copy to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReader.java
index 44a50dd42d..7ba1001a43 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReader.java
@@ -18,19 +18,20 @@
*/
package org.apache.hadoop.hdds.utils.db.managed;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.SstFileReader;
/**
- * Managed WriteBatch.
+ * Managed SstFileReader.
*/
-public class ManagedWriteBatch extends WriteBatch {
+public class ManagedSstFileReader extends ManagedObject<SstFileReader> {
- public ManagedWriteBatch() {
- super();
+ ManagedSstFileReader(SstFileReader original) {
+ super(original);
}
- public ManagedWriteBatch(byte[] data) {
- super(data);
+ public static ManagedSstFileReader managed(
+ SstFileReader reader) {
+ return new ManagedSstFileReader(reader);
}
@Override
@@ -38,4 +39,5 @@ public class ManagedWriteBatch extends WriteBatch {
ManagedRocksObjectUtils.assertClosed(this);
super.finalize();
}
+
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReaderIterator.java
similarity index 71%
copy from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
copy to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReaderIterator.java
index 44a50dd42d..0916e89571 100644
--- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileReaderIterator.java
@@ -18,19 +18,16 @@
*/
package org.apache.hadoop.hdds.utils.db.managed;
-import org.rocksdb.WriteBatch;
+import org.rocksdb.SstFileReaderIterator;
/**
- * Managed WriteBatch.
+ * Managed SstFileReaderIterator.
*/
-public class ManagedWriteBatch extends WriteBatch {
+public class ManagedSstFileReaderIterator
+ extends ManagedObject<SstFileReaderIterator> {
- public ManagedWriteBatch() {
- super();
- }
-
- public ManagedWriteBatch(byte[] data) {
- super(data);
+ ManagedSstFileReaderIterator(SstFileReaderIterator original) {
+ super(original);
}
@Override
@@ -38,4 +35,10 @@ public class ManagedWriteBatch extends WriteBatch {
ManagedRocksObjectUtils.assertClosed(this);
super.finalize();
}
+
+ public static ManagedSstFileReaderIterator managed(
+ SstFileReaderIterator iterator) {
+ return new ManagedSstFileReaderIterator(iterator);
+ }
+
}
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileWriter.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileWriter.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileWriter.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedSstFileWriter.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedStatistics.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedStatistics.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedStatistics.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedStatistics.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedTransactionLogIterator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedTransactionLogIterator.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedTransactionLogIterator.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedTransactionLogIterator.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteBatch.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteOptions.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteOptions.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteOptions.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedWriteOptions.java
diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/package-info.java
similarity index 100%
rename from hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/managed/package-info.java
rename to hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/utils/db/managed/package-info.java
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index e447dd7ba2..5e924be7f9 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -603,6 +603,11 @@ public final class OzoneConfigKeys {
OZONE_OM_SNAPSHOT_PRUNE_COMPACTION_DAG_DAEMON_RUN_INTERVAL_DEFAULT =
TimeUnit.HOURS.toMillis(1);
+ public static final String OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF =
+ "ozone.om.snapshot.force.full.diff";
+
+ public static final boolean OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT = false;
+
/**
* There is no need to instantiate this class.
*/
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 854f643a88..433001b822 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -3607,4 +3607,14 @@
production environments.
</description>
</property>
+
+ <property>
+ <name>ozone.om.snapshot.force.full.diff</name>
+ <value>false</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ If true, snapshot diff will always perform full diff (can be slow)
+ without using the optimised DAG based pruning approach
+ </description>
+ </property>
</configuration>
diff --git a/hadoop-hdds/framework/pom.xml b/hadoop-hdds/framework/pom.xml
index 4f82a355fd..7347e03366 100644
--- a/hadoop-hdds/framework/pom.xml
+++ b/hadoop-hdds/framework/pom.xml
@@ -109,10 +109,6 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.rocksdb</groupId>
- <artifactId>rocksdbjni</artifactId>
- </dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_dropwizard</artifactId>
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
index b74de855f1..e48340373f 100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java
@@ -35,11 +35,9 @@ import org.rocksdb.CompactionJobInfo;
import org.rocksdb.DBOptions;
import org.rocksdb.LiveFileMetaData;
import org.rocksdb.Options;
-import org.rocksdb.ReadOptions;
import org.rocksdb.RocksDB;
import org.rocksdb.RocksDBException;
import org.rocksdb.SstFileReader;
-import org.rocksdb.SstFileReaderIterator;
import org.rocksdb.TableProperties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -759,7 +757,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable {
* "/path/to/sstBackupDir/000060.sst"]
*/
public List<String> getSSTDiffListWithFullPath(
- DifferSnapshotInfo src, DifferSnapshotInfo dest) {
+ DifferSnapshotInfo src, DifferSnapshotInfo dest) throws IOException {
List<String> sstDiffList = getSSTDiffList(src, dest);
@@ -780,7 +778,7 @@ public class RocksDBCheckpointDiffer implements AutoCloseable {
* @return A list of SST files without extension. e.g. ["000050", "000060"]
*/
public synchronized List<String> getSSTDiffList(
- DifferSnapshotInfo src, DifferSnapshotInfo dest) {
+ DifferSnapshotInfo src, DifferSnapshotInfo dest) throws IOException {
// TODO: Reject or swap if dest is taken after src, once snapshot chain
// integration is done.
@@ -816,42 +814,26 @@ public class RocksDBCheckpointDiffer implements AutoCloseable {
}
if (src.getTablePrefixes() != null && !src.getTablePrefixes().isEmpty()) {
- filterRelevantSstFiles(fwdDAGDifferentFiles, src.getTablePrefixes());
+ filterRelevantSstFilesFullPath(fwdDAGDifferentFiles,
+ src.getTablePrefixes());
}
return new ArrayList<>(fwdDAGDifferentFiles);
}
- public void filterRelevantSstFiles(Set<String> inputFiles,
- Map<String, String> tableToPrefixMap) {
+ /**
+ * construct absolute sst file path first and
+ * filter the files.
+ */
+ public void filterRelevantSstFilesFullPath(Set<String> inputFiles,
+ Map<String, String> tableToPrefixMap) throws IOException {
for (Iterator<String> fileIterator =
inputFiles.iterator(); fileIterator.hasNext();) {
String filename = fileIterator.next();
String filepath = getAbsoluteSstFilePath(filename);
- try (SstFileReader sstFileReader = new SstFileReader(new Options())) {
- sstFileReader.open(filepath);
- TableProperties properties = sstFileReader.getTableProperties();
- String tableName = new String(properties.getColumnFamilyName(), UTF_8);
- if (tableToPrefixMap.containsKey(tableName)) {
- String prefix = tableToPrefixMap.get(tableName);
- SstFileReaderIterator iterator =
- sstFileReader.newIterator(new ReadOptions());
- iterator.seekToFirst();
- String firstKey = RocksDiffUtils
- .constructBucketKey(new String(iterator.key(), UTF_8));
- iterator.seekToLast();
- String lastKey = RocksDiffUtils
- .constructBucketKey(new String(iterator.key(), UTF_8));
- if (!RocksDiffUtils
- .isKeyWithPrefixPresent(prefix, firstKey, lastKey)) {
- fileIterator.remove();
- }
- } else {
- // entry from other tables
- fileIterator.remove();
- }
- } catch (RocksDBException e) {
- e.printStackTrace();
+ if (!RocksDiffUtils.doesSstFileContainKeyRange(filepath,
+ tableToPrefixMap)) {
+ fileIterator.remove();
}
}
}
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
index d6d25bbede..cca3eaf0e7 100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDiffUtils.java
@@ -18,13 +18,32 @@
package org.apache.ozone.rocksdiff;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReader;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedSstFileReaderIterator;
+import org.rocksdb.SstFileReader;
+import org.rocksdb.TableProperties;
+import org.rocksdb.Options;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RocksDBException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
/**
* Helper methods for snap-diff operations.
*/
public final class RocksDiffUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RocksDiffUtils.class);
+
private RocksDiffUtils() {
}
@@ -35,20 +54,56 @@ public final class RocksDiffUtils {
}
public static String constructBucketKey(String keyName) {
- if (!keyName.startsWith(OzoneConsts.OM_KEY_PREFIX)) {
- keyName = OzoneConsts.OM_KEY_PREFIX.concat(keyName);
+ if (!keyName.startsWith(OM_KEY_PREFIX)) {
+ keyName = OM_KEY_PREFIX.concat(keyName);
}
- String[] elements = keyName.split(OzoneConsts.OM_KEY_PREFIX);
+ String[] elements = keyName.split(OM_KEY_PREFIX);
String volume = elements[1];
String bucket = elements[2];
StringBuilder builder =
- new StringBuilder().append(OzoneConsts.OM_KEY_PREFIX).append(volume);
+ new StringBuilder().append(OM_KEY_PREFIX).append(volume);
if (StringUtils.isNotBlank(bucket)) {
- builder.append(OzoneConsts.OM_KEY_PREFIX).append(bucket);
+ builder.append(OM_KEY_PREFIX).append(bucket);
}
return builder.toString();
}
+ public static void filterRelevantSstFiles(Set<String> inputFiles,
+ Map<String, String> tableToPrefixMap) throws IOException {
+ for (Iterator<String> fileIterator =
+ inputFiles.iterator(); fileIterator.hasNext();) {
+ String filepath = fileIterator.next();
+ if (!RocksDiffUtils.doesSstFileContainKeyRange(filepath,
+ tableToPrefixMap)) {
+ fileIterator.remove();
+ }
+ }
+ }
+
+ public static boolean doesSstFileContainKeyRange(String filepath,
+ Map<String, String> tableToPrefixMap) throws IOException {
+ try (ManagedSstFileReader sstFileReader = ManagedSstFileReader.managed(
+ new SstFileReader(new Options()))) {
+ sstFileReader.get().open(filepath);
+ TableProperties properties = sstFileReader.get().getTableProperties();
+ String tableName = new String(properties.getColumnFamilyName(), UTF_8);
+ if (tableToPrefixMap.containsKey(tableName)) {
+ String prefix = tableToPrefixMap.get(tableName) + OM_KEY_PREFIX;
+ try (ManagedSstFileReaderIterator iterator =
+ ManagedSstFileReaderIterator.managed(sstFileReader.get()
+ .newIterator(new ReadOptions()))) {
+ iterator.get().seek(prefix.getBytes(UTF_8));
+ String seekResultKey = new String(iterator.get().key(), UTF_8);
+ return seekResultKey.startsWith(prefix);
+ }
+ }
+ return false;
+ } catch (RocksDBException e) {
+ LOG.error("Failed to read SST File ", e);
+ throw new IOException(e);
+ }
+ }
+
}
diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
index 871b27446e..f634222c63 100644
--- a/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
+++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/test/java/org/apache/ozone/rocksdiff/TestRocksDBCheckpointDiffer.java
@@ -330,7 +330,8 @@ public class TestRocksDBCheckpointDiffer {
/**
* Test SST differ.
*/
- void diffAllSnapshots(RocksDBCheckpointDiffer differ) {
+ void diffAllSnapshots(RocksDBCheckpointDiffer differ)
+ throws IOException {
final DifferSnapshotInfo src = snapshots.get(snapshots.size() - 1);
// Hard-coded expected output.
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
index c89f20c8c9..2414c6dfd4 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java
@@ -165,7 +165,7 @@ public class TestOMSnapshotDAG {
@Test
public void testDAGReconstruction()
- throws IOException, InterruptedException, TimeoutException {
+ throws IOException, InterruptedException, TimeoutException {
// Generate keys
RandomKeyGenerator randomKeyGenerator =
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index 6bb4f3f202..a5941fff9f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.utils.db.DBProfile;
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -94,6 +95,7 @@ public class TestOmSnapshot {
private static OzoneManagerProtocol writeClient;
private static BucketLayout bucketLayout = BucketLayout.LEGACY;
private static boolean enabledFileSystemPaths;
+ private static boolean forceFullSnapshotDiff;
private static ObjectStore store;
private static File metaDir;
private static OzoneManager leaderOzoneManager;
@@ -108,30 +110,34 @@ public class TestOmSnapshot {
@Parameterized.Parameters
public static Collection<Object[]> data() {
return Arrays.asList(
- new Object[]{OBJECT_STORE, false},
- new Object[]{FILE_SYSTEM_OPTIMIZED, false},
- new Object[]{BucketLayout.LEGACY, true});
+ new Object[]{OBJECT_STORE, false, false},
+ new Object[]{FILE_SYSTEM_OPTIMIZED, false, false},
+ new Object[]{BucketLayout.LEGACY, true, true});
}
public TestOmSnapshot(BucketLayout newBucketLayout,
- boolean newEnableFileSystemPaths) throws Exception {
+ boolean newEnableFileSystemPaths, boolean forceFullSnapDiff)
+ throws Exception {
// Checking whether 'newBucketLayout' and
// 'newEnableFileSystemPaths' flags represents next parameter
// index values. This is to ensure that initialize init() function
// will be invoked only at the beginning of every new set of
// Parameterized.Parameters.
if (TestOmSnapshot.enabledFileSystemPaths != newEnableFileSystemPaths ||
- TestOmSnapshot.bucketLayout != newBucketLayout) {
- setConfig(newBucketLayout, newEnableFileSystemPaths);
+ TestOmSnapshot.bucketLayout != newBucketLayout ||
+ TestOmSnapshot.forceFullSnapshotDiff != forceFullSnapDiff) {
+ setConfig(newBucketLayout, newEnableFileSystemPaths,
+ forceFullSnapDiff);
tearDown();
init();
}
}
private static void setConfig(BucketLayout newBucketLayout,
- boolean newEnableFileSystemPaths) {
+ boolean newEnableFileSystemPaths, boolean forceFullSnapDiff) {
TestOmSnapshot.enabledFileSystemPaths = newEnableFileSystemPaths;
TestOmSnapshot.bucketLayout = newBucketLayout;
+ TestOmSnapshot.forceFullSnapshotDiff = forceFullSnapDiff;
}
/**
@@ -145,6 +151,8 @@ public class TestOmSnapshot {
enabledFileSystemPaths);
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
bucketLayout.name());
+ conf.setBoolean(OzoneConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF,
+ forceFullSnapshotDiff);
conf.setEnum(HDDS_DB_PROFILE, DBProfile.TEST);
cluster = MiniOzoneCluster.newOMHABuilder(conf)
@@ -325,10 +333,10 @@ public class TestOmSnapshot {
}
}
OmKeyArgs keyArgs = genKeyArgs(snapshotKeyPrefix + key1);
-
+
OmKeyInfo omKeyInfo = writeClient.lookupKey(keyArgs);
assertEquals(omKeyInfo.getKeyName(), snapshotKeyPrefix + key1);
-
+
OmKeyInfo fileInfo = writeClient.lookupFile(keyArgs);
assertEquals(fileInfo.getKeyName(), snapshotKeyPrefix + key1);
@@ -615,6 +623,33 @@ public class TestOmSnapshot {
() -> store.snapshotDiff(nullstr, bucket, snap1, snap2));
}
+ @Test
+ public void testSnapDiffMultipleBuckets() throws Exception {
+ String volume = "vol-" + RandomStringUtils.randomNumeric(5);
+ String bucketName1 = "buck-" + RandomStringUtils.randomNumeric(5);
+ String bucketName2 = "buck-" + RandomStringUtils.randomNumeric(5);
+ store.createVolume(volume);
+ OzoneVolume volume1 = store.getVolume(volume);
+ volume1.createBucket(bucketName1);
+ volume1.createBucket(bucketName2);
+ OzoneBucket bucket1 = volume1.getBucket(bucketName1);
+ OzoneBucket bucket2 = volume1.getBucket(bucketName2);
+ // Create Key1 and take snapshot
+ String key1 = "key-1-";
+ key1 = createFileKey(bucket1, key1);
+ String snap1 = "snap" + RandomStringUtils.randomNumeric(5);
+ createSnapshot(volume, bucketName1, snap1);
+ // Create key in bucket2 and bucket1 and calculate diff
+ // Diff should not contain bucket2's key
+ createFileKey(bucket1, key1);
+ createFileKey(bucket2, key1);
+ String snap2 = "snap" + RandomStringUtils.randomNumeric(5);
+ createSnapshot(volume, bucketName1, snap2);
+ SnapshotDiffReport diff1 =
+ store.snapshotDiff(volume, bucketName1, snap1, snap2);
+ Assert.assertEquals(1, diff1.getDiffList().size());
+ }
+
/**
* Tests snapdiff when there are multiple sst files in the from & to
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 6dfb2d380f..1c1c683c90 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -78,7 +78,8 @@ public final class OmSnapshotManager implements AutoCloseable {
this.snapshotDiffDb =
createDbForSnapshotDiff(ozoneManager.getConfiguration());
- this.snapshotDiffManager = new SnapshotDiffManager(snapshotDiffDb, differ);
+ this.snapshotDiffManager = new SnapshotDiffManager(snapshotDiffDb, differ,
+ ozoneManager.getConfiguration());
// size of lru cache
int cacheSize = ozoneManager.getConfiguration().getInt(
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index 06c601a648..58006e108f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -22,7 +22,9 @@ import java.util.Iterator;
import java.util.UUID;
import org.apache.hadoop.hdds.utils.db.CodecRegistry;
import org.apache.hadoop.hdds.utils.db.IntegerCodec;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions;
import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -42,6 +44,7 @@ import org.apache.ozone.rocksdb.util.ManagedSstFileReader;
import org.apache.ozone.rocksdb.util.RdbUtil;
import org.apache.ozone.rocksdiff.DifferSnapshotInfo;
import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
+import org.apache.ozone.rocksdiff.RocksDiffUtils;
import org.jetbrains.annotations.NotNull;
import org.rocksdb.ColumnFamilyDescriptor;
import org.rocksdb.ColumnFamilyHandle;
@@ -72,10 +75,15 @@ public class SnapshotDiffManager {
private final ManagedRocksDB db;
private final CodecRegistry codecRegistry;
+ private OzoneConfiguration configuration;
+
+
public SnapshotDiffManager(ManagedRocksDB db,
- RocksDBCheckpointDiffer differ) {
+ RocksDBCheckpointDiffer differ,
+ OzoneConfiguration conf) {
this.db = db;
this.differ = differ;
+ this.configuration = conf;
this.codecRegistry = new CodecRegistry();
// Integers are used for indexing persistent list.
@@ -123,6 +131,7 @@ public class SnapshotDiffManager {
getTablePrefixes(snapshotOMMM, volumeName, bucketName));
}
+ @SuppressWarnings("checkstyle:methodlength")
public SnapshotDiffReport getSnapshotDiffReport(final String volume,
final String bucket,
final OmSnapshot fromSnapshot,
@@ -197,14 +206,22 @@ public class SnapshotDiffManager {
codecRegistry,
DiffReportEntry.class);
- final Table<String, OmKeyInfo> fsKeyTable = fromSnapshot
- .getMetadataManager().getKeyTable(bucketLayout);
- final Table<String, OmKeyInfo> tsKeyTable = toSnapshot
- .getMetadataManager().getKeyTable(bucketLayout);
+ final Table<String, OmKeyInfo> fsKeyTable =
+ fromSnapshot.getMetadataManager().getKeyTable(bucketLayout);
+ final Table<String, OmKeyInfo> tsKeyTable =
+ toSnapshot.getMetadataManager().getKeyTable(bucketLayout);
+
+ boolean useFullDiff = configuration.getBoolean(
+ OzoneConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF,
+ OzoneConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT);
+
+ Map<String, String> tablePrefixes =
+ getTablePrefixes(toSnapshot.getMetadataManager(), volume, bucket);
+
final Set<String> deltaFilesForKeyOrFileTable =
getDeltaFiles(fromSnapshot, toSnapshot,
- Collections.singletonList(fsKeyTable.getName()),
- fsInfo, tsInfo, volume, bucket);
+ Collections.singletonList(fsKeyTable.getName()), fsInfo, tsInfo,
+ useFullDiff, tablePrefixes);
addToObjectIdMap(fsKeyTable,
tsKeyTable,
@@ -212,7 +229,7 @@ public class SnapshotDiffManager {
objectIdToKeyNameMapForFromSnapshot,
objectIdToKeyNameMapForToSnapshot,
objectIDsToCheckMap,
- false);
+ tablePrefixes);
if (bucketLayout.isFileSystemOptimized()) {
// add to object ID map for directory.
@@ -222,16 +239,15 @@ public class SnapshotDiffManager {
toSnapshot.getMetadataManager().getDirectoryTable();
final Set<String> deltaFilesForDirTable =
getDeltaFiles(fromSnapshot, toSnapshot,
- Collections.singletonList(fsDirTable.getName()),
- fsInfo, tsInfo, volume, bucket);
-
+ Collections.singletonList(fsDirTable.getName()), fsInfo, tsInfo,
+ useFullDiff, tablePrefixes);
addToObjectIdMap(fsDirTable,
tsDirTable,
deltaFilesForDirTable,
objectIdToKeyNameMapForFromSnapshot,
objectIdToKeyNameMapForToSnapshot,
objectIDsToCheckMap,
- true);
+ tablePrefixes);
}
generateDiffReport(requestId,
@@ -281,7 +297,11 @@ public class SnapshotDiffManager {
PersistentMap<Long, String> oldObjIdToKeyMap,
PersistentMap<Long, String> newObjIdToKeyMap,
PersistentSet<Long> objectIDsToCheck,
- boolean isDirectoryTable) {
+ Map<String, String> tablePrefixes)
+ throws IOException {
+
+ boolean isDirectoryTable =
+ fsTable.getName().equals(OmMetadataManagerImpl.DIRECTORY_TABLE);
if (deltaFiles.isEmpty()) {
return;
@@ -292,7 +312,8 @@ public class SnapshotDiffManager {
try {
final WithObjectID oldKey = fsTable.get(key);
final WithObjectID newKey = tsTable.get(key);
- if (areKeysEqual(oldKey, newKey)) {
+ if (areKeysEqual(oldKey, newKey) || !isKeyInBucket(key, tablePrefixes,
+ fsTable.getName())) {
// We don't have to do anything.
return;
}
@@ -333,14 +354,16 @@ public class SnapshotDiffManager {
private Set<String> getDeltaFiles(OmSnapshot fromSnapshot,
OmSnapshot toSnapshot, List<String> tablesToLookUp,
SnapshotInfo fsInfo, SnapshotInfo tsInfo,
- String volume, String bucket)
+ boolean useFullDiff, Map<String, String> tablePrefixes)
throws RocksDBException, IOException {
// TODO: Refactor the parameter list
final Set<String> deltaFiles = new HashSet<>();
// Check if compaction DAG is available, use that if so
- if (differ != null && fsInfo != null && tsInfo != null) {
+ if (differ != null && fsInfo != null && tsInfo != null && !useFullDiff) {
+ String volume = fsInfo.getVolumeName();
+ String bucket = fsInfo.getBucketName();
// Construct DifferSnapshotInfo
final DifferSnapshotInfo fromDSI =
getDSIFromSI(fsInfo, fromSnapshot, volume, bucket);
@@ -385,6 +408,7 @@ public class SnapshotDiffManager {
deltaFiles.addAll(fromSnapshotFiles);
deltaFiles.addAll(toSnapshotFiles);
+ RocksDiffUtils.filterRelevantSstFiles(deltaFiles, tablePrefixes);
}
return deltaFiles;
@@ -553,4 +577,25 @@ public class SnapshotDiffManager {
}
return false;
}
+
+
+ /**
+ * check if the given key is in the bucket specified by tablePrefix map.
+ */
+ private boolean isKeyInBucket(String key, Map<String, String> tablePrefixes,
+ String tableName) {
+ String volumeBucketDbPrefix;
+ // In case of FSO - either File/Directory table
+ // the key Prefix would be volumeId/bucketId and
+ // in case of non-fso - volumeName/bucketName
+ if (tableName.equals(
+ OmMetadataManagerImpl.DIRECTORY_TABLE) || tableName.equals(
+ OmMetadataManagerImpl.FILE_TABLE)) {
+ volumeBucketDbPrefix =
+ tablePrefixes.get(OmMetadataManagerImpl.DIRECTORY_TABLE);
+ } else {
+ volumeBucketDbPrefix = tablePrefixes.get(OmMetadataManagerImpl.KEY_TABLE);
+ }
+ return key.startsWith(volumeBucketDbPrefix);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org