You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2023/03/18 08:53:21 UTC
[ozone] branch master updated: HDDS-8061. [Snapshot] Add SnapDiff client option to force a full diff (#4398)
This is an automated email from the ASF dual-hosted git repository.
siyao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1236414c1c HDDS-8061. [Snapshot] Add SnapDiff client option to force a full diff (#4398)
1236414c1c is described below
commit 1236414c1c46c67c92f37499b0b61921cdfab39a
Author: targetoee <10...@users.noreply.github.com>
AuthorDate: Sat Mar 18 16:53:13 2023 +0800
HDDS-8061. [Snapshot] Add SnapDiff client option to force a full diff (#4398)
---
.../apache/hadoop/ozone/client/ObjectStore.java | 6 ++--
.../ozone/client/protocol/ClientProtocol.java | 4 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 5 ++--
.../ozone/om/protocol/OzoneManagerProtocol.java | 4 ++-
...OzoneManagerProtocolClientSideTranslatorPB.java | 7 +++--
.../org/apache/hadoop/ozone/om/TestOmSnapshot.java | 32 ++++++++++++++--------
.../src/main/proto/OmClientProtocol.proto | 1 +
.../apache/hadoop/ozone/om/OmSnapshotManager.java | 5 ++--
.../org/apache/hadoop/ozone/om/OzoneManager.java | 5 ++--
.../ozone/om/snapshot/SnapshotDiffManager.java | 19 +++++++++----
.../protocolPB/OzoneManagerRequestHandler.java | 3 +-
.../hadoop/ozone/client/ClientProtocolStub.java | 3 +-
.../ozone/shell/snapshot/SnapshotDiffHandler.java | 8 +++++-
13 files changed, 70 insertions(+), 32 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
index 30e13b2f65..e8c9571be0 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
@@ -569,6 +569,7 @@ public class ObjectStore {
* @param toSnapshot The name of the ending snapshot
* @param token to get the index to return diff report from.
* @param pageSize maximum entries returned to the report.
+ * @param forceFullDiff request to force full diff, skipping DAG optimization
* @return the difference report between two snapshots
* @throws IOException in case of any exception while generating snapshot diff
*/
@@ -577,9 +578,10 @@ public class ObjectStore {
String fromSnapshot,
String toSnapshot,
String token,
- int pageSize)
+ int pageSize,
+ boolean forceFullDiff)
throws IOException {
return proxy.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot,
- token, pageSize);
+ token, pageSize, forceFullDiff);
}
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 3e4abd62ef..77fde60309 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -1059,11 +1059,13 @@ public interface ClientProtocol {
* @param toSnapshot The name of the ending snapshot
* @param token to get the index to return diff report from.
* @param pageSize maximum entries returned to the report.
+ * @param forceFullDiff request to force full diff, skipping DAG optimization
* @return the difference report between two snapshots
* @throws IOException in case of any exception while generating snapshot diff
*/
SnapshotDiffResponse snapshotDiff(String volumeName, String bucketName,
String fromSnapshot, String toSnapshot,
- String token, int pageSize)
+ String token, int pageSize,
+ boolean forceFullDiff)
throws IOException;
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 7676bc6dc4..cb920bcc00 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -984,14 +984,15 @@ public class RpcClient implements ClientProtocol {
String fromSnapshot,
String toSnapshot,
String token,
- int pageSize)
+ int pageSize,
+ boolean forceFullDiff)
throws IOException {
Preconditions.checkArgument(Strings.isNotBlank(volumeName),
"volume can't be null or empty.");
Preconditions.checkArgument(Strings.isNotBlank(bucketName),
"bucket can't be null or empty.");
return ozoneManagerClient.snapshotDiff(volumeName, bucketName,
- fromSnapshot, toSnapshot, token, pageSize);
+ fromSnapshot, toSnapshot, token, pageSize, forceFullDiff);
}
/**
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 19331b472c..b718e46309 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -705,6 +705,7 @@ public interface OzoneManagerProtocol
* @param toSnapshot The name of the ending snapshot
* @param token to get the index to return diff report from.
* @param pageSize maximum entries returned to the report.
+ * @param forceFullDiff request to force full diff, skipping DAG optimization
* @return the difference report between two snapshots
* @throws IOException in case of any exception while generating snapshot diff
*/
@@ -713,7 +714,8 @@ public interface OzoneManagerProtocol
String fromSnapshot,
String toSnapshot,
String token,
- int pageSize)
+ int pageSize,
+ boolean forceFullDiff)
throws IOException {
throw new UnsupportedOperationException("OzoneManager does not require " +
"this to be implemented");
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index d01d0d3ee7..2eb2b2dc85 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -1202,7 +1202,9 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
String fromSnapshot,
String toSnapshot,
String token,
- int pageSize) throws IOException {
+ int pageSize,
+ boolean forceFullDiff)
+ throws IOException {
final OzoneManagerProtocolProtos.SnapshotDiffRequest.Builder
requestBuilder =
OzoneManagerProtocolProtos.SnapshotDiffRequest.newBuilder()
@@ -1210,7 +1212,8 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
.setBucketName(bucketName)
.setFromSnapshot(fromSnapshot)
.setToSnapshot(toSnapshot)
- .setPageSize(pageSize);
+ .setPageSize(pageSize)
+ .setForceFullDiff(forceFullDiff);
if (!StringUtils.isBlank(token)) {
requestBuilder.setToken(token);
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
index f8eb9d3cd8..39c3750eb7 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshot.java
@@ -550,7 +550,7 @@ public class TestOmSnapshot {
SnapshotDiffResponse response;
do {
response = store.snapshotDiff(volume, bucket, fromSnapshot,
- toSnapshot, null, 0);
+ toSnapshot, null, 0, false);
Thread.sleep(response.getWaitTimeInMs());
} while (response.getJobStatus() != DONE);
@@ -574,11 +574,13 @@ public class TestOmSnapshot {
// Destination snapshot is invalid
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, snap1, snap2, null, 0));
+ () -> store.snapshotDiff(volume, bucket, snap1, snap2,
+ null, 0, false));
// From snapshot is invalid
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, snap2, snap1, null, 0));
+ () -> store.snapshotDiff(volume, bucket, snap2, snap1,
+ null, 0, false));
}
@Test
@@ -603,15 +605,18 @@ public class TestOmSnapshot {
// Bucket is nonexistent
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volumea, bucketb, snap1, snap2, null, 0));
+ () -> store.snapshotDiff(volumea, bucketb, snap1, snap2,
+ null, 0, false));
// Volume is nonexistent
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volumeb, bucketa, snap2, snap1, null, 0));
+ () -> store.snapshotDiff(volumeb, bucketa, snap2, snap1,
+ null, 0, false));
// Both volume and bucket are nonexistent
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volumeb, bucketb, snap2, snap1, null, 0));
+ () -> store.snapshotDiff(volumeb, bucketb, snap2, snap1,
+ null, 0, false));
}
@Test
@@ -633,17 +638,21 @@ public class TestOmSnapshot {
// Destination snapshot is empty
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, snap1, nullstr, null, 0));
+ () -> store.snapshotDiff(volume, bucket, snap1, nullstr,
+ null, 0, false));
// From snapshot is empty
LambdaTestUtils.intercept(OMException.class,
"KEY_NOT_FOUND",
- () -> store.snapshotDiff(volume, bucket, nullstr, snap1, null, 0));
+ () -> store.snapshotDiff(volume, bucket, nullstr, snap1,
+ null, 0, false));
// Bucket is empty
assertThrows(IllegalArgumentException.class,
- () -> store.snapshotDiff(volume, nullstr, snap1, snap2, null, 0));
+ () -> store.snapshotDiff(volume, nullstr, snap1, snap2,
+ null, 0, false));
// Volume is empty
assertThrows(IllegalArgumentException.class,
- () -> store.snapshotDiff(nullstr, bucket, snap1, snap2, null, 0));
+ () -> store.snapshotDiff(nullstr, bucket, snap1, snap2,
+ null, 0, false));
}
@Test
@@ -716,7 +725,8 @@ public class TestOmSnapshot {
createSnapshot(volumeName1, bucketName1, snap2); // 1.sst 2.sst 3.sst 4.sst
Assert.assertEquals(4, getKeyTableSstFiles().size());
SnapshotDiffReport diff1 =
- store.snapshotDiff(volumeName1, bucketName1, snap1, snap2, null, 0)
+ store.snapshotDiff(volumeName1, bucketName1, snap1, snap2,
+ null, 0, false)
.getSnapshotDiffReport();
Assert.assertEquals(1, diff1.getDiffList().size());
}
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 8a70f64361..73ed666c87 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1697,6 +1697,7 @@ message SnapshotDiffRequest {
optional string toSnapshot = 4;
optional string token = 5;
optional uint32 pageSize = 6;
+ optional bool forceFullDiff = 7;
}
message DeleteSnapshotRequest {
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
index 3f43fa623d..d794a6f30f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java
@@ -433,7 +433,8 @@ public final class OmSnapshotManager implements AutoCloseable {
final String fromSnapshot,
final String toSnapshot,
final String token,
- int pageSize)
+ int pageSize,
+ boolean forceFullDiff)
throws IOException {
// Validate fromSnapshot and toSnapshot
final SnapshotInfo fsInfo = getSnapshotInfo(volume, bucket, fromSnapshot);
@@ -452,7 +453,7 @@ public final class OmSnapshotManager implements AutoCloseable {
final OmSnapshot ts = snapshotCache.get(tsKey);
SnapshotDiffReport snapshotDiffReport =
snapshotDiffManager.getSnapshotDiffReport(volume, bucket, fs, ts,
- fsInfo, tsInfo, index, pageSize);
+ fsInfo, tsInfo, index, pageSize, forceFullDiff);
return new SnapshotDiffResponse(snapshotDiffReport, DONE, 0L);
} catch (ExecutionException | RocksDBException e) {
throw new IOException(e.getCause());
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 35361ef4bc..e6381162b5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -4489,10 +4489,11 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl
String fromSnapshot,
String toSnapshot,
String token,
- int pageSize)
+ int pageSize,
+ boolean forceFullDiff)
throws IOException {
return omSnapshotManager.getSnapshotDiffReport(volume, bucket,
- fromSnapshot, toSnapshot, token, pageSize);
+ fromSnapshot, toSnapshot, token, pageSize, forceFullDiff);
}
@Override // ReconfigureProtocol
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
index f2a47b9916..438ce18320 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java
@@ -179,7 +179,8 @@ public class SnapshotDiffManager {
final SnapshotInfo fsInfo,
final SnapshotInfo tsInfo,
final int index,
- final int pageSize)
+ final int pageSize,
+ final boolean forceFullDiff)
throws IOException, RocksDBException {
String diffJobKey = fsInfo.getSnapshotID() + DELIMITER +
tsInfo.getSnapshotID();
@@ -193,7 +194,7 @@ public class SnapshotDiffManager {
// This needs to be updated to queuing and job status base.
if (!jobExist) {
generateSnapshotDiffReport(jobId, volume, bucket, fromSnapshot,
- toSnapshot, fsInfo, tsInfo);
+ toSnapshot, fsInfo, tsInfo, forceFullDiff);
}
List<DiffReportEntry> diffReportList = new ArrayList<>();
@@ -241,7 +242,8 @@ public class SnapshotDiffManager {
final OmSnapshot fromSnapshot,
final OmSnapshot toSnapshot,
final SnapshotInfo fsInfo,
- final SnapshotInfo tsInfo)
+ final SnapshotInfo tsInfo,
+ final boolean forceFullDiff)
throws RocksDBException {
ColumnFamilyHandle fromSnapshotColumnFamily = null;
ColumnFamilyHandle toSnapshotColumnFamily = null;
@@ -300,6 +302,9 @@ public class SnapshotDiffManager {
boolean useFullDiff = configuration.getBoolean(
OzoneConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF,
OzoneConfigKeys.OZONE_OM_SNAPSHOT_FORCE_FULL_DIFF_DEFAULT);
+ if (forceFullDiff) {
+ useFullDiff = true;
+ }
Map<String, String> tablePrefixes =
getTablePrefixes(toSnapshot.getMetadataManager(), volume, bucket);
@@ -461,11 +466,13 @@ public class SnapshotDiffManager {
// End of Workaround
}
- if (deltaFiles.isEmpty()) {
+ if (useFullDiff || deltaFiles.isEmpty()) {
// If compaction DAG is not available (already cleaned up), fall back to
// the slower approach.
- LOG.warn("RocksDBCheckpointDiffer is not available, falling back to" +
- " slow path");
+ if (!useFullDiff) {
+ LOG.warn("RocksDBCheckpointDiffer is not available, falling back to" +
+ " slow path");
+ }
Set<String> fromSnapshotFiles =
RdbUtil.getSSTFilesForComparison(
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 973953d76f..03c31d0241 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -1226,7 +1226,8 @@ public class OzoneManagerRequestHandler implements RequestHandler {
snapshotDiffRequest.getFromSnapshot(),
snapshotDiffRequest.getToSnapshot(),
snapshotDiffRequest.getToken(),
- snapshotDiffRequest.getPageSize());
+ snapshotDiffRequest.getPageSize(),
+ snapshotDiffRequest.getForceFullDiff());
SnapshotDiffResponse.Builder builder = SnapshotDiffResponse.newBuilder()
.setJobStatus(response.getJobStatus().toProtobuf())
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 9495d2869c..ee62b84c0a 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -635,7 +635,8 @@ public class ClientProtocolStub implements ClientProtocol {
String fromSnapshot,
String toSnapshot,
String token,
- int pageSize)
+ int pageSize,
+ boolean forceFullDiff)
throws IOException {
return null;
}
diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
index 7690bd054c..7f1162cfa0 100644
--- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
+++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/snapshot/SnapshotDiffHandler.java
@@ -54,6 +54,12 @@ public class SnapshotDiffHandler extends Handler {
" (optional)")
private int pageSize;
+ @CommandLine.Option(names = {"--ffd", "--force-full-diff"},
+ description = "perform full diff of snapshot without using" +
+ " optimised DAG based pruning approach (optional)",
+ hidden = true)
+ private boolean forceFullDiff;
+
@Override
protected OzoneAddress getAddress() {
return snapshotPath.getValue();
@@ -71,7 +77,7 @@ public class SnapshotDiffHandler extends Handler {
try (PrintStream stream = out()) {
stream.print(client.getObjectStore()
.snapshotDiff(volumeName, bucketName, fromSnapshot, toSnapshot,
- token, pageSize));
+ token, pageSize, forceFullDiff));
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org