You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by si...@apache.org on 2026/10/08 18:54:04 UTC
(ozone) branch master updated: HDDS-16783. Support FileSystem#setVerifyChecksum in ofs and o3fs (#11444)
This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d68a272d3ee HDDS-16783. Support FileSystem#setVerifyChecksum in ofs and o3fs (#11444)
d68a272d3ee is described below
commit d68a272d3ee5086785d7a606c272e83983a22bd2
Author: Siyao Meng <50...@users.noreply.github.com>
AuthorDate: Thu Oct 8 11:53:58 2026 -0700
HDDS-16783. Support FileSystem#setVerifyChecksum in ofs and o3fs (#11444)
Co-authored-by: Claude Opus 5.5 <no...@anthropic.com>
---
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 2 +-
.../ozone/client/protocol/ClientProtocol.java | 8 ++++++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 5 +++++
.../fs/ozone/AbstractOzoneFileSystemTest.java | 5 +++++
.../ozone/AbstractRootedOzoneFileSystemTest.java | 5 +++++
.../hadoop/fs/ozone/OzoneFileSystemTestBase.java | 24 ++++++++++++++++++++++
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 5 +++++
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +++++
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 5 +++++
.../fs/ozone/BasicRootedOzoneFileSystem.java | 5 +++++
.../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 2 ++
11 files changed, 70 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index e88e6293907..ade379c1a24 100644
--- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -213,7 +213,7 @@ public class OzoneClientConfig {
description = "Ozone client to verify checksum of the checksum "
+ "blocksize data.",
tags = ConfigTag.CLIENT)
- private boolean checksumVerify = true;
+ private volatile boolean checksumVerify = true;
@Config(key = "ozone.client.max.ec.stripe.write.retries",
defaultValue = "10",
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index bb39664d868..96247bb2ad1 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -1639,6 +1639,14 @@ ListSnapshotDiffJobResponse listSnapshotDiffJobs(
void setTimes(OzoneObj obj, String keyName, long mtime, long atime)
throws IOException;
+ /**
+ * Set whether this client verifies checksums when reading. Overrides ozone.client.verify.checksum for this client
+ * only. Takes effect for input streams opened afterwards, streams that are already open may or may not pick it up.
+ * @param verifyChecksum false to skip checksum verification, e.g. when the application verifies its own checksums.
+ */
+ default void setVerifyChecksum(boolean verifyChecksum) {
+ }
+
/**
* Start the lease recovery of a file.
*
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 522fb17e1fb..40de69bc00b 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -3007,6 +3007,11 @@ public void setTimes(OzoneObj obj, String keyName, long mtime, long atime)
ozoneManagerClient.setTimes(builder.build(), mtime, atime);
}
+ @Override
+ public void setVerifyChecksum(boolean verifyChecksum) {
+ clientConfig.setChecksumVerify(verifyChecksum);
+ }
+
@Override
public LeaseKeyInfo recoverLease(String volumeName, String bucketName,
String keyName, boolean force)
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
index 302c3772c26..9b112e9de84 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
@@ -1466,6 +1466,11 @@ public void testSetTimes() throws Exception {
setTimes(ROOT);
}
+ @Test
+ void testSetVerifyChecksum() throws Exception {
+ setVerifyChecksum(cluster, new Path(ROOT, "testSetVerifyChecksum"));
+ }
+
@Test
public void testLoopInLinkBuckets() throws Exception {
String linksVolume = UUID.randomUUID().toString();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
index 0b31625e6f8..83661bd3d55 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
@@ -2258,6 +2258,11 @@ void testSetTimes() throws Exception {
setTimes(bucketPath1);
}
+ @Test
+ void testSetVerifyChecksum() throws Exception {
+ setVerifyChecksum(cluster, new Path(bucketPath, "testSetVerifyChecksum"));
+ }
+
@Test
public void testSetTimesForLinkedBucketPath() throws Exception {
// Create a file
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java
index dee6911eccd..344cb441d4f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTestBase.java
@@ -17,9 +17,12 @@
package org.apache.hadoop.fs.ozone;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_ACLS;
import static org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS;
import static org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
+import static org.apache.hadoop.hdds.utils.ClusterContainersUtil.corruptData;
+import static org.apache.hadoop.hdds.utils.ClusterContainersUtil.getContainerByID;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
@@ -50,7 +53,9 @@
import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.OzoneKeyDetails;
+import org.apache.hadoop.ozone.common.OzoneChecksumException;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -381,6 +386,25 @@ void setTimes(Path testPathRoot) throws Exception {
assertEquals(mtime, fileStatus.getModificationTime());
}
+ void setVerifyChecksum(MiniOzoneCluster cluster, Path path) throws Exception {
+ FileSystem fs = getFs();
+ byte[] data = "sample value".getBytes(UTF_8);
+ // Single replica, so that a read cannot fall back to a healthy one.
+ try (FSDataOutputStream stream = fs.create(path, (short) 1)) {
+ stream.write(data);
+ }
+ OzoneKeyDetails key = getKey(path, false);
+ assertEquals(1, key.getReplicationConfig().getRequiredNodes());
+ corruptData(cluster, getContainerByID(cluster, key.getOzoneKeyLocations().get(0).getContainerID()), key);
+
+ // Use a separate instance, the shared one must keep verifying.
+ try (FileSystem noChecksumFs = FileSystem.newInstance(fs.getUri(), fs.getConf())) {
+ noChecksumFs.setVerifyChecksum(false);
+ assertThat(ContractTestUtils.readDataset(noChecksumFs, path, data.length)).isNotEqualTo(data);
+ assertThrows(OzoneChecksumException.class, () -> ContractTestUtils.readDataset(fs, path, data.length));
+ }
+ }
+
void verifyListStatus(Path root, PathFilter filter) throws Exception {
Path parent = new Path(root, "testListStatus");
Path file1 = new Path(parent, "key1");
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index b733b461d48..513156e50ff 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -846,6 +846,11 @@ public void setTimes(String key, long mtime, long atime) throws IOException {
bucket.setTimes(key, mtime, atime);
}
+ @Override
+ public void setVerifyChecksum(boolean verifyChecksum) {
+ ozoneClient.getProxy().setVerifyChecksum(verifyChecksum);
+ }
+
@Override
public boolean isFileClosed(String pathStr) throws IOException {
incrementCounter(Statistic.INVOCATION_IS_FILE_CLOSED, 1);
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index 28cdb10dddb..2d569f7a68c 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -725,6 +725,11 @@ workingDir, getUsername(), true)
return statuses.toArray(new FileStatus[0]);
}
+ @Override
+ public void setVerifyChecksum(boolean verifyChecksum) {
+ adapter.setVerifyChecksum(verifyChecksum);
+ }
+
@Override
public void setWorkingDirectory(Path newDir) {
workingDir = newDir;
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index c7cf1f729ac..8540929de3b 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -1514,6 +1514,11 @@ private SnapshotDiffReportOzone getSnapshotDiffReportOnceComplete(
return snapshotDiffResponse.getSnapshotDiffReport();
}
+ @Override
+ public void setVerifyChecksum(boolean verifyChecksum) {
+ proxy.setVerifyChecksum(verifyChecksum);
+ }
+
@Override
public boolean isFileClosed(String pathStr) throws IOException {
incrementCounter(Statistic.INVOCATION_IS_FILE_CLOSED, 1);
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 5eb54a21335..30ed630b4a7 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -978,6 +978,11 @@ private List<FileStatusAdapter> listStatusAdapter(Path f, boolean lite) throws I
return statuses;
}
+ @Override
+ public void setVerifyChecksum(boolean verifyChecksum) {
+ adapter.setVerifyChecksum(verifyChecksum);
+ }
+
@Override
public void setWorkingDirectory(Path newDir) {
workingDir = newDir;
diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
index 57003f1b455..e398b4374d6 100644
--- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
+++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java
@@ -122,6 +122,8 @@ SnapshotDiffReport getSnapshotDiffReport(Path snapshotDir,
void setTimes(String key, long mtime, long atime) throws IOException;
+ void setVerifyChecksum(boolean verifyChecksum);
+
boolean isFileClosed(String pathStr) throws IOException;
boolean setSafeMode(SafeModeAction action, boolean isChecked)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org