You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by we...@apache.org on 2026/04/01 15:40:06 UTC
(ozone) branch master updated: HDDS-13919. S3 Conditional Writes (PutObject) (#9815)
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e9cceba8dfc HDDS-13919. S3 Conditional Writes (PutObject) (#9815)
e9cceba8dfc is described below
commit e9cceba8dfc4f5ed5a906261be263fb14e2548ef
Author: Peter Lee <pe...@gmail.com>
AuthorDate: Wed Apr 1 23:40:00 2026 +0800
HDDS-13919. S3 Conditional Writes (PutObject) (#9815)
---
.../apache/hadoop/ozone/client/OzoneBucket.java | 84 ++++++++++
.../hadoop/ozone/client/OzoneKeyDetails.java | 16 ++
.../ozone/client/protocol/ClientProtocol.java | 78 +++++++++
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 175 +++++++++++++--------
.../hadoop/ozone/om/exceptions/OMException.java | 4 +
.../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 32 ++++
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 38 +++++
...OzoneManagerProtocolClientSideTranslatorPB.java | 3 +
.../src/main/smoketest/s3/conditionalput.robot | 89 +++++++++++
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 112 +++++++++++++
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 102 ++++++++++++
.../ozone/client/rpc/OzoneRpcClientTests.java | 142 ++++++++++++++++-
.../src/main/proto/OmClientProtocol.proto | 14 ++
.../ozone/om/request/key/OMKeyCommitRequest.java | 19 +++
.../om/request/key/OMKeyCommitRequestWithFSO.java | 1 +
.../ozone/om/request/key/OMKeyCreateRequest.java | 16 ++
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 3 +
.../om/request/key/TestOMKeyCommitRequest.java | 137 ++++++++++++++++
.../om/request/key/TestOMKeyCreateRequest.java | 144 +++++++++++++++++
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 92 ++++++++++-
.../ozone/s3/endpoint/ObjectEndpointStreaming.java | 32 +++-
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 4 +
.../hadoop/ozone/client/ClientProtocolStub.java | 39 +++++
.../hadoop/ozone/client/OzoneBucketStub.java | 66 +++++++-
.../hadoop/ozone/s3/endpoint/TestObjectPut.java | 107 +++++++++++++
25 files changed, 1474 insertions(+), 75 deletions(-)
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
index 09cd189e8e4..2bde6939651 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
@@ -520,6 +520,44 @@ public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyG
return proxy.rewriteKey(volumeName, name, keyName, size, existingKeyGeneration, replicationConfig, metadata);
}
+ /**
+ * Creates a key only if it does not exist (S3 If-None-Match: * semantics).
+ *
+ * @param keyName Name of the key
+ * @param size Size of the data
+ * @param replicationConfig Replication configuration
+ * @param metadata custom key value metadata
+ * @param tags Tags used for S3 object tags
+ * @return OzoneOutputStream to which the data has to be written.
+ * @throws IOException
+ */
+ public OzoneOutputStream createKeyIfNotExists(String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ return proxy.createKeyIfNotExists(volumeName, name, keyName, size,
+ replicationConfig, metadata, tags);
+ }
+
+ /**
+ * Rewrites a key only if its ETag matches (S3 If-Match semantics).
+ *
+ * @param keyName Name of the key
+ * @param size Size of the data
+ * @param expectedETag The ETag value the existing key must have
+ * @param replicationConfig Replication configuration
+ * @param metadata custom key value metadata
+ * @param tags Tags used for S3 object tags
+ * @return OzoneOutputStream to which the data has to be written.
+ * @throws IOException
+ */
+ public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size,
+ String expectedETag, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags)
+ throws IOException {
+ return proxy.rewriteKeyIfMatch(volumeName, name, keyName, size,
+ expectedETag, replicationConfig, metadata, tags);
+ }
+
/**
* Creates a new key in the bucket, with default replication type RATIS and
* with replication factor THREE.
@@ -575,6 +613,52 @@ public OzoneDataStreamOutput createStreamKey(String key, long size,
replicationConfig, keyMetadata, tags);
}
+ /**
+ * Creates a key with datastream only if it does not exist already
+ * (S3 If-None-Match: * semantics).
+ *
+ * @param key Name of the key to be created.
+ * @param size Size of the data the key will point to.
+ * @param replicationConfig Replication configuration.
+ * @param keyMetadata Custom key metadata.
+ * @param tags Tags used for S3 object tags
+ * @return OzoneDataStreamOutput to which the data has to be written.
+ * @throws IOException
+ */
+ public OzoneDataStreamOutput createStreamKeyIfNotExists(String key, long size,
+ ReplicationConfig replicationConfig, Map<String, String> keyMetadata,
+ Map<String, String> tags) throws IOException {
+ if (replicationConfig == null) {
+ replicationConfig = defaultReplication;
+ }
+ return proxy.createStreamKeyIfNotExists(volumeName, name, key, size,
+ replicationConfig, keyMetadata, tags);
+ }
+
+ /**
+ * Rewrites a key with datastream only if its ETag matches
+ * (S3 If-Match semantics).
+ *
+ * @param key Name of the key to be rewritten.
+ * @param size Size of the data the key will point to.
+ * @param expectedETag The ETag value the existing key must have.
+ * @param replicationConfig Replication configuration.
+ * @param keyMetadata Custom key metadata.
+ * @param tags Tags used for S3 object tags
+ * @return OzoneDataStreamOutput to which the data has to be written.
+ * @throws IOException
+ */
+ public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String key, long size,
+ String expectedETag, ReplicationConfig replicationConfig,
+ Map<String, String> keyMetadata, Map<String, String> tags)
+ throws IOException {
+ if (replicationConfig == null) {
+ replicationConfig = defaultReplication;
+ }
+ return proxy.rewriteStreamKeyIfMatch(volumeName, name, key, size,
+ expectedETag, replicationConfig, keyMetadata, tags);
+ }
+
/**
* Reads an existing key from the bucket.
*
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
index 1bc1418d1f2..7514f2a7cd7 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java
@@ -23,6 +23,7 @@
import java.util.Map;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.ratis.util.function.CheckedSupplier;
@@ -107,4 +108,19 @@ public Long getGeneration() {
public OzoneInputStream getContent() throws IOException {
return this.contentSupplier.get();
}
+
+ public boolean hasEtag() {
+ return getMetadata().containsKey(OzoneConsts.ETAG);
+ }
+
+ public boolean isEtagEquals(String matchingETag) {
+ String currentETag = getMetadata().get(OzoneConsts.ETAG);
+ if (currentETag == null) {
+ return false;
+ }
+ if (matchingETag.equals("*")) {
+ return true;
+ }
+ return currentETag.equals(matchingETag);
+ }
}
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
index 24ddd782cd2..2542fda26e8 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
@@ -379,6 +379,44 @@ OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyNam
long size, long existingKeyGeneration, ReplicationConfig replicationConfig,
Map<String, String> metadata) throws IOException;
+ /**
+ * Creates a key only if it does not exist (S3 If-None-Match: * semantics).
+ *
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyName Name of the Key
+ * @param size Size of the data
+ * @param replicationConfig The replication configuration
+ * @param metadata custom key value metadata
+ * @param tags Tags used for S3 object tags
+ * @return {@link OzoneOutputStream}
+ * @throws OMException with KEY_ALREADY_EXISTS if key exists
+ */
+ OzoneOutputStream createKeyIfNotExists(String volumeName, String bucketName,
+ String keyName, long size, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags)
+ throws IOException;
+
+ /**
+ * Rewrites a key only if its ETag matches (S3 If-Match semantics).
+ *
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyName Name of the Key
+ * @param size Size of the data
+ * @param expectedETag The ETag value the existing key must have
+ * @param replicationConfig The replication configuration
+ * @param metadata custom key value metadata
+ * @param tags Tags used for S3 object tags
+ * @return {@link OzoneOutputStream}
+ * @throws OMException with ETAG_MISMATCH, ETAG_NOT_AVAILABLE, or KEY_NOT_FOUND
+ */
+ @SuppressWarnings("checkstyle:parameternumber")
+ OzoneOutputStream rewriteKeyIfMatch(String volumeName, String bucketName,
+ String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException;
+
/**
* Writes a key in an existing bucket.
* @param volumeName Name of the Volume
@@ -426,6 +464,46 @@ OzoneDataStreamOutput createStreamKey(String volumeName, String bucketName,
Map<String, String> metadata, Map<String, String> tags)
throws IOException;
+ /**
+ * Writes a key in an existing bucket only if it does not already exist
+ * (S3 If-None-Match: * semantics).
+ *
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyName Name of the Key
+ * @param size Size of the data
+ * @param replicationConfig The replication configuration
+ * @param metadata custom key value metadata
+ * @param tags Tags used for S3 object tags
+ * @return {@link OzoneDataStreamOutput}
+ * @throws OMException with KEY_ALREADY_EXISTS if key exists
+ */
+ OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException;
+
+ /**
+ * Writes a key in an existing bucket only if its ETag matches
+ * (S3 If-Match semantics).
+ *
+ * @param volumeName Name of the Volume
+ * @param bucketName Name of the Bucket
+ * @param keyName Name of the Key
+ * @param size Size of the data
+ * @param expectedETag The ETag value the existing key must have
+ * @param replicationConfig The replication configuration
+ * @param metadata custom key value metadata
+ * @param tags Tags used for S3 object tags
+ * @return {@link OzoneDataStreamOutput}
+ * @throws OMException with ETAG_MISMATCH, ETAG_NOT_AVAILABLE, or KEY_NOT_FOUND
+ */
+ @SuppressWarnings("checkstyle:parameternumber")
+ OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException;
+
/**
* Reads a key from an existing bucket.
* @param volumeName Name of the Volume
diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 641a63a28f9..f2440a9623f 100644
--- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -1369,36 +1369,11 @@ public OzoneOutputStream createKey(
String volumeName, String bucketName, String keyName, long size,
ReplicationConfig replicationConfig,
Map<String, String> metadata, Map<String, String> tags) throws IOException {
- createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig);
-
- if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) {
- if (tags != null && !tags.isEmpty()) {
- throw new IOException("OzoneManager does not support object tags");
- }
- }
-
String ownerName = getRealUserInfo().getShortUserName();
-
- OmKeyArgs.Builder builder = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .setDataSize(size)
- .setReplicationConfig(replicationConfig)
- .addAllMetadataGdpr(metadata)
- .addAllTags(tags)
- .setLatestVersionLocation(getLatestVersionLocation)
- .setOwnerName(ownerName);
-
- OpenKeySession openKey = ozoneManagerClient.openKey(builder.build());
- // For bucket with layout OBJECT_STORE, when create an empty file (size=0),
- // OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE,
- // which will cause S3G's atomic write length check to fail,
- // so reset size to 0 here.
- if (isS3GRequest.get() && size == 0) {
- openKey.getKeyInfo().setDataSize(size);
- }
- return createOutputStream(openKey);
+ OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName,
+ bucketName, keyName, size, replicationConfig, metadata, tags);
+ builder.setOwnerName(ownerName);
+ return openOutputStream(builder.build(), size);
}
@Override
@@ -1411,20 +1386,59 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String
Preconditions.checkArgument(existingKeyGeneration > 0,
"existingKeyGeneration must be positive, but was %s",
existingKeyGeneration);
+ OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName,
+ bucketName, keyName, size, replicationConfig, metadata,
+ Collections.emptyMap());
+ builder.setExpectedDataGeneration(existingKeyGeneration);
+ return openOutputStream(builder.build(), size);
+ }
- createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig);
+ @Override
+ public OzoneOutputStream createKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
+ throw new IOException(
+ "OzoneManager does not support atomic key creation.");
+ }
+ OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName,
+ bucketName, keyName, size, replicationConfig, metadata, tags);
+ builder.setExpectedDataGeneration(
+ OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS);
+ return openOutputStream(builder.build(), size);
+ }
- OmKeyArgs.Builder builder = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .setDataSize(size)
- .setReplicationConfig(replicationConfig)
- .addAllMetadataGdpr(metadata)
- .setLatestVersionLocation(getLatestVersionLocation)
- .setExpectedDataGeneration(existingKeyGeneration);
+ @Override
+ @SuppressWarnings("checkstyle:parameternumber")
+ public OzoneOutputStream rewriteKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
+ throw new IOException(
+ "OzoneManager does not support conditional key rewrite.");
+ }
+ OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(volumeName,
+ bucketName, keyName, size, replicationConfig, metadata, tags);
+ builder.setExpectedETag(expectedETag);
+ return openOutputStream(builder.build(), size);
+ }
- OpenKeySession openKey = ozoneManagerClient.openKey(builder.build());
+ private OmKeyArgs.Builder createWriteKeyArgsBuilder(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags)
+ throws IOException {
+ createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig);
+ validateObjectTagsSupport(tags);
+ return new OmKeyArgs.Builder(volumeName, bucketName, keyName, size,
+ replicationConfig, metadata, tags, getLatestVersionLocation);
+ }
+
+ private OzoneOutputStream openOutputStream(OmKeyArgs keyArgs, long size)
+ throws IOException {
+ OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs);
// For bucket with layout OBJECT_STORE, when create an empty file (size=0),
// OM will set DataSize to OzoneConfigKeys#OZONE_SCM_BLOCK_SIZE,
// which will cause S3G's atomic write length check to fail,
@@ -1435,6 +1449,15 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String
return createOutputStream(openKey);
}
+ private void validateObjectTagsSupport(Map<String, String> tags)
+ throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) {
+ if (tags != null && !tags.isEmpty()) {
+ throw new IOException("OzoneManager does not support object tags");
+ }
+ }
+ }
+
private void createKeyPreChecks(String volumeName, String bucketName, String keyName,
ReplicationConfig replicationConfig) throws IOException {
verifyVolumeName(volumeName);
@@ -1474,33 +1497,61 @@ public OzoneDataStreamOutput createStreamKey(
String volumeName, String bucketName, String keyName, long size,
ReplicationConfig replicationConfig,
Map<String, String> metadata, Map<String, String> tags) throws IOException {
- verifyVolumeName(volumeName);
- verifyBucketName(bucketName);
- if (checkKeyNameEnabled) {
- HddsClientUtils.verifyKeyName(keyName);
- }
- HddsClientUtils.checkNotNull(keyName);
+ OmKeyArgs.Builder builder = createStreamKeyArgsBuilder(
+ volumeName, bucketName, keyName, size, replicationConfig, metadata,
+ tags);
+ return openDataStreamOutput(builder.build());
+ }
- if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) {
- if (tags != null && !tags.isEmpty()) {
- throw new IOException("OzoneManager does not support object tags");
- }
+ @Override
+ public OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
+ throw new IOException(
+ "OzoneManager does not support atomic key creation.");
}
+ OmKeyArgs.Builder builder = createStreamKeyArgsBuilder(
+ volumeName, bucketName, keyName, size, replicationConfig, metadata,
+ tags);
+ builder.setExpectedDataGeneration(
+ OzoneConsts.EXPECTED_GEN_CREATE_IF_NOT_EXISTS);
+ return openDataStreamOutput(builder.build());
+ }
- String ownerName = getRealUserInfo().getShortUserName();
+ @Override
+ @SuppressWarnings("checkstyle:parameternumber")
+ public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ if (omVersion.compareTo(OzoneManagerVersion.ATOMIC_REWRITE_KEY) < 0) {
+ throw new IOException(
+ "OzoneManager does not support conditional key rewrite.");
+ }
+ OmKeyArgs.Builder builder = createStreamKeyArgsBuilder(
+ volumeName, bucketName, keyName, size, replicationConfig, metadata,
+ tags);
+ builder.setExpectedETag(expectedETag);
+ return openDataStreamOutput(builder.build());
+ }
- OmKeyArgs.Builder builder = new OmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .setDataSize(size)
- .setReplicationConfig(replicationConfig)
- .addAllMetadataGdpr(metadata)
- .addAllTags(tags)
- .setSortDatanodesInPipeline(true)
- .setOwnerName(ownerName);
+ private OmKeyArgs.Builder createStreamKeyArgsBuilder(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags)
+ throws IOException {
+ OmKeyArgs.Builder builder = createWriteKeyArgsBuilder(
+ volumeName, bucketName, keyName, size, replicationConfig, metadata,
+ tags);
+ builder.setOwnerName(getRealUserInfo().getShortUserName());
+ return builder;
+ }
- OpenKeySession openKey = ozoneManagerClient.openKey(builder.build());
+ private OzoneDataStreamOutput openDataStreamOutput(OmKeyArgs keyArgs)
+ throws IOException {
+ OpenKeySession openKey = ozoneManagerClient.openKey(keyArgs);
return createDataStreamOutput(openKey);
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
index 2b5b5559f92..dc9d16188c5 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java
@@ -275,5 +275,9 @@ public enum ResultCodes {
KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD,
TOO_MANY_SNAPSHOTS,
+
+ ETAG_MISMATCH,
+
+ ETAG_NOT_AVAILABLE,
}
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
index dfe0329fbe6..5d2de09c48e 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java
@@ -61,6 +61,7 @@ public final class OmKeyArgs extends WithMetadata implements Auditable {
// This allows a key to be created an committed atomically if the original has not
// been modified.
private Long expectedDataGeneration = null;
+ private final String expectedETag;
private OmKeyArgs(Builder b) {
super(b);
@@ -82,6 +83,7 @@ private OmKeyArgs(Builder b) {
this.ownerName = b.ownerName;
this.tags = b.tags.build();
this.expectedDataGeneration = b.expectedDataGeneration;
+ this.expectedETag = b.expectedETag;
}
public boolean getIsMultipartKey() {
@@ -164,6 +166,10 @@ public Long getExpectedDataGeneration() {
return expectedDataGeneration;
}
+ public String getExpectedETag() {
+ return expectedETag;
+ }
+
@Override
public Map<String, String> toAuditMap() {
Map<String, String> auditMap = new LinkedHashMap<>();
@@ -209,6 +215,9 @@ public KeyArgs toProtobuf() {
if (expectedDataGeneration != null) {
builder.setExpectedDataGeneration(expectedDataGeneration);
}
+ if (expectedETag != null) {
+ builder.setExpectedETag(expectedETag);
+ }
return builder.build();
}
@@ -234,11 +243,28 @@ public static class Builder extends WithMetadata.Builder {
private boolean forceUpdateContainerCacheFromSCM;
private final MapBuilder<String, String> tags;
private Long expectedDataGeneration = null;
+ private String expectedETag;
public Builder() {
this(AclListBuilder.empty());
}
+ @SuppressWarnings("checkstyle:parameternumber")
+ public Builder(String volumeName, String bucketName, String keyName,
+ long dataSize, ReplicationConfig replicationConfig,
+ Map<String, String> metadata, Map<String, String> tags,
+ boolean latestVersionLocation) {
+ this();
+ setVolumeName(volumeName)
+ .setBucketName(bucketName)
+ .setKeyName(keyName)
+ .setDataSize(dataSize)
+ .setReplicationConfig(replicationConfig)
+ .addAllMetadataGdpr(metadata)
+ .addAllTags(tags)
+ .setLatestVersionLocation(latestVersionLocation);
+ }
+
private Builder(AclListBuilder acls) {
this.acls = acls;
this.tags = MapBuilder.empty();
@@ -263,6 +289,7 @@ public Builder(OmKeyArgs obj) {
this.forceUpdateContainerCacheFromSCM =
obj.forceUpdateContainerCacheFromSCM;
this.expectedDataGeneration = obj.expectedDataGeneration;
+ this.expectedETag = obj.expectedETag;
this.tags = MapBuilder.of(obj.tags);
this.acls = AclListBuilder.of(obj.acls);
}
@@ -398,6 +425,11 @@ public Builder setExpectedDataGeneration(long generation) {
return this;
}
+ public Builder setExpectedETag(String eTag) {
+ this.expectedETag = eTag;
+ return this;
+ }
+
public OmKeyArgs build() {
return new OmKeyArgs(this);
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
index b0e26c49d69..e388df27b4b 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java
@@ -110,6 +110,7 @@ public final class OmKeyInfo extends WithParentObjectId
// This allows a key to be created an committed atomically if the original has not
// been modified.
private Long expectedDataGeneration = null;
+ private String expectedETag;
private OmKeyInfo(Builder b) {
super(b);
@@ -129,6 +130,7 @@ private OmKeyInfo(Builder b) {
this.ownerName = b.ownerName;
this.tags = b.tags.build();
this.expectedDataGeneration = b.expectedDataGeneration;
+ this.expectedETag = b.expectedETag;
}
private static Codec<OmKeyInfo> newCodec(boolean ignorePipeline) {
@@ -189,6 +191,14 @@ public Long getExpectedDataGeneration() {
return expectedDataGeneration;
}
+ public void setExpectedETag(String eTag) {
+ this.expectedETag = eTag;
+ }
+
+ public String getExpectedETag() {
+ return expectedETag;
+ }
+
public String getOwnerName() {
return ownerName;
}
@@ -492,6 +502,7 @@ public static class Builder extends WithParentObjectId.Builder<OmKeyInfo> {
private boolean isFile;
private final MapBuilder<String, String> tags;
private Long expectedDataGeneration = null;
+ private String expectedETag;
public Builder() {
this.acls = AclListBuilder.empty();
@@ -514,6 +525,7 @@ public Builder(OmKeyInfo obj) {
this.fileChecksum = obj.fileChecksum;
this.isFile = obj.isFile;
this.expectedDataGeneration = obj.expectedDataGeneration;
+ this.expectedETag = obj.expectedETag;
this.tags = MapBuilder.of(obj.tags);
obj.keyLocationVersions.forEach(keyLocationVersion ->
this.omKeyLocationInfoGroups.add(
@@ -685,6 +697,11 @@ public Builder setExpectedDataGeneration(Long existingGeneration) {
return this;
}
+ public Builder setExpectedETag(String eTag) {
+ this.expectedETag = eTag;
+ return this;
+ }
+
@Override
protected void validate() {
super.validate();
@@ -804,6 +821,9 @@ private KeyInfo getProtobuf(boolean ignorePipeline, String fullKeyName,
if (expectedDataGeneration != null) {
kb.setExpectedDataGeneration(expectedDataGeneration);
}
+ if (expectedETag != null) {
+ kb.setExpectedETag(expectedETag);
+ }
if (ownerName != null) {
kb.setOwnerName(ownerName);
}
@@ -857,6 +877,9 @@ public static Builder builderFromProtobuf(KeyInfo keyInfo) {
if (keyInfo.hasExpectedDataGeneration()) {
builder.setExpectedDataGeneration(keyInfo.getExpectedDataGeneration());
}
+ if (keyInfo.hasExpectedETag()) {
+ builder.setExpectedETag(keyInfo.getExpectedETag());
+ }
if (keyInfo.hasOwnerName()) {
builder.setOwnerName(keyInfo.getOwnerName());
@@ -997,6 +1020,21 @@ public boolean hasBlocks() {
return false;
}
+ public boolean hasEtag() {
+ return getMetadata().containsKey(OzoneConsts.ETAG);
+ }
+
+ public boolean isEtagEquals(String matchingETag) {
+ String currentETag = getMetadata().get(OzoneConsts.ETAG);
+ if (currentETag == null) {
+ return false;
+ }
+ if (matchingETag.equals("*")) {
+ return true;
+ }
+ return currentETag.equals(matchingETag);
+ }
+
public static boolean isKeyEmpty(@Nullable OmKeyInfo keyInfo) {
if (keyInfo == null) {
return true;
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index 20f0babab82..74c88edf070 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -744,6 +744,9 @@ public OpenKeySession openKey(OmKeyArgs args) throws IOException {
if (args.getExpectedDataGeneration() != null) {
keyArgs.setExpectedDataGeneration(args.getExpectedDataGeneration());
}
+ if (args.getExpectedETag() != null) {
+ keyArgs.setExpectedETag(args.getExpectedETag());
+ }
req.setKeyArgs(keyArgs.build());
diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot
new file mode 100644
index 00000000000..93bd25a43e2
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/s3/conditionalput.robot
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation S3 Conditional Put (If-None-Match / If-Match) tests
+Library OperatingSystem
+Library String
+Library Process
+Resource ../commonlib.robot
+Resource ./commonawslib.robot
+Test Timeout 5 minutes
+Suite Setup Setup s3 tests
+
+*** Variables ***
+${ENDPOINT_URL} http://s3g:9878
+${BUCKET} generated
+
+*** Test Cases ***
+
+Conditional Put If-None-Match Star Creates New Key
+ [Documentation] If-None-Match: * should succeed when key does not exist
+ ${key} = Set Variable condput-ifnonematch-new
+ Execute echo "test-content" > /tmp/${key}
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*"
+ Should contain ${result} ETag
+
+Conditional Put If-None-Match Star Fails For Existing Key
+ [Documentation] If-None-Match: * should fail with 412 when key already exists
+ ${key} = Set Variable condput-ifnonematch-existing
+ Execute echo "initial-content" > /tmp/${key}
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}
+ Should contain ${result} ETag
+ # Now try again with If-None-Match: *
+ ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-none-match "*"
+ Should contain ${result} PreconditionFailed
+
+Conditional Put If-Match With Correct ETag Succeeds
+ [Documentation] If-Match with correct ETag should succeed
+ ${key} = Set Variable condput-ifmatch-success
+ Execute echo "initial-content" > /tmp/${key}
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}
+ Should contain ${result} ETag
+ # Extract the ETag value
+ ${etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"')
+ # Rewrite with matching ETag
+ Execute echo "updated-content" > /tmp/${key}-updated
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}-updated --if-match ${etag}
+ Should contain ${result} ETag
+ ${new_etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"')
+ Should Not Be Equal ${new_etag} ${etag}
+ ${head_result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${key}
+ ${head_etag} = Evaluate __import__('json').loads(r'''${head_result}''')['ETag'].strip('"')
+ Should Be Equal ${head_etag} ${new_etag}
+
+Conditional Put If-Match With Wrong ETag Fails
+ [Documentation] If-Match with wrong ETag should fail with 412
+ ${key} = Set Variable condput-ifmatch-fail
+ Execute echo "initial-content" > /tmp/${key}
+ ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key}
+ Should contain ${result} ETag
+ ${etag} = Evaluate __import__('json').loads(r'''${result}''')['ETag'].strip('"')
+ # Try to rewrite with a wrong ETag
+ ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match wrong-etag
+ Should contain ${result} PreconditionFailed
+ ${head_result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${key}
+ ${head_etag} = Evaluate __import__('json').loads(r'''${head_result}''')['ETag'].strip('"')
+ Should Be Equal ${head_etag} ${etag}
+
+Conditional Put If-Match On Non-Existent Key Fails
+ [Documentation] If-Match on a key that does not exist should fail with 412
+ ${key} = Set Variable condput-ifmatch-nonexistent
+ Execute echo "test-content" > /tmp/${key}
+ ${result} = Execute AWSS3APICli and ignore error put-object --bucket ${BUCKET} --key ${key} --body /tmp/${key} --if-match some-etag
+ Should contain ${result} PreconditionFailed
+ ${head_result} = Execute AWSS3APICli and ignore error head-object --bucket ${BUCKET} --key ${key}
+ Should contain ${head_result} 404
+ Should contain ${head_result} Not Found
diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index 7fb2f885725..d8fbd1f1ce3 100644
--- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -25,6 +25,7 @@
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -377,6 +378,117 @@ public void testPutObject() {
assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag());
}
+ @Test
+ public void testPutObjectIfNoneMatch() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(bucketName);
+
+ InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectRequest request = new PutObjectRequest(
+ bucketName, keyName, is, new ObjectMetadata()).ifNoneMatch("*");
+
+ PutObjectResult putObjectResult = s3Client.putObject(request);
+ assertEquals("37b51d194a7513e45b56f6524f2d51f2", putObjectResult.getETag());
+ }
+
+ @Test
+ public void testPutObjectIfNoneMatchFail() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(bucketName);
+
+ InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
+
+ InputStream is2 = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectRequest request = new PutObjectRequest(
+ bucketName, keyName, is2, new ObjectMetadata()).ifNoneMatch("*");
+
+ AmazonServiceException ase = assertThrows(AmazonServiceException.class,
+ () -> s3Client.putObject(request));
+
+ assertEquals(ErrorType.Client, ase.getErrorType());
+ assertEquals(412, ase.getStatusCode());
+ assertEquals("PreconditionFailed", ase.getErrorCode());
+ }
+
+ @Test
+ public void testPutObjectIfMatch() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(bucketName);
+
+ InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
+ String etag = putObjectResult.getETag();
+
+ InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8));
+ PutObjectRequest request = new PutObjectRequest(
+ bucketName, keyName, is2, new ObjectMetadata()).ifMatch(etag);
+
+ PutObjectResult putObjectResult2 = s3Client.putObject(request);
+ assertNotNull(putObjectResult2.getETag());
+ assertNotEquals(etag, putObjectResult2.getETag());
+
+ ObjectMetadata updatedObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName);
+ assertEquals(putObjectResult2.getETag(), updatedObjectMetadata.getETag());
+ }
+
+ @Test
+ public void testPutObjectIfMatchFail() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(bucketName);
+
+ InputStream is = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectResult initialResult =
+ s3Client.putObject(bucketName, keyName, is, new ObjectMetadata());
+
+ InputStream is2 = new ByteArrayInputStream("bar2".getBytes(StandardCharsets.UTF_8));
+ PutObjectRequest request = new PutObjectRequest(
+ bucketName, keyName, is2, new ObjectMetadata()).ifMatch("wrong-etag");
+
+ AmazonServiceException ase = assertThrows(AmazonServiceException.class,
+ () -> s3Client.putObject(request));
+
+ assertEquals(ErrorType.Client, ase.getErrorType());
+ assertEquals(412, ase.getStatusCode());
+ assertEquals("PreconditionFailed", ase.getErrorCode());
+
+ ObjectMetadata existingObjectMetadata = s3Client.getObjectMetadata(bucketName, keyName);
+ assertEquals(initialResult.getETag(), existingObjectMetadata.getETag());
+ }
+
+ @Test
+ public void testPutObjectIfMatchMissingKeyFail() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ s3Client.createBucket(bucketName);
+
+ InputStream is = new ByteArrayInputStream("bar2".getBytes(
+ StandardCharsets.UTF_8));
+ PutObjectRequest request = new PutObjectRequest(
+ bucketName, keyName, is, new ObjectMetadata()).ifMatch("some-etag");
+
+ AmazonServiceException ase = assertThrows(AmazonServiceException.class,
+ () -> s3Client.putObject(request));
+
+ assertEquals(ErrorType.Client, ase.getErrorType());
+ assertEquals(412, ase.getStatusCode());
+ assertEquals("PreconditionFailed", ase.getErrorCode());
+
+ AmazonServiceException missingKey = assertThrows(AmazonServiceException.class,
+ () -> s3Client.getObject(bucketName, keyName));
+ assertEquals(ErrorType.Client, missingKey.getErrorType());
+ assertEquals(404, missingKey.getStatusCode());
+ assertEquals("NoSuchKey", missingKey.getErrorCode());
+ }
+
@Test
public void testPutObjectWithMD5Header() throws Exception {
final String bucketName = getBucketName();
diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
index 175dcea265f..35a53dd328b 100644
--- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
+++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java
@@ -25,6 +25,7 @@
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -236,6 +237,107 @@ public void testPutObject() {
assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", getObjectResponse.eTag());
}
+ @Test
+ public void testPutObjectIfNoneMatch() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(keyName)
+ .ifNoneMatch("*"),
+ RequestBody.fromString(content));
+
+ assertEquals("\"37b51d194a7513e45b56f6524f2d51f2\"", putObjectResponse.eTag());
+ }
+
+ @Test
+ public void testPutObjectIfNoneMatchFail() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ s3Client.putObject(b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content));
+
+ S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(keyName)
+ .ifNoneMatch("*"),
+ RequestBody.fromString(content)));
+
+ assertEquals(412, exception.statusCode());
+ assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
+ }
+
+ @Test
+ public void testPutObjectIfMatch() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ PutObjectResponse initialResponse = s3Client.putObject(b -> b.bucket(bucketName).key(keyName),
+ RequestBody.fromString(content));
+
+ PutObjectResponse putObjectResponse = s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(keyName)
+ .ifMatch(initialResponse.eTag()),
+ RequestBody.fromString("bar2"));
+
+ assertNotNull(putObjectResponse.eTag());
+ assertNotEquals(initialResponse.eTag(), putObjectResponse.eTag());
+
+ HeadObjectResponse headObjectResponse = s3Client.headObject(
+ b -> b.bucket(bucketName).key(keyName));
+ assertEquals(putObjectResponse.eTag(), headObjectResponse.eTag());
+ }
+
+ @Test
+ public void testPutObjectIfMatchFail() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ final String content = "bar";
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ PutObjectResponse initialResponse = s3Client.putObject(
+ b -> b.bucket(bucketName).key(keyName), RequestBody.fromString(content));
+
+ S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(keyName)
+ .ifMatch("wrong-etag"),
+ RequestBody.fromString("bar2")));
+
+ assertEquals(412, exception.statusCode());
+ assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
+
+ HeadObjectResponse headObjectResponse = s3Client.headObject(
+ b -> b.bucket(bucketName).key(keyName));
+ assertEquals(initialResponse.eTag(), headObjectResponse.eTag());
+ }
+
+ @Test
+ public void testPutObjectIfMatchMissingKeyFail() {
+ final String bucketName = getBucketName();
+ final String keyName = getKeyName();
+ s3Client.createBucket(b -> b.bucket(bucketName));
+
+ S3Exception exception = assertThrows(S3Exception.class, () -> s3Client.putObject(b -> b
+ .bucket(bucketName)
+ .key(keyName)
+ .ifMatch("some-etag"),
+ RequestBody.fromString("bar2")));
+
+ assertEquals(412, exception.statusCode());
+ assertEquals("PreconditionFailed", exception.awsErrorDetails().errorCode());
+ assertThrows(NoSuchKeyException.class, () -> s3Client.headObject(
+ b -> b.bucket(bucketName).key(keyName)));
+ }
+
@Test
public void testPutObjectEmpty() throws Exception {
final String bucketName = getBucketName();
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
index 378316c6b15..ec1d522b721 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java
@@ -42,6 +42,7 @@
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
import static org.apache.hadoop.ozone.client.OzoneClientTestUtils.assertKeyContent;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DIR_DELETING_SERVICE_INTERVAL;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_ALREADY_EXISTS;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.NO_SUCH_MULTIPART_UPLOAD_ERROR;
import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.PARTIAL_RENAME;
@@ -1482,6 +1483,125 @@ void cannotRewriteRenamedKey(BucketLayout layout) throws IOException {
assertThat(e).hasMessageContaining("not found");
}
+ @ParameterizedTest
+ @EnumSource
+ void testCreateKeyIfNotExistsSuccess(BucketLayout layout) throws IOException {
+ checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
+ OzoneBucket bucket = createBucket(layout);
+ String keyName = "create-if-not-exists-" + layout.name();
+ byte[] content = "test-content".getBytes(UTF_8);
+
+ try (OzoneOutputStream out = bucket.createKeyIfNotExists(
+ keyName, content.length,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ Collections.emptyMap(), Collections.emptyMap())) {
+ out.write(content);
+ }
+
+ assertKeyContent(bucket, keyName, content);
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testCreateKeyIfNotExistsFailsWhenKeyExists(BucketLayout layout) throws IOException {
+ checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
+ OzoneBucket bucket = createBucket(layout);
+ OzoneKeyDetails keyDetails = createTestKey(bucket);
+ byte[] newContent = "new-content".getBytes(UTF_8);
+
+ OMException e = assertThrows(OMException.class, () -> {
+ try (OzoneOutputStream out = bucket.createKeyIfNotExists(
+ keyDetails.getName(), newContent.length,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ Collections.emptyMap(), Collections.emptyMap())) {
+ out.write(newContent);
+ }
+ });
+ assertEquals(KEY_ALREADY_EXISTS, e.getResult());
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRewriteKeyIfMatchSuccess(BucketLayout layout) throws IOException {
+ checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
+ OzoneBucket bucket = createBucket(layout);
+ OzoneKeyDetails keyDetails = createTestKeyWithETag(bucket);
+ String etag = keyDetails.getMetadata().get(ETAG);
+ assertNotNull(etag, "Key should have an ETag");
+
+ byte[] newContent = "rewritten-content".getBytes(UTF_8);
+ Map<String, String> rewrittenMetadata = new HashMap<>(keyDetails.getMetadata());
+ String rewrittenETag = UUID.randomUUID().toString();
+ rewrittenMetadata.put(ETAG, rewrittenETag);
+ try (OzoneOutputStream out = bucket.rewriteKeyIfMatch(
+ keyDetails.getName(), newContent.length, etag,
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ rewrittenMetadata, Collections.emptyMap())) {
+ out.write(newContent);
+ }
+
+ assertKeyContent(bucket, keyDetails.getName(), newContent);
+ assertEquals(rewrittenETag, bucket.getKey(keyDetails.getName())
+ .getMetadata().get(ETAG));
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRewriteKeyIfMatchFailsWithWrongETag(BucketLayout layout) throws IOException {
+ checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
+ OzoneBucket bucket = createBucket(layout);
+ OzoneKeyDetails keyDetails = createTestKeyWithETag(bucket);
+ byte[] newContent = "rewritten-content".getBytes(UTF_8);
+
+ OMException e = assertThrows(OMException.class, () -> {
+ try (OzoneOutputStream out = bucket.rewriteKeyIfMatch(
+ keyDetails.getName(), newContent.length, "wrong-etag",
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ keyDetails.getMetadata(), Collections.emptyMap())) {
+ out.write(newContent);
+ }
+ });
+ assertEquals(OMException.ResultCodes.ETAG_MISMATCH, e.getResult());
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRewriteKeyIfMatchFailsWhenETagNotAvailable(BucketLayout layout)
+ throws IOException {
+ checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
+ OzoneBucket bucket = createBucket(layout);
+ OzoneKeyDetails keyDetails = createTestKey(bucket);
+ byte[] newContent = "rewritten-content".getBytes(UTF_8);
+
+ OMException e = assertThrows(OMException.class, () -> {
+ try (OzoneOutputStream out = bucket.rewriteKeyIfMatch(
+ keyDetails.getName(), newContent.length, "some-etag",
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ keyDetails.getMetadata(), Collections.emptyMap())) {
+ out.write(newContent);
+ }
+ });
+ assertEquals(OMException.ResultCodes.ETAG_NOT_AVAILABLE, e.getResult());
+ }
+
+ @ParameterizedTest
+ @EnumSource
+ void testRewriteKeyIfMatchFailsWhenKeyNotFound(BucketLayout layout) throws IOException {
+ checkFeatureEnable(OzoneManagerVersion.ATOMIC_REWRITE_KEY);
+ OzoneBucket bucket = createBucket(layout);
+ byte[] content = "content".getBytes(UTF_8);
+
+ OMException e = assertThrows(OMException.class, () -> {
+ try (OzoneOutputStream out = bucket.rewriteKeyIfMatch(
+ "nonexistent-key", content.length, "some-etag",
+ RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE),
+ Collections.emptyMap(), Collections.emptyMap())) {
+ out.write(content);
+ }
+ });
+ assertEquals(KEY_NOT_FOUND, e.getResult());
+ }
+
private static void rewriteKey(
OzoneBucket bucket, OzoneKeyDetails keyDetails, byte[] newContent
) throws IOException {
@@ -4483,9 +4603,29 @@ private OzoneKeyDetails createTestKey(
private OzoneKeyDetails createTestKey(
OzoneBucket bucket, String keyName, byte[] bytes
+ ) throws IOException {
+ return createTestKey(bucket, keyName, bytes, createTestKeyMetadata());
+ }
+
+ private OzoneKeyDetails createTestKeyWithETag(OzoneBucket bucket)
+ throws IOException {
+ Map<String, String> metadata = createTestKeyMetadata();
+ metadata.put(ETAG, UUID.randomUUID().toString());
+ return createTestKey(bucket, getTestName(),
+ UUID.randomUUID().toString().getBytes(UTF_8), metadata);
+ }
+
+ private static Map<String, String> createTestKeyMetadata() {
+ Map<String, String> metadata = new HashMap<>();
+ metadata.put("key", RandomStringUtils.secure().nextAscii(10));
+ return metadata;
+ }
+
+ private OzoneKeyDetails createTestKey(
+ OzoneBucket bucket, String keyName, byte[] bytes,
+ Map<String, String> metadata
) throws IOException {
RatisReplicationConfig replication = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE);
- Map<String, String> metadata = singletonMap("key", RandomStringUtils.secure().nextAscii(10));
try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, replication, metadata)) {
out.write(bytes);
}
diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index b6bed6b0198..0cf4dc1b35d 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -568,6 +568,10 @@ enum Status {
KEY_UNDER_LEASE_SOFT_LIMIT_PERIOD = 97;
TOO_MANY_SNAPSHOTS = 98;
+
+ ETAG_MISMATCH = 99;
+
+ ETAG_NOT_AVAILABLE = 100;
}
/**
@@ -1085,6 +1089,11 @@ message KeyArgs {
// This allows a key to be created an committed atomically if the original has not
// been modified.
optional uint64 expectedDataGeneration = 23;
+
+ // expectedETag, when set, indicates that the existing key must have
+ // the given ETag for the operation to succeed. This is used for
+ // S3 conditional writes with the If-Match header.
+ optional string expectedETag = 24;
}
message KeyLocation {
@@ -1176,6 +1185,11 @@ message KeyInfo {
// This allows a key to be created an committed atomically if the original has not
// been modified.
optional uint64 expectedDataGeneration = 22;
+
+ // expectedETag, when set, indicates that the existing key must have
+ // the given ETag for the operation to succeed. This is used for
+ // S3 conditional writes with the If-Match header.
+ optional string expectedETag = 23;
}
// KeyInfoProtoLight is a lightweight subset of KeyInfo message containing
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index 492d698db99..65b3394b5fc 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -309,6 +309,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
// Set the UpdateID to current transactionLogIndex
omKeyInfo = omKeyInfo.toBuilder()
.setExpectedDataGeneration(null)
+ .setExpectedETag(null)
.addAllMetadata(KeyValueUtil.getFromProtobuf(
commitKeyArgs.getMetadataList()))
.setUpdateID(trxnLogIndex)
@@ -641,6 +642,24 @@ protected void validateAtomicRewrite(OmKeyInfo existing, OmKeyInfo toCommit, Map
}
}
}
+
+ if (toCommit.getExpectedETag() != null) {
+ String expectedETag = toCommit.getExpectedETag();
+ auditMap.put("expectedETag", expectedETag);
+
+ if (existing == null) {
+ throw new OMException("Key not found for If-Match at commit",
+ OMException.ResultCodes.KEY_NOT_FOUND);
+ }
+ if (!existing.hasEtag()) {
+ throw new OMException("Key does not have an ETag at commit",
+ OMException.ResultCodes.ETAG_NOT_AVAILABLE);
+ }
+ if (!existing.isEtagEquals(expectedETag)) {
+ throw new OMException("ETag changed during write (concurrent modification)",
+ OMException.ResultCodes.ETAG_MISMATCH);
+ }
+ }
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
index 25b5a4b15d4..5aa4fbeb36f 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequestWithFSO.java
@@ -246,6 +246,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
// Optimistic locking validation has passed. Now set the rewrite fields to null so they are
// not persisted in the key table.
omKeyInfo.setExpectedDataGeneration(null);
+ omKeyInfo.setExpectedETag(null);
long correctedSpace = omKeyInfo.getReplicatedSize();
// if keyToDelete isn't null, usedNamespace shouldn't check and increase.
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
index 1298ff7426f..4d37a1d7aa7 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCreateRequest.java
@@ -496,5 +496,21 @@ protected void validateAtomicRewrite(OmKeyInfo dbKeyInfo, KeyArgs keyArgs)
}
}
}
+
+ if (keyArgs.hasExpectedETag()) {
+ String expectedETag = keyArgs.getExpectedETag();
+ if (dbKeyInfo == null) {
+ throw new OMException("Key not found for If-Match",
+ OMException.ResultCodes.KEY_NOT_FOUND);
+ }
+ if (!dbKeyInfo.hasEtag()) {
+ throw new OMException("Key does not have an ETag",
+ OMException.ResultCodes.ETAG_NOT_AVAILABLE);
+ }
+ if (!dbKeyInfo.isEtagEquals(expectedETag)) {
+ throw new OMException("ETag mismatch",
+ OMException.ResultCodes.ETAG_MISMATCH);
+ }
+ }
}
}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
index 7df2619e9e4..13dc904476d 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java
@@ -993,6 +993,9 @@ protected OmKeyInfo prepareFileInfo(
if (keyArgs.hasExpectedDataGeneration()) {
builder.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration());
}
+ if (keyArgs.hasExpectedETag()) {
+ builder.setExpectedETag(keyArgs.getExpectedETag());
+ }
return builder.build();
}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
index 652a7aa0fcf..1f40b0f72a0 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
@@ -350,6 +350,143 @@ public void testAtomicCreateIfNotExistsCommitKeyAlreadyExists() throws Exception
assertEquals(KEY_ALREADY_EXISTS, omClientResponse.getOMResponse().getStatus());
}
+ @Test
+ public void testCommitWithExpectedETagSuccess() throws Exception {
+ Table<String, OmKeyInfo> openKeyTable =
+ omMetadataManager.getOpenKeyTable(getBucketLayout());
+ Table<String, OmKeyInfo> closedKeyTable =
+ omMetadataManager.getKeyTable(getBucketLayout());
+
+ OMRequest modifiedOmRequest =
+ doPreExecute(createCommitKeyRequest());
+ OMKeyCommitRequest omKeyCommitRequest =
+ getOmKeyCommitRequest(modifiedOmRequest);
+ KeyArgs keyArgs =
+ modifiedOmRequest.getCommitKeyRequest().getKeyArgs();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, omKeyCommitRequest.getBucketLayout());
+
+ List<OmKeyLocationInfo> allocatedLocationList =
+ keyArgs.getKeyLocationsList().stream()
+ .map(OmKeyLocationInfo::getFromProtobuf)
+ .collect(Collectors.toList());
+
+ String expectedETag = "matching-etag";
+ OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig,
+ new OmKeyLocationInfoGroup(version, new ArrayList<>()));
+ omKeyInfoBuilder.setExpectedETag(expectedETag);
+
+ String openKey = addKeyToOpenKeyTable(allocatedLocationList,
+ omKeyInfoBuilder);
+ assertNotNull(openKeyTable.get(openKey));
+
+ // Add closed key with matching ETag
+ OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig,
+ new OmKeyLocationInfoGroup(version, new ArrayList<>()))
+ .addMetadata(OzoneConsts.ETAG, expectedETag).build();
+ closedKeyTable.put(getOzonePathKey(), closedKeyInfo);
+
+ OMClientResponse omClientResponse =
+ omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L);
+ assertEquals(OK, omClientResponse.getOMResponse().getStatus());
+
+ OmKeyInfo committedKey = closedKeyTable.get(getOzonePathKey());
+ assertNotNull(committedKey);
+ assertNull(committedKey.getExpectedETag());
+ }
+
+ @Test
+ public void testCommitWithExpectedETagMismatch() throws Exception {
+ Table<String, OmKeyInfo> openKeyTable =
+ omMetadataManager.getOpenKeyTable(getBucketLayout());
+ Table<String, OmKeyInfo> closedKeyTable =
+ omMetadataManager.getKeyTable(getBucketLayout());
+
+ OMRequest modifiedOmRequest =
+ doPreExecute(createCommitKeyRequest());
+ OMKeyCommitRequest omKeyCommitRequest =
+ getOmKeyCommitRequest(modifiedOmRequest);
+ KeyArgs keyArgs =
+ modifiedOmRequest.getCommitKeyRequest().getKeyArgs();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, omKeyCommitRequest.getBucketLayout());
+
+ List<OmKeyLocationInfo> allocatedLocationList =
+ keyArgs.getKeyLocationsList().stream()
+ .map(OmKeyLocationInfo::getFromProtobuf)
+ .collect(Collectors.toList());
+
+ OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig,
+ new OmKeyLocationInfoGroup(version, new ArrayList<>()));
+ omKeyInfoBuilder.setExpectedETag("expected-etag");
+
+ String openKey = addKeyToOpenKeyTable(allocatedLocationList,
+ omKeyInfoBuilder);
+ assertNotNull(openKeyTable.get(openKey));
+
+ // Add closed key with different ETag
+ OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig,
+ new OmKeyLocationInfoGroup(version, new ArrayList<>()))
+ .addMetadata(OzoneConsts.ETAG, "different-etag").build();
+ closedKeyTable.put(getOzonePathKey(), closedKeyInfo);
+
+ OMClientResponse omClientResponse =
+ omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L);
+ assertEquals(
+ OzoneManagerProtocolProtos.Status.ETAG_MISMATCH,
+ omClientResponse.getOMResponse().getStatus());
+ }
+
+ @Test
+ public void testCommitWithExpectedETagNoETagOnKey() throws Exception {
+ Table<String, OmKeyInfo> openKeyTable =
+ omMetadataManager.getOpenKeyTable(getBucketLayout());
+ Table<String, OmKeyInfo> closedKeyTable =
+ omMetadataManager.getKeyTable(getBucketLayout());
+
+ OMRequest modifiedOmRequest =
+ doPreExecute(createCommitKeyRequest());
+ OMKeyCommitRequest omKeyCommitRequest =
+ getOmKeyCommitRequest(modifiedOmRequest);
+ KeyArgs keyArgs =
+ modifiedOmRequest.getCommitKeyRequest().getKeyArgs();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, omKeyCommitRequest.getBucketLayout());
+
+ List<OmKeyLocationInfo> allocatedLocationList =
+ keyArgs.getKeyLocationsList().stream()
+ .map(OmKeyLocationInfo::getFromProtobuf)
+ .collect(Collectors.toList());
+
+ OmKeyInfo.Builder omKeyInfoBuilder = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig,
+ new OmKeyLocationInfoGroup(version, new ArrayList<>()));
+ omKeyInfoBuilder.setExpectedETag("expected-etag");
+
+ String openKey = addKeyToOpenKeyTable(allocatedLocationList,
+ omKeyInfoBuilder);
+ assertNotNull(openKeyTable.get(openKey));
+
+ // Add closed key WITHOUT ETag
+ OmKeyInfo closedKeyInfo = OMRequestTestUtils.createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig,
+ new OmKeyLocationInfoGroup(version, new ArrayList<>())).build();
+ closedKeyTable.put(getOzonePathKey(), closedKeyInfo);
+
+ OMClientResponse omClientResponse =
+ omKeyCommitRequest.validateAndUpdateCache(ozoneManager, 100L);
+ assertEquals(
+ OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE,
+ omClientResponse.getOMResponse().getStatus());
+ }
+
@Test
public void testValidateAndUpdateCacheWithUncommittedBlocks()
throws Exception {
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
index 52c9eeea07d..f9364d87710 100644
--- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java
@@ -230,6 +230,126 @@ public void testCreateKeyExpectedGenMismatchReturnsKeyGenerationMismatch(
assertNull(openKeyInfo);
}
+ @ParameterizedTest
+ @MethodSource("data")
+ public void testCreateWithExpectedETagKeyNotFound(
+ boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception {
+ when(ozoneManager.getOzoneLockProvider()).thenReturn(
+ new OzoneLockProvider(setKeyPathLock, setFileSystemPaths));
+
+ OMRequest modifiedOmRequest = doPreExecute(
+ createKeyRequestWithExpectedETag("some-etag"));
+ OMKeyCreateRequest omKeyCreateRequest =
+ getOMKeyCreateRequest(modifiedOmRequest);
+
+ addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager,
+ getBucketLayout());
+
+ OMClientResponse response =
+ omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L);
+ assertEquals(KEY_NOT_FOUND, response.getOMResponse().getStatus());
+ }
+
+ @ParameterizedTest
+ @MethodSource("data")
+ public void testCreateWithExpectedETagNoETagOnKey(
+ boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception {
+ when(ozoneManager.getOzoneLockProvider()).thenReturn(
+ new OzoneLockProvider(setKeyPathLock, setFileSystemPaths));
+
+ OMRequest modifiedOmRequest = doPreExecute(
+ createKeyRequestWithExpectedETag("some-etag"));
+ OMKeyCreateRequest omKeyCreateRequest =
+ getOMKeyCreateRequest(modifiedOmRequest);
+
+ addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager,
+ getBucketLayout());
+
+ // Create existing key without ETag metadata
+ OmKeyInfo existingKeyInfo = createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setUpdateID(1L).build();
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .put(getOzoneKey(), existingKeyInfo);
+
+ OMClientResponse response =
+ omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L);
+ assertEquals(
+ OzoneManagerProtocolProtos.Status.ETAG_NOT_AVAILABLE,
+ response.getOMResponse().getStatus());
+ }
+
+ @ParameterizedTest
+ @MethodSource("data")
+ public void testCreateWithExpectedETagMismatch(
+ boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception {
+ when(ozoneManager.getOzoneLockProvider()).thenReturn(
+ new OzoneLockProvider(setKeyPathLock, setFileSystemPaths));
+
+ OMRequest modifiedOmRequest = doPreExecute(
+ createKeyRequestWithExpectedETag("expected-etag"));
+ OMKeyCreateRequest omKeyCreateRequest =
+ getOMKeyCreateRequest(modifiedOmRequest);
+
+ addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager,
+ getBucketLayout());
+
+ // Create existing key with a different ETag
+ OmKeyInfo existingKeyInfo = createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setUpdateID(1L)
+ .addMetadata(OzoneConsts.ETAG, "different-etag")
+ .build();
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .put(getOzoneKey(), existingKeyInfo);
+
+ OMClientResponse response =
+ omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L);
+ assertEquals(
+ OzoneManagerProtocolProtos.Status.ETAG_MISMATCH,
+ response.getOMResponse().getStatus());
+ }
+
+ @ParameterizedTest
+ @MethodSource("data")
+ public void testCreateWithExpectedETagSuccess(
+ boolean setKeyPathLock, boolean setFileSystemPaths) throws Exception {
+ when(ozoneManager.getOzoneLockProvider()).thenReturn(
+ new OzoneLockProvider(setKeyPathLock, setFileSystemPaths));
+
+ String expectedETag = "matching-etag";
+ OMRequest modifiedOmRequest = doPreExecute(
+ createKeyRequestWithExpectedETag(expectedETag));
+ OMKeyCreateRequest omKeyCreateRequest =
+ getOMKeyCreateRequest(modifiedOmRequest);
+
+ addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager,
+ getBucketLayout());
+
+ // Create existing key with matching ETag
+ OmKeyInfo existingKeyInfo = createOmKeyInfo(
+ volumeName, bucketName, keyName, replicationConfig)
+ .setUpdateID(1L)
+ .addMetadata(OzoneConsts.ETAG, expectedETag)
+ .build();
+ omMetadataManager.getKeyTable(getBucketLayout())
+ .put(getOzoneKey(), existingKeyInfo);
+
+ long id = modifiedOmRequest.getCreateKeyRequest().getClientID();
+ OMClientResponse response =
+ omKeyCreateRequest.validateAndUpdateCache(ozoneManager, 100L);
+ assertEquals(OK, response.getOMResponse().getStatus());
+
+ // Verify open key was created with expectedETag
+ OmKeyInfo openKeyInfo = omMetadataManager.getOpenKeyTable(getBucketLayout())
+ .get(getOpenKey(id));
+ assertNotNull(openKeyInfo);
+ assertEquals(expectedETag, openKeyInfo.getExpectedETag());
+ // Creation time should remain the same on rewrite
+ assertEquals(existingKeyInfo.getCreationTime(),
+ openKeyInfo.getCreationTime());
+ }
+
@ParameterizedTest
@MethodSource("data")
public void testValidateAndUpdateCache(
@@ -909,6 +1029,30 @@ private OMRequest createKeyRequest(
.setCreateKeyRequest(createKeyRequest).build();
}
+ private OMRequest createKeyRequestWithExpectedETag(String expectedETag) {
+ KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
+ .setVolumeName(volumeName).setBucketName(bucketName)
+ .setKeyName(keyName).setIsMultipartKey(false)
+ .setFactor(
+ ((RatisReplicationConfig) replicationConfig)
+ .getReplicationFactor())
+ .setType(replicationConfig.getReplicationType())
+ .setLatestVersionLocation(true)
+ .setDataSize(100L);
+
+ if (expectedETag != null) {
+ keyArgs.setExpectedETag(expectedETag);
+ }
+
+ CreateKeyRequest createKeyRequest =
+ CreateKeyRequest.newBuilder().setKeyArgs(keyArgs).build();
+
+ return OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.CreateKey)
+ .setClientId(UUID.randomUUID().toString())
+ .setCreateKeyRequest(createKeyRequest).build();
+ }
+
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testKeyCreateWithFileSystemPathsEnabled(
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
index b18cf35d0d3..ad67ebd2590 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
@@ -187,8 +187,18 @@ public Response put(
throw newError(S3ErrorTable.NO_SUCH_BUCKET, bucketName, ex);
} else if (ex.getResult() == ResultCodes.FILE_ALREADY_EXISTS) {
throw newError(S3ErrorTable.NO_OVERWRITE, keyPath, ex);
+ } else if (ex.getResult() == ResultCodes.KEY_ALREADY_EXISTS) {
+ throw newError(PRECOND_FAILED, keyPath, ex);
+ } else if (ex.getResult() == ResultCodes.ETAG_MISMATCH) {
+ throw newError(PRECOND_FAILED, keyPath, ex);
+ } else if (ex.getResult() == ResultCodes.ETAG_NOT_AVAILABLE) {
+ throw newError(PRECOND_FAILED, keyPath, ex);
} else if (ex.getResult() == ResultCodes.INVALID_REQUEST) {
throw newError(S3ErrorTable.INVALID_REQUEST, keyPath);
+ } else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND
+ && getHeaders().getHeaderString(S3Consts.IF_MATCH_HEADER) != null) {
+ // If-Match failed because the key doesn't exist
+ throw newError(PRECOND_FAILED, keyPath, ex);
} else if (ex.getResult() == ResultCodes.KEY_NOT_FOUND) {
throw newError(S3ErrorTable.NO_SUCH_KEY, keyPath);
} else if (ex.getResult() == ResultCodes.NOT_SUPPORTED_OPERATION) {
@@ -230,7 +240,6 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr
copyHeader = getHeaders().getHeaderString(COPY_SOURCE_HEADER);
- // Normal put object
ReplicationConfig replicationConfig = getReplicationConfig(bucket);
boolean enableEC = false;
@@ -271,6 +280,38 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr
return Response.ok().status(HttpStatus.SC_OK).build();
}
+ String ifNoneMatch = getHeaders().getHeaderString(
+ S3Consts.IF_NONE_MATCH_HEADER);
+ String ifMatch = getHeaders().getHeaderString(
+ S3Consts.IF_MATCH_HEADER);
+
+ if (ifNoneMatch != null && StringUtils.isBlank(ifNoneMatch)) {
+ OS3Exception ex = newError(INVALID_REQUEST, keyPath);
+ ex.setErrorMessage("If-None-Match header cannot be empty.");
+ throw ex;
+ }
+ if (ifMatch != null && StringUtils.isBlank(ifMatch)) {
+ OS3Exception ex = newError(INVALID_REQUEST, keyPath);
+ ex.setErrorMessage("If-Match header cannot be empty.");
+ throw ex;
+ }
+
+ String ifNoneMatchTrimmed = ifNoneMatch == null ? null : ifNoneMatch.trim();
+ String ifMatchTrimmed = ifMatch == null ? null : ifMatch.trim();
+
+ if (ifNoneMatchTrimmed != null && ifMatchTrimmed != null) {
+ OS3Exception ex = newError(INVALID_REQUEST, keyPath);
+ ex.setErrorMessage("If-Match and If-None-Match cannot be specified together.");
+ throw ex;
+ }
+
+ if (ifNoneMatchTrimmed != null
+ && !"*".equals(stripQuotes(ifNoneMatchTrimmed))) {
+ OS3Exception ex = newError(INVALID_REQUEST, keyPath);
+ ex.setErrorMessage("Only If-None-Match: * is supported for conditional put.");
+ throw ex;
+ }
+
// Normal put object
S3ChunkInputStreamInfo chunkInputStreamInfo = getS3ChunkInputStreamInfo(body,
length, amzDecodedLength, keyPath);
@@ -287,15 +328,18 @@ Response handlePutRequest(ObjectRequestContext context, String keyPath, InputStr
perf.appendStreamMode();
Pair<String, Long> keyWriteResult = ObjectEndpointStreaming
.put(bucket, keyPath, length, replicationConfig, getChunkSize(),
- customMetadata, tags, multiDigestInputStream, getHeaders(), signatureInfo.isSignPayload(), perf);
+ customMetadata, tags, multiDigestInputStream, getHeaders(),
+ signatureInfo.isSignPayload(), perf, ifNoneMatchTrimmed,
+ ifMatchTrimmed);
md5Hash = keyWriteResult.getKey();
putLength = keyWriteResult.getValue();
} else {
final String amzContentSha256Header =
validateSignatureHeader(getHeaders(), keyPath, signatureInfo.isSignPayload());
- try (OzoneOutputStream output = getClientProtocol().createKey(
- volume.getName(), bucketName, keyPath, length, replicationConfig,
- customMetadata, tags)) {
+ try (OzoneOutputStream output = openKeyForPut(
+ volume.getName(), bucketName, keyPath, length,
+ replicationConfig, customMetadata, tags, ifNoneMatchTrimmed,
+ ifMatchTrimmed)) {
long metadataLatencyNs =
getMetrics().updatePutKeyMetadataStats(startNanos);
perf.appendMetaLatencyNanos(metadataLatencyNs);
@@ -1127,7 +1171,43 @@ private CopyObjectResponse copyObject(OzoneVolume volume,
}
}
}
-
+
+ /**
+ * Opens a key for put, applying conditional write logic based on
+ * If-None-Match and If-Match headers.
+ */
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private OzoneOutputStream openKeyForPut(String volumeName, String bucketName, String keyPath, long length,
+ ReplicationConfig replicationConfig, Map<String, String> customMetadata,
+ Map<String, String> tags, String ifNoneMatch, String ifMatch)
+ throws IOException {
+ if (ifNoneMatch != null && "*".equals(stripQuotes(ifNoneMatch.trim()))) {
+ return getClientProtocol().createKeyIfNotExists(
+ volumeName, bucketName, keyPath, length, replicationConfig,
+ customMetadata, tags);
+ } else if (ifMatch != null) {
+ String expectedETag = parseETag(ifMatch);
+ return getClientProtocol().rewriteKeyIfMatch(
+ volumeName, bucketName, keyPath, length, expectedETag,
+ replicationConfig, customMetadata, tags);
+ } else {
+ return getClientProtocol().createKey(
+ volumeName, bucketName, keyPath, length, replicationConfig,
+ customMetadata, tags);
+ }
+ }
+
+ /**
+ * Parses an ETag from a conditional header value, removing surrounding
+ * quotes if present.
+ */
+ static String parseETag(String headerValue) {
+ if (headerValue == null) {
+ return null;
+ }
+ return stripQuotes(headerValue.trim());
+ }
+
/** Request context shared among {@code ObjectOperationHandler}s. */
final class ObjectRequestContext extends S3RequestContext {
private final String bucketName;
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
index 767c11506dc..2eb2dd21f30 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpointStreaming.java
@@ -73,12 +73,13 @@ public static Pair<String, Long> put(
int chunkSize, Map<String, String> keyMetadata,
Map<String, String> tags, MultiDigestInputStream body,
HttpHeaders headers, boolean isSignedPayload,
- PerformanceStringBuilder perf)
+ PerformanceStringBuilder perf, String ifNoneMatch, String ifMatch)
throws IOException, OS3Exception {
try {
return putKeyWithStream(bucket, keyPath,
- length, chunkSize, replicationConfig, keyMetadata, tags, body, headers, isSignedPayload, perf);
+ length, chunkSize, replicationConfig, keyMetadata, tags, body,
+ headers, isSignedPayload, perf, ifNoneMatch, ifMatch);
} catch (IOException ex) {
LOG.error("Exception occurred in PutObject", ex);
if (ex instanceof OMException) {
@@ -113,14 +114,17 @@ public static Pair<String, Long> putKeyWithStream(
MultiDigestInputStream body,
HttpHeaders headers,
boolean isSignedPayload,
- PerformanceStringBuilder perf)
+ PerformanceStringBuilder perf,
+ String ifNoneMatch,
+ String ifMatch)
throws IOException, OS3Exception {
long startNanos = Time.monotonicNowNanos();
final String amzContentSha256Header = validateSignatureHeader(headers, keyPath, isSignedPayload);
long writeLen;
String md5Hash;
- try (OzoneDataStreamOutput streamOutput = bucket.createStreamKey(keyPath,
- length, replicationConfig, keyMetadata, tags)) {
+ try (OzoneDataStreamOutput streamOutput = openStreamKeyForPut(bucket,
+ keyPath, length, replicationConfig, keyMetadata, tags, ifNoneMatch,
+ ifMatch)) {
long metadataLatencyNs = METRICS.updatePutKeyMetadataStats(startNanos);
writeLen = writeToStreamOutput(streamOutput, body, bufferSize, length);
md5Hash = DatatypeConverter.printHexBinary(body.getMessageDigest(OzoneConsts.MD5_HASH).digest())
@@ -155,6 +159,24 @@ public static Pair<String, Long> putKeyWithStream(
return Pair.of(md5Hash, writeLen);
}
+ @SuppressWarnings("checkstyle:ParameterNumber")
+ private static OzoneDataStreamOutput openStreamKeyForPut(OzoneBucket bucket,
+ String keyPath, long length, ReplicationConfig replicationConfig,
+ Map<String, String> keyMetadata, Map<String, String> tags,
+ String ifNoneMatch, String ifMatch) throws IOException {
+ if (ifNoneMatch != null && "*".equals(ObjectEndpoint.parseETag(ifNoneMatch))) {
+ return bucket.createStreamKeyIfNotExists(keyPath, length,
+ replicationConfig, keyMetadata, tags);
+ }
+ if (ifMatch != null) {
+ return bucket.rewriteStreamKeyIfMatch(keyPath, length,
+ ObjectEndpoint.parseETag(ifMatch), replicationConfig, keyMetadata,
+ tags);
+ }
+ return bucket.createStreamKey(keyPath, length, replicationConfig,
+ keyMetadata, tags);
+ }
+
@SuppressWarnings("checkstyle:ParameterNumber")
public static long copyKeyWithStream(
OzoneBucket bucket,
diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
index 797ca1f3671..e90612cc56e 100644
--- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
+++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Consts.java
@@ -98,6 +98,10 @@ public final class S3Consts {
public static final String CHECKSUM_HEADER = "Content-MD5";
+ // Conditional request headers
+ public static final String IF_NONE_MATCH_HEADER = "If-None-Match";
+ public static final String IF_MATCH_HEADER = "If-Match";
+
//Never Constructed
private S3Consts() {
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
index 739babce1d0..32a969fddbc 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java
@@ -249,6 +249,45 @@ public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String
.rewriteKey(keyName, size, existingKeyGeneration, replicationConfig, metadata);
}
+ @Override
+ public OzoneOutputStream createKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ return getBucket(volumeName, bucketName)
+ .createKeyIfNotExists(keyName, size, replicationConfig, metadata, tags);
+ }
+
+ @Override
+ public OzoneOutputStream rewriteKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ return getBucket(volumeName, bucketName)
+ .rewriteKeyIfMatch(keyName, size, expectedETag, replicationConfig,
+ metadata, tags);
+ }
+
+ @Override
+ public OzoneDataStreamOutput createStreamKeyIfNotExists(String volumeName,
+ String bucketName, String keyName, long size,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ return getBucket(volumeName, bucketName)
+ .createStreamKeyIfNotExists(keyName, size, replicationConfig,
+ metadata, tags);
+ }
+
+ @Override
+ public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String volumeName,
+ String bucketName, String keyName, long size, String expectedETag,
+ ReplicationConfig replicationConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ return getBucket(volumeName, bucketName)
+ .rewriteStreamKeyIfMatch(keyName, size, expectedETag,
+ replicationConfig, metadata, tags);
+ }
+
@Override
public OzoneInputStream getKey(String volumeName, String bucketName,
String keyName) throws IOException {
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
index 8037f65fda1..75cc73aadf2 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java
@@ -104,7 +104,7 @@ public OzoneBucketStub build() {
return new OzoneBucketStub(this);
}
}
-
+
@Override
public OzoneOutputStream createKey(String key, long size) throws IOException {
return createKey(key, size,
@@ -192,6 +192,38 @@ public void close() throws IOException {
return new OzoneOutputStream(byteArrayOutputStream, null);
}
+ @Override
+ public OzoneOutputStream createKeyIfNotExists(String keyName, long size,
+ ReplicationConfig rConfig, Map<String, String> metadata,
+ Map<String, String> tags) throws IOException {
+ if (keyDetails.containsKey(keyName)) {
+ throw new OMException("Key already exists",
+ ResultCodes.KEY_ALREADY_EXISTS);
+ }
+ return createKey(keyName, size, rConfig, metadata, tags);
+ }
+
+ @Override
+ public OzoneOutputStream rewriteKeyIfMatch(String keyName, long size,
+ String expectedETag, ReplicationConfig rConfig,
+ Map<String, String> metadata, Map<String, String> tags)
+ throws IOException {
+ OzoneKeyDetails existing = keyDetails.get(keyName);
+ if (existing == null) {
+ throw new OMException("Key not found for If-Match",
+ ResultCodes.KEY_NOT_FOUND);
+ }
+ if (!existing.hasEtag()) {
+ throw new OMException("Key does not have an ETag",
+ ResultCodes.ETAG_NOT_AVAILABLE);
+ }
+ if (!existing.isEtagEquals(expectedETag)) {
+ throw new OMException("ETag mismatch",
+ ResultCodes.ETAG_MISMATCH);
+ }
+ return createKey(keyName, size, rConfig, metadata, tags);
+ }
+
@Override
public OzoneDataStreamOutput createStreamKey(String key, long size,
ReplicationConfig rConfig,
@@ -247,6 +279,38 @@ public void flush() throws IOException {
return new OzoneDataStreamOutputStub(byteBufferStreamOutput, key + size);
}
+ @Override
+ public OzoneDataStreamOutput createStreamKeyIfNotExists(String key, long size,
+ ReplicationConfig rConfig, Map<String, String> keyMetadata,
+ Map<String, String> tags) throws IOException {
+ if (keyDetails.containsKey(key)) {
+ throw new OMException("Key already exists",
+ ResultCodes.KEY_ALREADY_EXISTS);
+ }
+ return createStreamKey(key, size, rConfig, keyMetadata, tags);
+ }
+
+ @Override
+ public OzoneDataStreamOutput rewriteStreamKeyIfMatch(String key, long size,
+ String expectedETag, ReplicationConfig rConfig,
+ Map<String, String> keyMetadata, Map<String, String> tags)
+ throws IOException {
+ OzoneKeyDetails existing = keyDetails.get(key);
+ if (existing == null) {
+ throw new OMException("Key not found for If-Match",
+ ResultCodes.KEY_NOT_FOUND);
+ }
+ if (!existing.hasEtag()) {
+ throw new OMException("Key does not have an ETag",
+ ResultCodes.ETAG_NOT_AVAILABLE);
+ }
+ if (!existing.isEtagEquals(expectedETag)) {
+ throw new OMException("ETag mismatch",
+ ResultCodes.ETAG_MISMATCH);
+ }
+ return createStreamKey(key, size, rConfig, keyMetadata, tags);
+ }
+
@Override
public OzoneDataStreamOutput createMultipartStreamKey(String key,
long size,
diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
index c2456dd068f..e85b121aaaf 100644
--- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
+++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectPut.java
@@ -555,6 +555,113 @@ private HttpHeaders newMockHttpHeaders() {
return httpHeaders;
}
+ @Test
+ void testIfNoneMatchKeyDoesNotExistSuccess() throws Exception {
+ when(headers.getHeaderString("If-None-Match")).thenReturn("*");
+
+ assertSucceeds(() -> putObject(CONTENT));
+ assertKeyContent(bucket, KEY_NAME, CONTENT);
+ }
+
+ @Test
+ void testIfNoneMatchKeyExistsPreconditionFailed() throws Exception {
+ // First create the key
+ assertSucceeds(() -> putObject(CONTENT));
+
+ // Now try to create again with If-None-Match: *
+ when(headers.getHeaderString("If-None-Match")).thenReturn("*");
+
+ OS3Exception ex = assertErrorResponse(
+ S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT));
+ assertNotNull(ex);
+ }
+
+ @Test
+ void testIfMatchETagMatchesSuccess() throws Exception {
+ // First create the key to get an ETag
+ Response response = putObject(CONTENT);
+ String etag = response.getHeaderString(HttpHeaders.ETAG);
+ assertNotNull(etag);
+
+ // Now try to rewrite with matching ETag
+ when(headers.getHeaderString("If-Match")).thenReturn(etag);
+
+ assertSucceeds(() -> putObject("new-content"));
+ assertKeyContent(bucket, KEY_NAME, "new-content");
+ }
+
+ @Test
+ void testIfMatchETagMismatchPreconditionFailed() throws Exception {
+ // First create the key
+ assertSucceeds(() -> putObject(CONTENT));
+
+ // Try to rewrite with wrong ETag
+ when(headers.getHeaderString("If-Match")).thenReturn("\"wrong-etag\"");
+
+ OS3Exception ex = assertErrorResponse(
+ S3ErrorTable.PRECOND_FAILED, () -> putObject("new-content"));
+ assertNotNull(ex);
+ }
+
+ @Test
+ void testIfMatchKeyNotFoundPreconditionFailed() throws Exception {
+ // Try If-Match on a non-existent key
+ when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\"");
+
+ OS3Exception ex = assertErrorResponse(
+ S3ErrorTable.PRECOND_FAILED, () -> putObject(CONTENT));
+ assertNotNull(ex);
+ }
+
+ @Test
+ void testBothHeadersProvidedInvalidRequest() throws Exception {
+ when(headers.getHeaderString("If-None-Match")).thenReturn("*");
+ when(headers.getHeaderString("If-Match")).thenReturn("\"some-etag\"");
+
+ OS3Exception ex = assertErrorResponse(
+ INVALID_REQUEST, () -> putObject(CONTENT));
+ assertNotNull(ex);
+ assertThat(ex.getErrorMessage()).contains(
+ "If-Match and If-None-Match cannot be specified together");
+ }
+
+ @Test
+ void testBlankIfNoneMatchInvalidRequest() throws Exception {
+ when(headers.getHeaderString("If-None-Match")).thenReturn(" ");
+
+ OS3Exception ex = assertErrorResponse(
+ INVALID_REQUEST, () -> putObject(CONTENT));
+ assertThat(ex.getErrorMessage()).contains(
+ "If-None-Match header cannot be empty");
+ }
+
+ @Test
+ void testBlankIfMatchInvalidRequest() throws Exception {
+ when(headers.getHeaderString("If-Match")).thenReturn(" ");
+
+ OS3Exception ex = assertErrorResponse(
+ INVALID_REQUEST, () -> putObject(CONTENT));
+ assertThat(ex.getErrorMessage()).contains("If-Match header cannot be empty");
+ }
+
+ @Test
+ void testIfNoneMatchNotStarInvalidRequest() throws Exception {
+ when(headers.getHeaderString("If-None-Match")).thenReturn("\"etag\"");
+
+ OS3Exception ex = assertErrorResponse(
+ INVALID_REQUEST, () -> putObject(CONTENT));
+ assertThat(ex.getErrorMessage()).contains(
+ "Only If-None-Match: * is supported");
+ }
+
+ @Test
+ void testParseETag() {
+ assertEquals("abc123", ObjectEndpoint.parseETag("\"abc123\""));
+ assertEquals("abc123", ObjectEndpoint.parseETag("abc123"));
+ assertEquals("abc123", ObjectEndpoint.parseETag(" \"abc123\" "));
+ assertEquals(null, ObjectEndpoint.parseETag(null));
+ }
+
/** Put object at {@code bucketName}/{@code keyName} with pre-defined {@link #CONTENT}. */
private Response putObject(String bucketName, String keyName) throws IOException, OS3Exception {
return put(objectEndpoint, bucketName, keyName, CONTENT);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org