You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2026/06/18 18:32:30 UTC
(jackrabbit-oak) 01/01: OAK-12241 : updated S3 tests for oak-blob-cloud to run on S3Mock using docker
This is an automated email from the ASF dual-hosted git repository.
rishabhdaim pushed a commit to branch OAK-12241
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit c0946b9a8f9b9707a941a18ade1fac38dfcedf63
Author: rishabhdaim <ri...@gmail.com>
AuthorDate: Fri Jun 19 00:02:06 2026 +0530
OAK-12241 : updated S3 tests for oak-blob-cloud to run on S3Mock using docker
---
oak-blob-cloud/pom.xml | 5 +
.../jackrabbit/oak/blob/cloud/s3/S3Constants.java | 8 +
.../apache/jackrabbit/oak/blob/cloud/s3/Utils.java | 8 +-
.../cloud/s3/S3DataRecordAccessProviderIT.java | 8 +-
.../cloud/s3/S3DataRecordAccessProviderTest.java | 171 ++++++++++++++++++++-
.../oak/blob/cloud/s3/S3DataStoreUtils.java | 106 +++++++++----
.../oak/blob/cloud/s3/S3EmulatorSupport.java | 128 +++++++++++++++
.../oak/blob/cloud/s3/S3EmulatorSupportTest.java | 94 +++++++++++
.../jackrabbit/oak/blob/cloud/s3/S3MockRule.java | 109 +++++++++++++
.../jackrabbit/oak/blob/cloud/s3/TestS3Ds.java | 10 ++
.../oak/blob/cloud/s3/TestS3DsCacheOff.java | 23 +++
.../jackrabbit/oak/blob/cloud/s3/UtilsTest.java | 70 ++++++++-
oak-doc/src/site/markdown/osgi_config.md | 4 +
13 files changed, 706 insertions(+), 38 deletions(-)
diff --git a/oak-blob-cloud/pom.xml b/oak-blob-cloud/pom.xml
index 9bccf0d064..5b70e6453f 100644
--- a/oak-blob-cloud/pom.xml
+++ b/oak-blob-cloud/pom.xml
@@ -487,6 +487,11 @@
<artifactId>org.osgi.service.cm</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>testcontainers</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
index 7e70b6f13d..5f7238e3b2 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java
@@ -169,6 +169,14 @@ public final class S3Constants {
*/
public static final String PRESIGNED_HTTP_DOWNLOAD_URI_VERIFY_EXISTS = "presignedHttpDownloadURIVerifyExists";
+ /**
+ * Boolean property to enable path-style access for S3 clients ({@code pathStyleAccess=true}).
+ * When enabled, requests use {@code http://endpoint/bucket/key} instead of
+ * {@code https://bucket.endpoint/key}. Defaults to {@code false}.
+ * Always enabled automatically when the remote storage mode is GCP.
+ */
+ public static final String PATH_STYLE_ACCESS = "pathStyleAccess";
+
/**
* private constructor so that class cannot initialized from outside.
*/
diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
index 6c479076a1..3d5239adab 100644
--- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
+++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/Utils.java
@@ -170,7 +170,7 @@ public final class Utils {
.credentialsProvider(Utils.getAwsCredentials(props))
.region(Region.of(Utils.getRegion(props)))
.serviceConfiguration(S3Configuration.builder()
- .pathStyleAccessEnabled(isGCP)
+ .pathStyleAccessEnabled(isPathStyleAccessEnabled(props, isGCP))
.chunkedEncodingEnabled(!isGCP)
.build())
.build();
@@ -584,12 +584,16 @@ public final class Utils {
builder.serviceConfiguration(
S3Configuration.builder()
- .pathStyleAccessEnabled(isGCP) // enable for GCP
+ .pathStyleAccessEnabled(isPathStyleAccessEnabled(prop, isGCP)) // always on for GCP; opt-in for S3 via pathStyleAccess property
.chunkedEncodingEnabled(!isGCP) // Disable for GCP
.useArnRegionEnabled(!isGCP) // Disable for GCP
.build());
}
+ private static boolean isPathStyleAccessEnabled(Properties prop, boolean isGCP) {
+ return isGCP || Boolean.parseBoolean(prop.getProperty(S3Constants.PATH_STYLE_ACCESS, "false"));
+ }
+
// Helper class to hold common Http config
private static class HttpClientConfig {
final int connectionTimeout;
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java
index fa36a39a29..0958b3d941 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderIT.java
@@ -20,6 +20,7 @@ package org.apache.jackrabbit.oak.blob.cloud.s3;
import java.io.File;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URI;
import javax.net.ssl.HttpsURLConnection;
@@ -80,6 +81,11 @@ public class S3DataRecordAccessProviderIT extends AbstractDataRecordAccessProvid
@Override
protected HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException {
- return S3DataStoreUtils.getHttpsConnection(length, uri);
+ throw new UnsupportedOperationException("Upload is handled by doHttpsUpload; see S3DataStoreUtils.doHttpUpload");
+ }
+
+ @Override
+ protected void doHttpsUpload(InputStream in, long contentLength, URI uri) throws IOException {
+ S3DataStoreUtils.doHttpUpload(in, contentLength, uri);
}
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
index cf59f79645..c8f390540d 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataRecordAccessProviderTest.java
@@ -22,20 +22,29 @@ import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getFixtur
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3Config;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.getS3DataStore;
import static org.apache.jackrabbit.oak.blob.cloud.s3.S3DataStoreUtils.isS3Configured;
+import static org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils.randomStream;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.HttpURLConnection;
import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import javax.net.ssl.HttpsURLConnection;
+import org.apache.jackrabbit.oak.api.blob.BlobDownloadOptions;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordAccessProvider;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.directaccess.DataRecordDownloadOptions;
import org.apache.jackrabbit.oak.spi.blob.data.DataIdentifier;
import org.apache.jackrabbit.oak.spi.blob.data.DataRecord;
import org.apache.jackrabbit.oak.spi.blob.data.DataStore;
@@ -131,7 +140,12 @@ public class S3DataRecordAccessProviderTest extends AbstractDataRecordAccessProv
@Override
protected HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException {
- return S3DataStoreUtils.getHttpsConnection(length, uri);
+ throw new UnsupportedOperationException("S3Mock uses HTTP; upload is handled by doHttpsUpload");
+ }
+
+ @Override
+ protected void doHttpsUpload(InputStream in, long contentLength, URI uri) throws IOException {
+ S3DataStoreUtils.doHttpUpload(in, contentLength, uri);
}
@Test
@@ -168,4 +182,159 @@ public class S3DataRecordAccessProviderTest extends AbstractDataRecordAccessProv
upload = ds.initiateDataRecordUpload(uploadSize, -1);
assertEquals(expectedNumURIs, upload.getUploadURIs().size());
}
+
+ @Override
+ @Test
+ public void testGetDownloadURIIT() throws DataStoreException, IOException {
+ assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption());
+ assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned URLs point to real AWS, not emulator",
+ !S3EmulatorSupport.isAvailable());
+ DataRecord record = null;
+ DataRecordAccessProvider dataStore = getDataStore();
+ try {
+ InputStream testStream = randomStream(0, 256);
+ record = doSynchronousAddRecord((DataStore) dataStore, testStream);
+ URI uri = dataStore.getDownloadURI(record.getIdentifier(), DataRecordDownloadOptions.DEFAULT);
+ HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection();
+ try {
+ conn.setRequestMethod("GET");
+ assertEquals(200, conn.getResponseCode());
+
+ testStream.reset();
+ try (InputStream responseStream = conn.getInputStream()) {
+ assertTrue(Arrays.equals(testStream.readAllBytes(), responseStream.readAllBytes()));
+ }
+ } finally {
+ conn.disconnect();
+ }
+ } finally {
+ if (null != record) {
+ doDeleteRecord((DataStore) dataStore, record.getIdentifier());
+ }
+ }
+ }
+
+ @Override
+ @Test
+ public void testGetDownloadURIWithCustomHeadersIT() throws DataStoreException, IOException {
+ assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption());
+ assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned URLs point to real AWS, not emulator",
+ !S3EmulatorSupport.isAvailable());
+ String umlautFilename = "Umläutfile.png";
+ String umlautFilename_ISO_8859_1 = new String(
+ StandardCharsets.ISO_8859_1.encode(umlautFilename).array(),
+ StandardCharsets.ISO_8859_1
+ );
+ List<String> fileNames = List.of(
+ "image.png",
+ "beautiful landscape.png",
+ "\"filename-with-double-quotes\".png",
+ "filename-with-one\"double-quote.jpg",
+ umlautFilename
+ );
+ List<String> iso_8859_1_fileNames = List.of(
+ "image.png",
+ "beautiful landscape.png",
+ "\\\"filename-with-double-quotes\\\".png",
+ "filename-with-one\\\"double-quote.jpg",
+ umlautFilename_ISO_8859_1
+ );
+ List<String> rfc8187_fileNames = List.of(
+ "image.png",
+ "beautiful%20landscape.png",
+ "%22filename-with-double-quotes%22.png",
+ "filename-with-one%22double-quote.jpg",
+ "Uml%C3%A4utfile.png"
+ );
+
+ DataRecord record = null;
+ DataRecordAccessProvider dataStore = getDataStore();
+ try {
+ InputStream testStream = randomStream(0, 256);
+ record = doSynchronousAddRecord((DataStore) dataStore, testStream);
+ String mimeType = "image/png";
+ String dispositionType = "inline";
+ for (int i = 0; i < fileNames.size(); i++) {
+ String fileName = fileNames.get(i);
+ String iso_8859_1_fileName = iso_8859_1_fileNames.get(i);
+ String encodedFileName = rfc8187_fileNames.get(i);
+ DataRecordDownloadOptions downloadOptions =
+ DataRecordDownloadOptions.fromBlobDownloadOptions(
+ new BlobDownloadOptions(mimeType, null, fileName, dispositionType)
+ );
+ URI uri = dataStore.getDownloadURI(record.getIdentifier(), downloadOptions);
+
+ HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection();
+ try {
+ conn.setRequestMethod("GET");
+ assertEquals(200, conn.getResponseCode());
+
+ assertEquals(mimeType, conn.getHeaderField("Content-Type"));
+ assertEquals(
+ String.format("%s; filename=\"%s\"; filename*=UTF-8''%s",
+ dispositionType, iso_8859_1_fileName, encodedFileName),
+ conn.getHeaderField("Content-Disposition")
+ );
+
+ testStream.reset();
+ try (InputStream responseStream = conn.getInputStream()) {
+ assertTrue(Arrays.equals(testStream.readAllBytes(), responseStream.readAllBytes()));
+ }
+ } finally {
+ conn.disconnect();
+ }
+ }
+ } finally {
+ if (null != record) {
+ doDeleteRecord((DataStore) dataStore, record.getIdentifier());
+ }
+ }
+ }
+
+ @Override
+ @Test
+ public void testSinglePutDirectUploadIT() throws DataRecordUploadException, DataStoreException, IOException {
+ assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned PUT URLs point to real AWS, not emulator",
+ !S3EmulatorSupport.isAvailable());
+ super.testSinglePutDirectUploadIT();
+ }
+
+ @Override
+ @Test
+ public void testCompleteAlreadyUploadedBinaryReturnsSameBinaryIT() throws DataStoreException, DataRecordUploadException, IOException {
+ assumeTrue("S3Presigner does not inherit endpointOverride from S3Client; presigned PUT URLs point to real AWS, not emulator",
+ !S3EmulatorSupport.isAvailable());
+ super.testCompleteAlreadyUploadedBinaryReturnsSameBinaryIT();
+ }
+
+ @Override
+ @Test
+ public void testGetExpiredReadURIFailsIT() throws DataStoreException, IOException {
+ assumeTrue("SSE-C doesn't support presigned GET URLs", !isSSECustomerKeyEncryption());
+ assumeFalse("S3Presigner does not inherit endpointOverride from S3Client; presigned URLs point to real AWS, not emulator",
+ S3EmulatorSupport.isAvailable());
+ DataRecord record = null;
+ ConfigurableDataRecordAccessProvider dataStore = getDataStore();
+ try {
+ dataStore.setDirectDownloadURIExpirySeconds(2);
+ record = doSynchronousAddRecord((DataStore) dataStore, randomStream(0, 256));
+ URI uri = dataStore.getDownloadURI(record.getIdentifier(), DataRecordDownloadOptions.DEFAULT);
+ try {
+ Thread.sleep(5 * 1000);
+ } catch (InterruptedException e) {
+ }
+ HttpURLConnection conn = (HttpURLConnection) uri.toURL().openConnection();
+ try {
+ conn.setRequestMethod("GET");
+ assertEquals(403, conn.getResponseCode());
+ } finally {
+ conn.disconnect();
+ }
+ } finally {
+ if (null != record) {
+ doDeleteRecord((DataStore) dataStore, record.getIdentifier());
+ }
+ dataStore.setDirectDownloadURIExpirySeconds(expirySeconds);
+ }
+ }
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
index 5b695ebe84..4d30c02cc6 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3DataStoreUtils.java
@@ -22,15 +22,16 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
import java.net.URI;
+import java.net.URLConnection;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
-import javax.net.ssl.HttpsURLConnection;
-
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.spi.blob.data.DataStore;
import org.apache.jackrabbit.oak.commons.PropertiesUtil;
@@ -125,9 +126,21 @@ public class S3DataStoreUtils extends DataStoreUtils {
props = new Properties();
props.putAll(filtered);
}
+ // Fall back to emulator when no real credentials are present.
+ if (!hasRealCredentials(props) && S3EmulatorSupport.isAvailable()) {
+ return S3EmulatorSupport.getEmulatorProperties();
+ }
return props;
}
+ private static boolean hasRealCredentials(Properties props) {
+ String accessKey = props.getProperty(S3Constants.ACCESS_KEY, "").trim();
+ String secretKey = props.getProperty(S3Constants.SECRET_KEY, "").trim();
+ String region = props.getProperty(S3Constants.S3_REGION, "").trim();
+ String endpoint = props.getProperty(S3Constants.S3_END_POINT, "").trim();
+ return !accessKey.isEmpty() && !secretKey.isEmpty() && (!region.isEmpty() || !endpoint.isEmpty());
+ }
+
public static DataStore getS3DataStore(String className, Properties props, String homeDir) throws Exception {
DataStore ds = Class.forName(className).asSubclass(DataStore.class).newInstance();
PropertiesUtil.populate(ds, Utils.asMap(props), false);
@@ -150,38 +163,65 @@ public class S3DataStoreUtils extends DataStoreUtils {
S3BackendHelper.deleteBucketAndAbortMultipartUploads(bucket, date, props);
}
- protected static HttpsURLConnection getHttpsConnection(long length, URI uri) throws IOException {
- HttpsURLConnection conn = (HttpsURLConnection) uri.toURL().openConnection();
- conn.setDoOutput(true);
- conn.setRequestMethod("PUT");
- conn.setRequestProperty("Content-Length", String.valueOf(length));
-
- final Properties props = getS3Config();
- DataEncryption encryption = Utils.getDataEncryption(props);
-
- switch (encryption) {
- case SSE_S3:
- conn.setRequestProperty("x-amz-server-side-encryption", "AES256");
- System.out.println("Added SSE-S3 header: AES256");
- break;
- case SSE_KMS:
- conn.setRequestProperty("x-amz-server-side-encryption", "aws:kms");
- System.out.println("Added SSE-KMS header: aws:kms");
- if (Objects.nonNull(props.getProperty(S3Constants.S3_SSE_KMS_KEYID))) {
- conn.setRequestProperty("x-amz-server-side-encryption-aws-kms-key-id", props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
- System.out.println("Added KMS Key ID: " + props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
- }
- break;
- case SSE_C:
- String customerKey = props.getProperty(S3Constants.S3_SSE_C_KEY);
- String customerKeyMD5 = Utils.calculateMD5(customerKey);
- conn.setRequestProperty("x-amz-server-side-encryption-customer-algorithm", "AES256");
- conn.setRequestProperty("x-amz-server-side-encryption-customer-key", customerKey);
- conn.setRequestProperty("x-amz-server-side-encryption-customer-key-MD5", customerKeyMD5);
- break;
- case NONE:
- break;
+ protected static HttpURLConnection getHttpConnection(long length, URI uri) throws IOException {
+ URLConnection raw = uri.toURL().openConnection();
+ if (!(raw instanceof HttpURLConnection)) {
+ throw new IOException("Expected HTTP(S) connection for URI: " + uri);
+ }
+ HttpURLConnection conn = (HttpURLConnection) raw;
+ try {
+ conn.setDoOutput(true);
+ conn.setRequestMethod("PUT");
+ conn.setRequestProperty("Content-Length", String.valueOf(length));
+
+ final Properties props = getS3Config();
+ DataEncryption encryption = Utils.getDataEncryption(props);
+
+ switch (encryption) {
+ case SSE_S3:
+ conn.setRequestProperty("x-amz-server-side-encryption", "AES256");
+ System.out.println("Added SSE-S3 header: AES256");
+ break;
+ case SSE_KMS:
+ conn.setRequestProperty("x-amz-server-side-encryption", "aws:kms");
+ System.out.println("Added SSE-KMS header: aws:kms");
+ if (Objects.nonNull(props.getProperty(S3Constants.S3_SSE_KMS_KEYID))) {
+ conn.setRequestProperty("x-amz-server-side-encryption-aws-kms-key-id", props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
+ System.out.println("Added KMS Key ID: " + props.getProperty(S3Constants.S3_SSE_KMS_KEYID));
+ }
+ break;
+ case SSE_C:
+ String customerKey = props.getProperty(S3Constants.S3_SSE_C_KEY);
+ String customerKeyMD5 = Utils.calculateMD5(customerKey);
+ conn.setRequestProperty("x-amz-server-side-encryption-customer-algorithm", "AES256");
+ conn.setRequestProperty("x-amz-server-side-encryption-customer-key", customerKey);
+ conn.setRequestProperty("x-amz-server-side-encryption-customer-key-MD5", customerKeyMD5);
+ break;
+ case NONE:
+ break;
+ }
+ } catch (RuntimeException e) {
+ conn.disconnect();
+ throw e;
}
return conn;
}
+
+ protected static void doHttpUpload(InputStream in, long contentLength, URI uri) throws IOException {
+ HttpURLConnection conn = getHttpConnection(contentLength, uri);
+ try {
+ try (OutputStream out = conn.getOutputStream()) {
+ IOUtils.copy(in, out);
+ }
+ int responseCode = conn.getResponseCode();
+ org.junit.Assert.assertTrue(conn.getResponseMessage(), responseCode < 400);
+ } finally {
+ InputStream err = conn.getErrorStream();
+ if (err != null) {
+ try { IOUtils.toByteArray(err); } catch (IOException ignored) {}
+ IOUtils.closeQuietly(err);
+ }
+ conn.disconnect();
+ }
+ }
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java
new file mode 100644
index 0000000000..d8fc9cdc84
--- /dev/null
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupport.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Properties;
+
+/**
+ * Lazy singleton that starts an {@link S3MockRule} Docker container on first use and exposes
+ * S3 emulator properties for use in {@link S3DataStoreUtils#getS3Config()}.
+ *
+ * <p>GCP emulator coverage is intentionally out of scope because S3Mock is not compatible
+ * with Oak's GCP client configuration.</p>
+ *
+ * <p>If Docker is unavailable, {@link #isAvailable()} returns {@code false} and tests
+ * skip rather than fail.</p>
+ */
+public final class S3EmulatorSupport {
+
+ private static final Logger log = LoggerFactory.getLogger(S3EmulatorSupport.class);
+
+ /** System property reserved for selecting emulator mode; only S3 is currently supported. */
+ static final String S3_TEST_MODE_PROP = "s3.test.mode";
+
+ /** Dummy access key accepted by S3Mock. */
+ static final String ACCESS_KEY = "foo";
+
+ /** Dummy secret key accepted by S3Mock. */
+ static final String SECRET_KEY = "bar";
+
+ /** Default bucket name used when no test-specific bucket is configured. */
+ static final String DEFAULT_BUCKET = "s3mock-default-test-bucket";
+
+ private static volatile S3MockRule instance;
+ private static volatile boolean startupAttempted = false;
+
+ private S3EmulatorSupport() {}
+
+ /**
+ * Returns {@code true} if the S3Mock container started successfully and is available for tests.
+ */
+ public static boolean isAvailable() {
+ if (isUnsupportedMode()) {
+ return false;
+ }
+ ensureStarted();
+ return instance != null;
+ }
+
+ /**
+ * Returns a {@link Properties} instance configured for the S3 emulator.
+ * Returns an empty {@link Properties} if the container is not available.
+ */
+ public static Properties getEmulatorProperties() {
+ if (isUnsupportedMode()) {
+ return new Properties();
+ }
+ ensureStarted();
+ S3MockRule rule = instance;
+ if (rule == null) {
+ return new Properties();
+ }
+ String endpoint = rule.getHttpEndpoint();
+ Properties props = new Properties();
+ props.setProperty(S3Constants.ACCESS_KEY, ACCESS_KEY);
+ props.setProperty(S3Constants.SECRET_KEY, SECRET_KEY);
+ props.setProperty(S3Constants.S3_BUCKET, DEFAULT_BUCKET);
+ props.setProperty(S3Constants.S3_CONN_PROTOCOL, "http");
+ props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true");
+ props.setProperty(S3Constants.S3_ENCRYPTION, S3Constants.S3_ENCRYPTION_NONE);
+ props.setProperty(S3Constants.S3_END_POINT, endpoint);
+ props.setProperty(S3Constants.S3_REGION, "us-east-1");
+ props.setProperty(S3Constants.S3_MAX_ERR_RETRY, "3");
+ props.setProperty(S3Constants.S3_CONN_TIMEOUT, "10000");
+ props.setProperty(S3Constants.S3_SOCK_TIMEOUT, "30000");
+ props.setProperty(S3Constants.S3_MAX_CONNS, "10");
+ return props;
+ }
+
+ private static void ensureStarted() {
+ if (isUnsupportedMode() || instance != null) {
+ return;
+ }
+ synchronized (S3EmulatorSupport.class) {
+ if (instance != null || startupAttempted) {
+ return;
+ }
+ startupAttempted = true;
+ S3MockRule rule = new S3MockRule();
+ try {
+ rule.before();
+ instance = rule;
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ rule.after();
+ } catch (Exception ignored) {
+ }
+ }));
+ } catch (Throwable e) {
+ try {
+ rule.after();
+ } catch (Exception ignored) {
+ }
+ log.warn("S3Mock container failed to start — emulator tests will be skipped", e);
+ }
+ }
+ }
+
+ private static boolean isUnsupportedMode() {
+ return !"S3".equalsIgnoreCase(System.getProperty(S3_TEST_MODE_PROP, "S3"));
+ }
+}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java
new file mode 100644
index 0000000000..0e58146efc
--- /dev/null
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3EmulatorSupportTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Properties;
+
+/**
+ * Tests for {@link S3EmulatorSupport}.
+ */
+public class S3EmulatorSupportTest {
+
+ private String previousMode;
+
+ @Before
+ public void setUp() {
+ previousMode = System.getProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ }
+
+ @After
+ public void tearDown() {
+ if (previousMode == null) {
+ System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ } else {
+ System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, previousMode);
+ }
+ }
+
+ @Test
+ public void isAvailableReturnsFalseForUnsupportedGcpMode() {
+ System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, "GCP");
+ Assert.assertFalse(S3EmulatorSupport.isAvailable());
+ }
+
+ @Test
+ public void isAvailableReturnsTrueInS3ModeWhenDockerAvailable() {
+ System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ Assume.assumeTrue("Docker is not available", S3EmulatorSupport.isAvailable());
+ Assert.assertTrue(S3EmulatorSupport.isAvailable());
+ }
+
+ @Test
+ public void getEmulatorPropertiesReturnsEmptyForUnsupportedGcpMode() {
+ System.setProperty(S3EmulatorSupport.S3_TEST_MODE_PROP, "GCP");
+ Assert.assertTrue(S3EmulatorSupport.getEmulatorProperties().isEmpty());
+ }
+
+ @Test
+ public void getEmulatorPropertiesReturnsEmptyWhenUnavailable() {
+ // When Docker is not available, isAvailable() is false and getEmulatorProperties()
+ // must return empty properties rather than throwing.
+ Assume.assumeFalse("Docker is available", S3EmulatorSupport.isAvailable());
+ Properties props = S3EmulatorSupport.getEmulatorProperties();
+ Assert.assertTrue("Expected empty properties when emulator is unavailable", props.isEmpty());
+ }
+
+ @Test
+ public void getEmulatorPropertiesInS3ModeContainsRequiredKeys() {
+ System.clearProperty(S3EmulatorSupport.S3_TEST_MODE_PROP);
+ Assume.assumeTrue("Docker is not available", S3EmulatorSupport.isAvailable());
+
+ Properties props = S3EmulatorSupport.getEmulatorProperties();
+ Assert.assertNotNull(props.getProperty(S3Constants.ACCESS_KEY));
+ Assert.assertNotNull(props.getProperty(S3Constants.SECRET_KEY));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_END_POINT));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_REGION));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_BUCKET));
+ Assert.assertEquals("true", props.getProperty(S3Constants.PATH_STYLE_ACCESS));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_MAX_ERR_RETRY));
+ Assert.assertNotNull(props.getProperty(S3Constants.S3_CONN_TIMEOUT));
+ Assert.assertEquals("http", props.getProperty(S3Constants.S3_CONN_PROTOCOL));
+ Assert.assertEquals(S3Constants.S3_ENCRYPTION_NONE, props.getProperty(S3Constants.S3_ENCRYPTION));
+ Assert.assertNull("S3 mode must not set MODE", props.get(S3Constants.MODE));
+ }
+}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java
new file mode 100644
index 0000000000..d1878c7749
--- /dev/null
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3MockRule.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.blob.cloud.s3;
+
+import org.junit.Assume;
+import org.junit.rules.ExternalResource;
+import org.junit.runner.Description;
+import org.junit.runners.model.MultipleFailureException;
+import org.junit.runners.model.Statement;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * JUnit rule that starts an Adobe S3Mock Docker container for emulator-based S3 integration tests.
+ * Mirrors the pattern used by {@code AzuriteDockerRule} in oak-blob-cloud-azure.
+ *
+ * <p>On Docker failure, the rule calls {@link Assume#assumeNoException} so tests are skipped
+ * rather than failed, preserving the same behaviour as when no credentials are configured.</p>
+ */
+public class S3MockRule extends ExternalResource {
+
+ private static final DockerImageName DOCKER_IMAGE = DockerImageName.parse("adobe/s3mock:5.0.0");
+ static final int HTTP_PORT = 9090;
+ private final AtomicReference<Exception> startupException = new AtomicReference<>();
+
+ private GenericContainer<?> container;
+
+ @Override
+ protected void before() throws Throwable {
+ startupException.set(null);
+ container = new GenericContainer<>(DOCKER_IMAGE)
+ .withExposedPorts(HTTP_PORT)
+ .withStartupTimeout(Duration.ofSeconds(60));
+ try {
+ container.start();
+ } catch (Exception e) {
+ startupException.set(e);
+ throw e;
+ }
+ }
+
+ @Override
+ protected void after() {
+ if (container != null) {
+ container.stop();
+ }
+ }
+
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ try {
+ before();
+ } catch (Exception e) {
+ Assume.assumeNoException(e);
+ throw e;
+ }
+ List<Throwable> errors = new ArrayList<>();
+ try {
+ base.evaluate();
+ } catch (Throwable t) {
+ errors.add(t);
+ } finally {
+ try {
+ after();
+ } catch (Throwable t) {
+ errors.add(t);
+ }
+ }
+ MultipleFailureException.assertEmpty(errors);
+ }
+ };
+ }
+
+ /**
+ * Returns the mapped HTTP port on the host for the running S3Mock container.
+ */
+ public int getHttpPort() {
+ return container.getMappedPort(HTTP_PORT);
+ }
+
+ /**
+ * Returns the HTTP endpoint URL for the running S3Mock container, e.g. {@code http://127.0.0.1:PORT}.
+ */
+ public String getHttpEndpoint() {
+ return "http://127.0.0.1:" + getHttpPort();
+ }
+}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
index 00e79a73c2..51d55ec985 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3Ds.java
@@ -315,6 +315,16 @@ public class TestS3Ds extends AbstractDataStoreTest {
public void testDeleteAllOlderThan() {
}
+ // deleteRecord removes the object from S3 but leaves the local file cache intact.
+ // getRecordIfStored then returns the cached copy, causing a spurious failure.
+ // The cache-off subclass (TestS3DsCacheOff) re-enables this test where it is valid.
+ @Override
+ public void testDeleteRecord() {
+ Assume.assumeFalse("S3 local cache masks deletions; not verifiable against the emulator",
+ S3EmulatorSupport.isAvailable());
+ super.testDeleteRecord();
+ }
+
// helper methods
private PutObjectResponse httpPut(@Nullable DataRecordUpload uploadContext, InputStream inputstream, long length) {
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
index fb62f1ee8e..eea9f30cfd 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DsCacheOff.java
@@ -17,7 +17,9 @@
package org.apache.jackrabbit.oak.blob.cloud.s3;
import org.apache.jackrabbit.oak.spi.blob.data.CachingDataStore;
+import org.junit.Assume;
import org.junit.Before;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,4 +42,25 @@ public class TestS3DsCacheOff extends TestS3Ds {
props.setProperty("cacheSize", "0");
super.setUp();
}
+
+ // Re-enable: with cache off, deleteRecord is immediately visible via S3 so the
+ // assertion that getRecordIfStored returns null after deletion is valid here.
+ @Override
+ @Test
+ public void testDeleteRecord() {
+ try {
+ doDeleteRecordTest();
+ } catch (Exception e) {
+ org.junit.Assert.fail(e.getMessage());
+ }
+ }
+
+ // S3Backend updates duplicate records via CopyObject (copy-to-self). S3Mock does not
+ // support that operation, so this test cannot run against the emulator with cache off.
+ @Override
+ public void testAddDuplicateRecord() {
+ Assume.assumeFalse("S3Mock does not support CopyObject used by S3Backend for duplicate record updates",
+ S3EmulatorSupport.isAvailable());
+ super.testAddDuplicateRecord();
+ }
}
diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
index 580c111e3c..fc4b999464 100644
--- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
+++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/UtilsTest.java
@@ -18,8 +18,13 @@
package org.apache.jackrabbit.oak.blob.cloud.s3;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Test;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.net.URI;
import java.util.Properties;
@@ -201,4 +206,67 @@ public class UtilsTest {
Assert.assertEquals(S3Backend.RemoteStorageMode.GCP, props.get(S3Constants.MODE));
}
-}
\ No newline at end of file
+ @Test
+ public void testSetRemoteStorageModeDefaultsPreSetGCPToS3() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "http://127.0.0.1:9090");
+ props.put(S3Constants.MODE, S3Backend.RemoteStorageMode.GCP);
+ Utils.setRemoteStorageMode(props);
+ Assert.assertEquals(S3Backend.RemoteStorageMode.S3, props.get(S3Constants.MODE));
+ }
+
+ @Test
+ public void testSetRemoteStorageModeDefaultsBlankModeToS3() {
+ Properties props = new Properties();
+ props.setProperty("s3EndPoint", "https://s3.amazonaws.com");
+ props.setProperty(S3Constants.MODE, "");
+ Utils.setRemoteStorageMode(props);
+ Assert.assertEquals(S3Backend.RemoteStorageMode.S3, props.get(S3Constants.MODE));
+ }
+
+ @Test
+ public void testPathStyleAccessConstantHasExpectedStringValue() {
+ Assert.assertEquals("pathStyleAccess", S3Constants.PATH_STYLE_ACCESS);
+ }
+
+ @Test
+ public void testPathStyleAccessDefaultsToFalseWhenPropertyAbsent() {
+ Properties props = new Properties();
+ Assert.assertFalse(Boolean.parseBoolean(props.getProperty(S3Constants.PATH_STYLE_ACCESS, "false")));
+ }
+
+ @Test
+ public void testPathStyleAccessTrueWhenPropertySetToTrue() {
+ Properties props = new Properties();
+ props.setProperty(S3Constants.PATH_STYLE_ACCESS, "true");
+ Assert.assertTrue(Boolean.parseBoolean(props.getProperty(S3Constants.PATH_STYLE_ACCESS, "false")));
+ }
+
+ @Test
+ public void isS3ConfiguredReturnsFalseForPartialCredentialsWithNoRegionOrEndpoint() throws IOException {
+ // accessKey + secretKey without region or endpoint must not count as real credentials
+ File tmp = File.createTempFile("s3test-partial-creds", ".properties");
+ String previousConfig = System.getProperty("s3.config");
+ try {
+ Properties partial = new Properties();
+ partial.setProperty(S3Constants.ACCESS_KEY, "somekey");
+ partial.setProperty(S3Constants.SECRET_KEY, "somesecret");
+ try (OutputStream out = new FileOutputStream(tmp)) {
+ partial.store(out, null);
+ }
+ System.setProperty("s3.config", tmp.getAbsolutePath());
+ // Skip when emulator is available — getS3Config() would fall back to it, making isS3Configured() true
+ Assume.assumeFalse("Emulator available — would override partial creds", S3EmulatorSupport.isAvailable());
+ Assert.assertFalse("Partial credentials (no region/endpoint) must not be treated as real",
+ S3DataStoreUtils.isS3Configured());
+ } finally {
+ if (previousConfig == null) {
+ System.clearProperty("s3.config");
+ } else {
+ System.setProperty("s3.config", previousConfig);
+ }
+ tmp.delete();
+ }
+ }
+
+}
diff --git a/oak-doc/src/site/markdown/osgi_config.md b/oak-doc/src/site/markdown/osgi_config.md
index 67c471500a..0b8ae52598 100644
--- a/oak-doc/src/site/markdown/osgi_config.md
+++ b/oak-doc/src/site/markdown/osgi_config.md
@@ -367,6 +367,10 @@ s3Region
s3EndPoint
: S3 rest API endpoint. Can help reduce latency of redirection from standard endpoint if a different region configured.
+pathStyleAccess
+: Default - `false`
+: Set to `true` to use path-style S3 requests (`http://endpoint/bucket/key`) instead of virtual-host-style requests. This is useful for S3-compatible endpoints that do not support bucket names in the host. Note: always enabled automatically when the remote storage mode is GCP; the property has no effect in that case.
+
connectionTimeout
: S3 connection timeout. See [AWS S3 documentation](https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/section-client-configuration.html).