You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/08/04 01:59:03 UTC
(asterixdb) 03/05: [NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0795ad31de10f9018f0c330541e74923ba34f3ef
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Jul 26 12:31:06 2025 -0400
[NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage
MinIO does not accept lists on a . directory. Fix the
GLOBAL_TXN_DIR_NAME to use instead of . to avoid this
incompatibility
Ext-ref: MB-67814
Change-Id: I8e6ffb62ef1c412e263388b8dabb9f2d3bb66db5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20143
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../apache/asterix/api/common/LocalCloudUtil.java | 4 +-
.../api/common/LocalCloudUtilAdobeMock.java | 4 +-
.../apache/asterix/test/common/TestExecutor.java | 57 +++++++++++++---------
.../asterix/common/utils/StorageConstants.java | 2 +-
4 files changed, 39 insertions(+), 28 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
index 9e67c9217b..cb37cd05b4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
@@ -42,7 +42,7 @@ public class LocalCloudUtil {
private static final Logger LOGGER = LogManager.getLogger();
private static final int MOCK_SERVER_PORT = 8001;
- public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
+ public static final String MOCK_SERVER_ENDPOINT = "http://127.0.0.1:" + MOCK_SERVER_PORT;
public static final String CLOUD_STORAGE_BUCKET = "cloud-storage-container";
public static final String STORAGE_DUMMY_FILE = "storage/dummy.txt";
public static final String MOCK_SERVER_REGION = "us-west-2";
@@ -81,7 +81,7 @@ public class LocalCloudUtil {
LOGGER.info("S3 mock server started successfully");
S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server
builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
.endpointOverride(endpoint);
S3Client client = builder.build();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
index 1aefadfc66..ba7f8561f3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.api.common;
-import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_HOSTNAME;
+import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_ENDPOINT;
import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_REGION;
import java.net.URI;
@@ -79,7 +79,7 @@ public class LocalCloudUtilAdobeMock {
LOGGER.info("S3 mock server started successfully");
S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server
builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
.endpointOverride(endpoint);
S3Client client = builder.build();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 890e1882f6..1a50f44163 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -2419,21 +2419,7 @@ public class TestExecutor {
// For adapter placeholder, it means we have a template to replace
if (placeholder.getName().equals("adapter")) {
str = str.replace("%adapter%", placeholder.getValue());
-
- // Early terminate if there are no template place holders to replace
- if (noTemplateRequired(str)) {
- continue;
- }
-
- if (placeholder.getValue().equalsIgnoreCase("S3")) {
- str = applyS3Substitution(str, placeholders);
- } else if (placeholder.getValue().equalsIgnoreCase("AzureBlob")) {
- str = applyAzureSubstitution(str, placeholders);
- } else if (placeholder.getValue().equalsIgnoreCase("GCS")) {
- str = applyGCSSubstitution(str, placeholders);
- } else if (placeholder.getValue().equalsIgnoreCase("HDFS")) {
- str = applyHDFSSubstitution(str, placeholders);
- }
+ str = applyAdapterSubstitution(str, placeholder.getValue(), placeholders);
} else {
// Any other place holders, just replace with the value
str = str.replace("%" + placeholder.getName() + "%", placeholder.getValue());
@@ -2455,6 +2441,19 @@ public class TestExecutor {
return str;
}
+ protected String applyAdapterSubstitution(String str, String adapter, List<Placeholder> placeholders) {
+ if (adapter.equalsIgnoreCase("S3")) {
+ str = applyS3Substitution(str, placeholders);
+ } else if (adapter.equalsIgnoreCase("AzureBlob")) {
+ str = applyAzureSubstitution(str, placeholders);
+ } else if (adapter.equalsIgnoreCase("GCS")) {
+ str = applyGCSSubstitution(str, placeholders);
+ } else if (adapter.equalsIgnoreCase("HDFS")) {
+ str = applyHDFSSubstitution(str, placeholders);
+ }
+ return str;
+ }
+
protected String replaceExternalEndpoint(String str) {
return str.replace(BLOB_ENDPOINT_PLACEHOLDER, BLOB_ENDPOINT_DEFAULT);
}
@@ -2494,18 +2493,30 @@ public class TestExecutor {
str = setS3TemplateDefault(str);
}
- // Set to default if not replaced
- if (isReplaced && !hasRegion) {
- str = str.replace(TestConstants.S3_REGION_PLACEHOLDER, TestConstants.S3_REGION_DEFAULT);
- }
-
- if (isReplaced && !hasServiceEndpoint) {
- str = str.replace(TestConstants.S3_SERVICE_ENDPOINT_PLACEHOLDER, TestConstants.S3_SERVICE_ENDPOINT_DEFAULT);
- }
+ str = str.replace(TestConstants.S3_REGION_PLACEHOLDER, getS3RegionDefault());
+ str = str.replace(TestConstants.S3_SERVICE_ENDPOINT_PLACEHOLDER, getS3ServiceEndpointDefault());
+ str = str.replace(TestConstants.S3_ACCESS_KEY_ID_PLACEHOLDER, getS3AccessKeyIdDefault());
+ str = str.replace(TestConstants.S3_SECRET_ACCESS_KEY_PLACEHOLDER, getS3SecretAccessKeyDefault());
return str;
}
+ protected String getS3AccessKeyIdDefault() {
+ return TestConstants.S3_ACCESS_KEY_ID_DEFAULT;
+ }
+
+ protected String getS3SecretAccessKeyDefault() {
+ return TestConstants.S3_SECRET_ACCESS_KEY_DEFAULT;
+ }
+
+ protected String getS3ServiceEndpointDefault() {
+ return TestConstants.S3_SERVICE_ENDPOINT_DEFAULT;
+ }
+
+ protected String getS3RegionDefault() {
+ return TestConstants.S3_REGION_DEFAULT;
+ }
+
protected String setS3Template(String str) {
return str.replace("%template%", TestConstants.S3_TEMPLATE);
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index 627f170f73..0f18536550 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -33,7 +33,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManage
public class StorageConstants {
public static final String METADATA_TXN_NOWAL_DIR_NAME = "mtd-txn-logs";
- public static final String GLOBAL_TXN_DIR_NAME = ".";
+ public static final String GLOBAL_TXN_DIR_NAME = "";
public static final String STORAGE_ROOT_DIR_NAME = "storage";
public static final String APPLICATION_ROOT_DIR_NAME = APP_DIR_NAME;
public static final String INGESTION_LOGS_DIR_NAME = "ingestion_logs";