You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/08/04 01:59:00 UTC
(asterixdb) branch master updated (4fcc01028d -> 76a28495b2)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
from 4fcc01028d Reenable Iceberg unit tests
new f58dfc5db5 [NO ISSUE][OTH] Misc properties description fixes
new 0795ad31de [NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage
new 9c02ec70f2 [NO ISSUE][HYR][MISC] Allow a custom logger for thread dump at halt
new 620dd09ec7 Merge branch 'gerrit/trinity' into 'gerrit/phoenix'
new 76a28495b2 Merge branch 'gerrit/phoenix' into 'master'
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/asterix/api/common/LocalCloudUtil.java | 4 +-
.../api/common/LocalCloudUtilAdobeMock.java | 4 +-
.../apache/asterix/test/common/TestExecutor.java | 57 +++++++++++++---------
.../asterix/common/config/CloudProperties.java | 4 +-
.../asterix/common/utils/StorageConstants.java | 2 +-
.../java/org/apache/hyracks/util/ExitUtil.java | 10 ++--
6 files changed, 46 insertions(+), 35 deletions(-)
(asterixdb) 04/05: Merge branch 'gerrit/trinity' into 'gerrit/phoenix'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 620dd09ec7b722abdd20b9b008d79832495e8c83
Merge: 0795ad31de 9c02ec70f2
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Aug 1 16:34:50 2025 -0400
Merge branch 'gerrit/trinity' into 'gerrit/phoenix'
Ext-ref: MB-67819
Change-Id: Id75cdaad984c4ec9ae146bb33858281d0b79ba19
.../src/main/java/org/apache/hyracks/util/ExitUtil.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
(asterixdb) 03/05: [NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 0795ad31de10f9018f0c330541e74923ba34f3ef
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Jul 26 12:31:06 2025 -0400
[NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage
MinIO does not accept lists on a . directory. Fix the
GLOBAL_TXN_DIR_NAME to use instead of . to avoid this
incompatibility
Ext-ref: MB-67814
Change-Id: I8e6ffb62ef1c412e263388b8dabb9f2d3bb66db5
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20143
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
.../apache/asterix/api/common/LocalCloudUtil.java | 4 +-
.../api/common/LocalCloudUtilAdobeMock.java | 4 +-
.../apache/asterix/test/common/TestExecutor.java | 57 +++++++++++++---------
.../asterix/common/utils/StorageConstants.java | 2 +-
4 files changed, 39 insertions(+), 28 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
index 9e67c9217b..cb37cd05b4 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
@@ -42,7 +42,7 @@ public class LocalCloudUtil {
private static final Logger LOGGER = LogManager.getLogger();
private static final int MOCK_SERVER_PORT = 8001;
- public static final String MOCK_SERVER_HOSTNAME = "http://127.0.0.1:" + MOCK_SERVER_PORT;
+ public static final String MOCK_SERVER_ENDPOINT = "http://127.0.0.1:" + MOCK_SERVER_PORT;
public static final String CLOUD_STORAGE_BUCKET = "cloud-storage-container";
public static final String STORAGE_DUMMY_FILE = "storage/dummy.txt";
public static final String MOCK_SERVER_REGION = "us-west-2";
@@ -81,7 +81,7 @@ public class LocalCloudUtil {
LOGGER.info("S3 mock server started successfully");
S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server
builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
.endpointOverride(endpoint);
S3Client client = builder.build();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
index 1aefadfc66..ba7f8561f3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.api.common;
-import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_HOSTNAME;
+import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_ENDPOINT;
import static org.apache.asterix.api.common.LocalCloudUtil.MOCK_SERVER_REGION;
import java.net.URI;
@@ -79,7 +79,7 @@ public class LocalCloudUtilAdobeMock {
LOGGER.info("S3 mock server started successfully");
S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
+ URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server
builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
.endpointOverride(endpoint);
S3Client client = builder.build();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 890e1882f6..1a50f44163 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -2419,21 +2419,7 @@ public class TestExecutor {
// For adapter placeholder, it means we have a template to replace
if (placeholder.getName().equals("adapter")) {
str = str.replace("%adapter%", placeholder.getValue());
-
- // Early terminate if there are no template place holders to replace
- if (noTemplateRequired(str)) {
- continue;
- }
-
- if (placeholder.getValue().equalsIgnoreCase("S3")) {
- str = applyS3Substitution(str, placeholders);
- } else if (placeholder.getValue().equalsIgnoreCase("AzureBlob")) {
- str = applyAzureSubstitution(str, placeholders);
- } else if (placeholder.getValue().equalsIgnoreCase("GCS")) {
- str = applyGCSSubstitution(str, placeholders);
- } else if (placeholder.getValue().equalsIgnoreCase("HDFS")) {
- str = applyHDFSSubstitution(str, placeholders);
- }
+ str = applyAdapterSubstitution(str, placeholder.getValue(), placeholders);
} else {
// Any other place holders, just replace with the value
str = str.replace("%" + placeholder.getName() + "%", placeholder.getValue());
@@ -2455,6 +2441,19 @@ public class TestExecutor {
return str;
}
+ protected String applyAdapterSubstitution(String str, String adapter, List<Placeholder> placeholders) {
+ if (adapter.equalsIgnoreCase("S3")) {
+ str = applyS3Substitution(str, placeholders);
+ } else if (adapter.equalsIgnoreCase("AzureBlob")) {
+ str = applyAzureSubstitution(str, placeholders);
+ } else if (adapter.equalsIgnoreCase("GCS")) {
+ str = applyGCSSubstitution(str, placeholders);
+ } else if (adapter.equalsIgnoreCase("HDFS")) {
+ str = applyHDFSSubstitution(str, placeholders);
+ }
+ return str;
+ }
+
protected String replaceExternalEndpoint(String str) {
return str.replace(BLOB_ENDPOINT_PLACEHOLDER, BLOB_ENDPOINT_DEFAULT);
}
@@ -2494,18 +2493,30 @@ public class TestExecutor {
str = setS3TemplateDefault(str);
}
- // Set to default if not replaced
- if (isReplaced && !hasRegion) {
- str = str.replace(TestConstants.S3_REGION_PLACEHOLDER, TestConstants.S3_REGION_DEFAULT);
- }
-
- if (isReplaced && !hasServiceEndpoint) {
- str = str.replace(TestConstants.S3_SERVICE_ENDPOINT_PLACEHOLDER, TestConstants.S3_SERVICE_ENDPOINT_DEFAULT);
- }
+ str = str.replace(TestConstants.S3_REGION_PLACEHOLDER, getS3RegionDefault());
+ str = str.replace(TestConstants.S3_SERVICE_ENDPOINT_PLACEHOLDER, getS3ServiceEndpointDefault());
+ str = str.replace(TestConstants.S3_ACCESS_KEY_ID_PLACEHOLDER, getS3AccessKeyIdDefault());
+ str = str.replace(TestConstants.S3_SECRET_ACCESS_KEY_PLACEHOLDER, getS3SecretAccessKeyDefault());
return str;
}
+ protected String getS3AccessKeyIdDefault() {
+ return TestConstants.S3_ACCESS_KEY_ID_DEFAULT;
+ }
+
+ protected String getS3SecretAccessKeyDefault() {
+ return TestConstants.S3_SECRET_ACCESS_KEY_DEFAULT;
+ }
+
+ protected String getS3ServiceEndpointDefault() {
+ return TestConstants.S3_SERVICE_ENDPOINT_DEFAULT;
+ }
+
+ protected String getS3RegionDefault() {
+ return TestConstants.S3_REGION_DEFAULT;
+ }
+
protected String setS3Template(String str) {
return str.replace("%template%", TestConstants.S3_TEMPLATE);
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index 627f170f73..0f18536550 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -33,7 +33,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManage
public class StorageConstants {
public static final String METADATA_TXN_NOWAL_DIR_NAME = "mtd-txn-logs";
- public static final String GLOBAL_TXN_DIR_NAME = ".";
+ public static final String GLOBAL_TXN_DIR_NAME = "";
public static final String STORAGE_ROOT_DIR_NAME = "storage";
public static final String APPLICATION_ROOT_DIR_NAME = APP_DIR_NAME;
public static final String INGESTION_LOGS_DIR_NAME = "ingestion_logs";
(asterixdb) 01/05: [NO ISSUE][HYR][MISC] Allow a custom logger for thread dump at halt
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 9c02ec70f210e213eef3af73345ec3b7d54b2b62
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Jul 19 11:24:19 2025 -0400
[NO ISSUE][HYR][MISC] Allow a custom logger for thread dump at halt
Ext-ref: MB-67719
Change-Id: I412a13187ed4fcc25fe8283427d92b4a6af8b35d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20123
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../src/main/java/org/apache/hyracks/util/ExitUtil.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
index 7eb9fd4f8a..11ef6192a1 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ExitUtil.java
@@ -100,18 +100,18 @@ public class ExitUtil {
}
public static void halt(int status) {
- halt(status, Level.FATAL);
+ halt(status, Level.FATAL, LOGGER);
}
- public static synchronized void halt(int status, Level logLevel) {
+ public static synchronized void halt(int status, Level logLevel, Logger threadDumpLogger) {
try {
boolean interrupted = Thread.interrupted();
LOGGER.log(logLevel, "JVM halting with status {} (halting thread {}, interrupted {})", status,
Thread.currentThread(), interrupted);
Future<?> future = haltThreadDumpExecutor.submit(() -> {
- LOGGER.log(logLevel, "Thread dump at halt: {}", ThreadDumpUtil.takeDumpString());
- // try to give time for the log to be emitted...
- LogManager.shutdown();
+ threadDumpLogger.log(logLevel, "Thread dump at halt: {}", ThreadDumpUtil.takeDumpString());
+ // try to give time for the log to be emitted by asking the log manager to shut down...
+ LogManager.shutdown(false, true);
});
future.get(HALT_THREADDUMP_TIMEOUT_SECONDS, TimeUnit.SECONDS);
} catch (Exception e) {
(asterixdb) 02/05: [NO ISSUE][OTH] Misc properties description fixes
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit f58dfc5db5d709342f6bd2e4dc625474b2c59b6f
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Mon Jul 28 16:37:36 2025 -0700
[NO ISSUE][OTH] Misc properties description fixes
- user model changes: no
- storage format changes: no
- interface changes: no
Ext-ref: MB-65971
Change-Id: I2a9a3e7d09e25ce109b026826239a9ac1ac515fd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20148
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Ali Alsuliman <al...@gmail.com>
---
.../main/java/org/apache/asterix/common/config/CloudProperties.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index 87eace3f34..f556fcbb94 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -167,9 +167,9 @@ public class CloudProperties extends AbstractProperties {
return "The waiting time (in milliseconds) if a requesting thread failed to acquire a token if the"
+ " rate limit of cloud requests exceeded (default: 100, min: 1, and max: 5000)";
case CLOUD_MAX_WRITE_REQUESTS_PER_SECOND:
- return "The maximum number of write requests per second (default: 2500, 0 means unlimited)";
+ return "The maximum number of write requests per second (default: 250, 0 means unlimited)";
case CLOUD_MAX_READ_REQUESTS_PER_SECOND:
- return "The maximum number of read requests per second (default: 4000, 0 means unlimited)";
+ return "The maximum number of read requests per second (default: 1500, 0 means unlimited)";
case CLOUD_WRITE_BUFFER_SIZE:
return "The write buffer size in bytes. (default: 8MB, min: 5MB)";
case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
(asterixdb) 05/05: Merge branch 'gerrit/phoenix' into 'master'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 76a28495b23bc42cda02ef53d47052d61a961e93
Merge: 4fcc01028d 620dd09ec7
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Aug 3 12:18:46 2025 -0400
Merge branch 'gerrit/phoenix' into 'master'
* [NO ISSUE][HYR][MISC] Allow a custom logger for thread dump at halt
* [NO ISSUE][OTH] Misc properties description fixes
* [NO ISSUE][*DB][STO] Support MinIO S3 compatible cloud storage
Change-Id: I0d516f50152b1db145fe9dbad7441f92c7aa0abd
.../apache/asterix/api/common/LocalCloudUtil.java | 4 +-
.../api/common/LocalCloudUtilAdobeMock.java | 4 +-
.../apache/asterix/test/common/TestExecutor.java | 57 +++++++++++++---------
.../asterix/common/config/CloudProperties.java | 4 +-
.../asterix/common/utils/StorageConstants.java | 2 +-
.../java/org/apache/hyracks/util/ExitUtil.java | 10 ++--
6 files changed, 46 insertions(+), 35 deletions(-)