You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/10/02 10:45:30 UTC
(asterixdb) branch master updated (ca214efed2 -> 2b7efeae59)
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
from ca214efed2 [ASTERIXDB-3655][COMP] Disable index-only query plan when we have join
new 6991ae67e7 [ASTERIXDB-3643][EXT]: make path-style addressing configurable
new e7333eb78c [NO ISSUE][HYR][MISC] Utilize log4j-bom
new 72c799cb29 [NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify
new 4036181998 [ASTERIXDB-3647][TEST] Use InfiniteRetryPolicy
new af29e15fd0 Merge remote-tracking branch 'gerrit/neo' into trinity
new 7fd556ceca Merge branch 'trinity' into phoenix
new d22fc3f8c4 [ASTERIXDB-3653][EXT]: Properly handle errors when deleting for COPY TO
new 847a5b8f85 Merge branch 'gerrit/phoenix' into 'master'
new 2b7efeae59 Merge "Merge branch 'gerrit/phoenix' into 'master'"
The 9 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/asterix/api/common/LocalCloudUtil.java | 27 ++++++---
.../api/common/LocalCloudUtilAdobeMock.java | 9 +++
.../test/active/ActiveEventsListenerTest.java | 5 +-
.../asterix/test/active/InfiniteRetryPolicy.java | 30 +++++-----
.../apache/asterix/test/common/TestExecutor.java | 60 +++++++++++++++++--
.../asterix/test/runtime/LangExecutionUtil.java | 4 +-
.../boolean-value/external_dataset.000.ddl.sqlpp | 2 +-
.../boolean-value}/external_dataset.099.ddl.sqlpp | 0
.../external_dataset.000.ddl.sqlpp | 2 +-
.../external_dataset.099.ddl.sqlpp | 0
.../runtimets/testsuite_external_dataset_s3.xml | 13 +++-
.../apache/asterix/cloud/clients/ICloudClient.java | 9 +++
.../asterix/cloud/clients/UnstableCloudClient.java | 5 ++
.../cloud/clients/aws/s3/S3CloudClient.java | 17 ++++++
.../blobstorage/AzBlobStorageClientConfig.java | 16 +++--
.../blobstorage/AzBlobStorageCloudClient.java | 69 ++++++++++++++++++----
.../cloud/clients/google/gcs/GCSCloudClient.java | 51 ++++++++++------
.../AbstractCloudExternalFileWriterFactory.java | 3 +-
.../asterix/cloud/azure/LSMAzBlobStorageTest.java | 2 +-
.../asterix/external/util/aws/s3/S3AuthUtils.java | 35 +++++++++--
.../asterix/external/util/aws/s3/S3Constants.java | 1 +
.../asterix/test/server/AbstractExecutionIT.java | 4 +-
.../asterix/test/server/NCServiceExecutionIT.java | 4 +-
hyracks-fullstack/pom.xml | 16 ++---
24 files changed, 293 insertions(+), 91 deletions(-)
copy hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingSupplier.java => asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/InfiniteRetryPolicy.java (60%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/{cross-region => path-style-access}/boolean-value/external_dataset.000.ddl.sqlpp (97%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/{common/empty-string-definition => s3/path-style-access/boolean-value}/external_dataset.099.ddl.sqlpp (100%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/{cross-region/boolean-value => path-style-access/non-boolean-value}/external_dataset.000.ddl.sqlpp (96%)
copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/{common/empty-string-definition => s3/path-style-access/non-boolean-value}/external_dataset.099.ddl.sqlpp (100%)
(asterixdb) 01/09: [ASTERIXDB-3643][EXT]: make path-style addressing configurable
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 6991ae67e7401226df5cad6c7abfb11e8598c993
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Fri Sep 5 10:55:44 2025 +0300
[ASTERIXDB-3643][EXT]: make path-style addressing configurable
Ext-ref: MB-68398
Change-Id: I068f9caa9d9ea800d62954a1863b91d2074c29c6
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20296
Reviewed-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Tested-by: Hussain Towaileb <hu...@gmail.com>
---
.../boolean-value/external_dataset.000.ddl.sqlpp | 38 ++++++++++++++++++++++
.../boolean-value/external_dataset.099.ddl.sqlpp | 20 ++++++++++++
.../external_dataset.000.ddl.sqlpp | 38 ++++++++++++++++++++++
.../external_dataset.099.ddl.sqlpp | 20 ++++++++++++
.../runtimets/testsuite_external_dataset_s3.xml | 13 +++++++-
.../asterix/external/util/aws/s3/S3AuthUtils.java | 35 +++++++++++++++++---
.../asterix/external/util/aws/s3/S3Constants.java | 1 +
7 files changed, 159 insertions(+), 6 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000000..cac44314d4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-1"),
+ ("pathStyleAddressing"="true"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="json-data/reviews/single-line/json"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.099.ddl.sqlpp
new file mode 100644
index 0000000000..548e63267e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000000..5d19d3e4dd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-1"),
+ ("pathStyleAddressing"="non-boolean-value"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="json-data/reviews/single-line/json"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.099.ddl.sqlpp
new file mode 100644
index 0000000000..548e63267e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 7a07da00e5..5b1d265e20 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -1196,6 +1196,17 @@
<expected-error>Provided S3 region is not supported: 'some-new-region'</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="path-style-access/boolean-value">
+ <output-dir compare="Text">path-style-access/boolean-value</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="path-style-access/non-boolean-value">
+ <output-dir compare="Text">path-style-access/non-boolean-value</output-dir>
+ <expected-error>ASX1173: Invalid value for parameter 'pathStyleAddressing', allowed value(s): true, false</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/s3">
<compilation-unit name="cross-region/boolean-value">
<output-dir compare="Text">cross-region/boolean-value</output-dir>
@@ -1204,7 +1215,7 @@
<test-case FilePath="external-dataset/s3">
<compilation-unit name="cross-region/non-boolean-value">
<output-dir compare="Text">cross-region/non-boolean-value</output-dir>
- <expected-error>c</expected-error>
+ <expected-error>ASX1173: Invalid value for parameter 'crossRegion', allowed value(s): true, false</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/s3">
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
index 035415d9c9..90bb04f309 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
@@ -53,6 +53,7 @@ import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SESSION
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SIMPLE;
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_TEMPORARY;
import static org.apache.asterix.external.util.aws.s3.S3Constants.INSTANCE_PROFILE_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.PATH_STYLE_ADDRESSING_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.REGION_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.ROLE_ARN_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
@@ -160,6 +161,9 @@ public class S3AuthUtils {
}
}
+ boolean pathStyleAddressing =
+ validateAndGetPathStyleAddressing(configuration.get(PATH_STYLE_ADDRESSING_FIELD_NAME), serviceEndpoint);
+ builder.forcePathStyle(pathStyleAddressing);
return builder.build();
}
@@ -217,6 +221,25 @@ public class S3AuthUtils {
}
}
+ public static boolean validateAndGetPathStyleAddressing(String pathStyleAddressing, String endpoint)
+ throws CompilationException {
+ if (pathStyleAddressing == null) {
+ return endpoint != null && !endpoint.isEmpty();
+ }
+ validatePathStyleAddressing(pathStyleAddressing);
+ return Boolean.parseBoolean(pathStyleAddressing);
+ }
+
+ public static void validatePathStyleAddressing(String pathStyleAddressing) throws CompilationException {
+ if (pathStyleAddressing == null) {
+ return;
+ }
+ if (!"true".equalsIgnoreCase(pathStyleAddressing) && !"false".equalsIgnoreCase(pathStyleAddressing)) {
+ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, PATH_STYLE_ADDRESSING_FIELD_NAME,
+ "true, false");
+ }
+ }
+
public static boolean validateAndGetCrossRegion(String crossRegion) throws CompilationException {
if (crossRegion == null) {
return false;
@@ -385,11 +408,13 @@ public class S3AuthUtils {
jobConf.set(HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT);
}
- /*
- * This is to allow S3 definition to have path-style form. Should always be true to match the current
- * way we access files in S3
- */
- jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
+ boolean pathStyleAddressing =
+ validateAndGetPathStyleAddressing(configuration.get(PATH_STYLE_ADDRESSING_FIELD_NAME), serviceEndpoint);
+ if (pathStyleAddressing) {
+ jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
+ } else {
+ jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.FALSE);
+ }
/*
* Set the size of S3 connection pool to be the number of partitions
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index 807e887bf3..102c378cca 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -28,6 +28,7 @@ public class S3Constants {
// Authentication specific parameters
public static final String REGION_FIELD_NAME = "region";
+ public static final String PATH_STYLE_ADDRESSING_FIELD_NAME = "pathStyleAddressing";
public static final String CROSS_REGION_FIELD_NAME = "crossRegion";
public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
(asterixdb) 02/09: [ASTERIXDB-3647][TEST] Use InfiniteRetryPolicy
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 4036181998083682f7a5b7c07faf219e219aba3f
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Tue Sep 23 15:34:49 2025 -0700
[ASTERIXDB-3647][TEST] Use InfiniteRetryPolicy
This was replaced with instantiations of CountRetryPolciy,
which is similar but handles interrupts differently.
Ext-ref: MB-64900
Change-Id: I068f9caa9d9ea800d62954a1963291db074a29c3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20407
Reviewed-by: Ian Maxon <im...@apache.org>
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
---
.../test/active/ActiveEventsListenerTest.java | 5 +--
.../asterix/test/active/InfiniteRetryPolicy.java | 45 ++++++++++++++++++++++
2 files changed, 47 insertions(+), 3 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
index e5e33d00be..313e1fa065 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java
@@ -146,8 +146,7 @@ public class ActiveEventsListenerTest {
nodeControllers[0] = new TestNodeControllerActor(nodes[0], clusterController);
nodeControllers[1] = new TestNodeControllerActor(nodes[1], clusterController);
listener = new TestEventsListener(clusterController, nodeControllers, jobIdFactory, entityId,
- new ArrayList<>(allDatasets), statementExecutor, appCtx, hcc, locations,
- x -> new CountRetryPolicy(1000));
+ new ArrayList<>(allDatasets), statementExecutor, appCtx, hcc, locations, InfiniteRetryPolicy::new);
users = new TestUserActor[3];
users[0] = newUser("Till", appCtx);
users[1] = newUser("Mike", appCtx);
@@ -1540,7 +1539,7 @@ public class ActiveEventsListenerTest {
AlgebricksAbsolutePartitionConstraint locations = new AlgebricksAbsolutePartitionConstraint(nodes);
additionalListeners[i] = listener = new TestEventsListener(clusterController, nodeControllers, jobIdFactory,
entityId, new ArrayList<>(allDatasets), statementExecutor, ccAppCtx, hcc, locations,
- x -> new CountRetryPolicy(1000));
+ InfiniteRetryPolicy::new);
}
Action suspension = users[0].suspendAllActivities(handler);
suspension.sync();
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/InfiniteRetryPolicy.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/InfiniteRetryPolicy.java
new file mode 100644
index 0000000000..eb2440dc90
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/InfiniteRetryPolicy.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.active;
+
+import org.apache.asterix.active.IActiveEntityEventsListener;
+import org.apache.hyracks.util.IRetryPolicy;
+
+public class InfiniteRetryPolicy implements IRetryPolicy {
+
+ private final IActiveEntityEventsListener listener;
+
+ public InfiniteRetryPolicy(IActiveEntityEventsListener listener) {
+ this.listener = listener;
+ }
+
+ @Override
+ public boolean retry(Throwable failure) {
+ synchronized (listener) {
+ try {
+ listener.wait(5000); //NOSONAR this method is being called in a while loop
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
(asterixdb) 03/09: Merge remote-tracking branch 'gerrit/neo' into trinity
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit af29e15fd03ed0b4239e2b0d5ee1fc7f8191bdfd
Merge: 77704ebea4 4036181998
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Thu Sep 25 14:09:11 2025 -0700
Merge remote-tracking branch 'gerrit/neo' into trinity
Ext-ref: MB-64900
Change-Id: I068f9caa9d9ea800d62954a146329adb074a29c3
.../test/active/ActiveEventsListenerTest.java | 5 +--
.../asterix/test/active/InfiniteRetryPolicy.java | 45 ++++++++++++++++++++++
2 files changed, 47 insertions(+), 3 deletions(-)
(asterixdb) 07/09: [ASTERIXDB-3653][EXT]: Properly handle errors when deleting for COPY TO
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit d22fc3f8c4f41e7cb7becc601dd07f3ee0a042ae
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Tue Sep 30 18:29:52 2025 +0300
[ASTERIXDB-3653][EXT]: Properly handle errors when deleting for COPY TO
Ext-ref: MB-68654
Change-Id: Ib670a014a5e4378b0a63be740ead72d699a84bb1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20428
Tested-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../apache/asterix/cloud/clients/ICloudClient.java | 9 ++++
.../asterix/cloud/clients/UnstableCloudClient.java | 5 +++
.../cloud/clients/aws/s3/S3CloudClient.java | 17 ++++++++
.../blobstorage/AzBlobStorageCloudClient.java | 45 +++++++++++++------
.../cloud/clients/google/gcs/GCSCloudClient.java | 51 ++++++++++++++--------
.../AbstractCloudExternalFileWriterFactory.java | 3 +-
6 files changed, 97 insertions(+), 33 deletions(-)
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
index 0baebc7071..a8cb59b1dd 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/ICloudClient.java
@@ -120,6 +120,15 @@ public interface ICloudClient {
*/
void copy(String bucket, String srcPath, FileReference destPath) throws HyracksDataException;
+ /**
+ * Deletes an object at the specified bucket and path
+ *
+ * @param bucket bucket
+ * @param path path of object
+ * @throws HyracksDataException HyracksDataException
+ */
+ void deleteObject(String bucket, String path) throws HyracksDataException;
+
/**
* Deletes all objects at the specified bucket and paths
*
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
index d81419bfef..1160c12386 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/UnstableCloudClient.java
@@ -105,6 +105,11 @@ public class UnstableCloudClient implements ICloudClient {
cloudClient.copy(bucket, srcPath, destPath);
}
+ @Override
+ public void deleteObject(String bucket, String path) throws HyracksDataException {
+ cloudClient.deleteObject(bucket, path);
+ }
+
@Override
public void deleteObjects(String bucket, Collection<String> paths) throws HyracksDataException {
cloudClient.deleteObjects(bucket, paths);
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index 81b96d7862..c9fd485135 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@ -73,6 +73,7 @@ import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.model.CopyObjectRequest;
import software.amazon.awssdk.services.s3.model.Delete;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
@@ -237,6 +238,22 @@ public final class S3CloudClient implements ICloudClient {
}
}
+ @Override
+ public void deleteObject(String bucket, String path) throws HyracksDataException {
+ try {
+ if (path.isEmpty()) {
+ return;
+ }
+ guardian.checkWriteAccess(bucket, path);
+ profiler.objectDelete();
+ DeleteObjectRequest request =
+ DeleteObjectRequest.builder().bucket(bucket).key(config.getPrefix() + path).build();
+ s3Client.deleteObject(request);
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
+ }
+ }
+
@Override
public void deleteObjects(String bucket, Collection<String> paths) throws HyracksDataException {
if (paths.isEmpty()) {
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index bdb551b768..4a61f1caa9 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -258,6 +258,21 @@ public class AzBlobStorageCloudClient implements ICloudClient {
destBlobClient.beginCopy(srcBlobUrl, null);
}
+ @Override
+ public void deleteObject(String bucket, String path) throws HyracksDataException {
+ try {
+ if (path.isEmpty()) {
+ return;
+ }
+ guardian.checkWriteAccess(bucket, path);
+ profiler.objectDelete();
+ BlobClient blobClient = blobContainerClient.getBlobClient(config.getPrefix() + path);
+ blobClient.delete();
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
+ }
+ }
+
@Override
public void deleteObjects(String bucket, Collection<String> paths) throws HyracksDataException {
if (paths.isEmpty())
@@ -270,21 +285,25 @@ public class AzBlobStorageCloudClient implements ICloudClient {
for (List<String> batch : batchedBlobURLs) {
PagedIterable<Response<Void>> responses = blobBatchClient.deleteBlobs(batch, null);
Iterator<String> deletePathIter = paths.iterator();
- String deletedPath = null;
- try {
- for (Response<Void> response : responses) {
- deletedPath = deletePathIter.next();
- // The response.getStatusCode() method returns:
- // - 202 (Accepted) if the delete operation is successful
- // - exception if the delete operation fails
- int statusCode = response.getStatusCode();
- if (statusCode != SUCCESS_RESPONSE_CODE) {
- LOGGER.warn("Failed to delete blob: {} with status code: {} while deleting {}", deletedPath,
- statusCode, paths.toString());
+ String deletedPath;
+ String failedDeletedPath = null;
+ for (Response<Void> response : responses) {
+ deletedPath = deletePathIter.next();
+ // The response.getStatusCode() method returns:
+ // - 202 (Accepted) if the delete operation is successful
+ // - exception if the delete operation fails
+ int statusCode = response.getStatusCode();
+ if (statusCode != SUCCESS_RESPONSE_CODE) {
+ LOGGER.warn("Failed to delete blob: {} with status code: {} while deleting {}", deletedPath,
+ statusCode, paths.toString());
+ if (failedDeletedPath == null) {
+ failedDeletedPath = deletedPath;
}
}
- } catch (BlobStorageException e) {
- throw new RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE, e, "DELETE", deletedPath, paths.toString());
+ }
+ if (failedDeletedPath != null) {
+ throw new RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE, "DELETE", failedDeletedPath,
+ paths.toString());
}
}
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
index bd0a044f0d..48bb2cda05 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/google/gcs/GCSCloudClient.java
@@ -75,7 +75,7 @@ public class GCSCloudClient implements ICloudClient {
private final Storage gcsClient;
private final GCSClientConfig config;
private final ICloudGuardian guardian;
- private final IRequestProfilerLimiter profilerLimiter;
+ private final IRequestProfilerLimiter profiler;
private final int writeBufferSize;
public GCSCloudClient(GCSClientConfig config, Storage gcsClient, ICloudGuardian guardian) {
@@ -86,9 +86,9 @@ public class GCSCloudClient implements ICloudClient {
long profilerInterval = config.getProfilerLogInterval();
GCSRequestRateLimiter limiter = new GCSRequestRateLimiter(config);
if (profilerInterval > 0) {
- profilerLimiter = new CountRequestProfilerLimiter(profilerInterval, limiter);
+ profiler = new CountRequestProfilerLimiter(profilerInterval, limiter);
} else {
- profilerLimiter = new RequestLimiterNoOpProfiler(limiter);
+ profiler = new RequestLimiterNoOpProfiler(limiter);
}
guardian.setCloudClient(this);
}
@@ -104,18 +104,18 @@ public class GCSCloudClient implements ICloudClient {
@Override
public IRequestProfilerLimiter getProfilerLimiter() {
- return profilerLimiter;
+ return profiler;
}
@Override
public ICloudWriter createWriter(String bucket, String path, IWriteBufferProvider bufferProvider) {
- return new GCSWriter(bucket, config.getPrefix() + path, gcsClient, profilerLimiter, guardian, writeBufferSize);
+ return new GCSWriter(bucket, config.getPrefix() + path, gcsClient, profiler, guardian, writeBufferSize);
}
@Override
public Set<CloudFile> listObjects(String bucket, String path, FilenameFilter filter) {
guardian.checkReadAccess(bucket, path);
- profilerLimiter.objectsList();
+ profiler.objectsList();
Page<Blob> blobs = gcsClient.list(bucket, BlobListOption.prefix(config.getPrefix() + path),
BlobListOption.fields(Storage.BlobField.SIZE));
Set<CloudFile> files = new HashSet<>();
@@ -130,7 +130,7 @@ public class GCSCloudClient implements ICloudClient {
@Override
public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
guardian.checkReadAccess(bucket, path);
- profilerLimiter.objectGet();
+ profiler.objectGet();
BlobId blobId = BlobId.of(bucket, config.getPrefix() + path);
long readTo = offset + buffer.remaining();
int totalRead = 0;
@@ -152,7 +152,7 @@ public class GCSCloudClient implements ICloudClient {
@Override
public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
guardian.checkReadAccess(bucket, path);
- profilerLimiter.objectGet();
+ profiler.objectGet();
BlobId blobId = BlobId.of(bucket, config.getPrefix() + path);
try {
return gcsClient.readAllBytes(blobId);
@@ -167,7 +167,7 @@ public class GCSCloudClient implements ICloudClient {
@Override
public InputStream getObjectStream(String bucket, String path, long offset, long length) {
guardian.checkReadAccess(bucket, path);
- profilerLimiter.objectGet();
+ profiler.objectGet();
ReadChannel reader = null;
try {
reader = gcsClient.reader(bucket, config.getPrefix() + path).limit(offset + length);
@@ -181,7 +181,7 @@ public class GCSCloudClient implements ICloudClient {
@Override
public void write(String bucket, String path, byte[] data) {
guardian.checkWriteAccess(bucket, path);
- profilerLimiter.objectWrite();
+ profiler.objectWrite();
BlobInfo blobInfo = BlobInfo.newBuilder(bucket, config.getPrefix() + path).build();
gcsClient.create(blobInfo, data);
}
@@ -189,10 +189,10 @@ public class GCSCloudClient implements ICloudClient {
@Override
public void copy(String bucket, String srcPath, FileReference destPath) {
guardian.checkReadAccess(bucket, srcPath);
- profilerLimiter.objectsList();
+ profiler.objectsList();
Page<Blob> blobs = gcsClient.list(bucket, BlobListOption.prefix(config.getPrefix() + srcPath));
for (Blob blob : blobs.iterateAll()) {
- profilerLimiter.objectCopy();
+ profiler.objectCopy();
BlobId source = blob.getBlobId();
String targetName = destPath.getChildPath(IoUtil.getFileNameFromPath(source.getName()));
BlobId target = BlobId.of(bucket, targetName);
@@ -202,6 +202,21 @@ public class GCSCloudClient implements ICloudClient {
}
}
+ @Override
+ public void deleteObject(String bucket, String path) throws HyracksDataException {
+ try {
+ if (path.isEmpty()) {
+ return;
+ }
+ guardian.checkWriteAccess(bucket, path);
+ profiler.objectDelete();
+ BlobId blobId = BlobId.of(bucket, config.getPrefix() + path);
+ gcsClient.delete(blobId);
+ } catch (Exception ex) {
+ throw HyracksDataException.create(ex);
+ }
+ }
+
@Override
public void deleteObjects(String bucket, Collection<String> paths) throws HyracksDataException {
if (paths.isEmpty()) {
@@ -236,14 +251,14 @@ public class GCSCloudClient implements ICloudClient {
paths.toString());
}
}
- profilerLimiter.objectDelete();
+ profiler.objectDelete();
}
}
@Override
public long getObjectSize(String bucket, String path) {
guardian.checkReadAccess(bucket, path);
- profilerLimiter.objectGet();
+ profiler.objectGet();
Blob blob =
gcsClient.get(bucket, config.getPrefix() + path, Storage.BlobGetOption.fields(Storage.BlobField.SIZE));
if (blob == null) {
@@ -255,7 +270,7 @@ public class GCSCloudClient implements ICloudClient {
@Override
public boolean exists(String bucket, String path) {
guardian.checkReadAccess(bucket, path);
- profilerLimiter.objectGet();
+ profiler.objectGet();
Blob blob = gcsClient.get(bucket, config.getPrefix() + path,
Storage.BlobGetOption.fields(Storage.BlobField.values()));
return blob != null && blob.exists();
@@ -264,7 +279,7 @@ public class GCSCloudClient implements ICloudClient {
@Override
public boolean isEmptyPrefix(String bucket, String path) {
guardian.checkReadAccess(bucket, path);
- profilerLimiter.objectsList();
+ profiler.objectsList();
Page<Blob> blobs = gcsClient.list(bucket, BlobListOption.prefix(config.getPrefix() + path));
return !blobs.iterateAll().iterator().hasNext();
}
@@ -272,13 +287,13 @@ public class GCSCloudClient implements ICloudClient {
@Override
public IParallelDownloader createParallelDownloader(String bucket, IOManager ioManager)
throws HyracksDataException {
- return new GCSParallelDownloader(bucket, ioManager, config, profilerLimiter);
+ return new GCSParallelDownloader(bucket, ioManager, config, profiler);
}
@Override
public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
guardian.checkReadAccess(bucket, "/");
- profilerLimiter.objectsList();
+ profiler.objectsList();
Page<Blob> blobs = gcsClient.list(bucket, BlobListOption.fields(Storage.BlobField.SIZE));
ArrayNode objectsInfo = objectMapper.createArrayNode();
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
index c52e2b61c6..b6af3232b2 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
@@ -23,7 +23,6 @@ import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
import static org.apache.hyracks.cloud.util.CloudRetryableRequestUtil.runWithNoRetryOnInterruption;
import java.io.IOException;
-import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.Random;
@@ -179,7 +178,7 @@ abstract class AbstractCloudExternalFileWriterFactory<T extends Throwable> imple
}
} finally {
// Delete the written file
- runWithNoRetryOnInterruption(() -> testClient.deleteObjects(bucket, Collections.singleton(finalPath)));
+ runWithNoRetryOnInterruption(() -> testClient.deleteObject(bucket, finalPath));
}
}
}
(asterixdb) 09/09: Merge "Merge branch 'gerrit/phoenix' into 'master'"
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2b7efeae59480e8dd850c756219d50693f4d8a99
Merge: ca214efed2 847a5b8f85
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Thu Oct 2 10:38:40 2025 +0000
Merge "Merge branch 'gerrit/phoenix' into 'master'"
.../apache/asterix/api/common/LocalCloudUtil.java | 27 ++++++---
.../api/common/LocalCloudUtilAdobeMock.java | 9 +++
.../test/active/ActiveEventsListenerTest.java | 5 +-
.../asterix/test/active/InfiniteRetryPolicy.java | 45 ++++++++++++++
.../apache/asterix/test/common/TestExecutor.java | 60 +++++++++++++++++--
.../asterix/test/runtime/LangExecutionUtil.java | 4 +-
.../boolean-value/external_dataset.000.ddl.sqlpp | 38 ++++++++++++
.../boolean-value/external_dataset.099.ddl.sqlpp | 20 +++++++
.../external_dataset.000.ddl.sqlpp | 38 ++++++++++++
.../external_dataset.099.ddl.sqlpp | 20 +++++++
.../runtimets/testsuite_external_dataset_s3.xml | 13 +++-
.../apache/asterix/cloud/clients/ICloudClient.java | 9 +++
.../asterix/cloud/clients/UnstableCloudClient.java | 5 ++
.../cloud/clients/aws/s3/S3CloudClient.java | 17 ++++++
.../blobstorage/AzBlobStorageClientConfig.java | 16 +++--
.../blobstorage/AzBlobStorageCloudClient.java | 69 ++++++++++++++++++----
.../cloud/clients/google/gcs/GCSCloudClient.java | 51 ++++++++++------
.../AbstractCloudExternalFileWriterFactory.java | 3 +-
.../asterix/cloud/azure/LSMAzBlobStorageTest.java | 2 +-
.../asterix/external/util/aws/s3/S3AuthUtils.java | 35 +++++++++--
.../asterix/external/util/aws/s3/S3Constants.java | 1 +
.../asterix/test/server/AbstractExecutionIT.java | 4 +-
.../asterix/test/server/NCServiceExecutionIT.java | 4 +-
hyracks-fullstack/pom.xml | 16 ++---
24 files changed, 436 insertions(+), 75 deletions(-)
(asterixdb) 04/09: [NO ISSUE][HYR][MISC] Utilize log4j-bom
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e7333eb78c01b9674a319624047f1c270787c274
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Mon Sep 22 14:49:53 2025 -0400
[NO ISSUE][HYR][MISC] Utilize log4j-bom
Ext-ref: MB-68671
Change-Id: I60f493c12474b8352414ee42abf238f91a16dba9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20406
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
---
hyracks-fullstack/pom.xml | 16 ++++------------
1 file changed, 4 insertions(+), 12 deletions(-)
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index efc3d2ea20..2d871ba9ea 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -544,17 +544,14 @@
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
+ <artifactId>log4j-bom</artifactId>
<version>${log4j.version}</version>
+ <type>pom</type>
+ <scope>import</scope>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-core</artifactId>
- <version>${log4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-jul</artifactId>
+ <artifactId>log4j-jcl</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
@@ -612,11 +609,6 @@
<artifactId>slf4j-simple</artifactId>
<version>1.7.36</version>
</dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- <version>${log4j.version}</version>
- </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
(asterixdb) 06/09: Merge branch 'trinity' into phoenix
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 7fd556ceca44fdd96f17ca5b56a278e6eae69425
Merge: 72c799cb29 af29e15fd0
Author: Ian Maxon <ia...@maxons.email>
AuthorDate: Tue Sep 30 09:09:04 2025 -0700
Merge branch 'trinity' into phoenix
Ext-ref: MB-64900
Change-Id: I068f9caa9d9ea810d62954a19730219a334a21ae
.../test/active/ActiveEventsListenerTest.java | 5 +--
.../asterix/test/active/InfiniteRetryPolicy.java | 45 ++++++++++++++++++++++
2 files changed, 47 insertions(+), 3 deletions(-)
(asterixdb) 08/09: Merge branch 'gerrit/phoenix' into 'master'
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 847a5b8f85d1e2274eb75109633219723bb12f2b
Merge: 4d3a6678ad d22fc3f8c4
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Wed Oct 1 21:28:02 2025 -0400
Merge branch 'gerrit/phoenix' into 'master'
* [ASTERIXDB-3643][EXT]: make path-style addressing configurable
* [ASTERIXDB-3647][TEST] Use InfiniteRetryPolicy
* [NO ISSUE][HYR][MISC] Utilize log4j-bom
* [NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify
* [ASTERIXDB-3653][EXT]: Properly handle errors when deleting for COPY TO
Change-Id: I60e0ab66c5edcabb9d78de6db76162d913e30809
.../apache/asterix/api/common/LocalCloudUtil.java | 27 ++++++---
.../api/common/LocalCloudUtilAdobeMock.java | 9 +++
.../test/active/ActiveEventsListenerTest.java | 5 +-
.../asterix/test/active/InfiniteRetryPolicy.java | 45 ++++++++++++++
.../apache/asterix/test/common/TestExecutor.java | 60 +++++++++++++++++--
.../asterix/test/runtime/LangExecutionUtil.java | 4 +-
.../boolean-value/external_dataset.000.ddl.sqlpp | 38 ++++++++++++
.../boolean-value/external_dataset.099.ddl.sqlpp | 20 +++++++
.../external_dataset.000.ddl.sqlpp | 38 ++++++++++++
.../external_dataset.099.ddl.sqlpp | 20 +++++++
.../runtimets/testsuite_external_dataset_s3.xml | 13 +++-
.../apache/asterix/cloud/clients/ICloudClient.java | 9 +++
.../asterix/cloud/clients/UnstableCloudClient.java | 5 ++
.../cloud/clients/aws/s3/S3CloudClient.java | 17 ++++++
.../blobstorage/AzBlobStorageClientConfig.java | 16 +++--
.../blobstorage/AzBlobStorageCloudClient.java | 69 ++++++++++++++++++----
.../cloud/clients/google/gcs/GCSCloudClient.java | 51 ++++++++++------
.../AbstractCloudExternalFileWriterFactory.java | 3 +-
.../asterix/cloud/azure/LSMAzBlobStorageTest.java | 2 +-
.../asterix/external/util/aws/s3/S3AuthUtils.java | 35 +++++++++--
.../asterix/external/util/aws/s3/S3Constants.java | 1 +
.../asterix/test/server/AbstractExecutionIT.java | 4 +-
.../asterix/test/server/NCServiceExecutionIT.java | 4 +-
hyracks-fullstack/pom.xml | 16 ++---
24 files changed, 436 insertions(+), 75 deletions(-)
(asterixdb) 05/09: [NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify
Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 72c799cb29481eb9fb0d0283d95b678cdf610555
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Sep 27 18:13:36 2025 -0400
[NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify
Also, update test framework to drop created databases on cleanup
Ext-ref: MB-68204
Change-Id: If708d39892cb777d4e7f0eba606417bd82692ab2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20423
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../apache/asterix/api/common/LocalCloudUtil.java | 27 ++++++----
.../api/common/LocalCloudUtilAdobeMock.java | 9 ++++
.../apache/asterix/test/common/TestExecutor.java | 60 ++++++++++++++++++++--
.../asterix/test/runtime/LangExecutionUtil.java | 4 +-
.../blobstorage/AzBlobStorageClientConfig.java | 16 ++++--
.../blobstorage/AzBlobStorageCloudClient.java | 24 +++++++++
.../asterix/cloud/azure/LSMAzBlobStorageTest.java | 2 +-
.../asterix/test/server/AbstractExecutionIT.java | 4 +-
.../asterix/test/server/NCServiceExecutionIT.java | 4 +-
9 files changed, 129 insertions(+), 21 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
index ce424084a2..d33617b736 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
@@ -21,19 +21,16 @@ package org.apache.asterix.api.common;
import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.io.File;
-import java.net.URI;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import io.findify.s3mock.S3Mock;
-import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
@@ -76,11 +73,7 @@ public class LocalCloudUtil {
s3MockServer.start();
LOGGER.info("S3 mock server started successfully");
- S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server
- builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
- .endpointOverride(endpoint);
- S3Client client = builder.build();
+ S3Client client = LocalCloudUtilAdobeMock.getS3Client();
client.createBucket(CreateBucketRequest.builder().bucket(CLOUD_STORAGE_BUCKET).build());
LOGGER.info("Created bucket {} for cloud storage", CLOUD_STORAGE_BUCKET);
@@ -101,6 +94,22 @@ public class LocalCloudUtil {
return s3MockServer;
}
+ public static void recreateBucket(String bucketName, S3Client client) {
+ String verb = "Created";
+ try {
+ client.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
+ verb = "Recreated";
+ } catch (Exception e) {
+ // ignore any failure
+ }
+ try {
+ client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
+ LOGGER.info("{} bucket {}", verb, bucketName);
+ } finally {
+ client.close();
+ }
+ }
+
public static void stopS3MockServer() {
shutdownSilently();
// since they are running on same port, we need to shut down other mock server as well
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
index ba7f8561f3..8790632416 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
@@ -106,4 +106,13 @@ public class LocalCloudUtilAdobeMock {
s3Mock = null;
}
}
+
+ public static S3Client getS3Client() {
+ S3ClientBuilder builder = S3Client.builder();
+ URI endpoint = URI.create(MOCK_SERVER_ENDPOINT);
+ builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+ .endpointOverride(endpoint);
+ S3Client client = builder.build();
+ return client;
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index ee8599507c..58c92fb3ee 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -74,6 +74,7 @@ import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -84,6 +85,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.Queue;
+import java.util.SequencedSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -2728,17 +2730,38 @@ public class TestExecutor {
return stripLineComments(stripJavaComments(statement));
}
- public void cleanup(String testCase, List<String> badtestcases) throws Exception {
+ public final void cleanup(String testCase, SequencedSet<String> badtestcases) throws Exception {
+ cleanup(testCase, badtestcases, false);
+ }
+
+ public void cleanup(String testCase, SequencedSet<String> badtestcases, boolean skipClusterCleanup)
+ throws Exception {
+ if (skipClusterCleanup) {
+ return;
+ }
try {
List<Pair<String, DataverseName>> toBeDropped = new ArrayList<>();
listUserDefinedDataverses(toBeDropped);
if (!toBeDropped.isEmpty()) {
badtestcases.add(testCase);
- LOGGER.info("Last test left some garbage. Dropping dataverses: " + StringUtils.join(toBeDropped, ','));
+ LOGGER.info("Last test leaked scopes. Dropping scopes: {}",
+ toBeDropped.stream()
+ .map(pair -> (pair.getFirst() == null ? "" : (pair.getFirst() + SqlppStatementUtil.DOT))
+ + pair.getSecond().getCanonicalForm())
+ .collect(Collectors.joining(", ", "[", "]")));
+ // TODO(mblow): consider that dataverses can have references from views in other dataverses
for (Pair<String, DataverseName> dv : toBeDropped) {
dropDataverse(dv);
}
}
+ List<String> databases = listUserDefinedDatabases();
+ if (!databases.isEmpty()) {
+ LOGGER.info("Last test leaked databases. Dropping databases: " + databases);
+ badtestcases.add(testCase);
+ for (String db : databases) {
+ dropDatabase(db);
+ }
+ }
} catch (Throwable th) {
th.printStackTrace();
throw th;
@@ -2768,6 +2791,25 @@ public class TestExecutor {
}
}
+ protected List<String> listUserDefinedDatabases() throws Exception {
+ String query = "select value(DatabaseName) from Metadata.`Database` where Creator.Name != \"@sys\"";
+ URI uri = getEndpoint(Servlets.QUERY_SERVICE);
+ try {
+ InputStream resultStream = executeQueryService(query, OutputFormat.CLEAN_JSON, uri,
+ constructQueryParameters(query, OutputFormat.CLEAN_JSON, new ArrayList<Parameter>()), false,
+ StandardCharsets.UTF_8, code -> code == HttpStatus.SC_OK, false);
+ JsonNode result = extractResult(IOUtils.toString(resultStream, UTF_8));
+ List<String> databases = new ArrayList<>();
+ for (int i = 0; i < result.size(); i++) {
+ databases.add(result.get(i).asText());
+ }
+ return databases;
+ } catch (Exception e) {
+ // assuming databases are not enabled in this cluster
+ return Collections.emptyList();
+ }
+ }
+
protected void dropDataverse(Pair<String, DataverseName> dv) throws Exception {
StringBuilder dropStatement = new StringBuilder();
dropStatement.append("drop dataverse ");
@@ -2783,8 +2825,18 @@ public class TestExecutor {
ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON);
}
- protected void listDatasets(DataverseName dataverseName, List<Pair<String, DatasetConfig.DatasetType>> outDatasets)
- throws Exception {
+ protected void dropDatabase(String db) throws Exception {
+ StringBuilder dropStatement = new StringBuilder();
+ dropStatement.append("DROP DATABASE ");
+ SqlppStatementUtil.enclose(dropStatement, db);
+ dropStatement.append(";");
+ InputStream resultStream = executeQueryService(dropStatement.toString(), getEndpoint(Servlets.QUERY_SERVICE),
+ OutputFormat.CLEAN_JSON, UTF_8);
+ ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON);
+ }
+
+ protected void listDatasets(DataverseName dataverseName,
+ Collection<Pair<String, DatasetConfig.DatasetType>> outDatasets) throws Exception {
String query = "select d.DatasetName, d.DatasetType from Metadata.`Dataset` d where d.DataverseName = '"
+ dataverseName.getCanonicalForm() + "'";
InputStream resultStream = executeQueryService(query, getEndpoint(Servlets.QUERY_SERVICE),
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index d704d8e456..b7e8565897 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -30,7 +30,9 @@ import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.SequencedSet;
import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.app.external.IExternalUDFLibrarian;
@@ -54,7 +56,7 @@ public class LangExecutionUtil {
private static final boolean cleanupOnStart = true;
private static final boolean cleanupOnStop = true;
- private static final List<String> badTestCases = new ArrayList<>();
+ private static final SequencedSet<String> badTestCases = new LinkedHashSet<>();
private static TestExecutor testExecutor;
private static ExternalUDFLibrarian librarian;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
index f4536a18fe..889a6a42d8 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
@@ -29,9 +29,10 @@ import com.azure.identity.DefaultAzureCredential;
import com.azure.identity.DefaultAzureCredentialBuilder;
public class AzBlobStorageClientConfig {
- private final int writeBufferSize;
// Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id
static final int DELETE_BATCH_SIZE = 256;
+
+ private final int writeBufferSize;
private final String region;
private final String endpoint;
private final String prefix;
@@ -42,15 +43,16 @@ public class AzBlobStorageClientConfig {
private final long tokenAcquireTimeout;
private final int writeMaxRequestsPerSeconds;
private final int readMaxRequestsPerSeconds;
+ private final boolean storageDisableSSLVerify;
public AzBlobStorageClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, String bucket, int writeBufferSize) {
- this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, bucket, 1, 0, 0, writeBufferSize);
+ this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, bucket, 1, 0, 0, writeBufferSize, false);
}
public AzBlobStorageClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, String bucket, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
- int readMaxRequestsPerSeconds, int writeBufferSize) {
+ int readMaxRequestsPerSeconds, int writeBufferSize, boolean storageDisableSSLVerify) {
this.region = Objects.requireNonNull(region, "region");
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -61,6 +63,7 @@ public class AzBlobStorageClientConfig {
this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds;
this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds;
this.writeBufferSize = writeBufferSize;
+ this.storageDisableSSLVerify = storageDisableSSLVerify;
}
public static AzBlobStorageClientConfig of(CloudProperties cloudProperties) {
@@ -68,7 +71,8 @@ public class AzBlobStorageClientConfig {
cloudProperties.getStoragePrefix(), cloudProperties.isStorageAnonymousAuth(),
cloudProperties.getProfilerLogInterval(), cloudProperties.getStorageBucket(),
cloudProperties.getTokenAcquireTimeout(), cloudProperties.getWriteMaxRequestsPerSecond(),
- cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getWriteBufferSize());
+ cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getWriteBufferSize(),
+ cloudProperties.isStorageDisableSSLVerify());
}
public static AzBlobStorageClientConfig of(Map<String, String> configuration, int writeBufferSize) {
@@ -111,6 +115,10 @@ public class AzBlobStorageClientConfig {
return anonymousAuth;
}
+ public boolean isStorageDisableSSLVerify() {
+ return storageDisableSSLVerify;
+ }
+
public DefaultAzureCredential createCredentialsProvider() {
return new DefaultAzureCredentialBuilder().build();
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index d7b680eb9e..bdb551b768 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -58,6 +58,7 @@ import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
import com.azure.core.http.rest.PagedIterable;
import com.azure.core.http.rest.Response;
import com.azure.core.util.BinaryData;
@@ -79,6 +80,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import reactor.netty.http.client.HttpClient;
+
public class AzBlobStorageCloudClient implements ICloudClient {
private static final String BUCKET_ROOT_PATH = "";
public static final String AZURITE_ENDPOINT = "http://127.0.0.1:15055/devstoreaccount1/";
@@ -402,6 +408,24 @@ public class AzBlobStorageCloudClient implements ICloudClient {
blobServiceClientBuilder.endpoint(getEndpoint(config));
blobServiceClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS);
configCredentialsToAzClient(blobServiceClientBuilder, config);
+
+ // Disable SSL verification if the config property is set
+ if (config.isStorageDisableSSLVerify()) {
+ try {
+ // Create SSL context that trusts all certificates
+ SslContext sslContext =
+ SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build();
+
+ // Create a base Reactor Netty HttpClient with SSL verification disabled
+ HttpClient baseHttpClient = HttpClient.create().secure(sslSpec -> sslSpec.sslContext(sslContext));
+
+ // Configure the Azure HTTP client with the base client
+ blobServiceClientBuilder.httpClient(new NettyAsyncHttpClientBuilder(baseHttpClient).build());
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to disable SSL verification", e);
+ }
+ }
+
return blobServiceClientBuilder.buildClient();
}
diff --git a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
index 1f49fd9e75..0d5a9ba518 100644
--- a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
@@ -63,7 +63,7 @@ public class LSMAzBlobStorageTest extends AbstractLSMTest {
LOGGER.info("Az Blob Client created successfully");
int writeBufferSize = StorageUtil.getIntSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE);
AzBlobStorageClientConfig config = new AzBlobStorageClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "",
- true, 0, PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize);
+ true, 0, PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize, false);
CLOUD_CLIENT = new AzBlobStorageCloudClient(config, ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
index 35bf7e4e45..1360917490 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
@@ -21,8 +21,10 @@ import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
+import java.util.SequencedSet;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.IdentitiyResolverFactory;
@@ -61,7 +63,7 @@ public abstract class AbstractExecutionIT {
private static final String EXTERNAL_LIBRARY_TEST_GROUP = "lib";
- private static final List<String> badTestCases = new ArrayList<>();
+ private static final SequencedSet<String> badTestCases = new LinkedHashSet<>();
private static String reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath();
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
index ab4e19fd82..507a156b28 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
@@ -24,8 +24,10 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Random;
+import java.util.SequencedSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
@@ -135,7 +137,7 @@ public class NCServiceExecutionIT {
private final TestCaseContext tcCtx;
private static final TestExecutor testExecutor = new TestExecutor();
- private static final List<String> badTestCases = new ArrayList<>();
+ private static final SequencedSet<String> badTestCases = new LinkedHashSet<>();
private static HyracksVirtualCluster cluster;
private final KillCommand killType;
private static boolean clusterActive = false;