You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/10/02 10:45:35 UTC
(asterixdb) 05/09: [NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 72c799cb29481eb9fb0d0283d95b678cdf610555
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Sep 27 18:13:36 2025 -0400
[NO ISSUE][*DB][EXT] Update Azure Blob cloud client to support ssl !verify
Also, update test framework to drop created databases on cleanup
Ext-ref: MB-68204
Change-Id: If708d39892cb777d4e7f0eba606417bd82692ab2
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20423
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
Tested-by: Michael Blow <mb...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../apache/asterix/api/common/LocalCloudUtil.java | 27 ++++++----
.../api/common/LocalCloudUtilAdobeMock.java | 9 ++++
.../apache/asterix/test/common/TestExecutor.java | 60 ++++++++++++++++++++--
.../asterix/test/runtime/LangExecutionUtil.java | 4 +-
.../blobstorage/AzBlobStorageClientConfig.java | 16 ++++--
.../blobstorage/AzBlobStorageCloudClient.java | 24 +++++++++
.../asterix/cloud/azure/LSMAzBlobStorageTest.java | 2 +-
.../asterix/test/server/AbstractExecutionIT.java | 4 +-
.../asterix/test/server/NCServiceExecutionIT.java | 4 +-
9 files changed, 129 insertions(+), 21 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
index ce424084a2..d33617b736 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtil.java
@@ -21,19 +21,16 @@ package org.apache.asterix.api.common;
import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.io.File;
-import java.net.URI;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import io.findify.s3mock.S3Mock;
-import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.S3ClientBuilder;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.DeleteBucketRequest;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
@@ -76,11 +73,7 @@ public class LocalCloudUtil {
s3MockServer.start();
LOGGER.info("S3 mock server started successfully");
- S3ClientBuilder builder = S3Client.builder();
- URI endpoint = URI.create(MOCK_SERVER_ENDPOINT); // endpoint pointing to S3 mock server
- builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
- .endpointOverride(endpoint);
- S3Client client = builder.build();
+ S3Client client = LocalCloudUtilAdobeMock.getS3Client();
client.createBucket(CreateBucketRequest.builder().bucket(CLOUD_STORAGE_BUCKET).build());
LOGGER.info("Created bucket {} for cloud storage", CLOUD_STORAGE_BUCKET);
@@ -101,6 +94,22 @@ public class LocalCloudUtil {
return s3MockServer;
}
+ public static void recreateBucket(String bucketName, S3Client client) {
+ String verb = "Created";
+ try {
+ client.deleteBucket(DeleteBucketRequest.builder().bucket(bucketName).build());
+ verb = "Recreated";
+ } catch (Exception e) {
+ // ignore any failure
+ }
+ try {
+ client.createBucket(CreateBucketRequest.builder().bucket(bucketName).build());
+ LOGGER.info("{} bucket {}", verb, bucketName);
+ } finally {
+ client.close();
+ }
+ }
+
public static void stopS3MockServer() {
shutdownSilently();
// since they are running on same port, we need to shut down other mock server as well
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
index ba7f8561f3..8790632416 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/LocalCloudUtilAdobeMock.java
@@ -106,4 +106,13 @@ public class LocalCloudUtilAdobeMock {
s3Mock = null;
}
}
+
+ public static S3Client getS3Client() {
+ S3ClientBuilder builder = S3Client.builder();
+ URI endpoint = URI.create(MOCK_SERVER_ENDPOINT);
+ builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
+ .endpointOverride(endpoint);
+ S3Client client = builder.build();
+ return client;
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index ee8599507c..58c92fb3ee 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -74,6 +74,7 @@ import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -84,6 +85,7 @@ import java.util.Objects;
import java.util.Optional;
import java.util.OptionalInt;
import java.util.Queue;
+import java.util.SequencedSet;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -2728,17 +2730,38 @@ public class TestExecutor {
return stripLineComments(stripJavaComments(statement));
}
- public void cleanup(String testCase, List<String> badtestcases) throws Exception {
+ public final void cleanup(String testCase, SequencedSet<String> badtestcases) throws Exception {
+ cleanup(testCase, badtestcases, false);
+ }
+
+ public void cleanup(String testCase, SequencedSet<String> badtestcases, boolean skipClusterCleanup)
+ throws Exception {
+ if (skipClusterCleanup) {
+ return;
+ }
try {
List<Pair<String, DataverseName>> toBeDropped = new ArrayList<>();
listUserDefinedDataverses(toBeDropped);
if (!toBeDropped.isEmpty()) {
badtestcases.add(testCase);
- LOGGER.info("Last test left some garbage. Dropping dataverses: " + StringUtils.join(toBeDropped, ','));
+ LOGGER.info("Last test leaked scopes. Dropping scopes: {}",
+ toBeDropped.stream()
+ .map(pair -> (pair.getFirst() == null ? "" : (pair.getFirst() + SqlppStatementUtil.DOT))
+ + pair.getSecond().getCanonicalForm())
+ .collect(Collectors.joining(", ", "[", "]")));
+ // TODO(mblow): consider that dataverses can have references from views in other dataverses
for (Pair<String, DataverseName> dv : toBeDropped) {
dropDataverse(dv);
}
}
+ List<String> databases = listUserDefinedDatabases();
+ if (!databases.isEmpty()) {
+ LOGGER.info("Last test leaked databases. Dropping databases: " + databases);
+ badtestcases.add(testCase);
+ for (String db : databases) {
+ dropDatabase(db);
+ }
+ }
} catch (Throwable th) {
th.printStackTrace();
throw th;
@@ -2768,6 +2791,25 @@ public class TestExecutor {
}
}
+ protected List<String> listUserDefinedDatabases() throws Exception {
+ String query = "select value(DatabaseName) from Metadata.`Database` where Creator.Name != \"@sys\"";
+ URI uri = getEndpoint(Servlets.QUERY_SERVICE);
+ try {
+ InputStream resultStream = executeQueryService(query, OutputFormat.CLEAN_JSON, uri,
+ constructQueryParameters(query, OutputFormat.CLEAN_JSON, new ArrayList<Parameter>()), false,
+ StandardCharsets.UTF_8, code -> code == HttpStatus.SC_OK, false);
+ JsonNode result = extractResult(IOUtils.toString(resultStream, UTF_8));
+ List<String> databases = new ArrayList<>();
+ for (int i = 0; i < result.size(); i++) {
+ databases.add(result.get(i).asText());
+ }
+ return databases;
+ } catch (Exception e) {
+ // assuming databases are not enabled in this cluster
+ return Collections.emptyList();
+ }
+ }
+
protected void dropDataverse(Pair<String, DataverseName> dv) throws Exception {
StringBuilder dropStatement = new StringBuilder();
dropStatement.append("drop dataverse ");
@@ -2783,8 +2825,18 @@ public class TestExecutor {
ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON);
}
- protected void listDatasets(DataverseName dataverseName, List<Pair<String, DatasetConfig.DatasetType>> outDatasets)
- throws Exception {
+ protected void dropDatabase(String db) throws Exception {
+ StringBuilder dropStatement = new StringBuilder();
+ dropStatement.append("DROP DATABASE ");
+ SqlppStatementUtil.enclose(dropStatement, db);
+ dropStatement.append(";");
+ InputStream resultStream = executeQueryService(dropStatement.toString(), getEndpoint(Servlets.QUERY_SERVICE),
+ OutputFormat.CLEAN_JSON, UTF_8);
+ ResultExtractor.extract(resultStream, UTF_8, OutputFormat.CLEAN_JSON);
+ }
+
+ protected void listDatasets(DataverseName dataverseName,
+ Collection<Pair<String, DatasetConfig.DatasetType>> outDatasets) throws Exception {
String query = "select d.DatasetName, d.DatasetType from Metadata.`Dataset` d where d.DataverseName = '"
+ dataverseName.getCanonicalForm() + "'";
InputStream resultStream = executeQueryService(query, getEndpoint(Servlets.QUERY_SERVICE),
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index d704d8e456..b7e8565897 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -30,7 +30,9 @@ import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.SequencedSet;
import org.apache.asterix.app.external.ExternalUDFLibrarian;
import org.apache.asterix.app.external.IExternalUDFLibrarian;
@@ -54,7 +56,7 @@ public class LangExecutionUtil {
private static final boolean cleanupOnStart = true;
private static final boolean cleanupOnStop = true;
- private static final List<String> badTestCases = new ArrayList<>();
+ private static final SequencedSet<String> badTestCases = new LinkedHashSet<>();
private static TestExecutor testExecutor;
private static ExternalUDFLibrarian librarian;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
index f4536a18fe..889a6a42d8 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
@@ -29,9 +29,10 @@ import com.azure.identity.DefaultAzureCredential;
import com.azure.identity.DefaultAzureCredentialBuilder;
public class AzBlobStorageClientConfig {
- private final int writeBufferSize;
// Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id
static final int DELETE_BATCH_SIZE = 256;
+
+ private final int writeBufferSize;
private final String region;
private final String endpoint;
private final String prefix;
@@ -42,15 +43,16 @@ public class AzBlobStorageClientConfig {
private final long tokenAcquireTimeout;
private final int writeMaxRequestsPerSeconds;
private final int readMaxRequestsPerSeconds;
+ private final boolean storageDisableSSLVerify;
public AzBlobStorageClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, String bucket, int writeBufferSize) {
- this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, bucket, 1, 0, 0, writeBufferSize);
+ this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, bucket, 1, 0, 0, writeBufferSize, false);
}
public AzBlobStorageClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
long profilerLogInterval, String bucket, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
- int readMaxRequestsPerSeconds, int writeBufferSize) {
+ int readMaxRequestsPerSeconds, int writeBufferSize, boolean storageDisableSSLVerify) {
this.region = Objects.requireNonNull(region, "region");
this.endpoint = endpoint;
this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -61,6 +63,7 @@ public class AzBlobStorageClientConfig {
this.writeMaxRequestsPerSeconds = writeMaxRequestsPerSeconds;
this.readMaxRequestsPerSeconds = readMaxRequestsPerSeconds;
this.writeBufferSize = writeBufferSize;
+ this.storageDisableSSLVerify = storageDisableSSLVerify;
}
public static AzBlobStorageClientConfig of(CloudProperties cloudProperties) {
@@ -68,7 +71,8 @@ public class AzBlobStorageClientConfig {
cloudProperties.getStoragePrefix(), cloudProperties.isStorageAnonymousAuth(),
cloudProperties.getProfilerLogInterval(), cloudProperties.getStorageBucket(),
cloudProperties.getTokenAcquireTimeout(), cloudProperties.getWriteMaxRequestsPerSecond(),
- cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getWriteBufferSize());
+ cloudProperties.getReadMaxRequestsPerSecond(), cloudProperties.getWriteBufferSize(),
+ cloudProperties.isStorageDisableSSLVerify());
}
public static AzBlobStorageClientConfig of(Map<String, String> configuration, int writeBufferSize) {
@@ -111,6 +115,10 @@ public class AzBlobStorageClientConfig {
return anonymousAuth;
}
+ public boolean isStorageDisableSSLVerify() {
+ return storageDisableSSLVerify;
+ }
+
public DefaultAzureCredential createCredentialsProvider() {
return new DefaultAzureCredentialBuilder().build();
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index d7b680eb9e..bdb551b768 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -58,6 +58,7 @@ import org.apache.hyracks.control.nc.io.IOManager;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
import com.azure.core.http.rest.PagedIterable;
import com.azure.core.http.rest.Response;
import com.azure.core.util.BinaryData;
@@ -79,6 +80,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.netty.handler.ssl.SslContext;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import reactor.netty.http.client.HttpClient;
+
public class AzBlobStorageCloudClient implements ICloudClient {
private static final String BUCKET_ROOT_PATH = "";
public static final String AZURITE_ENDPOINT = "http://127.0.0.1:15055/devstoreaccount1/";
@@ -402,6 +408,24 @@ public class AzBlobStorageCloudClient implements ICloudClient {
blobServiceClientBuilder.endpoint(getEndpoint(config));
blobServiceClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS);
configCredentialsToAzClient(blobServiceClientBuilder, config);
+
+ // Disable SSL verification if the config property is set
+ if (config.isStorageDisableSSLVerify()) {
+ try {
+ // Create SSL context that trusts all certificates
+ SslContext sslContext =
+ SslContextBuilder.forClient().trustManager(InsecureTrustManagerFactory.INSTANCE).build();
+
+ // Create a base Reactor Netty HttpClient with SSL verification disabled
+ HttpClient baseHttpClient = HttpClient.create().secure(sslSpec -> sslSpec.sslContext(sslContext));
+
+ // Configure the Azure HTTP client with the base client
+ blobServiceClientBuilder.httpClient(new NettyAsyncHttpClientBuilder(baseHttpClient).build());
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to disable SSL verification", e);
+ }
+ }
+
return blobServiceClientBuilder.buildClient();
}
diff --git a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
index 1f49fd9e75..0d5a9ba518 100644
--- a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/azure/LSMAzBlobStorageTest.java
@@ -63,7 +63,7 @@ public class LSMAzBlobStorageTest extends AbstractLSMTest {
LOGGER.info("Az Blob Client created successfully");
int writeBufferSize = StorageUtil.getIntSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE);
AzBlobStorageClientConfig config = new AzBlobStorageClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "",
- true, 0, PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize);
+ true, 0, PLAYGROUND_CONTAINER, 1, 0, 0, writeBufferSize, false);
CLOUD_CLIENT = new AzBlobStorageCloudClient(config, ICloudGuardian.NoOpCloudGuardian.INSTANCE);
}
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
index 35bf7e4e45..1360917490 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/AbstractExecutionIT.java
@@ -21,8 +21,10 @@ import static org.apache.hyracks.util.file.FileUtil.joinPath;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Objects;
+import java.util.SequencedSet;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.IdentitiyResolverFactory;
@@ -61,7 +63,7 @@ public abstract class AbstractExecutionIT {
private static final String EXTERNAL_LIBRARY_TEST_GROUP = "lib";
- private static final List<String> badTestCases = new ArrayList<>();
+ private static final SequencedSet<String> badTestCases = new LinkedHashSet<>();
private static String reportPath = new File(joinPath("target", "failsafe-reports")).getAbsolutePath();
diff --git a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
index ab4e19fd82..507a156b28 100644
--- a/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
+++ b/asterixdb/asterix-server/src/test/java/org/apache/asterix/test/server/NCServiceExecutionIT.java
@@ -24,8 +24,10 @@ import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Random;
+import java.util.SequencedSet;
import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
@@ -135,7 +137,7 @@ public class NCServiceExecutionIT {
private final TestCaseContext tcCtx;
private static final TestExecutor testExecutor = new TestExecutor();
- private static final List<String> badTestCases = new ArrayList<>();
+ private static final SequencedSet<String> badTestCases = new LinkedHashSet<>();
private static HyracksVirtualCluster cluster;
private final KillCommand killType;
private static boolean clusterActive = false;