You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by dl...@apache.org on 2021/10/08 21:04:13 UTC
[asterixdb] branch master updated: [ASTERIXDB-2970]: azure blob -
remove connection string + add standalone parameters
This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 248fe07 [ASTERIXDB-2970]: azure blob - remove connection string + add standalone parameters
248fe07 is described below
commit 248fe07fc04063fe6fac90bc152fea789c2aef46
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Sun Oct 3 06:35:17 2021 +0300
[ASTERIXDB-2970]: azure blob - remove connection string + add standalone parameters
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Removed support for connectionString
- endpoint is now required
- Following are standalone parameters:
- accountName
- accountKey
- sharedAccessSignature
- managedIdentityId
- clientId
- clientSecret
- clientCertificate
- clientCertificatePassword
- tenantId
Change-Id: I8d9daf6859dc878369e1fd737f1ad04a6c6f308d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/13363
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
.../apache/asterix/test/common/TestConstants.java | 72 +++++++------
.../apache/asterix/test/common/TestExecutor.java | 55 +++++++---
.../AzureBlobStorageExternalDatasetTest.java | 17 +--
.../test.000.ddl.sqlpp | 2 +-
.../test.000.ddl.sqlpp | 2 +-
.../invalid-auth-methods/test.000.ddl.sqlpp | 2 +-
.../valid-auth-methods/test.000.ddl.sqlpp | 2 +-
...stsuite_external_dataset_azure_blob_storage.xml | 4 +-
.../external/util/ExternalDataConstants.java | 8 +-
.../asterix/external/util/ExternalDataUtils.java | 120 +++++++++++++--------
10 files changed, 173 insertions(+), 111 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
index 2d2a206..875b0f6 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
@@ -18,10 +18,8 @@
*/
package org.apache.asterix.test.common;
-import org.apache.asterix.test.external_dataset.microsoft.AzureBlobStorageExternalDatasetTest;
-
public class TestConstants {
- // AWS S3 constants and place holders
+ // AWS S3 constants and placeholders
public static final String S3_ACCESS_KEY_ID_PLACEHOLDER = "%accessKeyId%";
public static final String S3_ACCESS_KEY_ID_DEFAULT = "dummyAccessKey";
public static final String S3_SECRET_ACCESS_KEY_PLACEHOLDER = "%secretAccessKey%";
@@ -37,41 +35,47 @@ public class TestConstants {
+ "(\"secretAccessKey\"=\"" + S3_SECRET_ACCESS_KEY_DEFAULT + "\"),\n" + "(\"region\"=\"" + S3_REGION_DEFAULT
+ "\"),\n" + "(\"serviceEndpoint\"=\"" + S3_SERVICE_ENDPOINT_DEFAULT + "\")";
- // Azure blob storage constants and place holders
- // account name
- public static final String AZURE_ACCOUNT_NAME_PLACEHOLDER = "%azureblob-accountname%";
- public static final String AZURE_AZURITE_ACCOUNT_NAME_DEFAULT = "devstoreaccount1";
+ // Azure blob storage constants and placeholders
+ public static class Azure {
+ // account name
+ public static final String ACCOUNT_NAME_PLACEHOLDER = "%azureblob-accountname%";
+ public static final String AZURITE_ACCOUNT_NAME_DEFAULT = "devstoreaccount1";
+
+ // account key
+ public static final String ACCOUNT_KEY_PLACEHOLDER = "%azureblob-accountkey%";
+ public static final String AZURITE_ACCOUNT_KEY_DEFAULT =
+ "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+
+ // SAS token: this is generated and assigned at runtime at the start of the test
+ public static final String SAS_TOKEN_PLACEHOLDER = "%azureblob-sas%";
+ public static String sasToken = "";
+
+ // blob endpoint
+ public static final String BLOB_ENDPOINT_PLACEHOLDER = "%azureblob-endpoint%";
+ public static final String BLOB_ENDPOINT_DEFAULT = "http://localhost:10000/" + AZURITE_ACCOUNT_NAME_DEFAULT;
+
+ public static final String MANAGED_IDENTITY_ID_PLACEHOLDER = "%azureblob-managedidentityid%";
+ public static final String MANAGED_IDENTITY_ID_DEFAULT = "myManagedIdentityId";
- // account key
- public static final String AZURE_ACCOUNT_KEY_PLACEHOLDER = "%azureblob-accountkey%";
- public static final String AZURE_AZURITE_ACCOUNT_KEY_DEFAULT =
- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+ public static final String CLIENT_ID_PLACEHOLDER = "%azureblob-clientid%";
+ public static final String CLIENT_ID_DEFAULT = "myClientId";
- // SAS token: this is generated and assigned at runtime at the start of the test
- public static final String AZURE_SAS_TOKEN_PLACEHOLDER = "%azureblob-sas%";
- public static String sasToken = "";
+ public static final String CLIENT_SECRET_PLACEHOLDER = "%azureblob-clientsecret%";
+ public static final String CLIENT_SECRET_DEFAULT = "myClientSecret";
- // blob endpoint
- public static final String AZURE_BLOB_ENDPOINT_PLACEHOLDER = "%azureblob-endpoint%";
- public static final String AZURE_BLOB_ENDPOINT_DEFAULT =
- "http://localhost:10000/" + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT;
+ public static final String CLIENT_CERTIFICATE_PLACEHOLDER = "%azureblob-clientcertificate%";
+ public static final String CLIENT_CERTIFICATE_DEFAULT = "myClientCertificate";
- // connection string with account name & account key
- public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER =
- "%azureblob-connectionstringaccountkey%";
- public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY = "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER
- + ";AccountKey=" + AZURE_ACCOUNT_KEY_PLACEHOLDER + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER;
+ public static final String CLIENT_CERTIFICATE_PASSWORD_PLACEHOLDER = "%azureblob-clientcertificatepassword%";
+ public static final String CLIENT_CERTIFICATE_PASSWORD_DEFAULT = "myClientCertificatePassword";
- // connection string with account name & sas token
- public static final String AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER = "%azureblob-connectionstringsas%";
- public static final String AZURE_CONNECTION_STRING_SAS_TOKEN =
- "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER + ";SharedAccessSignature=" + AZURE_SAS_TOKEN_PLACEHOLDER
- + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER;
+ public static final String TENANT_ID_PLACEHOLDER = "%azureblob-tenantid%";
+ public static final String TENANT_ID_DEFAULT = "myTenantId";
- // azure template and default template
- public static final String AZURE_TEMPLATE = "(\"accountName\"=\"" + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT + "\"),\n"
- + "(\"accountKey\"=\"" + AZURE_AZURITE_ACCOUNT_KEY_DEFAULT + "\"),\n" + "(\"blobEndpoint\"=\""
- + AZURE_BLOB_ENDPOINT_PLACEHOLDER + "\")";
- public static final String AZURE_TEMPLATE_DEFAULT =
- "(\"connectionString\"=\"" + AzureBlobStorageExternalDatasetTest.CONNECTION_STRING + "\")";
+ // azure template and default template
+ public static final String TEMPLATE = "(\"accountName\"=\"" + AZURITE_ACCOUNT_NAME_DEFAULT + "\"),\n"
+ + "(\"accountKey\"=\"" + AZURITE_ACCOUNT_KEY_DEFAULT + "\"),\n" + "(\"endpoint\"=\""
+ + BLOB_ENDPOINT_PLACEHOLDER + "\")";
+ public static final String TEMPLATE_DEFAULT = TEMPLATE;
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 3a8ef36..bd618d5 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -19,6 +19,28 @@
package org.apache.asterix.test.common;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.asterix.test.common.TestConstants.Azure.ACCOUNT_KEY_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.ACCOUNT_NAME_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.AZURITE_ACCOUNT_KEY_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.AZURITE_ACCOUNT_NAME_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.BLOB_ENDPOINT_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.BLOB_ENDPOINT_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_CERTIFICATE_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_CERTIFICATE_PASSWORD_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_CERTIFICATE_PASSWORD_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_CERTIFICATE_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_ID_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_ID_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_SECRET_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.CLIENT_SECRET_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.MANAGED_IDENTITY_ID_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.MANAGED_IDENTITY_ID_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.SAS_TOKEN_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.TEMPLATE;
+import static org.apache.asterix.test.common.TestConstants.Azure.TEMPLATE_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.TENANT_ID_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.TENANT_ID_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.sasToken;
import static org.apache.hyracks.util.NetworkUtil.toHostPort;
import static org.apache.hyracks.util.file.FileUtil.canonicalize;
@@ -2301,21 +2323,22 @@ public class TestExecutor {
}
// This replaces specific external dataset placeholders
- str = str.replace(TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER,
- TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY);
- str = str.replace(TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER,
- TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN);
- str = str.replace(TestConstants.AZURE_ACCOUNT_NAME_PLACEHOLDER,
- TestConstants.AZURE_AZURITE_ACCOUNT_NAME_DEFAULT);
- str = str.replace(TestConstants.AZURE_ACCOUNT_KEY_PLACEHOLDER, TestConstants.AZURE_AZURITE_ACCOUNT_KEY_DEFAULT);
- str = str.replace(TestConstants.AZURE_SAS_TOKEN_PLACEHOLDER, TestConstants.sasToken);
+ str = str.replace(ACCOUNT_NAME_PLACEHOLDER, AZURITE_ACCOUNT_NAME_DEFAULT);
+ str = str.replace(ACCOUNT_KEY_PLACEHOLDER, AZURITE_ACCOUNT_KEY_DEFAULT);
+ str = str.replace(SAS_TOKEN_PLACEHOLDER, sasToken);
+ str = str.replace(MANAGED_IDENTITY_ID_PLACEHOLDER, MANAGED_IDENTITY_ID_DEFAULT);
+ str = str.replace(CLIENT_ID_PLACEHOLDER, CLIENT_ID_DEFAULT);
+ str = str.replace(CLIENT_SECRET_PLACEHOLDER, CLIENT_SECRET_DEFAULT);
+ str = str.replace(CLIENT_CERTIFICATE_PLACEHOLDER, CLIENT_CERTIFICATE_DEFAULT);
+ str = str.replace(CLIENT_CERTIFICATE_PASSWORD_PLACEHOLDER, CLIENT_CERTIFICATE_PASSWORD_DEFAULT);
+ str = str.replace(TENANT_ID_PLACEHOLDER, TENANT_ID_DEFAULT);
str = replaceExternalEndpoint(str);
return str;
}
protected String replaceExternalEndpoint(String str) {
- return str.replace(TestConstants.AZURE_BLOB_ENDPOINT_PLACEHOLDER, TestConstants.AZURE_BLOB_ENDPOINT_DEFAULT);
+ return str.replace(BLOB_ENDPOINT_PLACEHOLDER, BLOB_ENDPOINT_DEFAULT);
}
protected boolean noTemplateRequired(String str) {
@@ -2375,17 +2398,17 @@ public class TestExecutor {
protected String applyAzureSubstitution(String str, List<Placeholder> placeholders) {
boolean isReplaced = false;
- boolean hasBlobEndpoint = false;
+ boolean hasEndpoint = false;
for (Placeholder placeholder : placeholders) {
// Stop if all parameters are met
- if (hasBlobEndpoint) {
+ if (hasEndpoint) {
break;
- } else if (placeholder.getName().equals("blobEndpoint")) {
- hasBlobEndpoint = true;
+ } else if (placeholder.getName().equals("endpoint")) {
+ hasEndpoint = true;
isReplaced = true;
str = setAzureTemplate(str);
- str = str.replace(TestConstants.AZURE_BLOB_ENDPOINT_PLACEHOLDER, placeholder.getValue());
+ str = str.replace(BLOB_ENDPOINT_PLACEHOLDER, placeholder.getValue());
}
}
@@ -2398,11 +2421,11 @@ public class TestExecutor {
}
protected String setAzureTemplate(String str) {
- return str.replace("%template%", TestConstants.AZURE_TEMPLATE);
+ return str.replace("%template%", TEMPLATE);
}
protected String setAzureTemplateDefault(String str) {
- return str.replace("%template%", TestConstants.AZURE_TEMPLATE_DEFAULT);
+ return str.replace("%template%", TEMPLATE_DEFAULT);
}
protected void fail(boolean runDiagnostics, TestCaseContext testCaseCtx, CompilationUnit cUnit,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index 9e8614c..894b4bc 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -18,8 +18,10 @@
*/
package org.apache.asterix.test.external_dataset.microsoft;
-import static com.azure.storage.common.implementation.Constants.ConnectionStringConstants.EMULATOR_ACCOUNT_KEY;
-import static com.azure.storage.common.implementation.Constants.ConnectionStringConstants.EMULATOR_ACCOUNT_NAME;
+import static org.apache.asterix.test.common.TestConstants.Azure.AZURITE_ACCOUNT_KEY_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.AZURITE_ACCOUNT_NAME_DEFAULT;
+import static org.apache.asterix.test.common.TestConstants.Azure.BLOB_ENDPOINT_PLACEHOLDER;
+import static org.apache.asterix.test.common.TestConstants.Azure.sasToken;
import static org.apache.asterix.test.external_dataset.BinaryFileConverterUtil.BINARY_GEN_BASEDIR;
import static org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils.PARQUET_DEFINITION;
import static org.apache.hyracks.util.file.FileUtil.joinPath;
@@ -44,7 +46,6 @@ import java.util.Set;
import java.util.zip.GZIPOutputStream;
import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.test.common.TestConstants;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.external_dataset.ExternalDatasetTestUtils;
import org.apache.asterix.test.runtime.ExecutionTestUtil;
@@ -72,6 +73,7 @@ import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.models.PublicAccessType;
+import com.azure.storage.common.StorageSharedKeyCredential;
import com.azure.storage.common.sas.AccountSasPermission;
import com.azure.storage.common.sas.AccountSasResourceType;
import com.azure.storage.common.sas.AccountSasService;
@@ -120,8 +122,6 @@ public class AzureBlobStorageExternalDatasetTest {
private static final Set<String> fileNames = new HashSet<>();
// Create a BlobServiceClient object which will be used to create a container client
- public static final String CONNECTION_STRING = "AccountName=" + EMULATOR_ACCOUNT_NAME + ";" + "AccountKey="
- + EMULATOR_ACCOUNT_KEY + ";" + "BlobEndpoint=" + BLOB_SERVICE_ENDPOINT + "/" + EMULATOR_ACCOUNT_NAME + ";";
private static BlobServiceClient blobServiceClient;
private static BlobContainerClient playgroundContainer;
private static BlobContainerClient publicAccessContainer;
@@ -177,11 +177,14 @@ public class AzureBlobStorageExternalDatasetTest {
private static void createBlobServiceClient() {
LOGGER.info("Creating Azurite Blob Service client");
- blobServiceClient = new BlobServiceClientBuilder().connectionString(CONNECTION_STRING).buildClient();
+ BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
+ builder.credential(new StorageSharedKeyCredential(AZURITE_ACCOUNT_NAME_DEFAULT, AZURITE_ACCOUNT_KEY_DEFAULT));
+ builder.endpoint(BLOB_ENDPOINT_PLACEHOLDER);
+ blobServiceClient = builder.buildClient();
LOGGER.info("Azurite Blob Service client created successfully");
// Generate the SAS token for the SAS test cases
- TestConstants.sasToken = generateSasToken();
+ sasToken = generateSasToken();
// Create the container and upload some json files
PREPARE_PLAYGROUND_CONTAINER.run();
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
index 598831e..026696b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
@@ -27,7 +27,7 @@ create type test as open {
drop dataset test if exists;
CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
-("blobEndpoint"="%azureblob-endpoint%"),
+("endpoint"="%azureblob-endpoint%"),
("container"="public-access"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp
index 43db107..d8cbaa1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp
@@ -28,7 +28,7 @@ create type test as open {
// bad case: no auth method is provided
drop dataset test if exists;
CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
-("blobEndpoint"="%azureblob-endpoint%"),
+("endpoint"="%azureblob-endpoint%"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp
index b3c8bc4..f0225aa 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp
@@ -31,7 +31,7 @@ CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
("accountName"="%azureblob-accountname%"),
("%azureblob-credentialsname-1%"="%azureblob-credentialsvalue-1%"),
("%azureblob-credentialsname-2%"="%azureblob-credentialsvalue-2%"),
-("blobEndpoint"="%azureblob-endpoint%"),
+("endpoint"="%azureblob-endpoint%"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp
index afeaeae..01a2373 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp
@@ -29,7 +29,7 @@ drop dataset test if exists;
CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
("accountName"="%azureblob-accountname%"),
("%azureblob-credentialsname%"="%azureblob-credentialsvalue%"),
-("blobEndpoint"="%azureblob-endpoint%"),
+("endpoint"="%azureblob-endpoint%"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index d30d505..592f2ce 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -81,7 +81,7 @@
<test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
<compilation-unit name="anonymous-no-auth-no-endpoint">
<output-dir compare="Text">anonymous-no-auth-no-endpoint</output-dir>
- <expected-error>ASX1151: No authentication credentials provided, 'blobEndpoint' field is required for anonymous access</expected-error>
+ <expected-error>ASX1151: No authentication credentials provided, 'endpoint' field is required for anonymous access</expected-error>
</compilation-unit>
</test-case>
</test-group>
@@ -178,7 +178,7 @@
<test-case FilePath="external-dataset">
<compilation-unit name="common/invalid-endpoint">
<placeholder name="adapter" value="AZUREBLOB" />
- <placeholder name="blobEndpoint" value="http://^invalid-endpoint^" />
+ <placeholder name="endpoint" value="http://^invalid-endpoint^" />
<output-dir compare="Text">common/invalid-endpoint</output-dir>
<expected-error>External source error. java.net.URISyntaxException: Illegal character in authority at index 7: http://^invalid-endpoint^</expected-error>
</compilation-unit>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 391bb5d..9589212 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -349,15 +349,13 @@ public class ExternalDataConstants {
throw new AssertionError("do not instantiate");
}
- //ConnectionString prefixes
- public static final String ACCOUNT_KEY_PREFIX = "AccountKey=";
- public static final String SAS_KEY_PREFIX = "SharedAccessSignature=";
-
/*
* Asterix Configuration Keys
*/
- public static final String CONNECTION_STRING_FIELD_NAME = "connectionString";
public static final String MANAGED_IDENTITY_ID_FIELD_NAME = "managedIdentityId";
+ public static final String ACCOUNT_NAME_FIELD_NAME = "accountName";
+ public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey";
+ public static final String SHARED_ACCESS_SIGNATURE_FIELD_NAME = "sharedAccessSignature";
public static final String TENANT_ID_FIELD_NAME = "tenantId";
public static final String CLIENT_ID_FIELD_NAME = "clientId";
public static final String CLIENT_SECRET_FIELD_NAME = "clientSecret";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index ac02c92..fc77e6e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -23,6 +23,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.asterix.common.exceptions.ErrorCode.EXTERNAL_SOURCE_ERROR;
import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_NOT_ALLOWED_AT_SAME_TIME;
import static org.apache.asterix.common.exceptions.ErrorCode.PARAMETERS_REQUIRED;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_OR_PARAM_IF_PARAM_IS_PRESENT;
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.ACCESS_KEY_ID_FIELD_NAME;
@@ -37,18 +38,18 @@ import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOO
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_SESSION_TOKEN;
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.HADOOP_TEMP_ACCESS;
import static org.apache.asterix.external.util.ExternalDataConstants.AwsS3.SECRET_ACCESS_KEY_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_PREFIX;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CLIENT_CERTIFICATE_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CLIENT_ID_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CLIENT_SECRET_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ENDPOINT_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.HADOOP_AZURE_BLOB_PROTOCOL;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.HADOOP_AZURE_FS_ACCOUNT_KEY;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.HADOOP_AZURE_FS_SAS;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.MANAGED_IDENTITY_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SAS_KEY_PREFIX;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.TENANT_ID_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.GCS.JSON_CREDENTIALS_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ADAPTER_NAME_GCS;
@@ -83,7 +84,6 @@ import java.util.function.BiPredicate;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
-import java.util.stream.Stream;
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.CompilationException;
@@ -122,6 +122,7 @@ import org.apache.hyracks.dataflow.common.data.parsers.LongParserFactory;
import org.apache.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import org.apache.hyracks.util.StorageUtil;
+import com.azure.core.credential.AzureSasCredential;
import com.azure.identity.ClientCertificateCredentialBuilder;
import com.azure.identity.ClientSecretCredentialBuilder;
import com.azure.identity.ManagedIdentityCredentialBuilder;
@@ -130,6 +131,7 @@ import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.blob.models.BlobItem;
import com.azure.storage.blob.models.ListBlobsOptions;
+import com.azure.storage.common.StorageSharedKeyCredential;
import com.google.api.gax.paging.Page;
import com.google.auth.oauth2.ServiceAccountCredentials;
import com.google.cloud.storage.Blob;
@@ -1239,8 +1241,10 @@ public class ExternalDataUtils {
*/
public static BlobServiceClient buildAzureClient(Map<String, String> configuration)
throws CompilationException {
- String connectionString = configuration.get(CONNECTION_STRING_FIELD_NAME);
String managedIdentityId = configuration.get(MANAGED_IDENTITY_ID_FIELD_NAME);
+ String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
+ String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
+ String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
String tenantId = configuration.get(TENANT_ID_FIELD_NAME);
String clientId = configuration.get(CLIENT_ID_FIELD_NAME);
String clientSecret = configuration.get(CLIENT_SECRET_FIELD_NAME);
@@ -1251,32 +1255,63 @@ public class ExternalDataUtils {
// Client builder
BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
- // Connection string is used
- if (connectionString != null) {
- try {
- builder.connectionString(connectionString);
- } catch (Exception ex) {
- throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
+ // Endpoint is required
+ if (endpoint == null) {
+ throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME);
+ }
+ builder.endpoint(endpoint);
+
+ // Shared Key
+ if (accountName != null || accountKey != null) {
+ if (accountName == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_NAME_FIELD_NAME,
+ ACCOUNT_KEY_FIELD_NAME);
+ }
+
+ if (accountKey == null) {
+ throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCOUNT_KEY_FIELD_NAME,
+ ACCOUNT_NAME_FIELD_NAME);
+ }
+
+ Optional<String> provided = getFirstNotNull(configuration, SHARED_ACCESS_SIGNATURE_FIELD_NAME,
+ MANAGED_IDENTITY_ID_FIELD_NAME, CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ ACCOUNT_KEY_FIELD_NAME);
}
- } else if (isParquetFormat(configuration)) {
- //TODO(wail) support AAD for parquet
- throw new CompilationException(ErrorCode.UNSUPPORTED_AUTH_METHOD, "Azure Active Directory",
- ExternalDataConstants.FORMAT_PARQUET);
+ StorageSharedKeyCredential credential = new StorageSharedKeyCredential(accountName, accountKey);
+ builder.credential(credential);
}
- // TODO(htowaileb): Endpoint will be required for Data lake implementation
- if (endpoint != null) {
- builder.endpoint(endpoint);
+ // Shared access signature
+ if (sharedAccessSignature != null) {
+ Optional<String> provided = getFirstNotNull(configuration, MANAGED_IDENTITY_ID_FIELD_NAME,
+ CLIENT_ID_FIELD_NAME, CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME,
+ CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ }
+ AzureSasCredential credential = new AzureSasCredential(sharedAccessSignature);
+ builder.credential(credential);
}
- // Managed identity credentials
+ // Managed Identity auth
if (managedIdentityId != null) {
+ Optional<String> provided = getFirstNotNull(configuration, CLIENT_ID_FIELD_NAME,
+ CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME,
+ TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ MANAGED_IDENTITY_ID_FIELD_NAME);
+ }
builder.credential(new ManagedIdentityCredentialBuilder().clientId(managedIdentityId).build());
}
- // Active Directory authentication
+ // Client secret & certificate auth
if (clientId != null) {
- // Both (or neither) client secret and client certificate were provided, only one is allowed
+ // Both (or neither) client secret and client secret were provided, only one is allowed
if ((clientSecret == null) == (clientCertificate == null)) {
if (clientSecret != null) {
throw new CompilationException(PARAMETERS_NOT_ALLOWED_AT_SAME_TIME, CLIENT_SECRET_FIELD_NAME,
@@ -1293,10 +1328,10 @@ public class ExternalDataUtils {
CLIENT_ID_FIELD_NAME);
}
- // Client certificate is required if client certificate password is present
- if (clientCertificatePassword != null && clientCertificate == null) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, CLIENT_CERTIFICATE_FIELD_NAME,
- CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME);
+ // Client certificate password is not allowed if client secret is used
+ if (clientCertificatePassword != null && clientSecret != null) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT,
+ CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, CLIENT_SECRET_FIELD_NAME);
}
// Use AD authentication
@@ -1334,13 +1369,11 @@ public class ExternalDataUtils {
// If client id is not present, ensure client secret, certificate, tenant id and client certificate
// password are not present
if (clientId == null) {
- Optional<String> param = Stream
- .of(CLIENT_SECRET_FIELD_NAME, CLIENT_CERTIFICATE_FIELD_NAME, TENANT_ID_FIELD_NAME,
- CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME)
- .filter(field -> configuration.get(field) != null).findFirst();
- if (param.isPresent()) {
- throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, CLIENT_ID_FIELD_NAME,
- param.get());
+ Optional<String> provided = getFirstNotNull(configuration, CLIENT_SECRET_FIELD_NAME,
+ CLIENT_CERTIFICATE_FIELD_NAME, CLIENT_CERTIFICATE_PASSWORD_FIELD_NAME, TENANT_ID_FIELD_NAME);
+ if (provided.isPresent()) {
+ throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, provided.get(),
+ SHARED_ACCESS_SIGNATURE_FIELD_NAME);
}
}
@@ -1458,7 +1491,10 @@ public class ExternalDataUtils {
*/
public static void configureAzureHdfsJobConf(JobConf conf, Map<String, String> configuration, String endPoint) {
String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
- String connectionString = configuration.get(CONNECTION_STRING_FIELD_NAME);
+ String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
+ String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
+ String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
//Disable caching S3 FileSystem
HDFSUtils.disableHadoopFileSystemCache(conf, HADOOP_AZURE_BLOB_PROTOCOL);
@@ -1467,18 +1503,18 @@ public class ExternalDataUtils {
StringBuilder hadoopKey = new StringBuilder();
//Value for Hadoop configuration
String hadoopValue;
- if (connectionString != null) {
- if (connectionString.contains(ACCOUNT_KEY_PREFIX)) {
+ if (accountKey != null || sharedAccessSignature != null) {
+ if (accountKey != null) {
hadoopKey.append(HADOOP_AZURE_FS_ACCOUNT_KEY).append('.');
//Set only the AccountKey
- hadoopValue = extractKey(ACCOUNT_KEY_PREFIX, connectionString);
+ hadoopValue = accountKey;
} else {
//Use SAS for Hadoop FS as connectionString is provided
hadoopKey.append(HADOOP_AZURE_FS_SAS).append('.');
//Setting the container is required for SAS
hadoopKey.append(container).append('.');
//Set the connection string for SAS
- hadoopValue = extractKey(SAS_KEY_PREFIX, connectionString);
+ hadoopValue = sharedAccessSignature;
}
//Set the endPoint, which includes the AccountName
hadoopKey.append(endPoint);
@@ -1486,12 +1522,6 @@ public class ExternalDataUtils {
conf.set(hadoopKey.toString(), hadoopValue);
}
}
-
- private static String extractKey(String prefix, String connectionString) {
- int start = connectionString.indexOf(prefix) + prefix.length();
- int end = connectionString.indexOf(';', start);
- return connectionString.substring(start, end > 0 ? end : connectionString.length());
- }
}
public static class GCS {
@@ -1577,4 +1607,8 @@ public class ExternalDataUtils {
}
return maxArgSz;
}
+
+ private static Optional<String> getFirstNotNull(Map<String, String> configuration, String... parameters) {
+ return Arrays.stream(parameters).filter(field -> configuration.get(field) != null).findFirst();
+ }
}