You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ht...@apache.org on 2021/07/07 00:23:51 UTC
[asterixdb] 01/03: [ASTERIXDB-2919] Azure Blob external datasets:
add support to anonymous authentication
This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a9806215a4defc7d34c00b8473afd61099df4cbd
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Thu Jul 1 00:55:25 2021 +0300
[ASTERIXDB-2919] Azure Blob external datasets: add support to anonymous authentication
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Added support to creating external datasets with no
credentials (anonymous).
Change-Id: I19a161a3613371572fa26c786bab690e332fdd08
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11983
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <ti...@apache.org>
---
.../test/external_dataset/ExternalDatasetTestUtils.java | 4 ++--
.../microsoft/AzureBlobStorageExternalDatasetTest.java | 7 +++++++
.../test.000.ddl.sqlpp | 3 +--
.../test.099.ddl.sqlpp | 0
.../test.000.ddl.sqlpp | 3 +--
.../test.001.query.sqlpp} | 3 ++-
.../test.099.ddl.sqlpp | 0
.../test.000.ddl.sqlpp | 0
.../test.001.query.sqlpp} | 3 ++-
.../test.099.ddl.sqlpp | 0
.../result.001.adm | 1 +
.../testsuite_external_dataset_azure_blob_storage.xml | 17 ++++++++++++++---
.../org/apache/asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
.../apache/asterix/external/util/ExternalDataUtils.java | 13 +++++++------
15 files changed, 39 insertions(+), 17 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 3c129b4..c064281 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.test.external_dataset;
-import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.*;
+import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER;
import java.io.BufferedWriter;
import java.io.FileWriter;
@@ -344,4 +344,4 @@ public class ExternalDatasetTestUtils {
mixedDataLoader.upload(MIXED_DEFINITION + "json/NO-EXTENSION/" + "goodbye-world-2019", "{\"id\":" + 5 + "}");
mixedDataLoader.upload(MIXED_DEFINITION + "json/NO-EXTENSION/" + "goodbye-world-2020", "{\"id\":" + 6 + "}");
}
-}
\ No newline at end of file
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index 8372d87..2d41686 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -66,11 +66,13 @@ import org.junit.runners.Parameterized.Parameters;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.PublicAccessType;
import com.azure.storage.common.sas.AccountSasPermission;
import com.azure.storage.common.sas.AccountSasResourceType;
import com.azure.storage.common.sas.AccountSasService;
import com.azure.storage.common.sas.AccountSasSignatureValues;
+// TODO(Hussain): Need to run the test manually to ensure new tests (anonymous access) are working fine
@Ignore
@RunWith(Parameterized.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@@ -100,6 +102,7 @@ public class AzureBlobStorageExternalDatasetTest {
private static final String PLAYGROUND_CONTAINER = "playground";
private static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data
private static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
+ private static final String PUBLIC_ACCESS_CONTAINER = "public-access-container"; // requires no authentication
private static final String JSON_DEFINITION = "json-data/reviews/";
private static final String CSV_DEFINITION = "csv-data/reviews/";
private static final String TSV_DEFINITION = "tsv-data/reviews/";
@@ -116,6 +119,7 @@ public class AzureBlobStorageExternalDatasetTest {
+ "BlobEndpoint=" + BLOB_SERVICE_ENDPOINT + "/devstoreaccount1;";
private static BlobServiceClient blobServiceClient;
private static BlobContainerClient playgroundContainer;
+ private static BlobContainerClient publicAccessContainer;
protected TestCaseContext tcCtx;
@@ -195,6 +199,8 @@ public class AzureBlobStorageExternalDatasetTest {
LOGGER.info("creating container " + PLAYGROUND_CONTAINER);
playgroundContainer = blobServiceClient.createBlobContainer(PLAYGROUND_CONTAINER);
+ publicAccessContainer = blobServiceClient.createBlobContainer(PUBLIC_ACCESS_CONTAINER);
+ publicAccessContainer.setAccessPolicy(PublicAccessType.CONTAINER, null);
LOGGER.info("container " + PLAYGROUND_CONTAINER + " created successfully");
LOGGER.info("Adding JSON files");
@@ -342,6 +348,7 @@ public class AzureBlobStorageExternalDatasetTest {
// Load the data
playgroundContainer.getBlobClient(basePath + finalFileName).uploadFromFile(filePath.toString());
+ publicAccessContainer.getBlobClient(basePath + finalFileName).uploadFromFile(filePath.toString());
if (copyToSubLevels) {
playgroundContainer.getBlobClient(basePath + "level1a/" + finalFileName)
.uploadFromFile(filePath.toString());
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
similarity index 93%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
index 43db107..d6bc862 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
@@ -25,10 +25,9 @@ drop type test if exists;
create type test as open {
};
-// bad case: no auth method is provided
+// bad case: no auth method and no endpoint provided
drop dataset test if exists;
CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
-("blobEndpoint"="%azureblob-endpoint%"),
("container"="playground"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
index 43db107..598831e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
@@ -25,11 +25,10 @@ drop type test if exists;
create type test as open {
};
-// bad case: no auth method is provided
drop dataset test if exists;
CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
("blobEndpoint"="%azureblob-endpoint%"),
-("container"="playground"),
+("container"="public-access"),
("definition"="json-data/reviews/single-line/json"),
("format"="json")
);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp
index 548e632..8ec9cc0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp
@@ -17,4 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
\ No newline at end of file
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.099.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp
index 548e632..8ec9cc0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp
@@ -17,4 +17,5 @@
* under the License.
*/
-drop dataverse test if exists;
\ No newline at end of file
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.099.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index 1e302da..27ba148 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -68,9 +68,20 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
- <compilation-unit name="invalid-no-auth">
- <output-dir compare="Text">invalid-no-auth</output-dir>
- <expected-error>ASX1134: No authentication parameters provided</expected-error>
+ <compilation-unit name="anonymous-no-auth-public-access-allowed">
+ <output-dir compare="Text">anonymous-no-auth-public-access-allowed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="anonymous-no-auth-public-access-not-allowed">
+ <output-dir compare="Text">anonymous-no-auth-public-access-not-allowed</output-dir>
+ <expected-error>Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="anonymous-no-auth-no-endpoint">
+ <output-dir compare="Text">anonymous-no-auth-no-endpoint</output-dir>
+ <expected-error>ASX1151: No authentication credentials provided, 'blobEndpoint' field is required for anonymous access</expected-error>
</compilation-unit>
</test-case>
</test-group>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 4078ea2..e80dbfb 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -235,6 +235,7 @@ public enum ErrorCode implements IError {
CANNOT_DROP_OBJECT_DEPENDENT_EXISTS(1148),
ILLEGAL_FUNCTION_RECURSION(1149),
ILLEGAL_FUNCTION_USE(1150),
+ NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS(1151),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 18898bb..cd4a289 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -237,6 +237,7 @@
1148 = Cannot drop %1$s %2$s being used by %3$s %4$s
1149 = Illegal function recursion
1150 = Illegal use of function %1$s
+1151 = No authentication credentials provided, '%1$s' field is required for anonymous access
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8e94263..7872cef 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -899,7 +899,7 @@ public class ExternalDataUtils {
String blobEndpoint = configuration.get(BLOB_ENDPOINT_FIELD_NAME);
String endpointSuffix = configuration.get(ENDPOINT_SUFFIX_FIELD_NAME);
- // Constructor the connection string
+ // Construct the connection string
// Connection string format: name1=value1;name2=value2;....
StringBuilder connectionStringBuilder = new StringBuilder();
BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
@@ -926,17 +926,12 @@ public class ExternalDataUtils {
if (authMethodFound) {
throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED);
}
- authMethodFound = true;
// account name + shared access token
connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName)
.append(";").append(CONNECTION_STRING_SHARED_ACCESS_SIGNATURE).append("=")
.append(sharedAccessSignature).append(";");
}
- if (!authMethodFound) {
- throw new CompilationException(ErrorCode.NO_AUTH_METHOD_PROVIDED);
- }
-
// Add blobEndpoint and endpointSuffix if present, adjust any '/' as needed
if (blobEndpoint != null) {
connectionStringBuilder.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint)
@@ -954,6 +949,12 @@ public class ExternalDataUtils {
}
}
+ // No credentials or endpoint provided
+ if (connectionStringBuilder.length() == 0) {
+ throw new CompilationException(ErrorCode.NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS,
+ BLOB_ENDPOINT_FIELD_NAME);
+ }
+
try {
return builder.connectionString(connectionStringBuilder.toString()).buildClient();
} catch (Exception ex) {