You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by dl...@apache.org on 2021/01/27 01:07:47 UTC
[asterixdb] 01/11: [ASTERIXDB-2816][EXT] Support Azure SAS
authentication + connection string
This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 2eb7cebaddcbdcadeb535e16c17a4f2cbd041d60
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Wed Jan 13 20:57:26 2021 +0300
[ASTERIXDB-2816][EXT] Support Azure SAS authentication + connection string
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add support to passing shared access signature
and connection string as authentication methods.
Change-Id: I18d578465dbb03d129b02d2c3076db3d3f1df4aa
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9204
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
asterixdb/NOTICE | 2 +-
asterixdb/asterix-app/pom.xml | 4 ++
.../apache/asterix/test/common/TestConstants.java | 31 ++++++--
.../apache/asterix/test/common/TestExecutor.java | 28 ++++++--
.../AzureBlobStorageExternalDatasetTest.java | 18 +++++
.../invalid-auth-methods/test.000.ddl.sqlpp | 38 ++++++++++
.../invalid-auth-methods/test.099.ddl.sqlpp | 20 ++++++
.../invalid-no-auth/test.000.ddl.sqlpp | 35 +++++++++
.../invalid-no-auth/test.099.ddl.sqlpp | 20 ++++++
.../valid-auth-methods/test.000.ddl.sqlpp | 36 ++++++++++
.../valid-auth-methods/test.001.query.sqlpp | 21 ++++++
.../valid-auth-methods/test.099.ddl.sqlpp | 20 ++++++
.../auth-methods/valid-auth-methods/result.001.adm | 1 +
...stsuite_external_dataset_azure_blob_storage.xml | 56 +++++++++++++++
.../asterix/common/exceptions/ErrorCode.java | 2 +
.../src/main/resources/asx_errormsg/en.properties | 2 +
.../external/util/ExternalDataConstants.java | 6 +-
.../asterix/external/util/ExternalDataUtils.java | 84 ++++++++++++++++++----
asterixdb/pom.xml | 47 ++++++++++++
hyracks-fullstack/NOTICE | 2 +-
20 files changed, 445 insertions(+), 28 deletions(-)
diff --git a/asterixdb/NOTICE b/asterixdb/NOTICE
index b4729a8..4aabe27 100644
--- a/asterixdb/NOTICE
+++ b/asterixdb/NOTICE
@@ -1,5 +1,5 @@
Apache AsterixDB
-Copyright 2015-2020 The Apache Software Foundation
+Copyright 2015-2021 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 7f8085d..e309132 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -861,6 +861,10 @@
<artifactId>azure-storage-blob</artifactId>
</dependency>
<dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-common</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
index ef2ec6c..9188033 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
@@ -36,14 +36,37 @@ public class TestConstants {
+ "\"),\n" + "(\"serviceEndpoint\"=\"" + S3_SERVICE_ENDPOINT_DEFAULT + "\")";
// Azure blob storage constants and place holders
- public static final String AZURE_ACCOUNT_NAME_PLACEHOLDER = "%accountName%";
+ // account name
+ public static final String AZURE_ACCOUNT_NAME_PLACEHOLDER = "%azureblob-accountname%";
public static final String AZURE_AZURITE_ACCOUNT_NAME_DEFAULT = "devstoreaccount1";
- public static final String AZURE_ACCOUNT_KEY_PLACEHOLDER = "%accountKey%";
+
+ // account key
+ public static final String AZURE_ACCOUNT_KEY_PLACEHOLDER = "%azureblob-accountkey%";
public static final String AZURE_AZURITE_ACCOUNT_KEY_DEFAULT =
- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
- public static final String AZURE_BLOB_ENDPOINT_PLACEHOLDER = "%blobEndpoint%";
+ "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+
+ // SAS token: this is generated and assigned at runtime at the start of the test
+ public static final String AZURE_SAS_TOKEN_PLACEHOLDER = "%azureblob-sas%";
+ public static String sasToken = "";
+
+ // blob endpoint
+ public static final String AZURE_BLOB_ENDPOINT_PLACEHOLDER = "%azureblob-blobendpoint%";
public static final String AZURE_BLOB_ENDPOINT_DEFAULT =
"http://localhost:20000/" + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT;
+
+ // connection string with account name & account key
+ public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER =
+ "%azureblob-connectionstringaccountkey%";
+ public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY = "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER
+ + ";AccountKey=" + AZURE_ACCOUNT_KEY_PLACEHOLDER + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER;
+
+ // connection string with account name & sas token
+ public static final String AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER = "%azureblob-connectionstringsas%";
+ public static final String AZURE_CONNECTION_STRING_SAS_TOKEN =
+ "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER + ";SharedAccessSignature=" + AZURE_SAS_TOKEN_PLACEHOLDER
+ + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER;
+
+ // azure template and default template
public static final String AZURE_TEMPLATE = "(\"accountName\"=\"" + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT + "\"),\n"
+ "(\"accountKey\"=\"" + AZURE_AZURITE_ACCOUNT_KEY_DEFAULT + "\"),\n" + "(\"blobEndpoint\"=\""
+ AZURE_BLOB_ENDPOINT_PLACEHOLDER + "\")";
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 00d1034..e202ddf 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -2068,28 +2068,46 @@ public class TestExecutor {
}
protected String applyExternalDatasetSubstitution(String str, List<Placeholder> placeholders) {
+ // This replaces the full template of parameters depending on the adapter type
for (Placeholder placeholder : placeholders) {
+ // For adapter placeholder, it means we have a template to replace
if (placeholder.getName().equals("adapter")) {
str = str.replace("%adapter%", placeholder.getValue());
// Early terminate if there are no template place holders to replace
if (noTemplateRequired(str)) {
- return str;
+ continue;
}
if (placeholder.getValue().equalsIgnoreCase("S3")) {
- return applyS3Substitution(str, placeholders);
+ str = applyS3Substitution(str, placeholders);
} else if (placeholder.getValue().equalsIgnoreCase("AzureBlob")) {
- return applyAzureSubstitution(str, placeholders);
- } else {
- return str;
+ str = applyAzureSubstitution(str, placeholders);
}
+ } else {
+ // Any other place holders, just replace with the value
+ str = str.replace("%" + placeholder.getName() + "%", placeholder.getValue());
}
}
+ // This replaces specific external dataset placeholders
+ str = str.replace(TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER,
+ TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY);
+ str = str.replace(TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER,
+ TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN);
+ str = str.replace(TestConstants.AZURE_ACCOUNT_NAME_PLACEHOLDER,
+ TestConstants.AZURE_AZURITE_ACCOUNT_NAME_DEFAULT);
+ str = str.replace(TestConstants.AZURE_ACCOUNT_KEY_PLACEHOLDER, TestConstants.AZURE_AZURITE_ACCOUNT_KEY_DEFAULT);
+ str = str.replace(TestConstants.AZURE_SAS_TOKEN_PLACEHOLDER, TestConstants.sasToken);
+ str = replaceExternalEndpoint(str);
+
return str;
}
+ protected String replaceExternalEndpoint(String str) {
+ return str.replace(TestConstants.AZURE_BLOB_ENDPOINT_PLACEHOLDER, TestConstants.AZURE_BLOB_ENDPOINT_DEFAULT);
+ }
+
protected boolean noTemplateRequired(String str) {
return !str.contains("%template%");
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index da9f912..27a46c1 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -29,6 +29,8 @@ import java.net.InetSocketAddress;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
@@ -39,6 +41,7 @@ import java.util.Set;
import java.util.zip.GZIPOutputStream;
import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.test.common.TestConstants;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.runtime.ExecutionTestUtil;
import org.apache.asterix.test.runtime.LangExecutionUtil;
@@ -64,6 +67,10 @@ import org.junit.runners.Parameterized.Parameters;
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.sas.AccountSasPermission;
+import com.azure.storage.common.sas.AccountSasResourceType;
+import com.azure.storage.common.sas.AccountSasService;
+import com.azure.storage.common.sas.AccountSasSignatureValues;
@Ignore
@RunWith(Parameterized.class)
@@ -164,12 +171,23 @@ public class AzureBlobStorageExternalDatasetTest {
blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionString).buildClient();
LOGGER.info("Azurite Blob Service client created successfully");
+ // Generate the SAS token for the SAS test cases
+ TestConstants.sasToken = generateSasToken();
+
// Create the container and upload some json files
PREPARE_PLAYGROUND_CONTAINER.run();
PREPARE_FIXED_DATA_CONTAINER.run();
PREPARE_MIXED_DATA_CONTAINER.run();
}
+ private static String generateSasToken() {
+ OffsetDateTime expiry = OffsetDateTime.now().plus(1, ChronoUnit.YEARS);
+ AccountSasService service = AccountSasService.parse("b");
+ AccountSasPermission permission = AccountSasPermission.parse("acdlpruw");
+ AccountSasResourceType type = AccountSasResourceType.parse("cos");
+ return blobServiceClient.generateAccountSas(new AccountSasSignatureValues(expiry, permission, service, type));
+ }
+
/**
* Creates a container and fills it with some files for testing purpose.
*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp
new file mode 100644
index 0000000..c4ae026
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+// bad case: more than one authentication method is provided at once
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("accountName"="%azureblob-accountname%"),
+("%azureblob-credentialsname-1%"="%azureblob-credentialsvalue-1%"),
+("%azureblob-credentialsname-2%"="%azureblob-credentialsvalue-2%"),
+("blobEndpoint"="%azureblob-blobendpoint%"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
new file mode 100644
index 0000000..29fd14b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+// bad case: no auth method is provided
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("blobEndpoint"="%azureblob-blobendpoint%"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp
new file mode 100644
index 0000000..2a7eadd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("accountName"="%azureblob-accountname%"),
+("%azureblob-credentialsname%"="%azureblob-credentialsvalue%"),
+("blobEndpoint"="%azureblob-blobendpoint%"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp
new file mode 100644
index 0000000..8ec9cc0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index a715e7e..df60e60 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -18,6 +18,62 @@
! under the License.
!-->
<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="authentication">
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="accountKey" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-accountkey%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="sharedAccessSignature" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-sas%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-connectionstringaccountkey%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-connectionstringsas%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="invalid-auth-methods">
+ <placeholder name="azureblob-credentialsname-1" value="accountKey" />
+ <placeholder name="azureblob-credentialsvalue-1" value="%azureblob-accountkey%" />
+ <placeholder name="azureblob-credentialsname-2" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue-2" value="%azureblob-connectionstringaccountkey%" />
+ <output-dir compare="Text">invalid-auth-methods</output-dir>
+ <expected-error>ASX1133: Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="invalid-auth-methods">
+ <placeholder name="azureblob-credentialsname-1" value="sharedAccessSignature" />
+ <placeholder name="azureblob-credentialsvalue-1" value="%azureblob-sas%" />
+ <placeholder name="azureblob-credentialsname-2" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue-2" value="%azureblob-connectionstringaccountkey%" />
+ <output-dir compare="Text">invalid-auth-methods</output-dir>
+ <expected-error>ASX1133: Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="invalid-no-auth">
+ <output-dir compare="Text">invalid-no-auth</output-dir>
+ <expected-error>ASX1134: No authentication parameters provided</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="external-dataset">
<test-case FilePath="external-dataset">
<compilation-unit name="common/json/json">
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 6dbb46b..d662c8d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -222,6 +222,8 @@ public class ErrorCode {
public static final int ILLEGAL_RIGHT_OUTER_JOIN = 1130;
public static final int SYNONYM_EXISTS = 1131;
public static final int INVALID_HINT = 1132;
+ public static final int ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED = 1133;
+ public static final int NO_AUTH_METHOD_PROVIDED = 1134;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
public static final int UTIL_DATAFLOW_UTILS_TUPLE_TOO_LARGE = 3002;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index db8a70d..3eed1d8 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -219,6 +219,8 @@
1130 = Illegal use of RIGHT OUTER JOIN
1131 = A synonym with this name %1$s already exists
1132 = Invalid specification for hint %1$s. %2$s
+1133 = Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature
+1134 = No authentication parameters provided
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 326f53d..6be694b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -304,15 +304,17 @@ public class ExternalDataConstants {
throw new AssertionError("do not instantiate");
}
+ public static final String CONTAINER_NAME_FIELD_NAME = "container";
+ public static final String DEFINITION_FIELD_NAME = "definition";
+ public static final String CONNECTION_STRING_FIELD_NAME = "connectionString";
public static final String ACCOUNT_NAME_FIELD_NAME = "accountName";
public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey";
public static final String SHARED_ACCESS_SIGNATURE_FIELD_NAME = "sharedAccessSignature";
- public static final String CONTAINER_NAME_FIELD_NAME = "container";
- public static final String DEFINITION_FIELD_NAME = "definition";
public static final String BLOB_ENDPOINT_FIELD_NAME = "blobEndpoint";
public static final String ENDPOINT_SUFFIX_FIELD_NAME = "endpointSuffix";
// Connection string requires PascalCase (MyFieldFormat)
+ public static final String CONNECTION_STRING_CONNECTION_STRING = "ConnectionString";
public static final String CONNECTION_STRING_ACCOUNT_NAME = "AccountName";
public static final String CONNECTION_STRING_ACCOUNT_KEY = "AccountKey";
public static final String CONNECTION_STRING_SHARED_ACCESS_SIGNATURE = "SharedAccessSignature";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 44ae25f..537933e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,10 +18,17 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_KEY;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_BLOB_ENDPOINT;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ENDPOINT_SUFFIX;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_SHARED_ACCESS_SIGNATURE;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE;
@@ -819,25 +826,70 @@ public class ExternalDataUtils {
public static BlobServiceClient buildAzureClient(Map<String, String> configuration)
throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
- String accountName = configuration.get(ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME);
- String accountKey = configuration.get(ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME);
- String blobEndpoint = configuration.get(ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME);
- String endpointSuffix = configuration.get(ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME);
-
- // format: name1=value1;name2=value2;....
- // TODO(Hussain): This will be different when SAS (Shared Access Signature) is introduced
- StringBuilder connectionString = new StringBuilder();
- connectionString.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName).append(";");
- connectionString.append(CONNECTION_STRING_ACCOUNT_KEY).append("=").append(accountKey).append(";");
- connectionString.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint).append(";");
-
- if (endpointSuffix != null) {
- connectionString.append(CONNECTION_STRING_ENDPOINT_SUFFIX).append("=").append(endpointSuffix)
+ String connectionString = configuration.get(CONNECTION_STRING_FIELD_NAME);
+ String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
+ String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
+ String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ String blobEndpoint = configuration.get(BLOB_ENDPOINT_FIELD_NAME);
+ String endpointSuffix = configuration.get(ENDPOINT_SUFFIX_FIELD_NAME);
+
+ // Constructor the connection string
+ // Connection string format: name1=value1;name2=value2;....
+ StringBuilder connectionStringBuilder = new StringBuilder();
+ BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
+
+ boolean authMethodFound = false;
+
+ if (connectionString != null) {
+ // connection string
+ authMethodFound = true;
+ connectionStringBuilder.append(connectionString).append(";");
+ }
+
+ if (accountName != null && accountKey != null) {
+ if (authMethodFound) {
+ throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED);
+ }
+ authMethodFound = true;
+ // account name + account key
+ connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName)
+ .append(";").append(CONNECTION_STRING_ACCOUNT_KEY).append("=").append(accountKey).append(";");
+ }
+
+ if (accountName != null && sharedAccessSignature != null) {
+ if (authMethodFound) {
+ throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED);
+ }
+ authMethodFound = true;
+ // account name + shared access token
+ connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName)
+ .append(";").append(CONNECTION_STRING_SHARED_ACCESS_SIGNATURE).append("=")
+ .append(sharedAccessSignature).append(";");
+ }
+
+ if (!authMethodFound) {
+ throw new CompilationException(ErrorCode.NO_AUTH_METHOD_PROVIDED);
+ }
+
+ // Add blobEndpoint and endpointSuffix if present, adjust any '/' as needed
+ if (blobEndpoint != null) {
+ connectionStringBuilder.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint)
.append(";");
+ if (endpointSuffix != null) {
+ String endpointSuffixUpdated;
+ if (blobEndpoint.endsWith("/")) {
+ endpointSuffixUpdated =
+ endpointSuffix.startsWith("/") ? endpointSuffix.substring(1) : endpointSuffix;
+ } else {
+ endpointSuffixUpdated = endpointSuffix.startsWith("/") ? endpointSuffix : "/" + endpointSuffix;
+ }
+ connectionStringBuilder.append(CONNECTION_STRING_ENDPOINT_SUFFIX).append("=")
+ .append(endpointSuffixUpdated).append(";");
+ }
}
try {
- return new BlobServiceClientBuilder().connectionString(connectionString.toString()).buildClient();
+ return builder.connectionString(connectionStringBuilder.toString()).buildClient();
} catch (Exception ex) {
throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
@@ -876,6 +928,8 @@ public class ExternalDataUtils {
WarningUtil.forAsterix(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
collector.warn(warning);
}
+ } catch (CompilationException ex) {
+ throw ex;
} catch (Exception ex) {
throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index a6d4e7f..f8992d5 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1542,6 +1542,53 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-common</artifactId>
+ <version>${azurejavasdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-unix-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- Azure Blob Storage end -->
<dependency>
<groupId>org.mindrot</groupId>
diff --git a/hyracks-fullstack/NOTICE b/hyracks-fullstack/NOTICE
index 95fe98a..57c5843 100644
--- a/hyracks-fullstack/NOTICE
+++ b/hyracks-fullstack/NOTICE
@@ -1,5 +1,5 @@
Apache Hyracks and Algebricks
-Copyright 2015-2020 The Apache Software Foundation
+Copyright 2015-2021 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).