You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2025/09/05 10:07:01 UTC
Change in asterixdb[ionic]: [ASTERIXDB-XXXX][EXT]: Make path-access-style configurable
From Hussain Towaileb <hu...@gmail.com>:
Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20296 )
Change subject: [ASTERIXDB-XXXX][EXT]: Make path-access-style configurable
......................................................................
[ASTERIXDB-XXXX][EXT]: Make path-access-style configurable
Ext-ref: MB-68398
Change-Id: I068f9caa9d9ea800d62954a1863b91d2074c29c6
---
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.099.ddl.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.099.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.000.ddl.sqlpp
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
7 files changed, 159 insertions(+), 6 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/96/20296/1
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000..d6cc114
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-1"),
+ ("pathStyleAccess"="true"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="json-data/reviews/single-line/json"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/boolean-value/external_dataset.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.000.ddl.sqlpp
new file mode 100644
index 0000000..b77b8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+create external dataset test(test) using S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-1"),
+ ("pathStyleAccess"="non-boolean-value"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="json-data/reviews/single-line/json"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/path-style-access/non-boolean-value/external_dataset.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index a1ca803..7eb3f02 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -1178,6 +1178,17 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/s3">
+ <compilation-unit name="path-style-access/boolean-value">
+ <output-dir compare="Text">path-style-access/boolean-value</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/s3">
+ <compilation-unit name="path-style-access/non-boolean-value">
+ <output-dir compare="Text">path-style-access/non-boolean-value</output-dir>
+ <expected-error>ASX1173: Invalid value for parameter 'pathStyleAccess', allowed value(s): true, false</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/s3">
<compilation-unit name="cross-region/boolean-value">
<output-dir compare="Text">cross-region/boolean-value</output-dir>
</compilation-unit>
@@ -1185,7 +1196,7 @@
<test-case FilePath="external-dataset/s3">
<compilation-unit name="cross-region/non-boolean-value">
<output-dir compare="Text">cross-region/non-boolean-value</output-dir>
- <expected-error>c</expected-error>
+ <expected-error>ASX1173: Invalid value for parameter 'crossRegion', allowed value(s): true, false</expected-error>
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/s3">
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
index 035415d..bcc3605 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
@@ -53,6 +53,7 @@
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SIMPLE;
import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_TEMPORARY;
import static org.apache.asterix.external.util.aws.s3.S3Constants.INSTANCE_PROFILE_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.PATH_STYLE_ACCESS_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.REGION_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.ROLE_ARN_FIELD_NAME;
import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
@@ -137,11 +138,13 @@
String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
Region region = validateAndGetRegion(regionId);
+ boolean pathStyleAccess = validateAndGetPathStyleAccess(configuration.get(PATH_STYLE_ACCESS_FIELD_NAME));
boolean crossRegion = validateAndGetCrossRegion(configuration.get(CROSS_REGION_FIELD_NAME));
AwsCredentialsProvider credentialsProvider = buildCredentialsProvider(appCtx, configuration);
S3ClientBuilder builder = S3Client.builder();
builder.region(region);
+ builder.forcePathStyle(pathStyleAccess);
builder.crossRegionAccessEnabled(crossRegion);
builder.credentialsProvider(credentialsProvider);
@@ -217,6 +220,17 @@
}
}
+ public static boolean validateAndGetPathStyleAccess(String pathStyleAccess) throws CompilationException {
+ if (pathStyleAccess == null) {
+ return false;
+ }
+ if (!"true".equalsIgnoreCase(pathStyleAccess) && !"false".equalsIgnoreCase(pathStyleAccess)) {
+ throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, PATH_STYLE_ACCESS_FIELD_NAME,
+ "true, false");
+ }
+ return Boolean.parseBoolean(pathStyleAccess);
+ }
+
public static boolean validateAndGetCrossRegion(String crossRegion) throws CompilationException {
if (crossRegion == null) {
return false;
@@ -385,11 +399,12 @@
jobConf.set(HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT);
}
- /*
- * This is to allow S3 definition to have path-style form. Should always be true to match the current
- * way we access files in S3
- */
- jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
+ boolean pathStyleAccess = validateAndGetPathStyleAccess(configuration.get(PATH_STYLE_ACCESS_FIELD_NAME));
+ if (pathStyleAccess) {
+ jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
+ } else {
+ jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.FALSE);
+ }
/*
* Set the size of S3 connection pool to be the number of partitions
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index a67cd64..438cedc 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -25,6 +25,7 @@
// Authentication specific parameters
public static final String REGION_FIELD_NAME = "region";
+ public static final String PATH_STYLE_ACCESS_FIELD_NAME = "pathStyleAccess";
public static final String CROSS_REGION_FIELD_NAME = "crossRegion";
public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20296
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: ionic
Gerrit-Change-Id: I068f9caa9d9ea800d62954a1863b91d2074c29c6
Gerrit-Change-Number: 20296
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange