You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by br...@apache.org on 2024/11/19 15:38:53 UTC
(iceberg) branch 1.7.1-11504 created (now 325efa66cd)
This is an automated email from the ASF dual-hosted git repository.
bryanck pushed a change to branch 1.7.1-11504
in repository https://gitbox.apache.org/repos/asf/iceberg.git
at 325efa66cd 1.7.1 Support WASB scheme in ADLSFileIO
This branch includes the following new commits:
new 325efa66cd 1.7.1 Support WASB scheme in ADLSFileIO
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iceberg) 01/01: 1.7.1 Support WASB scheme in ADLSFileIO
Posted by br...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bryanck pushed a commit to branch 1.7.1-11504
in repository https://gitbox.apache.org/repos/asf/iceberg.git
commit 325efa66cd60c62da73b3a3a6c5c89e63d60b47f
Author: Bryan Keller <bk...@netflix.com>
AuthorDate: Tue Nov 19 07:38:36 2024 -0800
1.7.1 Support WASB scheme in ADLSFileIO
---
.../org/apache/iceberg/azure/AzureProperties.java | 13 +++++++++++-
.../apache/iceberg/azure/adlsv2/ADLSLocation.java | 23 ++++++++++++++--------
.../apache/iceberg/azure/AzurePropertiesTest.java | 10 ++++++----
.../iceberg/azure/adlsv2/ADLSLocationTest.java | 19 ++++++++++++++----
.../org/apache/iceberg/io/ResolvingFileIO.java | 4 +++-
5 files changed, 51 insertions(+), 18 deletions(-)
diff --git a/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java b/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java
index 2d363cbc52..a7f9885a47 100644
--- a/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java
+++ b/azure/src/main/java/org/apache/iceberg/azure/AzureProperties.java
@@ -77,6 +77,17 @@ public class AzureProperties implements Serializable {
return Optional.ofNullable(adlsWriteBlockSize);
}
+ /**
+ * Applies configuration to the {@link DataLakeFileSystemClientBuilder} to provide the endpoint
+ * and credentials required to create an instance of the client.
+ *
+ * <p>The default endpoint is constructed in the form {@code
+ * https://{account}.dfs.core.windows.net} and default credentials are provided via the {@link
+ * com.azure.identity.DefaultAzureCredential}.
+ *
+ * @param account the service account name
+ * @param builder the builder instance
+ */
public void applyClientConfiguration(String account, DataLakeFileSystemClientBuilder builder) {
String sasToken = adlsSasTokens.get(account);
if (sasToken != null && !sasToken.isEmpty()) {
@@ -93,7 +104,7 @@ public class AzureProperties implements Serializable {
if (connectionString != null && !connectionString.isEmpty()) {
builder.endpoint(connectionString);
} else {
- builder.endpoint("https://" + account);
+ builder.endpoint("https://" + account + ".dfs.core.windows.net");
}
}
}
diff --git a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSLocation.java b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSLocation.java
index e73093512b..d9b2112565 100644
--- a/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSLocation.java
+++ b/azure/src/main/java/org/apache/iceberg/azure/adlsv2/ADLSLocation.java
@@ -29,13 +29,20 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
*
* <p>Locations follow the conventions used by Hadoop's Azure support, i.e.
*
- * <pre>{@code abfs[s]://[<container>@]<storage account host>/<file path>}</pre>
+ * <pre>{@code abfs[s]://[<container>@]<storageAccount>.dfs.core.windows.net/<path>}</pre>
+ *
+ * or
+ *
+ * <pre>{@code wasb[s]://<container>@<storageAccount>.blob.core.windows.net/<path>}</pre>
+ *
+ * For compatibility, locations using the wasb scheme are also accepted but will use the Azure Data
+ * Lake Storage Gen2 REST APIs instead of the Blob Storage REST APIs.
*
* <p>See <a href="https://hadoop.apache.org/docs/stable/hadoop-azure/abfs.html">Hadoop Azure
* Support</a>
*/
class ADLSLocation {
- private static final Pattern URI_PATTERN = Pattern.compile("^abfss?://([^/?#]+)(.*)?$");
+ private static final Pattern URI_PATTERN = Pattern.compile("^(abfss?|wasbs?)://([^/?#]+)(.*)?$");
private final String storageAccount;
private final String container;
@@ -53,19 +60,19 @@ class ADLSLocation {
ValidationException.check(matcher.matches(), "Invalid ADLS URI: %s", location);
- String authority = matcher.group(1);
+ String authority = matcher.group(2);
String[] parts = authority.split("@", -1);
if (parts.length > 1) {
this.container = parts[0];
- this.storageAccount = parts[1];
+ String host = parts[1];
+ this.storageAccount = host.split("\\.", -1)[0];
} else {
this.container = null;
- this.storageAccount = authority;
+ this.storageAccount = authority.split("\\.", -1)[0];
}
- String uriPath = matcher.group(2);
- uriPath = uriPath == null ? "" : uriPath.startsWith("/") ? uriPath.substring(1) : uriPath;
- this.path = uriPath.split("\\?", -1)[0].split("#", -1)[0];
+ String uriPath = matcher.group(3);
+ this.path = uriPath == null ? "" : uriPath.startsWith("/") ? uriPath.substring(1) : uriPath;
}
/** Returns Azure storage account. */
diff --git a/azure/src/test/java/org/apache/iceberg/azure/AzurePropertiesTest.java b/azure/src/test/java/org/apache/iceberg/azure/AzurePropertiesTest.java
index 6b8287c44e..4f032d7ab1 100644
--- a/azure/src/test/java/org/apache/iceberg/azure/AzurePropertiesTest.java
+++ b/azure/src/test/java/org/apache/iceberg/azure/AzurePropertiesTest.java
@@ -97,11 +97,13 @@ public class AzurePropertiesTest {
@Test
public void testWithConnectionString() {
AzureProperties props =
- new AzureProperties(ImmutableMap.of("adls.connection-string.account1", "http://endpoint"));
+ new AzureProperties(
+ ImmutableMap.of(
+ "adls.connection-string.account1", "https://account1.dfs.core.usgovcloudapi.net"));
DataLakeFileSystemClientBuilder clientBuilder = mock(DataLakeFileSystemClientBuilder.class);
props.applyClientConfiguration("account1", clientBuilder);
- verify(clientBuilder).endpoint("http://endpoint");
+ verify(clientBuilder).endpoint("https://account1.dfs.core.usgovcloudapi.net");
}
@Test
@@ -111,7 +113,7 @@ public class AzurePropertiesTest {
DataLakeFileSystemClientBuilder clientBuilder = mock(DataLakeFileSystemClientBuilder.class);
props.applyClientConfiguration("account1", clientBuilder);
- verify(clientBuilder).endpoint("https://account1");
+ verify(clientBuilder).endpoint("https://account1.dfs.core.windows.net");
}
@Test
@@ -120,7 +122,7 @@ public class AzurePropertiesTest {
DataLakeFileSystemClientBuilder clientBuilder = mock(DataLakeFileSystemClientBuilder.class);
props.applyClientConfiguration("account", clientBuilder);
- verify(clientBuilder).endpoint("https://account");
+ verify(clientBuilder).endpoint("https://account.dfs.core.windows.net");
}
@Test
diff --git a/azure/src/test/java/org/apache/iceberg/azure/adlsv2/ADLSLocationTest.java b/azure/src/test/java/org/apache/iceberg/azure/adlsv2/ADLSLocationTest.java
index 867b54b4c7..e07fea7439 100644
--- a/azure/src/test/java/org/apache/iceberg/azure/adlsv2/ADLSLocationTest.java
+++ b/azure/src/test/java/org/apache/iceberg/azure/adlsv2/ADLSLocationTest.java
@@ -33,7 +33,18 @@ public class ADLSLocationTest {
String p1 = scheme + "://container@account.dfs.core.windows.net/path/to/file";
ADLSLocation location = new ADLSLocation(p1);
- assertThat(location.storageAccount()).isEqualTo("account.dfs.core.windows.net");
+ assertThat(location.storageAccount()).isEqualTo("account");
+ assertThat(location.container().get()).isEqualTo("container");
+ assertThat(location.path()).isEqualTo("path/to/file");
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"wasb", "wasbs"})
+ public void testWasbLocatonParsing(String scheme) {
+ String p1 = scheme + "://container@account.blob.core.windows.net/path/to/file";
+ ADLSLocation location = new ADLSLocation(p1);
+
+ assertThat(location.storageAccount()).isEqualTo("account");
assertThat(location.container().get()).isEqualTo("container");
assertThat(location.path()).isEqualTo("path/to/file");
}
@@ -43,7 +54,7 @@ public class ADLSLocationTest {
String p1 = "abfs://container@account.dfs.core.windows.net/path%20to%20file";
ADLSLocation location = new ADLSLocation(p1);
- assertThat(location.storageAccount()).isEqualTo("account.dfs.core.windows.net");
+ assertThat(location.storageAccount()).isEqualTo("account");
assertThat(location.container().get()).isEqualTo("container");
assertThat(location.path()).isEqualTo("path%20to%20file");
}
@@ -67,7 +78,7 @@ public class ADLSLocationTest {
String p1 = "abfs://account.dfs.core.windows.net/path/to/file";
ADLSLocation location = new ADLSLocation(p1);
- assertThat(location.storageAccount()).isEqualTo("account.dfs.core.windows.net");
+ assertThat(location.storageAccount()).isEqualTo("account");
assertThat(location.container().isPresent()).isFalse();
assertThat(location.path()).isEqualTo("path/to/file");
}
@@ -77,7 +88,7 @@ public class ADLSLocationTest {
String p1 = "abfs://container@account.dfs.core.windows.net";
ADLSLocation location = new ADLSLocation(p1);
- assertThat(location.storageAccount()).isEqualTo("account.dfs.core.windows.net");
+ assertThat(location.storageAccount()).isEqualTo("account");
assertThat(location.container().get()).isEqualTo("container");
assertThat(location.path()).isEqualTo("");
}
diff --git a/core/src/main/java/org/apache/iceberg/io/ResolvingFileIO.java b/core/src/main/java/org/apache/iceberg/io/ResolvingFileIO.java
index a858045aab..a8adf979f8 100644
--- a/core/src/main/java/org/apache/iceberg/io/ResolvingFileIO.java
+++ b/core/src/main/java/org/apache/iceberg/io/ResolvingFileIO.java
@@ -62,7 +62,9 @@ public class ResolvingFileIO implements HadoopConfigurable, DelegateFileIO {
"s3n", S3_FILE_IO_IMPL,
"gs", GCS_FILE_IO_IMPL,
"abfs", ADLS_FILE_IO_IMPL,
- "abfss", ADLS_FILE_IO_IMPL);
+ "abfss", ADLS_FILE_IO_IMPL,
+ "wasb", ADLS_FILE_IO_IMPL,
+ "wasbs", ADLS_FILE_IO_IMPL);
private final Map<String, DelegateFileIO> ioInstances = Maps.newConcurrentMap();
private final AtomicBoolean isClosed = new AtomicBoolean(false);