You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2025/11/20 14:20:00 UTC

[S] Change in asterixdb[phoenix]: [NO ISSUE] Fix parquet azure with dynamic prefix

From Preetham Poluparthi <pr...@apache.org>:

Preetham Poluparthi has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20593?usp=email )


Change subject: [NO ISSUE] Fix parquet azure with dynamic prefix
......................................................................

[NO ISSUE] Fix parquet azure with dynamic prefix

Ext-ref: MB-69466

Change-Id: I81c33dae177fb59256eeb82e04aef4db5533ee37
---
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
4 files changed, 42 insertions(+), 7 deletions(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/93/20593/1

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 2846cf1..0eb935e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -153,7 +153,9 @@
     public static final String KEY_ADAPTER_NAME_HTTP = "http_adapter";
     public static final String KEY_ADAPTER_NAME_AWS_S3 = "S3";
     public static final String KEY_ADAPTER_NAME_AZURE_BLOB = "AZUREBLOB";
+    public static final String KEY_ADAPTER_NAME_AZURE_BLOB_ALIAS = "AZURE_BLOB";
     public static final String KEY_ADAPTER_NAME_AZURE_DATA_LAKE = "AZUREDATALAKE";
+    public static final String KEY_ADAPTER_NAME_AZURE_DATA_LAKE_ALIAS = "AZURE_DATALAKE";
     public static final String KEY_ADAPTER_NAME_GCS = "GCS";
     public static final String KEY_ADAPTER_NAME_HDFS = "HDFS";
 
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index de31531..83c3a8b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -87,6 +87,8 @@
 import org.apache.asterix.external.util.aws.s3.S3Constants;
 import org.apache.asterix.external.util.aws.s3.S3Utils;
 import org.apache.asterix.external.util.azure.AzureConstants;
+import org.apache.asterix.external.util.azure.blob.BlobUtils;
+import org.apache.asterix.external.util.azure.datalake.DatalakeUtils;
 import org.apache.asterix.external.util.google.GCSConstants;
 import org.apache.asterix.external.util.google.GCSUtils;
 import org.apache.asterix.om.types.ARecordType;
@@ -1137,26 +1139,32 @@
         return configuration.getOrDefault(DEFINITION_FIELD_NAME, configuration.get(KEY_PATH));
     }
 
-    public static String getProtocolContainerPair(Map<String, String> configurations) {
+    public static String getProtocolContainerPair(Map<String, String> configurations) throws CompilationException {
         String container = configurations.getOrDefault(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME, "");
         String type = configurations.getOrDefault(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE, "");
         String protocol;
         switch (type) {
             case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
                 protocol = S3Constants.HADOOP_S3_PROTOCOL;
-                break;
+                return protocol + "://" + container + "/";
             case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB:
-            case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_DATA_LAKE:
+            case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB_ALIAS:
                 protocol = AzureConstants.HADOOP_AZURE_PROTOCOL;
-                break;
+                String blobEndpoint = BlobUtils.getEndpointFromClient(configurations);
+                return protocol + "://" + container + "@" + blobEndpoint + "/";
+            case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_DATA_LAKE:
+            case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_DATA_LAKE_ALIAS:
+                protocol = AzureConstants.HADOOP_AZURE_PROTOCOL;
+                String dataLakeEndpoint = DatalakeUtils.getEndpointFromClient(configurations);
+                return protocol + "://" + container + "@" + dataLakeEndpoint + "/";
             case ExternalDataConstants.KEY_ADAPTER_NAME_GCS:
                 protocol = GCSConstants.HADOOP_GCS_PROTOCOL;
-                break;
+                return protocol + "://" + container + "/";
             case ExternalDataConstants.KEY_ADAPTER_NAME_LOCALFS:
                 String path = getDefinitionOrPath(configurations);
                 String[] nodePathPair = path.trim().split("://");
                 protocol = nodePathPair[0];
-                break;
+                return protocol + "://" + container + "/";
             case ExternalDataConstants.KEY_ADAPTER_NAME_HDFS:
                 // Remove trailing slashes as prefixes/paths in hdfs start with a slash (absolute paths)
                 return configurations.get(ExternalDataConstants.KEY_HDFS_URL).replaceAll("/+$", "");
@@ -1164,7 +1172,6 @@
                 return "";
         }
 
-        return protocol + "://" + container + "/";
     }
 
     public static void validateType(Map<String, String> properties, ARecordType itemType) throws CompilationException {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
index 4860188..0c33f08 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob/BlobUtils.java
@@ -52,6 +52,7 @@
 import org.apache.asterix.external.util.ExternalDataPrefix;
 import org.apache.asterix.external.util.ExternalDataUtils;
 import org.apache.asterix.external.util.azure.AzureConstants;
+import org.apache.asterix.external.util.azure.AzureUtils;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -344,4 +345,16 @@
             throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
         }
     }
+
+    public static String getEndpointFromClient(Map<String, String> configuration) throws CompilationException {
+        String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
+        if (endpoint == null) {
+            throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME);
+        }
+
+        BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
+        builder.endpoint(endpoint);
+        return AzureUtils.extractEndPoint(builder.buildClient().getAccountUrl());
+    }
+
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
index 475ede3..212a677 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/datalake/DatalakeUtils.java
@@ -54,6 +54,7 @@
 import org.apache.asterix.external.util.ExternalDataPrefix;
 import org.apache.asterix.external.util.ExternalDataUtils;
 import org.apache.asterix.external.util.azure.AzureConstants;
+import org.apache.asterix.external.util.azure.AzureUtils;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.IWarningCollector;
 import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -315,4 +316,16 @@
             throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
         }
     }
+
+    public static String getEndpointFromClient(Map<String, String> configuration) throws CompilationException {
+        String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
+        if (endpoint == null) {
+            throw new CompilationException(PARAMETERS_REQUIRED, ENDPOINT_FIELD_NAME);
+        }
+
+        DataLakeServiceClientBuilder builder = new DataLakeServiceClientBuilder();
+        builder.endpoint(endpoint);
+        return AzureUtils.extractEndPoint(builder.buildClient().getAccountUrl());
+    }
+
 }

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20593?usp=email
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings?usp=email

Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: phoenix
Gerrit-Change-Id: I81c33dae177fb59256eeb82e04aef4db5533ee37
Gerrit-Change-Number: 20593
Gerrit-PatchSet: 1
Gerrit-Owner: Preetham Poluparthi <pr...@apache.org>