You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2021/06/12 09:55:39 UTC
[hadoop] branch trunk updated: HADOOP-17643 WASB : Make metadata
checks case insensitive (#2972)
This is an automated email from the ASF dual-hosted git repository.
surendralilhore pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 2cf952b HADOOP-17643 WASB : Make metadata checks case insensitive (#2972)
2cf952b is described below
commit 2cf952baf4d18d24d4c035ad3bd1ddc12ca8ba18
Author: Anoop Sam John <an...@gmail.com>
AuthorDate: Sat Jun 12 15:25:03 2021 +0530
HADOOP-17643 WASB : Make metadata checks case insensitive (#2972)
---
.../fs/azure/AzureNativeFileSystemStore.java | 76 +++++++++++++++-------
1 file changed, 52 insertions(+), 24 deletions(-)
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index c613468..69faebb 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -40,6 +40,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Locale;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
@@ -180,6 +181,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
*/
public static final String KEY_USE_LOCAL_SAS_KEY_MODE = "fs.azure.local.sas.key.mode";
+ /**
+ * Config to control case sensitive metadata key checks/retrieval. If this
+ * is false, blob metadata keys will be treated case insensitive.
+ */
+ private static final String KEY_BLOB_METADATA_KEY_CASE_SENSITIVE = "fs.azure.blob.metadata.key.case.sensitive";
private static final String PERMISSION_METADATA_KEY = "hdi_permission";
private static final String OLD_PERMISSION_METADATA_KEY = "asv_permission";
private static final String IS_FOLDER_METADATA_KEY = "hdi_isfolder";
@@ -353,6 +359,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
private String delegationToken;
+ private boolean metadataKeyCaseSensitive;
+
/** The error message template when container is not accessible. */
public static final String NO_ACCESS_TO_CONTAINER_MSG = "No credentials found for "
+ "account %s in the configuration, and its container %s is not "
@@ -574,6 +582,12 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
LOG.warn("Unable to initialize HBase root as an atomic rename directory.");
}
LOG.debug("Atomic rename directories: {} ", setToString(atomicRenameDirs));
+ metadataKeyCaseSensitive = conf
+ .getBoolean(KEY_BLOB_METADATA_KEY_CASE_SENSITIVE, true);
+ if (!metadataKeyCaseSensitive) {
+ LOG.info("{} configured as false. Blob metadata will be treated case insensitive.",
+ KEY_BLOB_METADATA_KEY_CASE_SENSITIVE);
+ }
}
/**
@@ -1618,15 +1632,24 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
blob.setMetadata(metadata);
}
- private static String getMetadataAttribute(CloudBlobWrapper blob,
+ private String getMetadataAttribute(HashMap<String, String> metadata,
String... keyAlternatives) {
- HashMap<String, String> metadata = blob.getMetadata();
if (null == metadata) {
return null;
}
for (String key : keyAlternatives) {
- if (metadata.containsKey(key)) {
- return metadata.get(key);
+ if (metadataKeyCaseSensitive) {
+ if (metadata.containsKey(key)) {
+ return metadata.get(key);
+ }
+ } else {
+ // See HADOOP-17643 for details on why this case insensitive metadata
+ // checks been added
+ for (Entry<String, String> entry : metadata.entrySet()) {
+ if (key.equalsIgnoreCase(entry.getKey())) {
+ return entry.getValue();
+ }
+ }
}
}
return null;
@@ -1650,7 +1673,7 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
}
private PermissionStatus getPermissionStatus(CloudBlobWrapper blob) {
- String permissionMetadataValue = getMetadataAttribute(blob,
+ String permissionMetadataValue = getMetadataAttribute(blob.getMetadata(),
PERMISSION_METADATA_KEY, OLD_PERMISSION_METADATA_KEY);
if (permissionMetadataValue != null) {
return PermissionStatusJsonSerializer.fromJSONString(
@@ -1698,19 +1721,32 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
}
- private static String getLinkAttributeValue(CloudBlobWrapper blob)
+ private String getLinkAttributeValue(CloudBlobWrapper blob)
throws UnsupportedEncodingException {
- String encodedLinkTarget = getMetadataAttribute(blob,
+ String encodedLinkTarget = getMetadataAttribute(blob.getMetadata(),
LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY,
OLD_LINK_BACK_TO_UPLOAD_IN_PROGRESS_METADATA_KEY);
return decodeMetadataAttribute(encodedLinkTarget);
}
- private static boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
+ private boolean retrieveFolderAttribute(CloudBlobWrapper blob) {
HashMap<String, String> metadata = blob.getMetadata();
- return null != metadata
- && (metadata.containsKey(IS_FOLDER_METADATA_KEY) || metadata
- .containsKey(OLD_IS_FOLDER_METADATA_KEY));
+ if (null != metadata) {
+ if (metadataKeyCaseSensitive) {
+ return metadata.containsKey(IS_FOLDER_METADATA_KEY)
+ || metadata.containsKey(OLD_IS_FOLDER_METADATA_KEY);
+ } else {
+ // See HADOOP-17643 for details on why this case insensitive metadata
+ // checks been added
+ for (String key : metadata.keySet()) {
+ if (key.equalsIgnoreCase(IS_FOLDER_METADATA_KEY)
+ || key.equalsIgnoreCase(OLD_IS_FOLDER_METADATA_KEY)) {
+ return true;
+ }
+ }
+ }
+ }
+ return false;
}
private static void storeVersionAttribute(CloudBlobContainerWrapper container) {
@@ -1725,18 +1761,9 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
container.setMetadata(metadata);
}
- private static String retrieveVersionAttribute(
- CloudBlobContainerWrapper container) {
- HashMap<String, String> metadata = container.getMetadata();
- if (metadata == null) {
- return null;
- } else if (metadata.containsKey(VERSION_METADATA_KEY)) {
- return metadata.get(VERSION_METADATA_KEY);
- } else if (metadata.containsKey(OLD_VERSION_METADATA_KEY)) {
- return metadata.get(OLD_VERSION_METADATA_KEY);
- } else {
- return null;
- }
+ private String retrieveVersionAttribute(CloudBlobContainerWrapper container) {
+ return getMetadataAttribute(container.getMetadata(), VERSION_METADATA_KEY,
+ OLD_VERSION_METADATA_KEY);
}
@Override
@@ -2231,7 +2258,8 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
CloudBlobWrapper blob = getBlobReference(key);
blob.downloadAttributes(getInstrumentedContext());
- String value = getMetadataAttribute(blob, ensureValidAttributeName(attribute));
+ String value = getMetadataAttribute(blob.getMetadata(),
+ ensureValidAttributeName(attribute));
value = decodeMetadataAttribute(value);
return value == null ? null : value.getBytes(METADATA_ENCODING);
} catch (Exception e) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org