You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/11/10 22:56:14 UTC
atlas git commit: ATLAS-2961: updated Hive hook to recognize
configuration 'hdfs_path.convert_to_lowercase'
Repository: atlas
Updated Branches:
refs/heads/branch-1.0 7b165cb91 -> fea8acbc6
ATLAS-2961: updated Hive hook to recognize configuration 'hdfs_path.convert_to_lowercase'
(cherry picked from commit a37aeda48aa4239cd110e665c837c730edb4b800)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/fea8acbc
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/fea8acbc
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/fea8acbc
Branch: refs/heads/branch-1.0
Commit: fea8acbc6790fa23fe5b3121b38a3c218c694c09
Parents: 7b165cb
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Nov 9 09:00:13 2018 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sat Nov 10 14:53:16 2018 -0800
----------------------------------------------------------------------
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 4 ++--
.../atlas/hive/hook/AtlasHiveHookContext.java | 4 ++++
.../org/apache/atlas/hive/hook/HiveHook.java | 7 ++++++
.../atlas/hive/hook/events/BaseHiveEvent.java | 24 ++++++++++++++++----
4 files changed, 33 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/fea8acbc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index dbb71ea..21ceda2 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -205,7 +205,7 @@ public class HiveMetaStoreBridge {
* @param hiveConf {@link HiveConf} for Hive component in the cluster
*/
public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf, AtlasClientV2 atlasClientV2) throws Exception {
- this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, true));
+ this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false));
}
/**
@@ -700,7 +700,7 @@ public class HiveMetaStoreBridge {
String nameServiceID = HdfsNameServiceResolver.getNameServiceIDForPath(pathUri);
Path path = new Path(pathUri);
- ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+ ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString());
ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, clusterName);
if (StringUtils.isNotEmpty(nameServiceID)) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/fea8acbc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index b467f4c..23cb853 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -85,6 +85,10 @@ public class AtlasHiveHookContext {
return hook.getClusterName();
}
+ public boolean isConvertHdfsPathToLowerCase() {
+ return hook.isConvertHdfsPathToLowerCase();
+ }
+
public boolean getSkipHiveColumnLineageHive20633() {
return hook.getSkipHiveColumnLineageHive20633();
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/fea8acbc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index a9570e1..0f48578 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -47,6 +47,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
public static final String CONF_PREFIX = "atlas.hook.hive.";
public static final String CONF_CLUSTER_NAME = "atlas.cluster.name";
+ public static final String HDFS_PATH_CONVERT_TO_LOWER_CASE = CONF_PREFIX + "hdfs_path.convert_to_lowercase";
public static final String HOOK_NAME_CACHE_ENABLED = CONF_PREFIX + "name.cache.enabled";
public static final String HOOK_NAME_CACHE_DATABASE_COUNT = CONF_PREFIX + "name.cache.database.count";
public static final String HOOK_NAME_CACHE_TABLE_COUNT = CONF_PREFIX + "name.cache.table.count";
@@ -59,6 +60,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private static final Map<String, HiveOperation> OPERATION_MAP = new HashMap<>();
private static final String clusterName;
+ private static final boolean convertHdfsPathToLowerCase;
private static final boolean nameCacheEnabled;
private static final int nameCacheDatabaseMaxCount;
private static final int nameCacheTableMaxCount;
@@ -75,6 +77,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
}
clusterName = atlasProperties.getString(CONF_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+ convertHdfsPathToLowerCase = atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false);
nameCacheEnabled = atlasProperties.getBoolean(HOOK_NAME_CACHE_ENABLED, true);
nameCacheDatabaseMaxCount = atlasProperties.getInt(HOOK_NAME_CACHE_DATABASE_COUNT, 10000);
nameCacheTableMaxCount = atlasProperties.getInt(HOOK_NAME_CACHE_TABLE_COUNT, 10000);
@@ -189,6 +192,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
return clusterName;
}
+ public boolean isConvertHdfsPathToLowerCase() {
+ return convertHdfsPathToLowerCase;
+ }
+
public boolean getSkipHiveColumnLineageHive20633() {
return skipHiveColumnLineageHive20633;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/fea8acbc/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 09f011c..e4537b4 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -145,6 +145,7 @@ public abstract class BaseHiveEvent {
public static final String HBASE_NAMESPACE_TABLE_DELIMITER = ":";
public static final String HBASE_PARAM_TABLE_NAME = "hbase.table.name";
public static final long MILLIS_CONVERT_FACTOR = 1000;
+ public static final String HDFS_PATH_PREFIX = "hdfs://";
public static final Map<Integer, String> OWNER_TYPE_TO_ENUM_VALUE = new HashMap<>();
@@ -505,7 +506,11 @@ public abstract class BaseHiveEvent {
protected AtlasEntity getPathEntity(Path path, AtlasEntityExtInfo extInfo) {
AtlasEntity ret;
- String strPath = path.toString().toLowerCase();
+ String strPath = path.toString();
+
+ if (strPath.startsWith(HDFS_PATH_PREFIX) && context.isConvertHdfsPathToLowerCase()) {
+ strPath = strPath.toLowerCase();
+ }
if (isS3Path(strPath)) {
String bucketName = path.toUri().getAuthority();
@@ -550,9 +555,15 @@ public abstract class BaseHiveEvent {
ret.setAttribute(ATTRIBUTE_NAMESERVICE_ID, nameServiceID);
}
+ String name = Path.getPathWithoutSchemeAndAuthority(path).toString();
+
+ if (strPath.startsWith(HDFS_PATH_PREFIX) && context.isConvertHdfsPathToLowerCase()) {
+ name = name.toLowerCase();
+ }
+
ret.setAttribute(ATTRIBUTE_PATH, attrPath);
ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, pathQualifiedName);
- ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+ ret.setAttribute(ATTRIBUTE_NAME, name);
ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, getClusterName());
context.putEntity(pathQualifiedName, ret);
@@ -688,7 +699,12 @@ public abstract class BaseHiveEvent {
}
protected String getQualifiedName(URI location) {
- String strPath = new Path(location).toString().toLowerCase();
+ String strPath = new Path(location).toString();
+
+ if (strPath.startsWith(HDFS_PATH_PREFIX) && context.isConvertHdfsPathToLowerCase()) {
+ strPath = strPath.toLowerCase();
+ }
+
String nameServiceID = HdfsNameServiceResolver.getNameServiceIDForPath(strPath);
String attrPath = StringUtils.isEmpty(nameServiceID) ? strPath : HdfsNameServiceResolver.getPathWithNameServiceID(strPath);
@@ -697,7 +713,7 @@ public abstract class BaseHiveEvent {
protected String getQualifiedName(String path) {
if (path.startsWith(HdfsNameServiceResolver.HDFS_SCHEME)) {
- return (path + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
+ return path + QNAME_SEP_CLUSTER_NAME + getClusterName();
}
return path.toLowerCase();