You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/11/10 22:52:30 UTC
atlas git commit: ATLAS-2961: updated Hive hook to recognize
configuration 'hdfs_path.convert_to_lowercase' [Forced Update!]
Repository: atlas
Updated Branches:
refs/heads/branch-0.8 4a72f2062 -> a37aeda48 (forced update)
ATLAS-2961: updated Hive hook to recognize configuration 'hdfs_path.convert_to_lowercase'
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a37aeda4
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a37aeda4
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a37aeda4
Branch: refs/heads/branch-0.8
Commit: a37aeda48aa4239cd110e665c837c730edb4b800
Parents: 8d7e94d
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Fri Nov 9 09:00:13 2018 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sat Nov 10 14:52:12 2018 -0800
----------------------------------------------------------------------
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 4 ++--
.../atlas/hive/hook/AtlasHiveHookContext.java | 4 ++++
.../org/apache/atlas/hive/hook/HiveHook.java | 7 +++++++
.../atlas/hive/hook/events/BaseHiveEvent.java | 22 +++++++++++++++-----
4 files changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index d6a1b37..2f6d1fa 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -204,7 +204,7 @@ public class HiveMetaStoreBridge {
* @param hiveConf {@link HiveConf} for Hive component in the cluster
*/
public HiveMetaStoreBridge(Configuration atlasProperties, HiveConf hiveConf, AtlasClientV2 atlasClientV2) throws Exception {
- this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, true));
+ this(atlasProperties.getString(HIVE_CLUSTER_NAME, DEFAULT_CLUSTER_NAME), Hive.get(hiveConf), atlasClientV2, atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false));
}
/**
@@ -690,7 +690,7 @@ public class HiveMetaStoreBridge {
AtlasEntity ret = new AtlasEntity(HDFS_PATH);
Path path = new Path(pathUri);
- ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+ ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString());
ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, clusterName);
ret.setAttribute(ATTRIBUTE_PATH, pathUri);
http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index b467f4c..23cb853 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -85,6 +85,10 @@ public class AtlasHiveHookContext {
return hook.getClusterName();
}
+ public boolean isConvertHdfsPathToLowerCase() {
+ return hook.isConvertHdfsPathToLowerCase();
+ }
+
public boolean getSkipHiveColumnLineageHive20633() {
return hook.getSkipHiveColumnLineageHive20633();
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 501eb63..7b60553 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -47,6 +47,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
public static final String CONF_PREFIX = "atlas.hook.hive.";
public static final String CONF_CLUSTER_NAME = "atlas.cluster.name";
+ public static final String HDFS_PATH_CONVERT_TO_LOWER_CASE = CONF_PREFIX + "hdfs_path.convert_to_lowercase";
public static final String HOOK_NAME_CACHE_ENABLED = CONF_PREFIX + "name.cache.enabled";
public static final String HOOK_NAME_CACHE_DATABASE_COUNT = CONF_PREFIX + "name.cache.database.count";
public static final String HOOK_NAME_CACHE_TABLE_COUNT = CONF_PREFIX + "name.cache.table.count";
@@ -59,6 +60,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private static final Map<String, HiveOperation> OPERATION_MAP = new HashMap<>();
private static final String clusterName;
+ private static final boolean convertHdfsPathToLowerCase;
private static final boolean nameCacheEnabled;
private static final int nameCacheDatabaseMaxCount;
private static final int nameCacheTableMaxCount;
@@ -75,6 +77,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
}
clusterName = atlasProperties.getString(CONF_CLUSTER_NAME, DEFAULT_CLUSTER_NAME);
+ convertHdfsPathToLowerCase = atlasProperties.getBoolean(HDFS_PATH_CONVERT_TO_LOWER_CASE, false);
nameCacheEnabled = atlasProperties.getBoolean(HOOK_NAME_CACHE_ENABLED, true);
nameCacheDatabaseMaxCount = atlasProperties.getInt(HOOK_NAME_CACHE_DATABASE_COUNT, 10000);
nameCacheTableMaxCount = atlasProperties.getInt(HOOK_NAME_CACHE_TABLE_COUNT, 10000);
@@ -189,6 +192,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
return clusterName;
}
+ public boolean isConvertHdfsPathToLowerCase() {
+ return convertHdfsPathToLowerCase;
+ }
+
public boolean getSkipHiveColumnLineageHive20633() {
return skipHiveColumnLineageHive20633;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/a37aeda4/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 9109a0f..2d90a15 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -124,6 +124,7 @@ public abstract class BaseHiveEvent {
public static final String ATTRIBUTE_EXPRESSION = "expression";
public static final String ATTRIBUTE_ALIASES = "aliases";
+ public static final String HDFS_PATH_PREFIX = "hdfs://";
public static final long MILLIS_CONVERT_FACTOR = 1000;
@@ -481,7 +482,14 @@ public abstract class BaseHiveEvent {
}
protected AtlasEntity getHDFSPathEntity(Path path) {
- String strPath = path.toString().toLowerCase();
+ String strPath = path.toString();
+ String name = Path.getPathWithoutSchemeAndAuthority(path).toString();
+
+ if (strPath.startsWith(HDFS_PATH_PREFIX) && context.isConvertHdfsPathToLowerCase()) {
+ strPath = strPath.toLowerCase();
+ name = name.toLowerCase();
+ }
+
String pathQualifiedName = getQualifiedName(strPath);
AtlasEntity ret = context.getEntity(pathQualifiedName);
@@ -490,7 +498,7 @@ public abstract class BaseHiveEvent {
ret.setAttribute(ATTRIBUTE_PATH, strPath);
ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, pathQualifiedName);
- ret.setAttribute(ATTRIBUTE_NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
+ ret.setAttribute(ATTRIBUTE_NAME, name);
ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, getClusterName());
context.putEntity(pathQualifiedName, ret);
@@ -625,14 +633,18 @@ public abstract class BaseHiveEvent {
}
protected String getQualifiedName(URI location) {
- String strPath = new Path(location).toString().toLowerCase();
+ String strPath = new Path(location).toString();
+
+ if (strPath.startsWith(HDFS_PATH_PREFIX) && context.isConvertHdfsPathToLowerCase()) {
+ strPath = strPath.toLowerCase();
+ }
return getQualifiedName(strPath);
}
protected String getQualifiedName(String path) {
- if (path.startsWith("hdfs://")) {
- return (path + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
+ if (path.startsWith(HDFS_PATH_PREFIX)) {
+ return path + QNAME_SEP_CLUSTER_NAME + getClusterName();
}
return path.toLowerCase();