You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by kb...@apache.org on 2020/02/12 06:38:21 UTC
[atlas] branch branch-2.0 updated: ATLAS-3606 :- Missing
HIVE_PROCESS attribute values in atlas.
This is an automated email from the ASF dual-hosted git repository.
kbhatt pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 16765a7 ATLAS-3606 :- Missing HIVE_PROCESS attribute values in atlas.
16765a7 is described below
commit 16765a78b6ca05c911226c74ea66ced1b33ffec8
Author: nixonrodrigues <ni...@apache.org>
AuthorDate: Tue Feb 11 17:40:52 2020 +0530
ATLAS-3606 :- Missing HIVE_PROCESS attribute values in atlas.
(cherry picked from commit 98b54160c5d2fd308a2f2be9269695a675a287f8)
Signed-off-by: kevalbhatt <kb...@apache.org>
---
.../org/apache/atlas/hive/hook/AtlasHiveHookContext.java | 4 ++++
.../src/main/java/org/apache/atlas/hive/hook/HiveHook.java | 9 +++++++--
.../org/apache/atlas/hive/hook/events/BaseHiveEvent.java | 14 +++++++++++---
3 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index 2e7a3c6..78290c0 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -236,6 +236,10 @@ public class AtlasHiveHookContext {
}
}
+ public boolean isHiveProcessPopulateDeprecatedAttributes() {
+ return hook.isHiveProcessPopulateDeprecatedAttributes();
+ }
+
private void init() {
String operation = hiveOperation.getOperationName();
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index e886b9b..dd425cc 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -60,6 +60,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
public static final String HOOK_NAME_CACHE_DATABASE_COUNT = CONF_PREFIX + "name.cache.database.count";
public static final String HOOK_NAME_CACHE_TABLE_COUNT = CONF_PREFIX + "name.cache.table.count";
public static final String HOOK_NAME_CACHE_REBUID_INTERVAL_SEC = CONF_PREFIX + "name.cache.rebuild.interval.seconds";
+ public static final String HOOK_HIVE_PROCESS_POPULATE_DEPRECATED_ATTRIBUTES = CONF_PREFIX + "hive_process.populate.deprecated.attributes";
public static final String HOOK_SKIP_HIVE_COLUMN_LINEAGE_HIVE_20633 = CONF_PREFIX + "skip.hive_column_lineage.hive-20633";
public static final String HOOK_SKIP_HIVE_COLUMN_LINEAGE_HIVE_20633_INPUTS_THRESHOLD = CONF_PREFIX + "skip.hive_column_lineage.hive-20633.inputs.threshold";
public static final String HOOK_HIVE_TABLE_IGNORE_PATTERN = CONF_PREFIX + "hive_table.ignore.pattern";
@@ -83,7 +84,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private static final List ignoreDummyDatabaseName;
private static final List ignoreDummyTableName;
private static final String ignoreValuesTmpTableNamePrefix;
-
+ private static final boolean hiveProcessPopulateDeprecatedAttributes;
private static HiveHookObjectNamesCache knownObjects = null;
private static String hostName;
@@ -99,7 +100,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
nameCacheRebuildIntervalSeconds = atlasProperties.getInt(HOOK_NAME_CACHE_REBUID_INTERVAL_SEC, 60 * 60); // 60 minutes default
skipHiveColumnLineageHive20633 = atlasProperties.getBoolean(HOOK_SKIP_HIVE_COLUMN_LINEAGE_HIVE_20633, false);
skipHiveColumnLineageHive20633InputsThreshold = atlasProperties.getInt(HOOK_SKIP_HIVE_COLUMN_LINEAGE_HIVE_20633_INPUTS_THRESHOLD, 15); // skip if avg # of inputs is > 15
-
+ hiveProcessPopulateDeprecatedAttributes = atlasProperties.getBoolean(HOOK_HIVE_PROCESS_POPULATE_DEPRECATED_ATTRIBUTES, false);
String[] patternHiveTablesToIgnore = atlasProperties.getStringArray(HOOK_HIVE_TABLE_IGNORE_PATTERN);
String[] patternHiveTablesToPrune = atlasProperties.getStringArray(HOOK_HIVE_TABLE_PRUNE_PATTERN);
@@ -272,6 +273,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
return ignoreValuesTmpTableNamePrefix;
}
+ public boolean isHiveProcessPopulateDeprecatedAttributes() {
+ return hiveProcessPopulateDeprecatedAttributes;
+ }
+
public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) {
PreprocessAction ret = PreprocessAction.NONE;
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 91accfa..d5ecf12 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -672,11 +672,19 @@ public abstract class BaseHiveEvent {
// mandatory attributes for hive process entity type.
ret.setAttribute(ATTRIBUTE_START_TIME, System.currentTimeMillis());
ret.setAttribute(ATTRIBUTE_END_TIME, System.currentTimeMillis());
- ret.setAttribute(ATTRIBUTE_USER_NAME, EMPTY_ATTRIBUTE_VALUE);
- ret.setAttribute(ATTRIBUTE_QUERY_TEXT, EMPTY_ATTRIBUTE_VALUE);
- ret.setAttribute(ATTRIBUTE_QUERY_ID, EMPTY_ATTRIBUTE_VALUE);
+ if (context.isHiveProcessPopulateDeprecatedAttributes()) {
+ ret.setAttribute(ATTRIBUTE_USER_NAME, getUserName());
+ ret.setAttribute(ATTRIBUTE_QUERY_TEXT, queryStr);
+ ret.setAttribute(ATTRIBUTE_QUERY_ID, getQueryId());
+ } else {
+ ret.setAttribute(ATTRIBUTE_USER_NAME, EMPTY_ATTRIBUTE_VALUE);
+ ret.setAttribute(ATTRIBUTE_QUERY_TEXT, EMPTY_ATTRIBUTE_VALUE);
+ ret.setAttribute(ATTRIBUTE_QUERY_ID, EMPTY_ATTRIBUTE_VALUE);
+ }
ret.setAttribute(ATTRIBUTE_QUERY_PLAN, "Not Supported");
ret.setAttribute(ATTRIBUTE_RECENT_QUERIES, Collections.singletonList(queryStr));
+ ret.setAttribute(ATTRIBUTE_CLUSTER_NAME, getMetadataNamespace());
+
return ret;
}