You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/05/03 04:12:19 UTC
[atlas] branch branch-1.0 updated: ATLAS-3033: Skip hive temp table
while getting Table object details from hiveContext.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new 594c998 ATLAS-3033: Skip hive temp table while getting Table object details from hiveContext.
594c998 is described below
commit 594c998037d55eb04b38c4c727d1f3419dfd4ec8
Author: nixonrodrigues <ni...@apache.org>
AuthorDate: Thu May 2 20:37:08 2019 -0700
ATLAS-3033: Skip hive temp table while getting Table object details from hiveContext.
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
(cherry picked from commit b5472ce313dcf24b0bffbe5201f0c95eec025848)
(cherry picked from commit 0323ce3825c63b06d7b0032edd10b73b84078a4a)
(cherry picked from commit 3e65639e2bf9155090a87bf05f16ef040ded14d1)
---
.../atlas/hive/hook/AtlasHiveHookContext.java | 13 +++++++++++
.../java/org/apache/atlas/hive/hook/HiveHook.java | 26 ++++++++++++++++++++++
.../atlas/hive/hook/events/BaseHiveEvent.java | 20 +++++++++++++----
3 files changed, 55 insertions(+), 4 deletions(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index 44c6437..58a73c3 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
@@ -102,6 +103,18 @@ public class AtlasHiveHookContext {
return hook.getPreprocessActionForHiveTable(qualifiedName);
}
+ public List getIgnoreDummyDatabaseName() {
+ return hook.getIgnoreDummyDatabaseName();
+ }
+
+ public List getIgnoreDummyTableName() {
+ return hook.getIgnoreDummyTableName();
+ }
+
+ public String getIgnoreValuesTmpTableNamePrefix() {
+ return hook.getIgnoreValuesTmpTableNamePrefix();
+ }
+
public String getQualifiedName(Database db) {
return (db.getName() + QNAME_SEP_CLUSTER_NAME).toLowerCase() + getClusterName();
}
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index ee02285..3456970 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -26,6 +26,7 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.shims.Utils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -81,6 +82,9 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private static final List<Pattern> hiveTablesToIgnore = new ArrayList<>();
private static final List<Pattern> hiveTablesToPrune = new ArrayList<>();
private static final Map<String, PreprocessAction> hiveTablesCache;
+ private static final List ignoreDummyDatabaseName;
+ private static final List ignoreDummyTableName;
+ private static final String ignoreValuesTmpTableNamePrefix;
private static HiveHookObjectNamesCache knownObjects = null;
@@ -134,6 +138,16 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
}
knownObjects = nameCacheEnabled ? new HiveHookObjectNamesCache(nameCacheDatabaseMaxCount, nameCacheTableMaxCount, nameCacheRebuildIntervalSeconds) : null;
+
+ List<String> defaultDummyDatabase = new ArrayList<>();
+ List<String> defaultDummyTable = new ArrayList<>();
+
+ defaultDummyDatabase.add(SemanticAnalyzer.DUMMY_DATABASE);
+ defaultDummyTable.add(SemanticAnalyzer.DUMMY_TABLE);
+
+ ignoreDummyDatabaseName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name", defaultDummyDatabase);
+ ignoreDummyTableName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name", defaultDummyTable);
+ ignoreValuesTmpTableNamePrefix = atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix", "Values__Tmp__Table__");
}
@@ -252,6 +266,18 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
return skipHiveColumnLineageHive20633InputsThreshold;
}
+ public List getIgnoreDummyDatabaseName() {
+ return ignoreDummyDatabaseName;
+ }
+
+ public List getIgnoreDummyTableName() {
+ return ignoreDummyTableName;
+ }
+
+ public String getIgnoreValuesTmpTableNamePrefix() {
+ return ignoreValuesTmpTableNamePrefix;
+ }
+
public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) {
PreprocessAction ret = PreprocessAction.NONE;
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 31346d0..b6754f7 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -238,17 +238,29 @@ public abstract class BaseHiveEvent {
switch (entity.getType()) {
case DATABASE: {
- Database db = getHive().getDatabase(entity.getDatabase().getName());
+ if (!context.getIgnoreDummyDatabaseName().contains(entity.getDatabase().getName())) {
+ Database db = getHive().getDatabase(entity.getDatabase().getName());
- ret = toDbEntity(db);
+ ret = toDbEntity(db);
+ }
}
break;
case TABLE:
case PARTITION: {
- Table table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName());
+ String dbName = entity.getTable().getDbName();
+ String tableName = entity.getTable().getTableName();
+ boolean skipTable = StringUtils.isNotEmpty(context.getIgnoreValuesTmpTableNamePrefix()) && tableName.toLowerCase().startsWith(context.getIgnoreValuesTmpTableNamePrefix());
+
+ if (!skipTable) {
+ skipTable = context.getIgnoreDummyTableName().contains(tableName) && context.getIgnoreDummyDatabaseName().contains(dbName);
+ }
- ret = toTableEntity(table, entityExtInfo);
+ if (!skipTable) {
+ Table table = getHive().getTable(dbName, tableName);
+
+ ret = toTableEntity(table, entityExtInfo);
+ }
}
break;