You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2021/06/24 19:02:23 UTC
[atlas] branch branch-2.0 updated: ATLAS-4339: Atlas should support
skip temporary tables using config property
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 756873e ATLAS-4339: Atlas should support skip temporary tables using config property
756873e is described below
commit 756873eabdd5fa8dbc47682c045b0dff381d4e52
Author: Radhika Kundam <rk...@cloudera.com>
AuthorDate: Wed Jun 23 14:18:28 2021 -0700
ATLAS-4339: Atlas should support skip temporary tables using config property
Signed-off-by: Sarath Subramanian <sa...@apache.org>
(cherry picked from commit 77b03aa1407ec6d78cc0fcaf9a06654191d42d73)
---
.../org/apache/atlas/hive/hook/AtlasHiveHookContext.java | 16 +++++++++++-----
.../main/java/org/apache/atlas/hive/hook/HiveHook.java | 15 +++++++++++----
.../apache/atlas/hive/hook/HiveMetastoreHookImpl.java | 4 ++--
.../org/apache/atlas/hive/hook/events/AlterTable.java | 2 +-
.../org/apache/atlas/hive/hook/events/BaseHiveEvent.java | 4 +++-
.../apache/atlas/hive/hook/events/CreateHiveProcess.java | 5 +----
.../org/apache/atlas/hive/hook/events/CreateTable.java | 5 +----
7 files changed, 30 insertions(+), 21 deletions(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index ab8e183..14cc2f2 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -58,19 +58,20 @@ public class AtlasHiveHookContext {
private boolean isSkippedInputEntity;
private boolean isSkippedOutputEntity;
+ private boolean skipTempTables;
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation, HookContext hiveContext,
- HiveHookObjectNamesCache knownObjects) throws Exception {
- this(hook, hiveOperation, hiveContext, knownObjects, null, null);
+ HiveHookObjectNamesCache knownObjects, boolean skipTempTables) throws Exception {
+ this(hook, hiveOperation, hiveContext, knownObjects, null, null, skipTempTables);
}
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation, HiveHookObjectNamesCache knownObjects,
- HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent) throws Exception {
- this(hook, hiveOperation, null, knownObjects, metastoreHook, listenerEvent);
+ HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent, boolean skipTempTables) throws Exception {
+ this(hook, hiveOperation, null, knownObjects, metastoreHook, listenerEvent, skipTempTables);
}
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation, HookContext hiveContext, HiveHookObjectNamesCache knownObjects,
- HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent) throws Exception {
+ HiveMetastoreHook metastoreHook, ListenerEvent listenerEvent, boolean skipTempTables) throws Exception {
this.hook = hook;
this.hiveOperation = hiveOperation;
this.hiveContext = hiveContext;
@@ -79,6 +80,7 @@ public class AtlasHiveHookContext {
this.metastoreHook = metastoreHook;
this.metastoreEvent = listenerEvent;
this.metastoreHandler = (listenerEvent != null) ? metastoreEvent.getIHMSHandler() : null;
+ this.skipTempTables = skipTempTables;
init();
}
@@ -131,6 +133,10 @@ public class AtlasHiveHookContext {
}
}
+ public boolean isSkipTempTables() {
+ return skipTempTables;
+ }
+
public LineageInfo getLineageInfo() {
return hiveContext != null ? hiveContext.getLinfo() : null;
}
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index bf500ca..2db970a 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -70,7 +70,8 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
public static final String HOOK_HIVE_TABLE_PRUNE_PATTERN = CONF_PREFIX + "hive_table.prune.pattern";
public static final String HOOK_HIVE_TABLE_CACHE_SIZE = CONF_PREFIX + "hive_table.cache.size";
public static final String HOOK_HIVE_IGNORE_DDL_OPERATIONS = CONF_PREFIX + "hs2.ignore.ddl.operations";
- public static final String HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX + "hs2.filter.entity.additional.types.to.retain";
+ public static final String HOOK_HIVE_FILTER_ENTITY_ADDITIONAL_TYPES_TO_RETAIN = CONF_PREFIX + "hs2.filter.entity.additional.types.to.retain";
+ public static final String HOOK_HIVE_SKIP_TEMP_TABLES = CONF_PREFIX + "skip.temp.tables";
public static final String DEFAULT_HOST_NAME = "localhost";
private static final Map<String, HiveOperation> OPERATION_MAP = new HashMap<>();
@@ -93,6 +94,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private static final boolean hiveProcessPopulateDeprecatedAttributes;
private static HiveHookObjectNamesCache knownObjects = null;
private static String hostName;
+ private static boolean skipTempTables = true;
static {
for (HiveOperation hiveOperation : HiveOperation.values()) {
@@ -154,6 +156,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
ignoreDummyDatabaseName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.database.name", defaultDummyDatabase);
ignoreDummyTableName = atlasProperties.getList("atlas.hook.hive.ignore.dummy.table.name", defaultDummyTable);
ignoreValuesTmpTableNamePrefix = atlasProperties.getString("atlas.hook.hive.ignore.values.tmp.table.name.prefix", "Values__Tmp__Table__");
+ skipTempTables = atlasProperties.getBoolean(HOOK_HIVE_SKIP_TEMP_TABLES, true);
try {
hostName = InetAddress.getLocalHost().getHostName();
@@ -177,7 +180,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
try {
HiveOperation oper = OPERATION_MAP.get(hookContext.getOperationName());
- AtlasHiveHookContext context = new AtlasHiveHookContext(this, oper, hookContext, getKnownObjects());
+ AtlasHiveHookContext context = new AtlasHiveHookContext(this, oper, hookContext, getKnownObjects(), isSkipTempTables());
BaseHiveEvent event = null;
switch (oper) {
@@ -196,7 +199,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
break;
case CREATETABLE:
- event = new CreateTable(context, true);
+ event = new CreateTable(context);
break;
case DROPTABLE:
@@ -213,7 +216,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
case EXPORT:
case IMPORT:
case QUERY:
- event = new CreateHiveProcess(context, true);
+ event = new CreateHiveProcess(context);
break;
case ALTERTABLE_FILEFORMAT:
@@ -292,6 +295,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
return hiveProcessPopulateDeprecatedAttributes;
}
+ public static boolean isSkipTempTables() {
+ return skipTempTables;
+ }
+
public PreprocessAction getPreprocessActionForHiveTable(String qualifiedName) {
PreprocessAction ret = PreprocessAction.NONE;
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
index 3c0f0c1..a371d44 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveMetastoreHookImpl.java
@@ -115,7 +115,7 @@ public class HiveMetastoreHookImpl extends MetaStoreEventListener {
try {
HiveOperation oper = operContext.getOperation();
- AtlasHiveHookContext context = new AtlasHiveHookContext(hiveHook, oper, hiveHook.getKnownObjects(), this, listenerEvent);
+ AtlasHiveHookContext context = new AtlasHiveHookContext(hiveHook, oper, hiveHook.getKnownObjects(), this, listenerEvent, hiveHook.isSkipTempTables());
BaseHiveEvent event = null;
switch (oper) {
@@ -132,7 +132,7 @@ public class HiveMetastoreHookImpl extends MetaStoreEventListener {
break;
case CREATETABLE:
- event = new CreateTable(context, true);
+ event = new CreateTable(context);
break;
case DROPTABLE:
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
index e164370..d2f09cc 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/AlterTable.java
@@ -29,7 +29,7 @@ import java.util.List;
public class AlterTable extends CreateTable {
public AlterTable(AtlasHiveHookContext context) {
- super(context, true);
+ super(context);
}
@Override
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index aefe9e9..8f6cbcc 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -165,6 +165,7 @@ public abstract class BaseHiveEvent {
public static final Map<Integer, String> OWNER_TYPE_TO_ENUM_VALUE = new HashMap<>();
+ protected final boolean skipTempTables;
static {
OWNER_TYPE_TO_ENUM_VALUE.put(1, "USER");
@@ -176,7 +177,8 @@ public abstract class BaseHiveEvent {
protected BaseHiveEvent(AtlasHiveHookContext context) {
- this.context = context;
+ this.context = context;
+ this.skipTempTables = context.isSkipTempTables();
}
public AtlasHiveHookContext getContext() {
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index e5295f4..5787c93 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -49,12 +49,9 @@ import java.util.Set;
public class CreateHiveProcess extends BaseHiveEvent {
private static final Logger LOG = LoggerFactory.getLogger(CreateHiveProcess.class);
- private final boolean skipTempTables;
- public CreateHiveProcess(AtlasHiveHookContext context, boolean skipTempTables) {
+ public CreateHiveProcess(AtlasHiveHookContext context) {
super(context);
-
- this.skipTempTables = skipTempTables;
}
@Override
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
index 63fc894..91611de 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateTable.java
@@ -42,12 +42,9 @@ import static org.apache.hadoop.hive.ql.plan.HiveOperation.*;
public class CreateTable extends BaseHiveEvent {
private static final Logger LOG = LoggerFactory.getLogger(CreateTable.class);
- private final boolean skipTempTables;
- public CreateTable(AtlasHiveHookContext context, boolean skipTempTables) {
+ public CreateTable(AtlasHiveHookContext context) {
super(context);
-
- this.skipTempTables = skipTempTables;
}
@Override