You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2020/01/21 16:36:46 UTC
[atlas] branch master updated: ATLAS-3590: Hive hook should ignore
capturing lineage for temporary tables in CTAS queries
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 47f2e77 ATLAS-3590: Hive hook should ignore capturing lineage for temporary tables in CTAS queries
47f2e77 is described below
commit 47f2e77bf798547e652e16cf10adaa818ded1e3e
Author: Sarath Subramanian <sa...@apache.org>
AuthorDate: Thu Jan 16 16:51:14 2020 -0800
ATLAS-3590: Hive hook should ignore capturing lineage for temporary tables in CTAS queries
---
.../src/main/java/org/apache/atlas/hive/hook/HiveHook.java | 2 +-
.../java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java | 10 +++++++---
.../org/apache/atlas/hive/hook/events/CreateHiveProcess.java | 9 ++++++---
3 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 5b1f61b..e886b9b 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -201,7 +201,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
case EXPORT:
case IMPORT:
case QUERY:
- event = new CreateHiveProcess(context);
+ event = new CreateHiveProcess(context, true);
break;
case ALTERTABLE_FILEFORMAT:
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 02e27d5..60dc114 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -230,14 +230,14 @@ public abstract class BaseHiveEvent {
}
}
- protected AtlasEntity getInputOutputEntity(Entity entity, AtlasEntityExtInfo entityExtInfo) throws Exception {
+ protected AtlasEntity getInputOutputEntity(Entity entity, AtlasEntityExtInfo entityExtInfo, boolean skipTempTables) throws Exception {
AtlasEntity ret = null;
switch(entity.getType()) {
case TABLE:
case PARTITION:
case DFS_DIR: {
- ret = toAtlasEntity(entity, entityExtInfo);
+ ret = toAtlasEntity(entity, entityExtInfo, skipTempTables);
}
break;
}
@@ -245,7 +245,7 @@ public abstract class BaseHiveEvent {
return ret;
}
- protected AtlasEntity toAtlasEntity(Entity entity, AtlasEntityExtInfo entityExtInfo) throws Exception {
+ protected AtlasEntity toAtlasEntity(Entity entity, AtlasEntityExtInfo entityExtInfo, boolean skipTempTables) throws Exception {
AtlasEntity ret = null;
switch (entity.getType()) {
@@ -271,6 +271,10 @@ public abstract class BaseHiveEvent {
}
if (!skipTable) {
+ skipTable = skipTempTables && entity.getTable().isTemporary();
+ }
+
+ if (!skipTable) {
Table table = getHive().getTable(dbName, tableName);
ret = toTableEntity(table, entityExtInfo);
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index 82df576..659417d 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -49,9 +49,12 @@ import java.util.Set;
public class CreateHiveProcess extends BaseHiveEvent {
private static final Logger LOG = LoggerFactory.getLogger(CreateHiveProcess.class);
+ private final boolean skipTempTables;
- public CreateHiveProcess(AtlasHiveHookContext context) {
+ public CreateHiveProcess(AtlasHiveHookContext context, boolean skipTempTables) {
super(context);
+
+ this.skipTempTables = skipTempTables;
}
@Override
@@ -84,7 +87,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
continue;
}
- AtlasEntity entity = getInputOutputEntity(input, ret);
+ AtlasEntity entity = getInputOutputEntity(input, ret, skipTempTables);
if (!input.isDirect()) {
continue;
@@ -104,7 +107,7 @@ public class CreateHiveProcess extends BaseHiveEvent {
continue;
}
- AtlasEntity entity = getInputOutputEntity(output, ret);
+ AtlasEntity entity = getInputOutputEntity(output, ret, skipTempTables);
if (entity != null) {
outputs.add(entity);