You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/05/03 04:45:19 UTC
[atlas] branch branch-2.0 updated: ATLAS-3188: Regression: Simple
insert queries are being captured in ATLAS_HOOK topic
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 41e1404 ATLAS-3188: Regression: Simple insert queries are being captured in ATLAS_HOOK topic
41e1404 is described below
commit 41e14045d2dadffc71d1fbd44e40b19d52caa2a1
Author: Sarath Subramanian <ss...@hortonworks.com>
AuthorDate: Thu May 2 21:35:49 2019 -0700
ATLAS-3188: Regression: Simple insert queries are being captured in ATLAS_HOOK topic
(cherry picked from commit fcca791b60364eb0fd5ca3b74c0f52f8f781cf64)
---
.../atlas/hive/hook/AtlasHiveHookContext.java | 36 +++++++++++++++++++---
.../atlas/hive/hook/events/BaseHiveEvent.java | 2 ++
.../atlas/hive/hook/events/CreateHiveProcess.java | 13 ++++++--
3 files changed, 45 insertions(+), 6 deletions(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index 5c56a39..1b7a718 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -26,10 +26,7 @@ import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.hive.metastore.IHMSHandler;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.events.*;
-import org.apache.hadoop.hive.ql.hooks.HookContext;
-import org.apache.hadoop.hive.ql.hooks.LineageInfo;
-import org.apache.hadoop.hive.ql.hooks.ReadEntity;
-import org.apache.hadoop.hive.ql.hooks.WriteEntity;
+import org.apache.hadoop.hive.ql.hooks.*;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
@@ -58,6 +55,9 @@ public class AtlasHiveHookContext {
private final ListenerEvent metastoreEvent;
private final IHMSHandler metastoreHandler;
+ private boolean isSkippedInputEntity;
+ private boolean isSkippedOutputEntity;
+
public AtlasHiveHookContext(HiveHook hook, HiveOperation hiveOperation, HookContext hiveContext,
HiveHookObjectNamesCache knownObjects) throws Exception {
this(hook, hiveOperation, hiveContext, knownObjects, null, null);
@@ -102,6 +102,34 @@ public class AtlasHiveHookContext {
return hiveContext != null ? hiveContext.getOutputs() : Collections.emptySet();
}
+ public boolean isSkippedInputEntity() {
+ return isSkippedInputEntity;
+ }
+
+ public boolean isSkippedOutputEntity() {
+ return isSkippedOutputEntity;
+ }
+
+ public void registerSkippedEntity(Entity entity) {
+ if (entity instanceof ReadEntity) {
+ registerSkippedInputEntity();
+ } else if (entity instanceof WriteEntity) {
+ registerSkippedOutputEntity();
+ }
+ }
+
+ public void registerSkippedInputEntity() {
+ if (!isSkippedInputEntity) {
+ isSkippedInputEntity = true;
+ }
+ }
+
+ public void registerSkippedOutputEntity() {
+ if (!isSkippedOutputEntity) {
+ isSkippedOutputEntity = true;
+ }
+ }
+
public LineageInfo getLineageInfo() {
return hiveContext != null ? hiveContext.getLinfo() : null;
}
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index bb39fdf..79b700b 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -259,6 +259,8 @@ public abstract class BaseHiveEvent {
Table table = getHive().getTable(dbName, tableName);
ret = toTableEntity(table, entityExtInfo);
+ } else {
+ context.registerSkippedEntity(entity);
}
}
break;
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
index 019deb7..daebcf5 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateHiveProcess.java
@@ -25,7 +25,6 @@ import org.apache.atlas.model.notification.HookNotification;
import org.apache.atlas.model.notification.HookNotification.EntityCreateRequestV2;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hive.ql.hooks.Entity;
-import org.apache.hadoop.hive.ql.hooks.HookContext;
import org.apache.hadoop.hive.ql.hooks.LineageInfo;
import org.apache.hadoop.hive.ql.hooks.LineageInfo.BaseColumnInfo;
import org.apache.hadoop.hive.ql.hooks.LineageInfo.Dependency;
@@ -112,7 +111,17 @@ public class CreateHiveProcess extends BaseHiveEvent {
}
}
- if (!inputs.isEmpty() || !outputs.isEmpty()) {
+ boolean skipProcess = inputs.isEmpty() && outputs.isEmpty();
+
+ if (!skipProcess) {
+ if (inputs.isEmpty() && context.isSkippedInputEntity()) {
+ skipProcess = true;
+ } else if (outputs.isEmpty() && context.isSkippedOutputEntity()) {
+ skipProcess = true;
+ }
+ }
+
+ if (!skipProcess) {
AtlasEntity process = getHiveProcessEntity(inputs, outputs);
ret.addEntity(process);