You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by pi...@apache.org on 2022/04/12 07:14:51 UTC
[atlas] branch branch-2.0 updated: ATLAS-4571 : Impala Hook : query-string is too large to store Asset.name (solr.StrField).
This is an automated email from the ASF dual-hosted git repository.
pinal pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new c4bc80b22 ATLAS-4571 : Impala Hook : query-string is too large to store Asset.name (solr.StrField).
c4bc80b22 is described below
commit c4bc80b22d397cbeeffdc3f85a837fc7670d6487
Author: snehal.ambavkar <sn...@cloudera.com>
AuthorDate: Tue Apr 12 10:40:43 2022 +0530
ATLAS-4571 : Impala Hook : query-string is too large to store Asset.name (solr.StrField).
Signed-off-by: Pinal Shah <pi...@freestoneinfotech.com>
(cherry picked from commit 742eeba4021d1f9972e8bf572693474fd8d7b367)
---
.../atlas/impala/hook/events/BaseImpalaEvent.java | 16 ++++---
.../apache/atlas/impala/ImpalaLineageToolIT.java | 51 ++++++++++++++++++++++
.../atlas/impala/hook/ImpalaLineageHookIT.java | 11 +++++
3 files changed, 71 insertions(+), 7 deletions(-)
diff --git a/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/events/BaseImpalaEvent.java b/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/events/BaseImpalaEvent.java
index 32efb8321..3a13d0c41 100644
--- a/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/events/BaseImpalaEvent.java
+++ b/addons/impala-bridge/src/main/java/org/apache/atlas/impala/hook/events/BaseImpalaEvent.java
@@ -530,11 +530,12 @@ public abstract class BaseImpalaEvent {
Long startTime = getQueryStartTime();
Long endTime = getQueryEndTime();
+ String qualifiedName = getQualifiedName(inputs, outputs);
- ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, getQualifiedName(inputs, outputs));
+ ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, qualifiedName);
ret.setAttribute(ATTRIBUTE_INPUTS, getObjectIds(inputs));
- ret.setAttribute(ATTRIBUTE_OUTPUTS, getObjectIds(outputs));
- ret.setAttribute(ATTRIBUTE_NAME, queryStr);
+ ret.setAttribute(ATTRIBUTE_OUTPUTS, getObjectIds(outputs));
+ ret.setAttribute(ATTRIBUTE_NAME, qualifiedName);
ret.setAttribute(ATTRIBUTE_OPERATION_TYPE, context.getImpalaOperationType());
// We are setting an empty value to these attributes, since now we have a new entity type called impala process
@@ -561,11 +562,12 @@ public abstract class BaseImpalaEvent {
Long startTime = getQueryStartTime();
Long endTime = getQueryEndTime();
+ String qualifiedName = impalaProcess.getAttribute(ATTRIBUTE_QUALIFIED_NAME).toString() +
+ QNAME_SEP_PROCESS + startTime.toString() +
+ QNAME_SEP_PROCESS + endTime.toString();
- ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, impalaProcess.getAttribute(ATTRIBUTE_QUALIFIED_NAME).toString() +
- QNAME_SEP_PROCESS + startTime.toString() +
- QNAME_SEP_PROCESS + endTime.toString());
- ret.setAttribute(ATTRIBUTE_NAME, queryStr + QNAME_SEP_PROCESS + startTime);
+ ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, qualifiedName);
+ ret.setAttribute(ATTRIBUTE_NAME, qualifiedName);
ret.setAttribute(ATTRIBUTE_START_TIME, startTime);
ret.setAttribute(ATTRIBUTE_END_TIME, endTime);
ret.setAttribute(ATTRIBUTE_USER_NAME, getUserName());
diff --git a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java
index 53e9b1224..4f8c2da3b 100644
--- a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java
+++ b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/ImpalaLineageToolIT.java
@@ -17,6 +17,9 @@
*/
package org.apache.atlas.impala;
+import static org.apache.atlas.impala.hook.AtlasImpalaHookContext.QNAME_SEP_PROCESS;
+import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_END_TIME;
+import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_NAME;
import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_QUERY_TEXT;
import java.util.ArrayList;
@@ -31,6 +34,7 @@ import org.testng.Assert;
import org.testng.annotations.Test;
import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_DDL_QUERIES;
+import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_START_TIME;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
@@ -85,10 +89,14 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
String queryString = "create view db_1.view_1 as select count, id from db_1.table_1";
AtlasEntity processEntity1 = validateProcess(processQFName, queryString);
AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, queryString);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
AtlasObjectId process1 = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -151,6 +159,11 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -284,6 +297,11 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -341,6 +359,12 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -397,6 +421,11 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -454,6 +483,11 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -504,13 +538,18 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
CLUSTER_NAME + AtlasImpalaHookContext.QNAME_SEP_PROCESS + createTime2;
String processQFName = "QUERY:" + sourceQFName.toLowerCase() + "->:INSERT:" + targetQFName.toLowerCase();
+
String queryString = "insert into table " + dbName + "." + targetTableName + " (count, id) select count, id from " + dbName + "." + sourceTableName;
AtlasEntity processEntity1 = validateProcess(processQFName, queryString);
AtlasEntity processExecutionEntity1 = validateProcessExecution(processEntity1, queryString);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
AtlasObjectId process1 = toAtlasObjectId(processExecutionEntity1.getRelationshipAttribute(
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
@@ -587,6 +626,12 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
String errorMessage = String.format("process query text '%s' does not match expected value of '%s' or '%s'", entityQueryText, queryString, queryString2);
Assert.assertTrue(false, errorMessage);
}
+
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity.getAttribute(ATTRIBUTE_NAME).toString());
}
String guid = assertTableIsRegistered(dbName, targetTableName);
@@ -645,6 +690,12 @@ public class ImpalaLineageToolIT extends ImpalaLineageITBase {
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ String processExecutionQFName = processQFName + QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+
String guid = assertTableIsRegistered(dbName, targetTableName);
AtlasEntity entity = atlasClientV2.getEntityByGuid(guid).getEntity();
List ddlQueries = (List) entity.getRelationshipAttribute(ATTRIBUTE_DDL_QUERIES);
diff --git a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java
index 56d74fee3..e0277c884 100644
--- a/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java
+++ b/addons/impala-bridge/src/test/java/org/apache/atlas/impala/hook/ImpalaLineageHookIT.java
@@ -34,9 +34,15 @@ import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;
+
+import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_START_TIME;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
+import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_END_TIME;
+import static org.apache.atlas.impala.hook.events.BaseImpalaEvent.ATTRIBUTE_NAME;
+
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -149,6 +155,11 @@ public class ImpalaLineageHookIT extends ImpalaLineageITBase {
BaseImpalaEvent.ATTRIBUTE_PROCESS));
Assert.assertEquals(process1.getGuid(), processEntity1.getGuid());
Assert.assertEquals(numberOfProcessExecutions(processEntity1), 1);
+ String processExecutionQFName = processQFName + AtlasImpalaHookContext.QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_START_TIME).toString() +
+ AtlasImpalaHookContext.QNAME_SEP_PROCESS + processExecutionEntity1.getAttribute(ATTRIBUTE_END_TIME).toString();
+
+ assertEquals(processQFName, processEntity1.getAttribute(ATTRIBUTE_NAME).toString());
+ assertEquals(processExecutionQFName, processExecutionEntity1.getAttribute(ATTRIBUTE_NAME).toString());
// check DDL entity
String viewId = assertTableIsRegistered(viewName);