You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/07/20 13:03:05 UTC
[2/5] incubator-atlas git commit: ATLAS-639 Exception for lineage
request (svimal2106 via shwethags)
ATLAS-639 Exception for lineage request (svimal2106 via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/67acb9d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/67acb9d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/67acb9d6
Branch: refs/heads/master
Commit: 67acb9d6edf3c62ddc63299bb7bb304b950b7f7d
Parents: f672aae
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Wed Jul 20 12:11:08 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Wed Jul 20 12:11:08 2016 +0530
----------------------------------------------------------------------
.../org/apache/atlas/hive/hook/HiveHookIT.java | 4 +-
release-log.txt | 1 +
.../org/apache/atlas/query/GremlinQuery.scala | 5 +-
.../org/apache/atlas/BaseRepositoryTest.java | 10 ++
.../discovery/DataSetLineageServiceTest.java | 16 +++
.../GraphBackedDiscoveryServiceTest.java | 100 +++++++++----------
6 files changed, 82 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index a15aa49..573f7f5 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -1620,7 +1620,7 @@ public class HiveHookIT {
expectedProps.put("testPropKey2", "testPropValue2");
//Add another property
query = String.format(fmtQuery, entityName, SET_OP, getSerializedProps(expectedProps));
- runCommand(query);
+ runCommandWithDelay(query, 1000);
verifyEntityProperties(entityType, entityName, expectedProps, false);
if (entityType != Entity.Type.DATABASE) {
@@ -1628,7 +1628,7 @@ public class HiveHookIT {
//Unset all the props
StringBuilder sb = new StringBuilder("'");
query = String.format(fmtQuery, entityName, UNSET_OP, Joiner.on("','").skipNulls().appendTo(sb, expectedProps.keySet()).append('\''));
- runCommand(query);
+ runCommandWithDelay(query, 1000);
verifyEntityProperties(entityType, entityName, expectedProps, true);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 40de319..ef9f67f 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES:
ALL CHANGES:
+ATLAS-639 Exception for lineage request (svimal2106 via shwethags)
ATLAS-1022 Update typesystem wiki with details (yhemanth via shwethags)
ATLAS-1021 Update Atlas architecture wiki (yhemanth via sumasai)
ATLAS-957 Atlas is not capturing topologies that have $ in the data payload (shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
index d76b173..ee221c9 100755
--- a/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/GremlinQuery.scala
@@ -280,9 +280,10 @@ class GremlinTranslator(expr: Expression,
val inputQry = genQuery(input, inSelect)
val loopingPathGExpr = genQuery(loopExpr, inSelect)
val loopGExpr = s"""loop("${input.asInstanceOf[AliasExpression].alias}")"""
- val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{true}"
+ val untilCriteria = if (t.isDefined) s"{it.loops < ${t.get.value}}" else "{it.path.contains(it.object)?false:true}"
val loopObjectGExpr = gPersistenceBehavior.loopObjectExpression(input.dataType)
- s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}"""
+ val enablePathExpr = s".enablePath()"
+ s"""${inputQry}.${loopingPathGExpr}.${loopGExpr}${untilCriteria}${loopObjectGExpr}${enablePathExpr}"""
}
case BackReference(alias, _, _) =>
if (inSelect) gPersistenceBehavior.fieldPrefixInSelect else s"""back("$alias")"""
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
index 4d3f165..73d9fc5 100644
--- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
@@ -224,6 +224,16 @@ public class BaseRepositoryTest {
table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed",
salesFactColumns, "Metric");
+ Id circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
+
+ Id circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("circularLineage1", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable1),
+ ImmutableList.of(circularLineageTable2), "create table as select ", "plan", "id", "graph", "ETL");
+
+ loadProcess("circularLineage2", "hive query for daily summary", "John ETL", ImmutableList.of(circularLineageTable2),
+ ImmutableList.of(circularLineageTable1), "create table as select ", "plan", "id", "graph", "ETL");
+
loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL", ImmutableList.of(salesFact, timeDim),
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
index 89e9b41..460a88f 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/DataSetLineageServiceTest.java
@@ -176,6 +176,22 @@ public class DataSetLineageServiceTest extends BaseRepositoryTest {
}
@Test
+ public void testCircularLineage() throws Exception{
+ JSONObject results = new JSONObject(lineageService.getInputsGraph("table2"));
+ assertNotNull(results);
+ System.out.println("inputs graph = " + results);
+
+ JSONObject values = results.getJSONObject("values");
+ assertNotNull(values);
+
+ final JSONObject vertices = values.getJSONObject("vertices");
+ Assert.assertEquals(vertices.length(), 2);
+
+ final JSONObject edges = values.getJSONObject("edges");
+ Assert.assertEquals(edges.length(), 4);
+ }
+
+ @Test
public void testGetInputsGraphForEntity() throws Exception {
ITypedReferenceableInstance entity =
repository.getEntityDefinition(HIVE_TABLE_TYPE, "name", "sales_fact_monthly_mv");
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/67acb9d6/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index c27894e..313a2ae 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -316,7 +316,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_db hive_db.name = \"Reporting\"", 1},
{"hive_db where hive_db.name=\"Reporting\" select name, owner", 1},
{"hive_db has name", 3},
- {"hive_db, hive_table", 8},
+ {"hive_db, hive_table", 10},
{"View is JdbcAccess", 2},
{"hive_db as db1, hive_table where db1.name = \"Reporting\"", 0}, //Not working - ATLAS-145
// - Final working query -> discoveryService.searchByGremlin("L:{_var_0 = [] as Set;g.V().has(\"__typeName\", \"hive_db\").fill(_var_0);g.V().has(\"__superTypeNames\", \"hive_db\").fill(_var_0);_var_0._().as(\"db1\").in(\"__hive_table.db\").back(\"db1\").and(_().has(\"hive_db.name\", T.eq, \"Reporting\")).toList()}")
@@ -324,20 +324,20 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_db, hive_process has name"}, //Invalid query
{"hive_db where hive_db.name=\"Reporting\" and hive_db.createTime < " + System.currentTimeMillis()}
*/
- {"from hive_table", 8},
- {"hive_table", 8},
+ {"from hive_table", 10},
+ {"hive_table", 10},
{"hive_table isa Dimension", 3},
- {"hive_column where hive_column isa PII", 6},
+ {"hive_column where hive_column isa PII", 8},
{"View is Dimension" , 2},
// {"hive_column where hive_column isa PII select hive_column.name", 6}, //Not working - ATLAS-175
- {"hive_column select hive_column.name", 29},
- {"hive_column select name", 29},
- {"hive_column where hive_column.name=\"customer_id\"", 4},
- {"from hive_table select hive_table.name", 8},
+ {"hive_column select hive_column.name", 37},
+ {"hive_column select name",37},
+ {"hive_column where hive_column.name=\"customer_id\"", 6},
+ {"from hive_table select hive_table.name", 10},
{"hive_db where (name = \"Reporting\")", 1},
{"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1},
{"hive_db where hive_db is JdbcAccess", 0}, //Not supposed to work
- {"hive_db hive_table", 8},
+ {"hive_db hive_table", 10},
{"hive_db where hive_db has name", 3},
{"hive_db as db1 hive_table where (db1.name = \"Reporting\")", 0}, //Not working -> ATLAS-145
{"hive_db where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ", 1},
@@ -359,9 +359,9 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
// trait searches
{"Dimension", 5},
{"JdbcAccess", 2},
- {"ETL", 3},
- {"Metric", 5},
- {"PII", 6},
+ {"ETL", 5},
+ {"Metric", 9},
+ {"PII", 8},
{"`Log Data`", 4},
/* Lineage queries are fired through ClosureQuery and are tested through HiveLineageJerseyResourceIt in webapp module.
@@ -388,7 +388,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
return new Object[][]{
{"hive_column limit 10 ", 10},
{"hive_column select hive_column.name limit 10 ", 10},
- {"hive_column select hive_column.name withPath", 29},
+ {"hive_column select hive_column.name withPath", 37},
{"hive_column select hive_column.name limit 10 withPath", 10},
{"from hive_db", 3},
{"from hive_db limit 2", 2},
@@ -405,10 +405,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_db has name limit 2 offset 1", 2},
{"hive_db has name limit 10 offset 1", 2},
{"hive_db has name limit 10 offset 0", 3},
- {"hive_db, hive_table", 8},
+ {"hive_db, hive_table", 10},
{"hive_db, hive_table limit 5", 5},
{"hive_db, hive_table limit 5 offset 0", 5},
- {"hive_db, hive_table limit 5 offset 5", 3},
+ {"hive_db, hive_table limit 5 offset 5", 5},
{"View is JdbcAccess", 2},
{"View is JdbcAccess limit 1", 1},
@@ -416,13 +416,13 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_db as db1, hive_table where db1.name = \"Reporting\"", 0}, //Not working - ATLAS-145
- {"from hive_table", 8},
+ {"from hive_table", 10},
{"from hive_table limit 5", 5},
- {"from hive_table limit 5 offset 5", 3},
+ {"from hive_table limit 5 offset 5", 5},
- {"hive_table", 8},
+ {"hive_table", 10},
{"hive_table limit 5", 5},
- {"hive_table limit 5 offset 5", 3},
+ {"hive_table limit 5 offset 5", 5},
{"hive_table isa Dimension", 3},
{"hive_table isa Dimension limit 2", 2},
@@ -430,10 +430,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_table isa Dimension limit 2 offset 1", 2},
{"hive_table isa Dimension limit 3 offset 1", 2},
- {"hive_column where hive_column isa PII", 6},
+ {"hive_column where hive_column isa PII", 8},
{"hive_column where hive_column isa PII limit 5", 5},
{"hive_column where hive_column isa PII limit 5 offset 1", 5},
- {"hive_column where hive_column isa PII limit 5 offset 5", 1},
+ {"hive_column where hive_column isa PII limit 5 offset 5", 3},
{"View is Dimension" , 2},
@@ -441,31 +441,31 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"View is Dimension limit 1 offset 1" , 1},
{"View is Dimension limit 10 offset 1" , 1},
- {"hive_column select hive_column.name", 29},
+ {"hive_column select hive_column.name", 37},
{"hive_column select hive_column.name limit 5", 5},
- {"hive_column select hive_column.name limit 5 offset 28", 1},
+ {"hive_column select hive_column.name limit 5 offset 36", 1},
- {"hive_column select name", 29},
+ {"hive_column select name", 37},
{"hive_column select name limit 5", 5},
- {"hive_column select name limit 5 offset 28 ", 1},
+ {"hive_column select name limit 5 offset 36 ", 1},
- {"hive_column where hive_column.name=\"customer_id\"", 4},
+ {"hive_column where hive_column.name=\"customer_id\"", 6},
{"hive_column where hive_column.name=\"customer_id\" limit 2", 2},
{"hive_column where hive_column.name=\"customer_id\" limit 2 offset 1", 2},
- {"hive_column where hive_column.name=\"customer_id\" limit 10 offset 3", 1},
+ {"hive_column where hive_column.name=\"customer_id\" limit 10 offset 3", 3},
- {"from hive_table select hive_table.name", 8},
+ {"from hive_table select hive_table.name", 10},
{"from hive_table select hive_table.name limit 5", 5},
- {"from hive_table select hive_table.name limit 5 offset 5", 3},
+ {"from hive_table select hive_table.name limit 5 offset 5", 5},
{"hive_db where (name = \"Reporting\")", 1},
{"hive_db where (name = \"Reporting\") limit 10", 1},
{"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1", 1},
{"hive_db where (name = \"Reporting\") select name as _col_0, owner as _col_1 limit 10", 1},
{"hive_db where hive_db is JdbcAccess", 0}, //Not supposed to work
- {"hive_db hive_table", 8},
+ {"hive_db hive_table", 10},
{"hive_db hive_table limit 5", 5},
- {"hive_db hive_table limit 5 offset 5", 3},
+ {"hive_db hive_table limit 5 offset 5", 5},
{"hive_db where hive_db has name", 3},
{"hive_db where hive_db has name limit 5", 3},
{"hive_db where hive_db has name limit 2 offset 0", 2},
@@ -499,24 +499,24 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"JdbcAccess limit 2 offset 1", 1},
{"JdbcAccess limit 1", 1},
- {"ETL", 3},
+ {"ETL", 5},
{"ETL limit 2", 2},
{"ETL limit 1", 1},
{"ETL limit 1 offset 0", 1},
{"ETL limit 2 offset 1", 2},
- {"Metric", 5},
- {"Metric limit 10", 5},
+ {"Metric", 9},
+ {"Metric limit 10", 9},
{"Metric limit 2", 2},
- {"Metric limit 10 offset 1", 4},
+ {"Metric limit 10 offset 1", 8},
- {"PII", 6},
- {"PII limit 10", 6},
+ {"PII", 8},
+ {"PII limit 10", 8},
{"PII limit 2", 2},
- {"PII limit 10 offset 1", 5},
-
+ {"PII limit 10 offset 1", 7},
+
{"`Log Data`", 4},
{"`Log Data` limit 3", 3},
{"`Log Data` limit 10 offset 2", 2},
@@ -553,10 +553,10 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_db where hive_db.name=\"Reporting\" select name, owner orderby 'hive_db.name' ", 1, "name", isAscending},
{"hive_db has name orderby 'hive_db.owner' limit 10 offset 0", 3, "owner", isAscending},
- {"from hive_table orderby 'hive_table.owner'", 8, "owner", isAscending},
+ {"from hive_table orderby 'hive_table.owner'", 10, "owner", isAscending},
{"from hive_table orderby 'hive_table.owner' limit 8", 8, "owner", isAscending},
- {"hive_table orderby 'hive_table.owner'", 8, "owner", isAscending},
+ {"hive_table orderby 'hive_table.owner'", 10, "owner", isAscending},
{"hive_table orderby 'hive_table.owner' limit 8", 8, "owner", isAscending},
{"hive_table orderby 'hive_table.owner' limit 8 offset 0", 8, "owner", isAscending},
{"hive_table orderby 'hive_table.owner' desc limit 8 offset 0", 8, "owner", !isAscending},
@@ -566,29 +566,29 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
{"hive_table isa Dimension orderby 'hive_table.owner' limit 3 offset 0", 3, "owner", isAscending},
{"hive_table isa Dimension orderby 'hive_table.owner' desc limit 3 offset 0", 3, "owner", !isAscending},
- {"hive_column where hive_column isa PII orderby 'hive_column.name'", 6, "name", isAscending},
+ {"hive_column where hive_column isa PII orderby 'hive_column.name'", 8, "name", isAscending},
{"hive_column where hive_column isa PII orderby 'hive_column.name' limit 5", 5, "name", isAscending},
{"hive_column where hive_column isa PII orderby 'hive_column.name' limit 5 offset 1", 5, "name", isAscending},
{"hive_column where hive_column isa PII orderby 'hive_column.name' desc limit 5 offset 1", 5, "name", !isAscending},
- {"hive_column select hive_column.name orderby 'hive_column.name' ", 29, "hive_column.name", isAscending},
+ {"hive_column select hive_column.name orderby 'hive_column.name' ", 37, "hive_column.name", isAscending},
{"hive_column select hive_column.name orderby 'hive_column.name' limit 5", 5, "hive_column.name", isAscending},
{"hive_column select hive_column.name orderby 'hive_column.name' desc limit 5", 5, "hive_column.name", !isAscending},
- {"hive_column select hive_column.name orderby 'hive_column.name' limit 5 offset 28", 1, "hive_column.name", isAscending},
+ {"hive_column select hive_column.name orderby 'hive_column.name' limit 5 offset 36", 1, "hive_column.name", isAscending},
- {"hive_column select name orderby 'hive_column.name'", 29, "name", isAscending},
+ {"hive_column select name orderby 'hive_column.name'", 37, "name", isAscending},
{"hive_column select name orderby 'hive_column.name' limit 5", 5, "name", isAscending},
- {"hive_column select name orderby 'hive_column.name' desc", 29, "name", !isAscending},
+ {"hive_column select name orderby 'hive_column.name' desc", 37, "name", !isAscending},
- {"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name'", 4, "name", isAscending},
+ {"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name'", 6, "name", isAscending},
{"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name' limit 2", 2, "name", isAscending},
{"hive_column where hive_column.name=\"customer_id\" orderby 'hive_column.name' limit 2 offset 1", 2, "name", isAscending},
- {"from hive_table select owner orderby 'hive_table.owner'", 8, "owner", isAscending},
+ {"from hive_table select owner orderby 'hive_table.owner'", 10, "owner", isAscending},
{"from hive_table select owner orderby 'hive_table.owner' limit 5", 5, "owner", isAscending},
{"from hive_table select owner orderby 'hive_table.owner' desc limit 5", 5, "owner", !isAscending},
- {"from hive_table select owner orderby 'hive_table.owner' limit 5 offset 5", 3, "owner", isAscending},
+ {"from hive_table select owner orderby 'hive_table.owner' limit 5 offset 5", 5, "owner", isAscending},
{"hive_db where (name = \"Reporting\") orderby 'hive_db.name'", 1, "name", isAscending},
{"hive_db where (name = \"Reporting\") orderby 'hive_db.name' limit 10", 1, "name", isAscending},
@@ -617,7 +617,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
//Test if proeprty is not defined. it should not fail the query
- {"hive_table orderby 'hive_table.owner_notdefined'", 8, null, isAscending},
+ {"hive_table orderby 'hive_table.owner_notdefined'", 10, null, isAscending},
};
}