You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/05/12 06:32:51 UTC
incubator-atlas git commit: ATLAS-738 Add query ability on system
properties like guid, state, createdtime etc (shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 915595780 -> bfd5f5caa
ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/bfd5f5ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/bfd5f5ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/bfd5f5ca
Branch: refs/heads/master
Commit: bfd5f5caa609b9084ec816deb4c6bf11eec18bab
Parents: 9155957
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Thu May 12 12:02:46 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Thu May 12 12:02:46 2016 +0530
----------------------------------------------------------------------
release-log.txt | 1 +
.../org/apache/atlas/repository/Constants.java | 16 ++++
.../graph/GraphBackedMetadataRepository.java | 3 +
.../org/apache/atlas/query/TypeUtils.scala | 6 ++
.../GraphBackedDiscoveryServiceTest.java | 98 +++++++++++++-------
.../atlas/typesystem/types/AttributeInfo.java | 2 +-
.../atlas/typesystem/types/utils/TypesUtil.java | 14 +++
7 files changed, 106 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 7e3149c..66064d6 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -20,6 +20,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-738 Add query ability on system properties like guid, state, createdtime etc (shwethags)
ATLAS-692 Create abstraction layer for graph databases (jnhagelb via yhemanth)
ATLAS-689 Migrate Atlas-Storm integration to use Storm 1.0 dependencies. (svimal2106 via yhemanth)
ATLAS-754 InstanceSerialization does not serialize Reference in the values array of Reference.(harishjp via sumasai)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/Constants.java b/repository/src/main/java/org/apache/atlas/repository/Constants.java
index 0832c81..893f1b6 100755
--- a/repository/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/repository/src/main/java/org/apache/atlas/repository/Constants.java
@@ -18,6 +18,10 @@
package org.apache.atlas.repository;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+
public final class Constants {
/**
@@ -62,6 +66,18 @@ public final class Constants {
public static final String TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "timestamp";
public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp";
+ public static AttributeInfo getAttributeInfoForSystemAttributes(String field) {
+ switch (field) {
+ case STATE_PROPERTY_KEY:
+ case GUID_PROPERTY_KEY:
+ return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
+
+ case TIMESTAMP_PROPERTY_KEY:
+ case MODIFICATION_TIMESTAMP_PROPERTY_KEY:
+ return TypesUtil.newAttributeInfo(field, DataTypes.LONG_TYPE);
+ }
+ return null;
+ }
/**
* search backing index name.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index de29e86..3604277 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -105,6 +105,9 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
+ if (aInfo.name.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX)) {
+ return aInfo.name;
+ }
return GraphHelper.getQualifiedFieldName(dataType, aInfo.name);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
index 552cc32..5a64c53 100755
--- a/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/TypeUtils.scala
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger
import org.apache.atlas.AtlasException
import org.apache.atlas.query.Expressions.{PathExpression, SelectExpression}
+import org.apache.atlas.repository.Constants
import org.apache.atlas.typesystem.types.DataTypes.{ArrayType, PrimitiveType, TypeCategory}
import org.apache.atlas.typesystem.types._
@@ -203,6 +204,11 @@ object TypeUtils {
return Some(FieldInfo(typ,fMap.get.fields.get(id)))
}
+ val systemField = Constants.getAttributeInfoForSystemAttributes(id)
+ if (systemField != null) {
+ return Some(FieldInfo(systemField.dataType(), systemField))
+ }
+
try {
val FIELD_QUALIFIER(clsNm, rest) = id
val idTyp = typSystem.getDataType(classOf[IDataType[_]], clsNm)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index 9cc501f..3b50dfb 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -56,6 +56,8 @@ import java.util.Map;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
@@ -94,32 +96,62 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
}
@Test
+ public void testSearchBySystemProperties() throws Exception {
+ //system property in select
+ String dslQuery = "from Department select __guid";
+
+ String jsonResults = discoveryService.searchByDSL(dslQuery);
+ assertNotNull(jsonResults);
+
+ JSONObject results = new JSONObject(jsonResults);
+ assertEquals(results.length(), 3);
+
+ JSONArray rows = results.getJSONArray("rows");
+ assertNotNull(rows);
+ assertEquals(rows.length(), 1);
+ assertNotNull(rows.getJSONObject(0).getString("__guid"));
+
+ //system property in where clause
+ String guid = rows.getJSONObject(0).getString("__guid");
+ dslQuery = "Department where __guid = '" + guid + "' and __state = 'ACTIVE'";
+ jsonResults = discoveryService.searchByDSL(dslQuery);
+ assertNotNull(jsonResults);
+
+ results = new JSONObject(jsonResults);
+ assertEquals(results.length(), 3);
+
+ rows = results.getJSONArray("rows");
+ assertNotNull(rows);
+ assertEquals(rows.length(), 1);
+ }
+
+ @Test
public void testSearchByDSLReturnsEntity() throws Exception {
String dslQuery = "from Department";
String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
+ assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
+ assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
- Assert.assertNotNull(query);
+ assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
+ assertNotNull(dataType);
String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
- Assert.assertEquals(typeName, "Department");
+ assertNotNull(typeName);
+ assertEquals(typeName, "Department");
JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertEquals(rows.length(), 1);
+ assertNotNull(rows);
+ assertEquals(rows.length(), 1);
//Assert that entity state is set in the result entities
String entityState = rows.getJSONObject(0).getJSONObject("$id$").getString("state");
- Assert.assertEquals(entityState, Id.EntityState.ACTIVE.name());
+ assertEquals(entityState, Id.EntityState.ACTIVE.name());
}
@Test(expectedExceptions = Throwable.class)
@@ -167,11 +199,11 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
Assert.assertTrue(resultList.size() > 0);
for (Map<String, Object> vertexProps : resultList) {
Object object = vertexProps.get(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY);
- Assert.assertNotNull(object);
+ assertNotNull(object);
Long timestampAsLong = Long.valueOf((String)object);
Assert.assertTrue(timestampAsLong > 1420070400000L);
object = vertexProps.get(Constants.TIMESTAMP_PROPERTY_KEY);
- Assert.assertNotNull(object);
+ assertNotNull(object);
}
}
@@ -490,23 +522,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testSearchByDSLQueriesWithOrderBy(String dslQuery, Integer expectedNumRows, String orderBy, boolean ascending) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
+ assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
+ assertEquals(results.length(), 3);
Object query = results.get("query");
- Assert.assertNotNull(query);
+ assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
+ assertNotNull(dataType);
String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
+ assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
+ assertNotNull(rows);
+ assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
List<String> returnedList = new ArrayList<String> ();
for (int i = 0; i < rows.length(); i++) {
JSONObject row = rows.getJSONObject(i);
@@ -546,23 +578,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testSearchByDSLQueries(String dslQuery, Integer expectedNumRows) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
+ assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
+ assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
- Assert.assertNotNull(query);
+ assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
+ assertNotNull(dataType);
String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
+ assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
+ assertNotNull(rows);
+ assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
@@ -570,23 +602,23 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
public void testSearchByDSLQueriesWithLimit(String dslQuery, Integer expectedNumRows) throws Exception {
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
+ assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
- Assert.assertEquals(results.length(), 3);
+ assertEquals(results.length(), 3);
System.out.println("results = " + results);
Object query = results.get("query");
- Assert.assertNotNull(query);
+ assertNotNull(query);
JSONObject dataType = results.getJSONObject("dataType");
- Assert.assertNotNull(dataType);
+ assertNotNull(dataType);
String typeName = dataType.getString("typeName");
- Assert.assertNotNull(typeName);
+ assertNotNull(typeName);
JSONArray rows = results.getJSONArray("rows");
- Assert.assertNotNull(rows);
- Assert.assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
+ assertNotNull(rows);
+ assertEquals(rows.length(), expectedNumRows.intValue()); // some queries may not have any results
System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
}
@@ -609,7 +641,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
String dslQuery = "from D where a = 1";
String jsonResults = discoveryService.searchByDSL(dslQuery);
- Assert.assertNotNull(jsonResults);
+ assertNotNull(jsonResults);
JSONObject results = new JSONObject(jsonResults);
System.out.println("results = " + results);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
index 330d1cb..e748579 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
@@ -38,7 +38,7 @@ public class AttributeInfo {
public final String reverseAttributeName;
private IDataType dataType;
- AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws AtlasException {
+ public AttributeInfo(TypeSystem t, AttributeDefinition def, Map<String, IDataType> tempTypes) throws AtlasException {
this.name = def.name;
this.dataType =
(tempTypes != null && tempTypes.containsKey(def.dataTypeName)) ? tempTypes.get(def.dataTypeName) :
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bfd5f5ca/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
index 2ea90a3..ef8448d 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
@@ -21,8 +21,10 @@ package org.apache.atlas.typesystem.types.utils;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
@@ -32,6 +34,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
import scala.collection.JavaConversions;
/**
@@ -100,4 +103,15 @@ public class TypesUtil {
return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
}
+
+ private static final TypeSystem ts = TypeSystem.getInstance();
+
+ public static AttributeInfo newAttributeInfo(String attribute, IDataType type) {
+ try {
+ return new AttributeInfo(ts, new AttributeDefinition(attribute, type.getName(), Multiplicity.REQUIRED,
+ false, null), null);
+ } catch (AtlasException e) {
+ throw new RuntimeException(e);
+ }
+ }
}