You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ve...@apache.org on 2015/05/13 23:28:17 UTC
[36/50] [abbrv] incubator-atlas git commit: fixed test failures -
GraphBackedMetadataRepositoryTest#testFullTextSearch, DSLTest.test1
fixed test failures - GraphBackedMetadataRepositoryTest#testFullTextSearch, DSLTest.test1
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4b2a4cae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4b2a4cae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4b2a4cae
Branch: refs/remotes/origin/master
Commit: 4b2a4caed572fb358cddb37ce1b183072e08a0a6
Parents: 8e73ed2
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Mon May 4 14:59:11 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Mon May 4 14:59:22 2015 +0530
----------------------------------------------------------------------
.../graph/GraphBackedMetadataRepository.java | 71 +++++++++-----------
.../metadata/repository/graph/GraphHelper.java | 2 +-
.../GraphBackedMetadataRepositoryTest.java | 14 ++--
.../hadoop/metadata/tools/dsl/package.scala | 2 +-
.../hadoop/metadata/tools/dsl/DSLTest.scala | 2 +-
.../metadata/typesystem/types/TypeSystem.java | 8 +--
6 files changed, 44 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
index d6a3123..d273787 100755
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepository.java
@@ -473,9 +473,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Id id = typedInstance.getId();
Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
String fullText = getFullTextForVertex(instanceVertex, true);
- instanceVertex.setProperty(Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
- LOG.debug("Adding {} for {} = {}", Constants.ENTITY_TEXT_PROPERTY_KEY,
- instanceVertex, fullText);
+ addProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
}
}
@@ -658,8 +656,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
Map<String, AttributeInfo> fields,
Map<Id, Vertex> idToVertexMap) throws MetadataException {
- LOG.debug("Mapping instance {} to vertex {} for fields {}",
- typedInstance.getTypeName(), instanceVertex, fields);
+ LOG.debug("Mapping instance {} of {} to vertex {}",
+ typedInstance, typedInstance.getTypeName(), instanceVertex);
for (AttributeInfo attributeInfo : fields.values()) {
final IDataType dataType = attributeInfo.dataType();
mapAttributesToVertex(id, typedInstance, instanceVertex,
@@ -672,9 +670,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Map<Id, Vertex> idToVertexMap,
AttributeInfo attributeInfo,
IDataType dataType) throws MetadataException {
- LOG.debug("mapping attributeInfo {}", attributeInfo);
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
final String propertyName = getQualifiedName(typedInstance, attributeInfo);
- if (typedInstance.get(attributeInfo.name) == null) {
+ if (attrValue == null) {
return;
}
@@ -684,8 +683,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
break;
case ENUM:
- instanceVertex.setProperty(propertyName,
- typedInstance.getInt(attributeInfo.name));
+ addProperty(instanceVertex, propertyName, typedInstance.getInt(attributeInfo.name));
break;
case ARRAY:
@@ -749,7 +747,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
buffer.setLength(buffer.length() - 1);
// for dereference on way out
- instanceVertex.setProperty(propertyName, buffer.toString());
+ addProperty(instanceVertex, propertyName, buffer.toString());
}
private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance,
@@ -778,7 +776,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
buffer.setLength(buffer.length() - 1);
// for dereference on way out
- instanceVertex.setProperty(propertyName, buffer.toString());
+ addProperty(instanceVertex, propertyName, buffer.toString());
}
private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex,
@@ -792,7 +790,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
switch (elementType.getTypeCategory()) {
case PRIMITIVE:
case ENUM:
- instanceVertex.setProperty(propertyNameWithSuffix, value);
+ addProperty(instanceVertex, propertyNameWithSuffix, value);
return propertyNameWithSuffix;
case ARRAY:
@@ -851,7 +849,8 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Vertex structInstanceVertex = GraphHelper.createVertexWithoutIdentity(
titanGraph, structInstance.getTypeName(), id,
Collections.<String>emptySet()); // no super types for struct type
- LOG.debug("created vertex {} for struct {}", structInstanceVertex, attributeInfo.name);
+ LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name,
+ structInstance);
// map all the attributes to this newly created vertex
mapInstanceToVertex(id, structInstance, structInstanceVertex,
@@ -895,47 +894,43 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
private void mapPrimitiveToVertex(ITypedInstance typedInstance,
Vertex instanceVertex,
AttributeInfo attributeInfo) throws MetadataException {
- LOG.debug("Adding primitive {} to v {}", attributeInfo, instanceVertex);
- if (typedInstance.get(attributeInfo.name) == null) {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ if (attrValue == null) {
return; // add only if instance has this attribute
}
final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
-
+ Object propertyValue = null;
if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getString(attributeInfo.name));
+ propertyValue = typedInstance.getString(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getShort(attributeInfo.name));
+ propertyValue = typedInstance.getShort(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getInt(attributeInfo.name));
+ propertyValue = typedInstance.getInt(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getBigInt(attributeInfo.name));
+ propertyValue = typedInstance.getBigInt(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getBoolean(attributeInfo.name));
+ propertyValue = typedInstance.getBoolean(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getByte(attributeInfo.name));
+ propertyValue = typedInstance.getByte(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getLong(attributeInfo.name));
+ propertyValue = typedInstance.getLong(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getFloat(attributeInfo.name));
+ propertyValue = typedInstance.getFloat(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getDouble(attributeInfo.name));
+ propertyValue = typedInstance.getDouble(attributeInfo.name);
} else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- instanceVertex.setProperty(vertexPropertyName,
- typedInstance.getBigDecimal(attributeInfo.name));
+ propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
}
+ addProperty(instanceVertex, vertexPropertyName, propertyValue);
}
}
+ private void addProperty(Vertex vertex, String propertyName, Object value) {
+ LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
+ vertex.setProperty(propertyName, value);
+ }
+
public final class GraphToTypedInstanceMapper {
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid,
@@ -985,7 +980,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
public void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
AttributeInfo attributeInfo) throws MetadataException {
- LOG.debug("mapping attributeInfo = " + attributeInfo);
+ LOG.debug("mapping attributeInfo {}", attributeInfo.name);
final IDataType dataType = attributeInfo.dataType();
final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
@@ -1226,7 +1221,7 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
ITypedStruct structInstance = structType.createInstance();
typedInstance.set(attributeInfo.name, structInstance);
- String relationshipLabel = getQualifiedName(structType, attributeInfo.name);
+ String relationshipLabel = getQualifiedName(typedInstance, attributeInfo);
LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
index 970bd81..ac9e2fa 100755
--- a/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/hadoop/metadata/repository/graph/GraphHelper.java
@@ -85,7 +85,7 @@ public final class GraphHelper {
public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex,
String edgeLabel) {
- LOG.debug("Adding edge for {} -> struct label {} -> v{}",
+ LOG.debug("Adding edge for {} -> label {} -> {}",
fromVertex, edgeLabel, toVertex);
return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
index 89609db..842677b 100755
--- a/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
+++ b/repository/src/test/java/org/apache/hadoop/metadata/repository/graph/GraphBackedMetadataRepositoryTest.java
@@ -334,11 +334,6 @@ public class GraphBackedMetadataRepositoryTest {
String response = discoveryService.searchByFullText("john");
Assert.assertNotNull(response);
JSONArray results = new JSONArray(response);
- System.out.println("Found the following results");
- for (int i = 0 ; i < results.length(); i++) {
- JSONObject myrow = results.getJSONObject(i);
- System.out.println(myrow.toString());
- }
Assert.assertEquals(results.length(), 1);
JSONObject row = (JSONObject) results.get(0);
Assert.assertEquals(row.get("typeName"), "Person");
@@ -346,11 +341,10 @@ public class GraphBackedMetadataRepositoryTest {
//person in hr department who lives in santa clara
response = discoveryService.searchByFullText("Jane AND santa AND clara");
Assert.assertNotNull(response);
- // todo: enable this - temporarily commented this as its failing
-// results = new JSONArray(response);
-// Assert.assertEquals(results.length(), 1);
-// row = (JSONObject) results.get(0);
-// Assert.assertEquals(row.get("typeName"), "Manager");
+ results = new JSONArray(response);
+ Assert.assertEquals(results.length(), 1);
+ row = (JSONObject) results.get(0);
+ Assert.assertEquals(row.get("typeName"), "Manager");
//search for person in hr department whose name starts is john/jahn
response = discoveryService.searchByFullText("hr AND (john OR jahn)");
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala
index de62551..ba09342 100755
--- a/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala
+++ b/tools/src/main/scala/org/apache/hadoop/metadata/tools/dsl/package.scala
@@ -36,7 +36,7 @@ import scala.collection.JavaConversions._
package object dsl {
val defFormat = new DefaultFormats {
- override protected def dateFormatter = new SimpleDateFormat("yyyy-MM-dd")
+ override protected def dateFormatter = TypeSystem.getInstance().getDateFormat;
override val typeHints = NoTypeHints
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
----------------------------------------------------------------------
diff --git a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala b/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
index 47db8fe..1698d40 100755
--- a/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
+++ b/tools/src/test/scala/org/apache/hadoop/metadata/tools/dsl/DSLTest.scala
@@ -120,7 +120,7 @@ class DSLTest {
Assert.assertEquals(s"${i.o.asInstanceOf[java.util.Map[_, _]].keySet}", "[b, a]")
// 5. Serialize mytype instance to Json
- Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-02\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}")
+ Assert.assertEquals(s"${pretty(render(i))}", "{\n \"$typeName$\":\"mytype\",\n \"e\":1," + "\n \"n\":[1,1.100000000000000088817841970012523233890533447265625],\n \"h\":1.0,\n \"b\":true,\n \"k\":1,\n \"j\":1,\n \"d\":2,\n \"m\":[1,1],\n \"g\":1,\n \"a\":1,\n \"i\":1.0,\n \"c\":1,\n \"l\":\"2014-12-03\",\n \"f\":1,\n \"o\":{\n \"b\":2.0,\n \"a\":1.0\n }\n}")
}
@Test def test2 {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4b2a4cae/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
index c04016e..61c60dc 100755
--- a/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
+++ b/typesystem/src/main/java/org/apache/hadoop/metadata/typesystem/types/TypeSystem.java
@@ -42,10 +42,10 @@ import java.util.concurrent.ConcurrentHashMap;
@InterfaceAudience.Private
public class TypeSystem {
private static final TypeSystem INSTANCE = new TypeSystem();
- public static ThreadLocal<DateFormat> dateFormat = new ThreadLocal() {
+ private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal() {
@Override
- public DateFormat initialValue() {
- DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ public SimpleDateFormat initialValue() {
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
return dateFormat;
}
@@ -294,7 +294,7 @@ public class TypeSystem {
return eT;
}
- public DateFormat getDateFormat() {
+ public SimpleDateFormat getDateFormat() {
return dateFormat.get();
}