You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2015/12/08 07:40:07 UTC
[1/5] incubator-atlas git commit: ATLAS-47 Entity mutations for
complex types (sumasai via shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 6c3f0964b -> 51656991f
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java
new file mode 100644
index 0000000..8a28e38
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeExistsException.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.exception;
+
+import org.apache.atlas.AtlasException;
+
+public class TypeExistsException extends AtlasException {
+ public TypeExistsException(String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java
new file mode 100644
index 0000000..3654a4b
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/exception/TypeNotFoundException.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.exception;
+
+import org.apache.atlas.AtlasException;
+
+/**
+ * A simple wrapper for 404.
+ */
+public class TypeNotFoundException extends AtlasException {
+ public TypeNotFoundException() {
+ }
+
+ public TypeNotFoundException(String message) {
+ super(message);
+ }
+
+ public TypeNotFoundException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public TypeNotFoundException(Throwable cause) {
+ super(cause);
+ }
+
+ public TypeNotFoundException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
index deb15b5..d3b9a33 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/DownCastStructInstance.java
@@ -52,6 +52,11 @@ public class DownCastStructInstance implements IStruct {
fieldMapping.set(this, attrName, val);
}
+ @Override
+ public void setNull(String attrName) throws AtlasException {
+ throw new UnsupportedOperationException("unset on attributes are not allowed");
+ }
+
/*
* Use only for json serialization
* @nonpublic
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
index 641146a..20c2f91 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/Id.java
@@ -20,12 +20,16 @@ package org.apache.atlas.typesystem.persistence;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.types.FieldMapping;
+import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
import java.util.Date;
import java.util.Map;
import java.util.UUID;
@@ -37,6 +41,8 @@ public class Id implements ITypedReferenceableInstance {
public final int version;
public Id(String id, int version, String className) {
+ ParamChecker.notEmpty(className, "id");
+ ParamChecker.notEmpty(className, "className");
this.id = id;
this.className = className;
this.version = version;
@@ -248,4 +254,12 @@ public class Id implements ITypedReferenceableInstance {
public void setString(String attrName, String val) throws AtlasException {
throw new AtlasException("Get/Set not supported on an Id object");
}
+
+ @Override
+ public String getSignatureHash(MessageDigest digester) throws AtlasException {
+ digester.update(id.getBytes(Charset.forName("UTF-8")));
+ digester.update(className.getBytes(Charset.forName("UTF-8")));
+ byte[] digest = digester.digest();
+ return MD5Utils.toString(digest);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
index 911a5f4..0fa4666 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/ReferenceableInstance.java
@@ -24,10 +24,14 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.FieldMapping;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.security.MessageDigest;
import java.util.Date;
/*
@@ -75,7 +79,7 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
* @nopub
* @param id
*/
- void replaceWithNewId(Id id) {
+ public void replaceWithNewId(Id id) {
this.id = id;
}
@@ -92,4 +96,12 @@ public class ReferenceableInstance extends StructInstance implements ITypedRefer
throw new RuntimeException(me);
}
}
+
+ @Override
+ public String getSignatureHash(MessageDigest digester) throws AtlasException {
+ ClassType classType = TypeSystem.getInstance().getDataType(ClassType.class, getTypeName());
+ classType.updateSignatureHash(digester, this);
+ byte[] digest = digester.digest();
+ return MD5Utils.toString(digest);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
index ea4a3cb..16c3a24 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/persistence/StructInstance.java
@@ -29,12 +29,15 @@ import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.FieldMapping;
+import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.ValueConversionException;
+import org.apache.atlas.utils.MD5Utils;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.security.MessageDigest;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@@ -229,6 +232,30 @@ public class StructInstance implements ITypedStruct {
}
int nullPos = fieldMapping.fieldNullPos.get(attrName);
nullFlags[nullPos] = true;
+
+ int pos = fieldMapping.fieldPos.get(attrName);
+
+ if (i.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ bigIntegers[pos] = null;
+ } else if (i.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ bigDecimals[pos] = null;
+ } else if (i.dataType() == DataTypes.DATE_TYPE) {
+ dates[pos] = null;
+ } else if (i.dataType() == DataTypes.STRING_TYPE) {
+ strings[pos] = null;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
+ arrays[pos] = null;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
+ maps[pos] = null;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.STRUCT
+ || i.dataType().getTypeCategory() == DataTypes.TypeCategory.TRAIT) {
+ structs[pos] = null;
+ } else if (i.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ ids[pos] = null;
+ referenceables[pos] = null;
+ } else {
+ throw new AtlasException(String.format("Unknown datatype %s", i.dataType()));
+ }
}
/*
@@ -729,4 +756,12 @@ public class StructInstance implements ITypedStruct {
throw new RuntimeException(me);
}
}
+
+ @Override
+ public String getSignatureHash(MessageDigest digester) throws AtlasException {
+ StructType structType = TypeSystem.getInstance().getDataType(StructType.class, getTypeName());
+ structType.updateSignatureHash(digester, this);
+ byte[] digest = digester.digest();
+ return MD5Utils.toString(digest);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
index 2fea1be..29c3450 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeDefinition.java
@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.utils.ParamChecker;
public final class AttributeDefinition {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
index cdfbf07..ac758fa 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ClassType.java
@@ -34,6 +34,8 @@ import org.apache.atlas.typesystem.persistence.StructInstance;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -123,9 +125,9 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
r != null ? createInstanceWithTraits(id, r, r.getTraits().toArray(new String[0])) :
createInstance(id);
- if (id != null && id.isAssigned()) {
- return tr;
- }
+// if (id != null && id.isAssigned()) {
+// return tr;
+// }
for (Map.Entry<String, AttributeInfo> e : fieldMapping.fields.entrySet()) {
String attrKey = e.getKey();
@@ -214,4 +216,24 @@ public class ClassType extends HierarchicalType<ClassType, IReferenceableInstanc
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
+
+ @Override
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ if( !(val instanceof ITypedReferenceableInstance)) {
+ throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
+ }
+ digester.update(getName().getBytes(Charset.forName("UTF-8")));
+
+ if(fieldMapping.fields != null && val != null) {
+ IReferenceableInstance typedValue = (IReferenceableInstance) val;
+ if(fieldMapping.fields.values() != null) {
+ for (AttributeInfo aInfo : fieldMapping.fields.values()) {
+ Object attrVal = typedValue.get(aInfo.name);
+ if (attrVal != null) {
+ aInfo.dataType().updateSignatureHash(digester, attrVal);
+ }
+ }
+ }
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
index 4c55ce7..afd1682 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/DataTypes.java
@@ -29,10 +29,13 @@ import org.apache.commons.lang3.StringUtils;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
import java.text.ParseException;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
public class DataTypes {
@@ -95,6 +98,14 @@ public class DataTypes {
return nullValue();
}
+
+ @Override
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ if ( val != null ) {
+ digester.update(val.toString().getBytes(Charset.forName("UTF-8")));
+ }
+ }
+
}
public static class BooleanType extends PrimitiveType<Boolean> {
@@ -161,6 +172,13 @@ public class DataTypes {
public Byte nullValue() {
return 0;
}
+
+ @Override
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ if ( val != null ) {
+ digester.update(((Byte) val).byteValue());
+ }
+ }
}
public static class ShortType extends PrimitiveType<Short> {
@@ -508,6 +526,7 @@ public class DataTypes {
} else if (val instanceof Iterator) {
it = (Iterator) val;
}
+
if (it != null) {
ImmutableCollection.Builder b = m.isUnique ? ImmutableSet.builder() : ImmutableList.builder();
while (it.hasNext()) {
@@ -557,6 +576,15 @@ public class DataTypes {
public TypeCategory getTypeCategory() {
return TypeCategory.ARRAY;
}
+
+ @Override
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ IDataType elemType = getElemType();
+ List vals = (List) val;
+ for (Object listElem : vals) {
+ elemType.updateSignatureHash(digester, listElem);
+ }
+ }
}
public static class MapType extends AbstractDataType<ImmutableMap<?, ?>> {
@@ -586,7 +614,7 @@ public class DataTypes {
}
protected void setValueType(IDataType valueType) {
- this.keyType = valueType;
+ this.valueType = valueType;
}
@Override
@@ -605,7 +633,8 @@ public class DataTypes {
Map.Entry e = it.next();
b.put(keyType.convert(e.getKey(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
- Multiplicity.REQUIRED), valueType.convert(e.getValue(),
+ Multiplicity.REQUIRED),
+ valueType.convert(e.getValue(),
TypeSystem.getInstance().allowNullsInCollections() ? Multiplicity.OPTIONAL :
Multiplicity.REQUIRED));
}
@@ -657,6 +686,17 @@ public class DataTypes {
public TypeCategory getTypeCategory() {
return TypeCategory.MAP;
}
+
+ @Override
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ IDataType keyType = getKeyType();
+ IDataType valueType = getValueType();
+ Map vals = (Map) val;
+ for (Object key : vals.keySet()) {
+ keyType.updateSignatureHash(digester, key);
+ valueType.updateSignatureHash(digester, vals.get(key));
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
index 1439303..b751307 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
@@ -23,6 +23,9 @@ import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasException;
import scala.math.BigInt;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
+
public class EnumType extends AbstractDataType<EnumValue> {
public final TypeSystem typeSystem;
@@ -80,7 +83,7 @@ public class EnumType extends AbstractDataType<EnumValue> {
public void validateUpdate(IDataType newType) throws TypeUpdateException {
super.validateUpdate(newType);
- EnumType enumType = (EnumType)newType;
+ EnumType enumType = (EnumType) newType;
for (EnumValue enumValue : values()) {
//The old enum value should be part of new enum definition as well
if (!enumType.valueMap.containsKey(enumValue.value)) {
@@ -96,6 +99,12 @@ public class EnumType extends AbstractDataType<EnumValue> {
}
}
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ if (val != null) {
+ digester.update(fromValue((String) val).toString().getBytes(Charset.forName("UTF-8")));
+ }
+ }
+
public EnumValue fromOrdinal(int o) {
return ordinalMap.get(o);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
index 7afc52e..aca1a41 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumTypeDefinition.java
@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
index d22e7e0..d75259b 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumValue.java
@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.utils.ParamChecker;
public class EnumValue {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
index d9b1b34..293014e 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
@@ -20,6 +20,8 @@ package org.apache.atlas.typesystem.types;
import org.apache.atlas.AtlasException;
+import java.security.MessageDigest;
+
public interface IDataType<T> {
String getName();
@@ -30,5 +32,7 @@ public interface IDataType<T> {
void output(T val, Appendable buf, String prefix) throws AtlasException;
void validateUpdate(IDataType newType) throws TypeUpdateException;
+
+ void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
index a54dabc..74d7f7c 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/Multiplicity.java
@@ -25,8 +25,8 @@ public final class Multiplicity {
public static final Multiplicity OPTIONAL = new Multiplicity(0, 1, false);
public static final Multiplicity REQUIRED = new Multiplicity(1, 1, false);
- public static final Multiplicity COLLECTION = new Multiplicity(1, Integer.MAX_VALUE, false);
- public static final Multiplicity SET = new Multiplicity(1, Integer.MAX_VALUE, true);
+ public static final Multiplicity COLLECTION = new Multiplicity(0, Integer.MAX_VALUE, false);
+ public static final Multiplicity SET = new Multiplicity(0, Integer.MAX_VALUE, true);
public final int lower;
public final int upper;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
index fb4f79f..db87cf9 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/ObjectGraphWalker.java
@@ -214,5 +214,11 @@ public class ObjectGraphWalker {
this.aInfo = aInfo;
this.value = value;
}
+
+ @Override
+ public String toString(){
+ StringBuilder string = new StringBuilder().append(instance).append(aInfo).append(value);
+ return string.toString();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
index 1a40484..ba053d2 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
@@ -22,6 +22,8 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
@@ -193,6 +195,24 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
handler.output(s, buf, prefix);
}
+ @Override
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ if( !(val instanceof ITypedStruct)) {
+ throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
+ }
+ digester.update(getName().getBytes(Charset.forName("UTF-8")));
+
+ if(fieldMapping.fields != null && val != null) {
+ IStruct typedValue = (IStruct) val;
+ for (AttributeInfo aInfo : fieldMapping.fields.values()) {
+ Object attrVal = typedValue.get(aInfo.name);
+ if(attrVal != null) {
+ aInfo.dataType().updateSignatureHash(digester, attrVal);
+ }
+ }
+ }
+ }
+
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
index e4bd28d..e47c4e5 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructTypeDefinition.java
@@ -18,7 +18,7 @@
package org.apache.atlas.typesystem.types;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.utils.ParamChecker;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
index d23d247..f5a3875 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TraitType.java
@@ -23,6 +23,8 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
+import java.nio.charset.Charset;
+import java.security.MessageDigest;
import java.util.List;
import java.util.Map;
@@ -68,6 +70,24 @@ public class TraitType extends HierarchicalType<TraitType, IStruct>
}
@Override
+ public void updateSignatureHash(MessageDigest digester, Object val) throws AtlasException {
+ if( !(val instanceof ITypedStruct)) {
+ throw new IllegalArgumentException("Unexpected value type " + val.getClass().getSimpleName() + ". Expected instance of ITypedStruct");
+ }
+ digester.update(getName().getBytes(Charset.forName("UTF-8")));
+
+ if(fieldMapping.fields != null && val != null) {
+ IStruct typedValue = (IStruct) val;
+ for (AttributeInfo aInfo : fieldMapping.fields.values()) {
+ Object attrVal = typedValue.get(aInfo.name);
+ if(attrVal != null) {
+ aInfo.dataType().updateSignatureHash(digester, attrVal);
+ }
+ }
+ }
+ }
+
+ @Override
public List<String> getNames(AttributeInfo info) {
return infoToNameMap.get(info);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
index d9b1edb..c0b0698 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
@@ -22,10 +22,10 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Multimap;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.TypeExistsException;
-import org.apache.atlas.TypeNotFoundException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.exception.TypeExistsException;
+import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import javax.inject.Singleton;
import java.lang.reflect.Constructor;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
index e38772d..c5aa6e8 100755
--- a/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
+++ b/typesystem/src/main/scala/org/apache/atlas/typesystem/json/Serialization.scala
@@ -29,18 +29,22 @@ import org.json4s.native.Serialization._
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
-class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => ( {
- case JDecimal(e) => e.bigDecimal
-}, {
- case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
-}
+class BigDecimalSerializer extends CustomSerializer[java.math.BigDecimal](format => (
+ {
+ case JDecimal(e) => e.bigDecimal
+ },
+ {
+ case e: java.math.BigDecimal => JDecimal(new BigDecimal(e))
+ }
))
-class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => ( {
- case JInt(e) => e.bigInteger
-}, {
- case e: java.math.BigInteger => JInt(new BigInt(e))
-}
+class BigIntegerSerializer extends CustomSerializer[java.math.BigInteger](format => (
+ {
+ case JInt(e) => e.bigInteger
+ },
+ {
+ case e: java.math.BigInteger => JInt(new BigInt(e))
+ }
))
class IdSerializer extends CustomSerializer[Id](format => ( {
@@ -292,12 +296,19 @@ object Serialization {
read[ReferenceableInstance](jsonStr)
}
- def traitFromJson(jsonStr: String): ITypedInstance = {
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+ def traitFromJson(jsonStr: String): ITypedInstance = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ read[StructInstance](jsonStr)
+ }
- read[StructInstance](jsonStr)
- }
+ def arrayFromJson(jsonStr: String): ITypedInstance = {
+ implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
+ new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
+
+ read[StructInstance](jsonStr)
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
index 703f4ee..c735ecd 100755
--- a/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
+++ b/webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
@@ -86,7 +86,7 @@ public class AuditFilter implements Filter {
LOG.debug("Audit: {}/{} performed request {} {} ({}) at time {}", who, fromAddress, whatRequest, whatURL,
whatAddrs, whenISO9601);
- audit(who, fromAddress, fromHost, whatURL, whatAddrs, whenISO9601);
+ audit(who, fromAddress, whatRequest, fromHost, whatURL, whatAddrs, whenISO9601);
}
private String getUserFromRequest(HttpServletRequest httpRequest) {
@@ -95,9 +95,9 @@ public class AuditFilter implements Filter {
return userFromRequest == null ? "UNKNOWN" : userFromRequest;
}
- private void audit(String who, String fromAddress, String fromHost, String whatURL, String whatAddrs,
+ private void audit(String who, String fromAddress, String whatRequest, String fromHost, String whatURL, String whatAddrs,
String whenISO9601) {
- AUDIT_LOG.info("Audit: {}/{}-{} performed request {} ({}) at time {}", who, fromAddress, fromHost, whatURL,
+ AUDIT_LOG.info("Audit: {}/{}-{} performed request {} {} ({}) at time {}", who, fromAddress, fromHost, whatRequest, whatURL,
whatAddrs, whenISO9601);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
index 21ea05f..2ee0027 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/EntityResource.java
@@ -21,11 +21,13 @@ package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.ParamChecker;
-import org.apache.atlas.TypeNotFoundException;
-import org.apache.atlas.repository.EntityExistsException;
-import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.typesystem.exception.EntityExistsException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.exception.TypeNotFoundException;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.lang.StringUtils;
@@ -85,7 +87,6 @@ public class EntityResource {
this.metadataService = metadataService;
}
-
/**
* Submits the entity definitions (instances).
* The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any
@@ -138,6 +139,175 @@ public class EntityResource {
}
/**
+ * Complete update of a set of entities - the values not specified will be replaced with null/removed
+ * Adds/Updates given entities identified by its GUID or unique attribute
+ * @return response payload as json
+ */
+ @PUT
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response updateEntities(@Context HttpServletRequest request) {
+ try {
+ final String entities = Servlets.getRequestPayload(request);
+ LOG.debug("updating entities {} ", AtlasClient.toString(new JSONArray(entities)));
+
+ final String guids = metadataService.updateEntities(entities);
+
+ JSONObject response = new JSONObject();
+ response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
+ response.put(AtlasClient.GUID, new JSONArray(guids));
+ response.put(AtlasClient.DEFINITION, metadataService.getEntityDefinition(new JSONArray(guids).getString(0)));
+
+ return Response.ok(response).build();
+ } catch(EntityExistsException e) {
+ LOG.error("Unique constraint violation", e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
+ } catch (ValueConversionException ve) {
+ LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
+ throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
+ } catch (AtlasException | IllegalArgumentException e) {
+ LOG.error("Unable to persist entity instance", e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to persist entity instance", e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Adds/Updates given entity identified by its unique attribute( entityType, attributeName and value)
+ * Updates support only partial update of an entity - Adds/updates any new values specified
+ * Updates do not support removal of attribute values
+ *
+ * @param entityType the entity type
+ * @param attribute the unique attribute used to identify the entity
+ * @param value the unique attributes value
+ * @param request The updated entity json
+ * @return response payload as json
+ * The body contains the JSONArray of entity json. The service takes care of de-duping the entities based on any
+ * unique attribute for the give type.
+ */
+ @POST
+ @Path("qualifiedName")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response updateByUniqueAttribute(@QueryParam("type") String entityType,
+ @QueryParam("property") String attribute,
+ @QueryParam("value") String value, @Context HttpServletRequest request) {
+ try {
+ String entities = Servlets.getRequestPayload(request);
+
+ LOG.debug("Partially updating entity by unique attribute {} {} {} {} ", entityType, attribute, value, entities);
+
+ Referenceable updatedEntity =
+ InstanceSerialization.fromJsonReferenceable(entities, true);
+ final String guid = metadataService.updateEntityByUniqueAttribute(entityType, attribute, value, updatedEntity);
+
+ JSONObject response = new JSONObject();
+ response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
+ response.put(AtlasClient.GUID, guid);
+ return Response.ok(response).build();
+ } catch (ValueConversionException ve) {
+ LOG.error("Unable to persist entity instance due to a desrialization error ", ve);
+ throw new WebApplicationException(Servlets.getErrorResponse(ve.getCause(), Response.Status.BAD_REQUEST));
+ } catch(EntityExistsException e) {
+ LOG.error("Unique constraint violation", e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with type={} and qualifiedName={} does not exist", entityType, value, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (AtlasException | IllegalArgumentException e) {
+ LOG.error("Unable to create/update entity {}" + entityType + ":" + attribute + "." + value, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to update entity {}" + entityType + ":" + attribute + "." + value, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Updates entity identified by its GUID
+ * Support Partial update of an entity - Adds/updates any new values specified
+ * Does not support removal of attribute values
+ *
+ * @param guid
+ * @param request The updated entity json
+ * @return
+ */
+ @POST
+ @Path("{guid}")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public Response updateEntityByGuid(@PathParam("guid") String guid, @QueryParam("property") String attribute,
+ @Context HttpServletRequest request) {
+ if (StringUtils.isEmpty(attribute)) {
+ return updateEntityPartialByGuid(guid, request);
+ } else {
+ return updateEntityAttributeByGuid(guid, attribute, request);
+ }
+ }
+
+ private Response updateEntityPartialByGuid(String guid, HttpServletRequest request) {
+ try {
+ ParamChecker.notEmpty(guid, "Guid property cannot be null");
+ final String entityJson = Servlets.getRequestPayload(request);
+ LOG.debug("partially updating entity for guid {} : {} ", guid, entityJson);
+
+ Referenceable updatedEntity =
+ InstanceSerialization.fromJsonReferenceable(entityJson, true);
+ metadataService.updateEntityPartialByGuid(guid, updatedEntity);
+
+ JSONObject response = new JSONObject();
+ response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with GUID={} does not exist", guid, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (AtlasException | IllegalArgumentException e) {
+ LOG.error("Unable to update entity {}", guid, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to update entity {}", guid, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
+ * Supports Partial updates
+ * Adds/Updates given entity specified by its GUID
+ * Supports updation of only simple primitive attributes like strings, ints, floats, enums, class references and
+ * does not support updation of complex types like arrays, maps
+ * @param guid entity id
+ * @param property property to add
+ * @postbody property's value
+ * @return response payload as json
+ */
+ private Response updateEntityAttributeByGuid(String guid, String property, HttpServletRequest request) {
+ try {
+ Preconditions.checkNotNull(property, "Entity property cannot be null");
+ String value = Servlets.getRequestPayload(request);
+ Preconditions.checkNotNull(value, "Entity value cannot be null");
+
+ metadataService.updateEntityAttributeByGuid(guid, property, value);
+
+ JSONObject response = new JSONObject();
+ response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
+ response.put(AtlasClient.GUID, guid);
+
+ return Response.ok(response).build();
+ } catch (EntityNotFoundException e) {
+ LOG.error("An entity with GUID={} does not exist", guid, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
+ } catch (AtlasException | IllegalArgumentException e) {
+ LOG.error("Unable to add property {} to entity id {}", property, guid, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
+ } catch (Throwable e) {
+ LOG.error("Unable to add property {} to entity id {}", property, guid, e);
+ throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
+ }
+ }
+
+ /**
* Fetch the complete definition of an entity given its GUID.
*
* @param guid GUID for the entity
@@ -265,39 +435,6 @@ public class EntityResource {
}
}
- /**
- * Adds property to the given entity id
- * @param guid entity id
- * @param property property to add
- * @param value property's value
- * @return response payload as json
- */
- @PUT
- @Path("{guid}")
- @Consumes(Servlets.JSON_MEDIA_TYPE)
- @Produces(Servlets.JSON_MEDIA_TYPE)
- public Response update(@PathParam("guid") String guid, @QueryParam("property") String property,
- @QueryParam("value") String value) {
- try {
- Preconditions.checkNotNull(property, "Entity property cannot be null");
- Preconditions.checkNotNull(value, "Entity value cannot be null");
-
- metadataService.updateEntity(guid, property, value);
-
- JSONObject response = new JSONObject();
- response.put(AtlasClient.REQUEST_ID, Thread.currentThread().getName());
- return Response.ok(response).build();
- } catch (EntityNotFoundException e) {
- LOG.error("An entity with GUID={} does not exist", guid, e);
- throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
- } catch (AtlasException | IllegalArgumentException e) {
- LOG.error("Unable to add property {} to entity id {}", property, guid, e);
- throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
- } catch (Throwable e) {
- LOG.error("Unable to add property {} to entity id {}", property, guid, e);
- throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
- }
- }
// Trait management functions
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
index 7bcaf6b..9b3fbc9 100644
--- a/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/HiveLineageResource.java
@@ -19,10 +19,10 @@
package org.apache.atlas.web.resources;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.LineageService;
-import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
index 35f171f..45502b7 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/MetadataDiscoveryResource.java
@@ -20,7 +20,7 @@ package org.apache.atlas.web.resources;
import com.google.common.base.Preconditions;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.DiscoveryException;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.web.util.Servlets;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
index 1f0b98a..3b90248 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
@@ -21,8 +21,8 @@ package org.apache.atlas.web.resources;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.TypeExistsException;
import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.web.util.Servlets;
import org.codehaus.jettison.json.JSONArray;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java b/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
index 3d4c715..8c6b616 100755
--- a/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
+++ b/webapp/src/main/java/org/apache/atlas/web/util/Servlets.java
@@ -19,7 +19,7 @@
package org.apache.atlas.web.util;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
index 6929961..4d2cce7 100644
--- a/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
+++ b/webapp/src/test/java/org/apache/atlas/notification/EntityNotificationIT.java
@@ -118,7 +118,7 @@ public class EntityNotificationIT extends BaseResourceIT {
final String guid = tableId._getId();
- serviceClient.updateEntity(guid, property, newValue);
+ serviceClient.updateEntityAttribute(guid, property, newValue);
waitForNotification(MAX_WAIT_TIME);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
index e03f618..e5af26c 100644
--- a/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
+++ b/webapp/src/test/java/org/apache/atlas/notification/NotificationHookConsumerIT.java
@@ -30,7 +30,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@Guice(modules = NotificationModule.class)
-public class NotificationHookConsumerIT extends BaseResourceIT{
+public class NotificationHookConsumerIT extends BaseResourceIT {
@Inject
private NotificationInterface kafka;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
index 291ef48..361cece 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/BaseResourceIT.java
@@ -42,6 +42,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.web.util.Servlets;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.RandomStringUtils;
@@ -83,7 +84,11 @@ public abstract class BaseResourceIT {
protected void createType(TypesDef typesDef) throws Exception {
HierarchicalTypeDefinition<ClassType> sampleType = typesDef.classTypesAsJavaList().get(0);
- if (serviceClient.getType(sampleType.typeName) == null) {
+ try {
+ serviceClient.getType(sampleType.typeName);
+ LOG.info("Types already exist. Skipping type creation");
+ } catch(AtlasServiceException ase) {
+ //Expected if type doesnt exist
String typesAsJSON = TypesSerialization.toJson(typesDef);
createType(typesAsJSON);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
index 380f280..7337eaf 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/EntityJerseyResourceIT.java
@@ -52,7 +52,10 @@ import org.testng.annotations.Test;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.Response;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
/**
@@ -174,7 +177,6 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmitEntityWithBadDateFormat() throws Exception {
-
try {
Referenceable tableInstance = createHiveTableInstance("db" + randomString(), "table" + randomString());
tableInstance.set("lastAccessTime", "2014-07-11");
@@ -191,8 +193,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
final String guid = tableId._getId();
//add property
String description = "bar table - new desc";
- ClientResponse clientResponse = addProperty(guid, "description", description);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+ addProperty(guid, "description", description);
String entityRef = getEntityDefinition(getEntityDefinition(guid));
Assert.assertNotNull(entityRef);
@@ -200,13 +201,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
tableInstance.set("description", description);
//invalid property for the type
- clientResponse = addProperty(guid, "invalid_property", "bar table");
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.BAD_REQUEST.getStatusCode());
+ try {
+ addProperty(guid, "invalid_property", "bar table");
+ Assert.fail("Expected AtlasServiceException");
+ } catch (AtlasServiceException e) {
+ Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
+ }
//non-string property, update
String currentTime = String.valueOf(System.currentTimeMillis());
- clientResponse = addProperty(guid, "createTime", currentTime);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+ addProperty(guid, "createTime", currentTime);
entityRef = getEntityDefinition(getEntityDefinition(guid));
Assert.assertNotNull(entityRef);
@@ -222,12 +226,16 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Assert.fail();
}
- @Test(dependsOnMethods = "testSubmitEntity", expectedExceptions = IllegalArgumentException.class)
+ @Test(dependsOnMethods = "testSubmitEntity")
public void testAddNullPropertyValue() throws Exception {
final String guid = tableId._getId();
//add property
- addProperty(guid, "description", null);
- Assert.fail();
+ try {
+ addProperty(guid, "description", null);
+ Assert.fail("Expected AtlasServiceException");
+ } catch(AtlasServiceException e) {
+ Assert.assertEquals(e.getStatus().getStatusCode(), Response.Status.BAD_REQUEST.getStatusCode());
+ }
}
@Test(dependsOnMethods = "testSubmitEntity")
@@ -242,8 +250,7 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
//Add reference property
final String guid = tableId._getId();
- ClientResponse clientResponse = addProperty(guid, "db", dbId);
- Assert.assertEquals(clientResponse.getStatus(), Response.Status.OK.getStatusCode());
+ addProperty(guid, "db", dbId);
}
@Test(dependsOnMethods = "testSubmitEntity")
@@ -264,11 +271,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
InstanceSerialization.fromJsonReferenceable(definition, true);
}
- private ClientResponse addProperty(String guid, String property, String value) {
- WebResource resource = service.path(ENTITIES).path(guid);
-
- return resource.queryParam("property", property).queryParam("value", value).accept(Servlets.JSON_MEDIA_TYPE)
- .type(Servlets.JSON_MEDIA_TYPE).method(HttpMethod.PUT, ClientResponse.class);
+ private void addProperty(String guid, String property, String value) throws AtlasServiceException {
+ serviceClient.updateEntityAttribute(guid, property, value);
}
private ClientResponse getEntityDefinition(String guid) {
@@ -547,4 +551,82 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
Assert.assertEquals(getReferenceable.get(attrName), attrValue);
}
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testPartialUpdate() throws Exception {
+ final List<Referenceable> columns = new ArrayList<>();
+ Map<String, Object> values = new HashMap<>();
+ values.put("name", "col1");
+ values.put("dataType", "string");
+ values.put("comment", "col1 comment");
+
+ Referenceable ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
+ columns.add(ref);
+ Referenceable tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
+ put("columns", columns);
+ }});
+
+ LOG.debug("Updating entity= " + tableUpdated);
+ serviceClient.updateEntity(tableId._getId(), tableUpdated);
+
+ ClientResponse response = getEntityDefinition(tableId._getId());
+ String definition = getEntityDefinition(response);
+ Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
+ List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns");
+
+ Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
+
+ //Update by unique attribute
+ values.put("dataType", "int");
+ ref = new Referenceable(BaseResourceIT.COLUMN_TYPE, values);
+ columns.set(0, ref);
+ tableUpdated = new Referenceable(BaseResourceIT.HIVE_TABLE_TYPE, new HashMap<String, Object>() {{
+ put("columns", columns);
+ }});
+
+ LOG.debug("Updating entity= " + tableUpdated);
+ serviceClient.updateEntity(BaseResourceIT.HIVE_TABLE_TYPE, "name", (String) tableInstance.get("name"),
+ tableUpdated);
+
+ response = getEntityDefinition(tableId._getId());
+ definition = getEntityDefinition(response);
+ getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
+ refs = (List<Referenceable>) getReferenceable.get("columns");
+
+ Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
+ Assert.assertEquals(refs.get(0).get("dataType"), "int");
+
+ }
+
+ @Test(dependsOnMethods = "testSubmitEntity")
+ public void testCompleteUpdate() throws Exception {
+ final List<Referenceable> columns = new ArrayList<>();
+ Map<String, Object> values1 = new HashMap<>();
+ values1.put("name", "col3");
+ values1.put("dataType", "string");
+ values1.put("comment", "col3 comment");
+
+ Map<String, Object> values2 = new HashMap<>();
+ values2.put("name", "col4");
+ values2.put("dataType", "string");
+ values2.put("comment", "col4 comment");
+
+ Referenceable ref1 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values1);
+ Referenceable ref2 = new Referenceable(BaseResourceIT.COLUMN_TYPE, values2);
+ columns.add(ref1);
+ columns.add(ref2);
+ tableInstance.set("columns", columns);
+
+ LOG.debug("Replacing entity= " + tableInstance);
+ serviceClient.updateEntities(tableInstance);
+
+ ClientResponse response = getEntityDefinition(tableId._getId());
+ String definition = getEntityDefinition(response);
+ Referenceable getReferenceable = InstanceSerialization.fromJsonReferenceable(definition, true);
+ List<Referenceable> refs = (List<Referenceable>) getReferenceable.get("columns");
+ Assert.assertEquals(refs.size(), 2);
+
+ Assert.assertTrue(refs.get(0).equalsContents(columns.get(0)));
+ Assert.assertTrue(refs.get(1).equalsContents(columns.get(1)));
+ }
}
[4/5] incubator-atlas git commit: ATLAS-47 Entity mutations for
complex types (sumasai via shwethags)
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index 6a620b5..fe1e576 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -19,11 +19,7 @@
package org.apache.atlas.repository.graph;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.thinkaurelius.titan.core.SchemaViolationException;
import com.thinkaurelius.titan.core.TitanGraph;
-import com.thinkaurelius.titan.core.TitanProperty;
-import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
@@ -31,42 +27,25 @@ import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.EntityExistsException;
-import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
-import org.apache.atlas.typesystem.IReferenceableInstance;
-import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
-import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.exception.EntityExistsException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumValue;
-import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.ObjectGraphWalker;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
-import java.math.BigDecimal;
-import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.List;
-import java.util.Map;
/**
* An implementation backed by a Graph database provided
@@ -76,19 +55,19 @@ import java.util.Map;
public class GraphBackedMetadataRepository implements MetadataRepository {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
- private static final String FULL_TEXT_DELIMITER = " ";
- private static final String EDGE_LABEL_PREFIX = "__";
- private final TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper();
- private final GraphToTypedInstanceMapper graphToInstanceMapper = new GraphToTypedInstanceMapper();
+ private final GraphToTypedInstanceMapper graphToInstanceMapper;
+
+ private static TypeSystem typeSystem = TypeSystem.getInstance();
+
+ private static final GraphHelper graphHelper = GraphHelper.getInstance();
- private final TypeSystem typeSystem;
private final TitanGraph titanGraph;
@Inject
- public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) throws AtlasException {
- this.typeSystem = TypeSystem.getInstance();
+ public GraphBackedMetadataRepository(GraphProvider<TitanGraph> graphProvider) {
this.titanGraph = graphProvider.get();
+ this.graphToInstanceMapper = new GraphToTypedInstanceMapper(titanGraph);
}
public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
@@ -116,42 +95,31 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
public String getTraitLabel(IDataType<?> dataType, String traitName) {
- return getTraitLabel(dataType.getName(), traitName);
+ return GraphHelper.getTraitLabel(dataType.getName(), traitName);
}
@Override
public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
- return getQualifiedName(dataType, aInfo.name);
+ return GraphHelper.getQualifiedFieldName(dataType, aInfo.name);
}
public String getFieldNameInVertex(IDataType<?> dataType, String attrName) throws AtlasException {
- return getQualifiedName(dataType, attrName);
- }
- @Override
- public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
- return getEdgeLabel(dataType.getName(), aInfo.name);
+ return GraphHelper.getQualifiedFieldName(dataType, attrName);
}
- public String getEdgeLabel(String typeName, String attrName) {
- return EDGE_LABEL_PREFIX + typeName + "." + attrName;
- }
-
- public String getTraitLabel(String typeName, String attrName) {
- return typeName + "." + attrName;
- }
-
- public String getEdgeLabel(ITypedInstance typedInstance, AttributeInfo aInfo) throws AtlasException {
- IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
- return getEdgeLabel(dataType, aInfo);
+ @Override
+ public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
+ return GraphHelper.getEdgeLabel(dataType, aInfo);
}
@Override
@GraphTransaction
public String[] createEntities(ITypedReferenceableInstance... entities) throws RepositoryException,
- EntityExistsException {
+ EntityExistsException {
LOG.info("adding entities={}", entities);
try {
- return instanceToGraphMapper.mapTypedInstanceToGraph(entities);
+ TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
+ return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities);
} catch (EntityExistsException e) {
throw e;
} catch (AtlasException e) {
@@ -161,10 +129,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
- public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException {
+ public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException {
LOG.info("Retrieving entity with guid={}", guid);
- Vertex instanceVertex = getVertexForGUID(guid);
+ Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
try {
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
@@ -175,35 +143,17 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
- public ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value)
+ public ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value)
throws AtlasException {
LOG.info("Retrieving entity with type={} and {}={}", entityType, attribute, value);
IDataType type = typeSystem.getDataType(IDataType.class, entityType);
String propertyKey = getFieldNameInVertex(type, attribute);
- Vertex instanceVertex = getVertexForProperty(propertyKey, value);
+ Vertex instanceVertex = graphHelper.getVertexForProperty(propertyKey, value);
String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
}
-
- private Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
- return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid);
- }
-
- private Vertex getVertexForProperty(String propertyKey, Object value) throws EntityNotFoundException {
- Vertex instanceVertex = GraphHelper.findVertex(titanGraph, propertyKey, value);
- if (instanceVertex == null) {
- LOG.debug("Could not find a vertex with {}={}", propertyKey, value);
- throw new EntityNotFoundException("Could not find an entity in the repository with " + propertyKey + "="
- + value);
- } else {
- LOG.debug("Found a vertex {} with {}={}", instanceVertex, propertyKey, value);
- }
-
- return instanceVertex;
- }
-
@Override
@GraphTransaction
public List<String> getEntityList(String entityType) throws RepositoryException {
@@ -234,18 +184,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@GraphTransaction
public List<String> getTraitNames(String guid) throws AtlasException {
LOG.info("Retrieving trait names for entity={}", guid);
- Vertex instanceVertex = getVertexForGUID(guid);
- return getTraitNames(instanceVertex);
+ Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
+ return GraphHelper.getTraitNames(instanceVertex);
}
- public List<String> getTraitNames(Vertex entityVertex) {
- ArrayList<String> traits = new ArrayList<>();
- for (TitanProperty property : ((TitanVertex) entityVertex).getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
- traits.add((String) property.getValue());
- }
-
- return traits;
- }
/**
* Adds a new trait to an existing entity represented by a guid.
@@ -262,14 +204,15 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
LOG.info("Adding a new trait={} for entity={}", traitName, guid);
try {
- Vertex instanceVertex = getVertexForGUID(guid);
+ Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
// add the trait instance as a new vertex
- final String typeName = getTypeName(instanceVertex);
+ final String typeName = GraphHelper.getTypeName(instanceVertex);
+
+ TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
+ instanceToGraphMapper.mapTraitInstanceToVertex(traitInstance,
+ typeSystem.getDataType(ClassType.class, typeName), instanceVertex);
- instanceToGraphMapper
- .mapTraitInstanceToVertex(traitInstance, getIdFromVertex(typeName, instanceVertex), typeSystem.getDataType(ClassType.class, typeName),
- instanceVertex, Collections.<Id, Vertex>emptyMap());
// update the traits in entity once adding trait instance is successful
GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
@@ -290,19 +233,19 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
*/
@Override
@GraphTransaction
- public void deleteTrait(String guid, String traitNameToBeDeleted) throws RepositoryException {
+ public void deleteTrait(String guid, String traitNameToBeDeleted) throws EntityNotFoundException, RepositoryException {
LOG.info("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
try {
- Vertex instanceVertex = getVertexForGUID(guid);
+ Vertex instanceVertex = graphHelper.getVertexForGUID(guid);
- List<String> traitNames = getTraitNames(instanceVertex);
+ List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
if (!traitNames.contains(traitNameToBeDeleted)) {
throw new EntityNotFoundException(
"Could not find trait=" + traitNameToBeDeleted + " in the repository for entity: " + guid);
}
- final String entityTypeName = getTypeName(instanceVertex);
- String relationshipLabel = getTraitLabel(entityTypeName, traitNameToBeDeleted);
+ final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
+ String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted);
Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
if (results.hasNext()) { // there should only be one edge for this label
final Edge traitEdge = results.next();
@@ -319,8 +262,6 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
updateTraits(instanceVertex, traitNames);
}
}
- } catch (RepositoryException e) {
- throw e;
} catch (Exception e) {
throw new RepositoryException(e);
}
@@ -338,961 +279,26 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
@Override
@GraphTransaction
- public void updateEntity(String guid, String property, String value) throws RepositoryException {
- LOG.info("Adding property {} for entity guid {}", property, guid);
-
+ public String[] updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException {
+ LOG.info("updating entity {}", entitiesUpdated);
try {
- Vertex instanceVertex = getVertexForGUID(guid);
-
- LOG.debug("Found a vertex {} for guid {}", instanceVertex, guid);
- String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
- ClassType type = typeSystem.getDataType(ClassType.class, typeName);
- AttributeInfo attributeInfo = type.fieldMapping.fields.get(property);
- if (attributeInfo == null) {
- throw new AtlasException("Invalid property " + property + " for entity " + typeName);
- }
-
- DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
- ITypedReferenceableInstance instance = type.createInstance();
- if (attrTypeCategory == DataTypes.TypeCategory.PRIMITIVE) {
- instance.set(property, value);
- } else if (attrTypeCategory == DataTypes.TypeCategory.CLASS) {
-
- // Disconnect any existing reference to the previous reference target.
- disconnectClassReference(instanceVertex, attributeInfo, instance);
-
- Id id = new Id(value, 0, attributeInfo.dataType().getName());
- instance.set(property, id);
- } else {
- throw new RepositoryException("Update of " + attrTypeCategory + " is not supported");
- }
-
- instanceToGraphMapper
- .mapAttributesToVertex(getIdFromVertex(typeName, instanceVertex), instance, instanceVertex,
- new HashMap<Id, Vertex>(), attributeInfo, attributeInfo.dataType());
- } catch (RepositoryException e) {
- throw e;
- } catch (Exception e) {
+ TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
+ return instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
+ entitiesUpdated);
+ } catch (AtlasException e) {
throw new RepositoryException(e);
}
}
- private void disconnectClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
- ITypedReferenceableInstance instance) throws AtlasException {
-
- String edgeLabel = getEdgeLabel(instance, attributeInfo);
- Iterable<Edge> edges = instanceVertex.getEdges(Direction.OUT, edgeLabel);
- if (edges != null) {
- Iterator<Edge> it = edges.iterator();
- if (it.hasNext()) {
- titanGraph.removeEdge(it.next());
- }
- }
- }
-
- public Id getIdFromVertex(String dataTypeName, Vertex vertex) {
- return new Id(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
- vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName);
- }
-
- String getTypeName(Vertex instanceVertex) {
- return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
- }
-
-
- String getQualifiedName(ITypedInstance typedInstance, AttributeInfo attributeInfo) throws AtlasException {
- IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
- return getQualifiedName(dataType, attributeInfo.name);
- }
-
- public static String getQualifiedName(IDataType dataType, String attributeName) throws AtlasException {
- return dataType.getTypeCategory() == DataTypes.TypeCategory.STRUCT ? dataType.getName() + "." + attributeName
- // else class or trait
- : ((HierarchicalType) dataType).getQualifiedName(attributeName);
- }
-
- private final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
-
- public final Map<Id, IReferenceableInstance> idToInstanceMap;
- public final Map<Id, Vertex> idToVertexMap;
-
- public EntityProcessor() {
- idToInstanceMap = new LinkedHashMap<>();
- idToVertexMap = new HashMap<>();
- }
-
- public void cleanUp() {
- idToInstanceMap.clear();
- }
-
- @Override
- public void processNode(ObjectGraphWalker.Node nd) throws AtlasException {
- IReferenceableInstance ref = null;
- Id id = null;
-
- if (nd.attributeName == null) {
- ref = (IReferenceableInstance) nd.instance;
- id = ref.getId();
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (nd.value != null && (nd.value instanceof Id)) {
- id = (Id) nd.value;
- }
- }
-
- if (id != null) {
- if (id.isUnassigned()) {
- if (ref != null) {
- if (idToInstanceMap.containsKey(id)) { // Oops
- throw new RepositoryException(
- String.format("Unexpected internal error: Id %s processed again", id));
- }
-
- idToInstanceMap.put(id, ref);
- }
- }
- }
- }
-
- private List<ITypedReferenceableInstance> createVerticesForClassType(
- List<ITypedReferenceableInstance> typedInstances) throws AtlasException {
-
- List<ITypedReferenceableInstance> instancesCreated = new ArrayList<>();
- for (ITypedReferenceableInstance typedInstance : typedInstances) {
- final Id id = typedInstance.getId();
- if (!idToVertexMap.containsKey(id)) {
- Vertex instanceVertex;
- if (id.isAssigned()) { // has a GUID
- instanceVertex = getVertexForGUID(id.id);
- } else {
- //Check if there is already an instance with the same unique attribute value
- ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
- instanceVertex = instanceToGraphMapper.getVertexForInstanceByUniqueAttribute(classType, typedInstance);
- if (instanceVertex == null) {
- instanceVertex = GraphHelper.createVertexWithIdentity(titanGraph, typedInstance,
- classType.getAllSuperTypeNames());
- instancesCreated.add(typedInstance);
-
- instanceToGraphMapper.mapInstanceToVertex(id, typedInstance, instanceVertex,
- classType.fieldMapping().fields, idToVertexMap, true);
-
- }
- }
-
- idToVertexMap.put(id, instanceVertex);
- }
- }
- return instancesCreated;
- }
- }
-
- private final class TypedInstanceToGraphMapper {
-
- private String[] mapTypedInstanceToGraph(ITypedReferenceableInstance[] typedInstances)
- throws AtlasException {
- EntityProcessor entityProcessor = new EntityProcessor();
- List<String> guids = new ArrayList<>();
- for (ITypedReferenceableInstance typedInstance : typedInstances) {
- try {
- LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
- entityProcessor.cleanUp();
- new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
- } catch (AtlasException me) {
- throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
- }
-
- List<ITypedReferenceableInstance> newTypedInstances = discoverInstances(entityProcessor);
- List<ITypedReferenceableInstance> instancesCreated =
- entityProcessor.createVerticesForClassType(newTypedInstances);
-
- for (ITypedReferenceableInstance instance : instancesCreated) {
- try {
- //new vertex, set all the properties
- addDiscoveredInstance(entityProcessor, instance);
- } catch(SchemaViolationException e) {
- throw new EntityExistsException(typedInstance, e);
- }
- }
-
- addFullTextProperty(entityProcessor, instancesCreated);
-
- //Return guid for
- Vertex instanceVertex = entityProcessor.idToVertexMap.get(typedInstance.getId());
- String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- guids.add(guid);
- }
- return guids.toArray(new String[guids.size()]);
- }
-
- private void addFullTextProperty(EntityProcessor entityProcessor,
- List<ITypedReferenceableInstance> newTypedInstances) throws AtlasException {
-
- for (ITypedReferenceableInstance typedInstance : newTypedInstances) { // Traverse
- Id id = typedInstance.getId();
- Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
- String fullText = getFullTextForVertex(instanceVertex, true);
- GraphHelper.setProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
- }
- }
-
- private String getFullTextForVertex(Vertex instanceVertex, boolean followReferences) throws AtlasException {
- String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- ITypedReferenceableInstance typedReference =
- graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
- String fullText = getFullTextForInstance(typedReference, followReferences);
- StringBuilder fullTextBuilder =
- new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
-
- List<String> traits = typedReference.getTraits();
- for (String traitName : traits) {
- String traitText = getFullTextForInstance((ITypedInstance) typedReference.getTrait(traitName), false);
- fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER)
- .append(traitText);
- }
- return fullTextBuilder.toString();
- }
-
- private String getFullTextForAttribute(IDataType type, Object value, boolean followReferences)
- throws AtlasException {
- switch (type.getTypeCategory()) {
- case PRIMITIVE:
- return String.valueOf(value);
-
- case ENUM:
- return ((EnumValue) value).value;
-
- case ARRAY:
- StringBuilder fullText = new StringBuilder();
- IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
- List list = (List) value;
-
- for (Object element : list) {
- String elemFullText = getFullTextForAttribute(elemType, element, false);
- if (StringUtils.isNotEmpty(elemFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
- }
- }
- return fullText.toString();
-
- case MAP:
- fullText = new StringBuilder();
- IDataType keyType = ((DataTypes.MapType) type).getKeyType();
- IDataType valueType = ((DataTypes.MapType) type).getValueType();
- Map map = (Map) value;
-
- for (Object entryObj : map.entrySet()) {
- Map.Entry entry = (Map.Entry) entryObj;
- String keyFullText = getFullTextForAttribute(keyType, entry.getKey(), false);
- if (StringUtils.isNotEmpty(keyFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
- }
- String valueFullText = getFullTextForAttribute(valueType, entry.getValue(), false);
- if (StringUtils.isNotEmpty(valueFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
- }
- }
- return fullText.toString();
-
- case CLASS:
- if (followReferences) {
- String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
- Vertex refVertex = getVertexForGUID(refGuid);
- return getFullTextForVertex(refVertex, false);
- }
- break;
-
- case STRUCT:
- if (followReferences) {
- return getFullTextForInstance((ITypedInstance) value, false);
- }
- break;
-
- default:
- throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
-
- }
- return null;
- }
-
- private String getFullTextForInstance(ITypedInstance typedInstance, boolean followReferences)
- throws AtlasException {
- StringBuilder fullText = new StringBuilder();
- for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
- Object attrValue = typedInstance.get(attributeInfo.name);
- if (attrValue == null) {
- continue;
- }
-
- String attrFullText = getFullTextForAttribute(attributeInfo.dataType(), attrValue, followReferences);
- if (StringUtils.isNotEmpty(attrFullText)) {
- fullText =
- fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER)
- .append(attrFullText);
- }
- }
- return fullText.toString();
- }
-
- /**
- * Step 2: Traverse oldIdToInstance map create newInstances :
- * List[ITypedReferenceableInstance]
- * - create a ITypedReferenceableInstance.
- * replace any old References ( ids or object references) with new Ids.
- */
- private List<ITypedReferenceableInstance> discoverInstances(EntityProcessor entityProcessor)
- throws RepositoryException {
- List<ITypedReferenceableInstance> newTypedInstances = new ArrayList<>();
- for (IReferenceableInstance transientInstance : entityProcessor.idToInstanceMap.values()) {
- LOG.debug("Discovered instance {}", transientInstance.getTypeName());
- try {
- ClassType cT = typeSystem.getDataType(ClassType.class, transientInstance.getTypeName());
- ITypedReferenceableInstance newInstance = cT.convert(transientInstance, Multiplicity.REQUIRED);
- newTypedInstances.add(newInstance);
- } catch (AtlasException me) {
- throw new RepositoryException(
- String.format("Failed to create Instance(id = %s", transientInstance.getId()), me);
- }
- }
-
- //Reverse the list to create the entities in dependency order
- return Lists.reverse(newTypedInstances);
- }
-
- /**
- * For the given type, finds an unique attribute and checks if there is an existing instance with the same
- * unique value
- * @param classType
- * @param instance
- * @return
- * @throws AtlasException
- */
- Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance)
- throws AtlasException {
- for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
- if (attributeInfo.isUnique) {
- String propertyKey = getFieldNameInVertex(classType, attributeInfo);
- try {
- return getVertexForProperty(propertyKey, instance.get(attributeInfo.name));
- } catch(EntityNotFoundException e) {
- //Its ok if there is no entity with the same unique value
- }
- }
- }
-
- return null;
- }
-
- private void addDiscoveredInstance(EntityProcessor entityProcessor, ITypedReferenceableInstance typedInstance)
- throws AtlasException {
- LOG.debug("Adding typed instance {}", typedInstance.getTypeName());
-
- Id id = typedInstance.getId();
- if (id == null) { // oops
- throw new RepositoryException("id cannot be null");
- }
-
- Vertex instanceVertex = entityProcessor.idToVertexMap.get(id);
-
- // add the attributes for the instance
- ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
- final Map<String, AttributeInfo> fields = classType.fieldMapping().fields;
-
- mapInstanceToVertex(id, typedInstance, instanceVertex, fields, entityProcessor.idToVertexMap, false);
-
- for (String traitName : typedInstance.getTraits()) {
- LOG.debug("mapping trait {}", traitName);
- GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
- ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
-
- // add the attributes for the trait instance
- mapTraitInstanceToVertex(traitInstance, typedInstance.getId(), classType, instanceVertex,
- entityProcessor.idToVertexMap);
- }
- }
-
- private void mapInstanceToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
- Map<String, AttributeInfo> fields, Map<Id, Vertex> idToVertexMap, boolean mapOnlyUniqueAttributes)
- throws AtlasException {
- LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(),
- instanceVertex);
- for (AttributeInfo attributeInfo : fields.values()) {
- if (mapOnlyUniqueAttributes && !attributeInfo.isUnique) {
- continue;
- }
-
- final IDataType dataType = attributeInfo.dataType();
- mapAttributesToVertex(id, typedInstance, instanceVertex, idToVertexMap, attributeInfo, dataType);
- }
- }
-
- private void mapAttributesToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
- Map<Id, Vertex> idToVertexMap, AttributeInfo attributeInfo, IDataType dataType) throws AtlasException {
- Object attrValue = typedInstance.get(attributeInfo.name);
- LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
- final String propertyName = getQualifiedName(typedInstance, attributeInfo);
- String edgeLabel = getEdgeLabel(typedInstance, attributeInfo);
- if (attrValue == null) {
- return;
- }
-
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- mapPrimitiveToVertex(typedInstance, instanceVertex, attributeInfo);
- break;
-
- case ENUM:
- //handles both int and string for enum
- EnumValue enumValue =
- (EnumValue) dataType.convert(typedInstance.get(attributeInfo.name), Multiplicity.REQUIRED);
- GraphHelper.setProperty(instanceVertex, propertyName, enumValue.value);
- break;
-
- case ARRAY:
- mapArrayCollectionToVertex(id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
- break;
-
- case MAP:
- mapMapCollectionToVertex(id, typedInstance, instanceVertex, attributeInfo, idToVertexMap);
- break;
-
- case STRUCT:
- Vertex structInstanceVertex =
- mapStructInstanceToVertex(id, (ITypedStruct) typedInstance.get(attributeInfo.name),
- attributeInfo, idToVertexMap);
- // add an edge to the newly created vertex from the parent
- GraphHelper.addEdge(titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
- break;
-
- case TRAIT:
- // do NOTHING - this is taken care of earlier
- break;
-
- case CLASS:
- mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel, (ITypedReferenceableInstance) attrValue);
- break;
-
- default:
- throw new IllegalArgumentException("Unknown type category: " + dataType.getTypeCategory());
- }
- }
-
- private void mapArrayCollectionToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
- AttributeInfo attributeInfo, Map<Id, Vertex> idToVertexMap) throws AtlasException {
- LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
- attributeInfo.name);
- List list = (List) typedInstance.get(attributeInfo.name);
- if (list == null || list.isEmpty()) {
- return;
- }
-
- String propertyName = getQualifiedName(typedInstance, attributeInfo);
- IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
-
- List<String> values = new ArrayList<>(list.size());
- for (int index = 0; index < list.size(); index++) {
- String entryId =
- mapCollectionEntryToVertex(id, instanceVertex, attributeInfo, idToVertexMap, elementType,
- list.get(index), propertyName);
- values.add(entryId);
- }
-
- // for dereference on way out
- GraphHelper.setProperty(instanceVertex, propertyName, values);
- }
-
- private void mapMapCollectionToVertex(Id id, ITypedInstance typedInstance, Vertex instanceVertex,
- AttributeInfo attributeInfo, Map<Id, Vertex> idToVertexMap) throws AtlasException {
- LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
- attributeInfo.name);
- @SuppressWarnings("unchecked") Map<Object, Object> collection =
- (Map<Object, Object>) typedInstance.get(attributeInfo.name);
- if (collection == null || collection.isEmpty()) {
- return;
- }
-
- String propertyName = getQualifiedName(typedInstance, attributeInfo);
- IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
- for (Map.Entry entry : collection.entrySet()) {
- String myPropertyName = propertyName + "." + entry.getKey().toString();
- String value = mapCollectionEntryToVertex(id, instanceVertex, attributeInfo, idToVertexMap, elementType,
- entry.getValue(), myPropertyName);
- GraphHelper.setProperty(instanceVertex, myPropertyName, value);
- }
-
- // for dereference on way out
- GraphHelper.setProperty(instanceVertex, propertyName, new ArrayList(collection.keySet()));
- }
-
- private String mapCollectionEntryToVertex(Id id, Vertex instanceVertex, AttributeInfo attributeInfo,
- Map<Id, Vertex> idToVertexMap, IDataType elementType, Object value, String propertyName)
- throws AtlasException {
- final String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
- switch (elementType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- return value.toString();
-
- case ARRAY:
- case MAP:
- case TRAIT:
- // do nothing
- return null;
-
- case STRUCT:
- Vertex structInstanceVertex =
- mapStructInstanceToVertex(id, (ITypedStruct) value, attributeInfo, idToVertexMap);
- // add an edge to the newly created vertex from the parent
- Edge structElementEdge =
- GraphHelper.addEdge(titanGraph, instanceVertex, structInstanceVertex, edgeLabel);
- return structElementEdge.getId().toString();
-
- case CLASS:
- return mapClassReferenceAsEdge(instanceVertex, idToVertexMap, edgeLabel,
- (ITypedReferenceableInstance) value);
-
- default:
- throw new IllegalArgumentException("Unknown type category: " + elementType.getTypeCategory());
- }
- }
-
- private String mapClassReferenceAsEdge(Vertex instanceVertex, Map<Id, Vertex> idToVertexMap, String propertyKey,
- ITypedReferenceableInstance typedReference) throws AtlasException {
- if (typedReference != null) {
- Vertex referenceVertex;
- Id id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
- if (id.isAssigned()) {
- referenceVertex = getVertexForGUID(id.id);
- } else {
- referenceVertex = idToVertexMap.get(id);
- }
-
- if (referenceVertex != null) {
- // Add an edge to the class vertex from the instance.
- Edge edge = GraphHelper.addEdge(titanGraph, instanceVertex, referenceVertex, propertyKey);
- return String.valueOf(edge.getId());
- }
- }
-
- return null;
- }
-
- private Vertex mapStructInstanceToVertex(Id id, ITypedStruct structInstance, AttributeInfo attributeInfo,
- Map<Id, Vertex> idToVertexMap) throws AtlasException {
- // add a new vertex for the struct or trait instance
- Vertex structInstanceVertex = GraphHelper
- .createVertexWithoutIdentity(titanGraph, structInstance.getTypeName(), id,
- Collections.<String>emptySet()); // no super types for struct type
- LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name,
- structInstance);
-
- // map all the attributes to this newly created vertex
- mapInstanceToVertex(id, structInstance, structInstanceVertex, structInstance.fieldMapping().fields,
- idToVertexMap, false);
-
- return structInstanceVertex;
- }
-
- private void mapTraitInstanceToVertex(ITypedStruct traitInstance, Id typedInstanceId,
- IDataType entityType, Vertex parentInstanceVertex, Map<Id, Vertex> idToVertexMap)
- throws AtlasException {
- // add a new vertex for the struct or trait instance
- final String traitName = traitInstance.getTypeName();
- Vertex traitInstanceVertex = GraphHelper
- .createVertexWithoutIdentity(titanGraph, traitInstance.getTypeName(), typedInstanceId,
- typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
- LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
-
- // map all the attributes to this newly created vertex
- mapInstanceToVertex(typedInstanceId, traitInstance, traitInstanceVertex,
- traitInstance.fieldMapping().fields, idToVertexMap, false);
-
- // add an edge to the newly created vertex from the parent
- String relationshipLabel = getTraitLabel(entityType, traitName);
- GraphHelper.addEdge(titanGraph, parentInstanceVertex, traitInstanceVertex, relationshipLabel);
- }
-
- private void mapPrimitiveToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
- AttributeInfo attributeInfo) throws AtlasException {
- Object attrValue = typedInstance.get(attributeInfo.name);
- if (attrValue == null) {
- return; // add only if instance has this attribute
- }
-
- final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
- Object propertyValue = null;
- if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
- propertyValue = typedInstance.getString(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
- propertyValue = typedInstance.getShort(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
- propertyValue = typedInstance.getInt(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
- propertyValue = typedInstance.getBigInt(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
- propertyValue = typedInstance.getBoolean(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
- propertyValue = typedInstance.getByte(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
- propertyValue = typedInstance.getLong(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
- propertyValue = typedInstance.getFloat(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
- propertyValue = typedInstance.getDouble(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
- } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
- final Date dateVal = typedInstance.getDate(attributeInfo.name);
- //Convert Property value to Long while persisting
- propertyValue = dateVal.getTime();
- }
- GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
- }
- }
-
- public final class GraphToTypedInstanceMapper {
-
- public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, Vertex instanceVertex)
- throws AtlasException {
-
- LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid);
- String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
- List<String> traits = getTraitNames(instanceVertex);
-
- Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
- LOG.debug("Created id {} for instance type {}", id, typeName);
-
- ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
- ITypedReferenceableInstance typedInstance =
- classType.createInstance(id, traits.toArray(new String[traits.size()]));
-
- mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
- mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
-
- return typedInstance;
- }
-
- private void mapVertexToInstanceTraits(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
- List<String> traits) throws AtlasException {
- for (String traitName : traits) {
- LOG.debug("mapping trait {} to instance", traitName);
- TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
- mapVertexToTraitInstance(instanceVertex, typedInstance, traitName, traitType);
- }
- }
-
- public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
- Map<String, AttributeInfo> fields) throws AtlasException {
-
- LOG.debug("Mapping vertex {} to instance {} for fields", instanceVertex, typedInstance.getTypeName(),
- fields);
- for (AttributeInfo attributeInfo : fields.values()) {
- mapVertexToAttribute(instanceVertex, typedInstance, attributeInfo);
- }
- }
-
-
- private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo) throws AtlasException {
- LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
- final IDataType dataType = attributeInfo.dataType();
- final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
-
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- mapVertexToPrimitive(instanceVertex, typedInstance, attributeInfo);
- break; // add only if vertex has this attribute
-
- case ENUM:
- if (instanceVertex.getProperty(vertexPropertyName) == null) {
- return;
- }
-
- typedInstance.set(attributeInfo.name,
- dataType.convert(instanceVertex.<String>getProperty(vertexPropertyName),
- Multiplicity.REQUIRED));
- break;
-
- case ARRAY:
- mapVertexToArrayInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
- break;
-
- case MAP:
- mapVertexToMapInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
- break;
-
- case STRUCT:
- mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
- break;
-
- case TRAIT:
- // do NOTHING - handled in class
- break;
-
- case CLASS:
- String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
- Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo, relationshipLabel,
- attributeInfo.dataType());
- typedInstance.set(attributeInfo.name, idOrInstance);
- break;
-
- default:
- break;
- }
- }
-
- private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
- String relationshipLabel, IDataType dataType) throws AtlasException {
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
- if (results.hasNext()) {
- final Vertex referenceVertex = results.next().getVertex(Direction.IN);
- if (referenceVertex != null) {
- final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid);
- if (attributeInfo.isComposite) {
- //Also, when you retrieve a type's instance, you get the complete object graph of the composites
- LOG.debug("Found composite, mapping vertex to instance");
- return mapGraphToTypedInstance(guid, referenceVertex);
- } else {
- Id referenceId =
- new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
- dataType.getName());
- LOG.debug("Found non-composite, adding id {} ", referenceId);
- return referenceId;
- }
- }
- }
-
- return null;
- }
-
- @SuppressWarnings("unchecked")
- private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo, String propertyName) throws AtlasException {
- LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
- List list = instanceVertex.getProperty(propertyName);
- if (list == null || list.size() == 0) {
- return;
- }
- DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
- final IDataType elementType = arrayType.getElemType();
-
- ArrayList values = new ArrayList();
- for (Object listElement : list) {
- values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, listElement,
- propertyName));
- }
-
- typedInstance.set(attributeInfo.name, values);
- }
-
- private Object mapVertexToCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
- IDataType elementType, Object value, String propertyName) throws AtlasException {
- String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
- switch (elementType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- return value;
-
- case ARRAY:
- case MAP:
- case TRAIT:
- // do nothing
- break;
-
- case STRUCT:
- return getStructInstanceFromVertex(instanceVertex, elementType, attributeInfo.name, edgeLabel,
- (String) value);
-
- case CLASS:
- return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
-
- default:
- break;
- }
-
- throw new IllegalArgumentException();
- }
-
- @SuppressWarnings("unchecked")
- private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo, final String propertyName) throws AtlasException {
- LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
- List<String> keys = instanceVertex.getProperty(propertyName);
- if (keys == null || keys.size() == 0) {
- return;
- }
- DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
- final IDataType valueType = mapType.getValueType();
-
- HashMap values = new HashMap();
- for (String key : keys) {
- String keyPropertyName = propertyName + "." + key;
- Object keyValue = instanceVertex.getProperty(keyPropertyName);
- values.put(key,
- mapVertexToCollectionEntry(instanceVertex, attributeInfo, valueType, keyValue, propertyName));
- }
-
- typedInstance.set(attributeInfo.name, values);
- }
-
- private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, IDataType elemType,
- String attributeName, String relationshipLabel, String edgeId) throws AtlasException {
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- if (edgeId.equals(String.valueOf(edge.getId()))) {
- Vertex structInstanceVertex = edge.getVertex(Direction.IN);
- LOG.debug("mapping vertex {} to struct {}", structInstanceVertex, attributeName);
-
- if (structInstanceVertex != null) {
- LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
- elemType.getName());
- StructType structType = typeSystem.getDataType(StructType.class, elemType.getName());
- ITypedStruct structInstance = structType.createInstance();
- mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
- return structInstance;
- }
-
- break;
- }
- }
-
- return null;
- }
-
- private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
- String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException {
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- if (edgeId.equals(String.valueOf(edge.getId()))) {
- final Vertex referenceVertex = edge.getVertex(Direction.IN);
- if (referenceVertex != null) {
- final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel,
- guid);
- if (attributeInfo.isComposite) {
- //Also, when you retrieve a type's instance, you get the complete object graph of the composites
- LOG.debug("Found composite, mapping vertex to instance");
- return mapGraphToTypedInstance(guid, referenceVertex);
- } else {
- Id referenceId =
- new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
- dataType.getName());
- LOG.debug("Found non-composite, adding id {} ", referenceId);
- return referenceId;
- }
- }
-
- break;
- }
- }
-
- return null;
- }
-
- private void mapVertexToStructInstance(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo) throws AtlasException {
- LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
- StructType structType = typeSystem.getDataType(StructType.class, attributeInfo.dataType().getName());
- ITypedStruct structInstance = structType.createInstance();
- typedInstance.set(attributeInfo.name, structInstance);
-
- String relationshipLabel = getEdgeLabel(typedInstance, attributeInfo);
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
- if (structInstanceVertex != null) {
- LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
- structInstance.getTypeName());
- mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
- break;
- }
- }
- }
-
- private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
- String traitName, TraitType traitType) throws AtlasException {
- ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
-
- mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(), traitName, traitType, traitInstance);
- }
-
- private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName, String traitName,
- TraitType traitType, ITypedStruct traitInstance) throws AtlasException {
- String relationshipLabel = getTraitLabel(typedInstanceTypeName, traitName);
- LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
- for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
- final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
- if (traitInstanceVertex != null) {
- LOG.debug("Found trait instance vertex {}, mapping to instance {} ", traitInstanceVertex,
- traitInstance.getTypeName());
- mapVertexToInstance(traitInstanceVertex, traitInstance, traitType.fieldMapping().fields);
- break;
- }
- }
- }
-
- private void mapVertexToPrimitive(Vertex instanceVertex, ITypedInstance typedInstance,
- AttributeInfo attributeInfo) throws AtlasException {
- LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
- final String vertexPropertyName = getQualifiedName(typedInstance, attributeInfo);
- if (instanceVertex.getProperty(vertexPropertyName) == null) {
- return;
- }
-
- if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
- typedInstance.setString(attributeInfo.name, instanceVertex.<String>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
- typedInstance.setShort(attributeInfo.name, instanceVertex.<Short>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
- typedInstance.setInt(attributeInfo.name, instanceVertex.<Integer>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
- typedInstance.setBigInt(attributeInfo.name, instanceVertex.<BigInteger>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
- typedInstance.setBoolean(attributeInfo.name, instanceVertex.<Boolean>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
- typedInstance.setByte(attributeInfo.name, instanceVertex.<Byte>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
- typedInstance.setLong(attributeInfo.name, instanceVertex.<Long>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
- typedInstance.setFloat(attributeInfo.name, instanceVertex.<Float>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
- typedInstance.setDouble(attributeInfo.name, instanceVertex.<Double>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
- typedInstance
- .setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
- } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
- final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName);
- typedInstance.setDate(attributeInfo.name, new Date(dateVal));
- }
- }
-
- public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
- final Edge edge = titanGraph.getEdge(edgeId);
- if(edge != null) {
- final Vertex referredVertex = edge.getVertex(Direction.IN);
- if (referredVertex != null) {
- switch (referredType.getTypeCategory()) {
- case STRUCT:
- LOG.debug("Found struct instance vertex {}, mapping to instance {} ", referredVertex,
- referredType.getName());
- StructType structType = (StructType)referredType;
- ITypedStruct instance = structType.createInstance();
- Map<String, AttributeInfo> fields = structType.fieldMapping().fields;
- mapVertexToInstance(referredVertex, instance, fields);
- return instance;
- case CLASS:
- //TODO isComposite handling for class loads
- final String guid = referredVertex.getProperty(Constants.GUID_PROPERTY_KEY);
- Id referenceId =
- new Id(guid, referredVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
- referredType.getName());
- return referenceId;
- default:
- throw new UnsupportedOperationException("Loading " + referredType.getTypeCategory() + " is not supported");
- }
- }
- }
- return null;
+ @Override
+ @GraphTransaction
+ public void updatePartial(ITypedReferenceableInstance entity) throws RepositoryException {
+ LOG.info("updating entity {}", entity);
+ try {
+ TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper);
+ instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity);
+ } catch (AtlasException e) {
+ throw new RepositoryException(e);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index aa532c4..9ac2819 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -19,19 +19,33 @@
package org.apache.atlas.repository.graph;
import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.TitanProperty;
import com.thinkaurelius.titan.core.TitanVertex;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.Set;
import java.util.UUID;
@@ -41,26 +55,41 @@ import java.util.UUID;
public final class GraphHelper {
private static final Logger LOG = LoggerFactory.getLogger(GraphHelper.class);
+ public static final String EDGE_LABEL_PREFIX = "__";
- private GraphHelper() {
+ private static final TypeSystem typeSystem = TypeSystem.getInstance();
+
+ private static final GraphHelper INSTANCE = new GraphHelper(TitanGraphProvider.getGraphInstance());
+
+ private TitanGraph titanGraph;
+
+ private GraphHelper(TitanGraph titanGraph) {
+ this.titanGraph = titanGraph;
}
- public static Vertex createVertexWithIdentity(Graph graph, ITypedReferenceableInstance typedInstance,
+ public static GraphHelper getInstance() {
+ return INSTANCE;
+ }
+
+ public Vertex createVertexWithIdentity(ITypedReferenceableInstance typedInstance,
Set<String> superTypeNames) {
- final Vertex vertexWithIdentity = createVertexWithoutIdentity(graph, typedInstance.getTypeName(),
+ final Vertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(),
typedInstance.getId(), superTypeNames);
// add identity
final String guid = UUID.randomUUID().toString();
setProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid);
+ // add version information
+ setProperty(vertexWithIdentity, Constants.VERSION_PROPERTY_KEY, typedInstance.getId().version);
+
return vertexWithIdentity;
}
- public static Vertex createVertexWithoutIdentity(Graph graph, String typeName, Id typedInstanceId,
- Set<String> superTypeNames) {
- LOG.debug("Creating vertex for type {} id {}", typeName, typedInstanceId._getId());
- final Vertex vertexWithoutIdentity = graph.addVertex(null);
+ public Vertex createVertexWithoutIdentity(String typeName, Id typedInstanceId, Set<String> superTypeNames) {
+ LOG.debug("Creating vertex for type {} id {}", typeName,
+ typedInstanceId != null ? typedInstanceId._getId() : null);
+ final Vertex vertexWithoutIdentity = titanGraph.addVertex(null);
// add type information
setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
@@ -70,22 +99,19 @@ public final class GraphHelper {
addProperty(vertexWithoutIdentity, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
- // add version information
- setProperty(vertexWithoutIdentity, Constants.VERSION_PROPERTY_KEY, typedInstanceId.version);
-
// add timestamp information
setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, System.currentTimeMillis());
return vertexWithoutIdentity;
}
- public static Edge addEdge(TitanGraph titanGraph, Vertex fromVertex, Vertex toVertex, String edgeLabel) {
+ public Edge addEdge(Vertex fromVertex, Vertex toVertex, String edgeLabel) {
LOG.debug("Adding edge for {} -> label {} -> {}", fromVertex, edgeLabel, toVertex);
return titanGraph.addEdge(null, fromVertex, toVertex, edgeLabel);
}
- public static Vertex findVertex(TitanGraph titanGraph, String propertyKey, Object value) {
+ public Vertex findVertex(String propertyKey, Object value) {
LOG.debug("Finding vertex for {}={}", propertyKey, value);
GraphQuery query = titanGraph.query().has(propertyKey, value);
@@ -94,6 +120,20 @@ public final class GraphHelper {
return results.hasNext() ? results.next() : null;
}
+ public static Iterable<Edge> getOutGoingEdgesByLabel(Vertex instanceVertex, String edgeLabel) {
+ if(instanceVertex != null && edgeLabel != null) {
+ return instanceVertex.getEdges(Direction.OUT, edgeLabel);
+ }
+ return null;
+ }
+
+ public Edge getOutGoingEdgeById(String edgeId) {
+ if(edgeId != null) {
+ return titanGraph.getEdge(edgeId);
+ }
+ return null;
+ }
+
public static String vertexString(final Vertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
@@ -110,7 +150,17 @@ public final class GraphHelper {
public static void setProperty(Vertex vertex, String propertyName, Object value) {
LOG.debug("Setting property {} = \"{}\" to vertex {}", propertyName, value, vertex);
- vertex.setProperty(propertyName, value);
+ Object existValue = vertex.getProperty(propertyName);
+ if(value == null || (value instanceof Collection && ((Collection) value).isEmpty())) {
+ if(existValue != null) {
+ LOG.info("Removing property - {} value from vertex {}", propertyName, vertex);
+ vertex.removeProperty(propertyName);
+ }
+ } else {
+ if (!value.equals(existValue)) {
+ vertex.setProperty(propertyName, value);
+ }
+ }
}
public static void addProperty(Vertex vertex, String propertyName, Object value) {
@@ -118,7 +168,108 @@ public final class GraphHelper {
((TitanVertex)vertex).addProperty(propertyName, value);
}
-/*
+ public Edge removeRelation(String edgeId, boolean cascade) {
+ LOG.debug("Removing edge with id {}", edgeId);
+ final Edge edge = titanGraph.getEdge(edgeId);
+ titanGraph.removeEdge(edge);
+ LOG.info("Removed edge {}", edge);
+ if (cascade) {
+ Vertex referredVertex = edge.getVertex(Direction.IN);
+ titanGraph.removeVertex(referredVertex);
+ LOG.info("Removed vertex {}", referredVertex);
+ }
+ return edge;
+ }
+
+ public Vertex getVertexForGUID(String guid) throws EntityNotFoundException {
+ return getVertexForProperty(Constants.GUID_PROPERTY_KEY, guid);
+ }
+
+
+ public Vertex getVertexForProperty(String propertyKey, Object value) throws EntityNotFoundException {
+ Vertex instanceVertex = findVertex(propertyKey, value);
+ if (instanceVertex == null) {
+ LOG.debug("Could not find a vertex with {}={}", propertyKey, value);
+ throw new EntityNotFoundException("Could not find an entity in the repository with " + propertyKey + "="
+ + value);
+ } else {
+ LOG.debug("Found a vertex {} with {}={}", instanceVertex, propertyKey, value);
+ }
+
+ return instanceVertex;
+ }
+
+ public static String getQualifiedFieldName(ITypedInstance typedInstance, AttributeInfo attributeInfo) throws AtlasException {
+ IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
+ return getQualifiedFieldName(dataType, attributeInfo.name);
+ }
+
+ public static String getQualifiedFieldName(IDataType dataType, String attributeName) throws AtlasException {
+ return dataType.getTypeCategory() == DataTypes.TypeCategory.STRUCT ? dataType.getName() + "." + attributeName
+ // else class or trait
+ : ((HierarchicalType) dataType).getQualifiedName(attributeName);
+ }
+
+ public static String getTraitLabel(String typeName, String attrName) {
+ return typeName + "." + attrName;
+ }
+
+ public static List<String> getTraitNames(Vertex entityVertex) {
+ ArrayList<String> traits = new ArrayList<>();
+ for (TitanProperty property : ((TitanVertex) entityVertex).getProperties(Constants.TRAIT_NAMES_PROPERTY_KEY)) {
+ traits.add((String) property.getValue());
+ }
+
+ return traits;
+ }
+
+ public static String getEdgeLabel(ITypedInstance typedInstance, AttributeInfo aInfo) throws AtlasException {
+ IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
+ return getEdgeLabel(dataType, aInfo);
+ }
+
+ public static String getEdgeLabel(IDataType dataType, AttributeInfo aInfo) throws AtlasException {
+ return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedFieldName(dataType, aInfo.name);
+ }
+
+ public static Id getIdFromVertex(String dataTypeName, Vertex vertex) {
+ return new Id(vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY),
+ vertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), dataTypeName);
+ }
+
+ public static String getTypeName(Vertex instanceVertex) {
+ return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ }
+
+ /**
+ * For the given type, finds an unique attribute and checks if there is an existing instance with the same
+ * unique value
+ *
+ * @param classType
+ * @param instance
+ * @return
+ * @throws AtlasException
+ */
+ public Vertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance)
+ throws AtlasException {
+
+ Vertex result = null;
+ for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
+ if (attributeInfo.isUnique) {
+ String propertyKey = getQualifiedFieldName(classType, attributeInfo.name);
+ try {
+ result = getVertexForProperty(propertyKey, instance.get(attributeInfo.name));
+ LOG.debug("Found vertex by unique attribute : " + propertyKey + "=" + instance.get(attributeInfo.name));
+ } catch (EntityNotFoundException e) {
+ //Its ok if there is no entity with the same unique value
+ }
+ }
+ }
+
+ return result;
+ }
+
+
public static void dumpToLog(final Graph graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
@@ -132,5 +283,4 @@ public final class GraphHelper {
}
LOG.debug("*******************Graph Dump****************************");
}
-*/
}
\ No newline at end of file
[3/5] incubator-atlas git commit: ATLAS-47 Entity mutations for
complex types (sumasai via shwethags)
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
new file mode 100644
index 0000000..1fabafa
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
@@ -0,0 +1,419 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public final class GraphToTypedInstanceMapper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GraphToTypedInstanceMapper.class);
+ private static TypeSystem typeSystem = TypeSystem.getInstance();
+ private final TitanGraph titanGraph;
+
+ public GraphToTypedInstanceMapper(TitanGraph titanGraph) {
+ this.titanGraph = titanGraph;
+ }
+
+ public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, Vertex instanceVertex)
+ throws AtlasException {
+
+ LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid);
+ String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ List<String> traits = GraphHelper.getTraitNames(instanceVertex);
+
+ Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
+ LOG.debug("Created id {} for instance type {}", id, typeName);
+
+ ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
+ ITypedReferenceableInstance typedInstance =
+ classType.createInstance(id, traits.toArray(new String[traits.size()]));
+
+ mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
+ mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
+
+ return typedInstance;
+ }
+
+ private void mapVertexToInstanceTraits(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
+ List<String> traits) throws AtlasException {
+ for (String traitName : traits) {
+ LOG.debug("mapping trait {} to instance", traitName);
+ TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
+ mapVertexToTraitInstance(instanceVertex, typedInstance, traitName, traitType);
+ }
+ }
+
+ public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ Map<String, AttributeInfo> fields) throws AtlasException {
+
+ LOG.debug("Mapping vertex {} to instance {} for fields", instanceVertex, typedInstance.getTypeName(),
+ fields);
+ for (AttributeInfo attributeInfo : fields.values()) {
+ mapVertexToAttribute(instanceVertex, typedInstance, attributeInfo);
+ }
+ }
+
+
+ private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws AtlasException {
+ LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
+ final IDataType dataType = attributeInfo.dataType();
+ final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+
+ switch (dataType.getTypeCategory()) {
+ case PRIMITIVE:
+ mapVertexToPrimitive(instanceVertex, typedInstance, attributeInfo);
+ break; // add only if vertex has this attribute
+
+ case ENUM:
+ if (instanceVertex.getProperty(vertexPropertyName) == null) {
+ return;
+ }
+
+ typedInstance.set(attributeInfo.name,
+ dataType.convert(instanceVertex.<String>getProperty(vertexPropertyName),
+ Multiplicity.REQUIRED));
+ break;
+
+ case ARRAY:
+ mapVertexToArrayInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
+ break;
+
+ case MAP:
+ mapVertexToMapInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
+ break;
+
+ case STRUCT:
+ mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
+ break;
+
+ case TRAIT:
+ // do NOTHING - handled in class
+ break;
+
+ case CLASS:
+ String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
+ Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo, relationshipLabel,
+ attributeInfo.dataType());
+ if (idOrInstance != null) {
+ typedInstance.set(attributeInfo.name, idOrInstance);
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
+ String relationshipLabel, IDataType dataType) throws AtlasException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
+ if (results.hasNext()) {
+ final Vertex referenceVertex = results.next().getVertex(Direction.IN);
+ if (referenceVertex != null) {
+ final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid);
+ if (attributeInfo.isComposite) {
+ //Also, when you retrieve a type's instance, you get the complete object graph of the composites
+ LOG.debug("Found composite, mapping vertex to instance");
+ return mapGraphToTypedInstance(guid, referenceVertex);
+ } else {
+ Id referenceId =
+ new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ dataType.getName());
+ LOG.debug("Found non-composite, adding id {} ", referenceId);
+ return referenceId;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo, String propertyName) throws AtlasException {
+ LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
+ List list = instanceVertex.getProperty(propertyName);
+ if (list == null || list.size() == 0) {
+ return;
+ }
+ DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
+ final IDataType elementType = arrayType.getElemType();
+
+ String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
+ ArrayList values = new ArrayList();
+ for (int index = 0; index < list.size(); index++) {
+ values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, list.get(index),
+ edgeLabel));
+ }
+
+ if (values.size() > 0) {
+ typedInstance.set(attributeInfo.name, values);
+ }
+ }
+
+ private Object mapVertexToCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
+ IDataType elementType, Object value, String edgeLabel) throws AtlasException {
+ switch (elementType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ return value;
+
+ case ARRAY:
+ case MAP:
+ case TRAIT:
+ // do nothing
+ break;
+
+ case STRUCT:
+ return getStructInstanceFromVertex(instanceVertex, elementType, attributeInfo.name, edgeLabel,
+ (String) value);
+
+ case CLASS:
+ return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
+
+ default:
+ break;
+ }
+
+ throw new IllegalArgumentException();
+ }
+
+ @SuppressWarnings("unchecked")
+ private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo, final String propertyName) throws AtlasException {
+ LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
+ List<String> keys = instanceVertex.getProperty(propertyName);
+ if (keys == null || keys.size() == 0) {
+ return;
+ }
+ DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
+ final IDataType valueType = mapType.getValueType();
+
+ HashMap values = new HashMap();
+ for (String key : keys) {
+ final String keyPropertyName = propertyName + "." + key;
+ final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + keyPropertyName;
+ final Object keyValue = instanceVertex.getProperty(keyPropertyName);
+ Object mapValue = mapVertexToCollectionEntry(instanceVertex, attributeInfo, valueType, keyValue, edgeLabel);
+ if (mapValue != null) {
+ values.put(key, mapValue);
+ }
+ }
+
+ if (!values.isEmpty()) {
+ typedInstance.set(attributeInfo.name, values);
+ }
+ }
+
+ private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, IDataType elemType,
+ String attributeName, String relationshipLabel, String edgeId) throws AtlasException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ if (edgeId.equals(String.valueOf(edge.getId()))) {
+ Vertex structInstanceVertex = edge.getVertex(Direction.IN);
+ LOG.debug("mapping vertex {} to struct {}", structInstanceVertex, attributeName);
+
+ if (structInstanceVertex != null) {
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
+ elemType.getName());
+ StructType structType = typeSystem.getDataType(StructType.class, elemType.getName());
+ ITypedStruct structInstance = structType.createInstance();
+ mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
+ return structInstance;
+ }
+
+ break;
+ }
+ }
+
+ return null;
+ }
+
+ private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
+ String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ if (edgeId.equals(String.valueOf(edge.getId()))) {
+ final Vertex referenceVertex = edge.getVertex(Direction.IN);
+ if (referenceVertex != null) {
+ final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel,
+ guid);
+ if (attributeInfo.isComposite) {
+ //Also, when you retrieve a type's instance, you get the complete object graph of the composites
+ LOG.debug("Found composite, mapping vertex to instance");
+ return mapGraphToTypedInstance(guid, referenceVertex);
+ } else {
+ Id referenceId =
+ new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ dataType.getName());
+ LOG.debug("Found non-composite, adding id {} ", referenceId);
+ return referenceId;
+ }
+ }
+
+ break;
+ }
+ }
+
+ return null;
+ }
+
+ private void mapVertexToStructInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws AtlasException {
+ LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
+ StructType structType = typeSystem.getDataType(StructType.class, attributeInfo.dataType().getName());
+ ITypedStruct structInstance = null;
+
+ String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ final Iterable<Edge> edges = instanceVertex.getEdges(Direction.OUT, relationshipLabel);
+ if (edges.iterator().hasNext()) {
+ structInstance = structType.createInstance();
+ typedInstance.set(attributeInfo.name, structInstance);
+ }
+
+ for (Edge edge : edges) {
+ final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
+ if (structInstanceVertex != null) {
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
+ structInstance.getTypeName());
+ mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
+ break;
+ }
+ }
+ }
+
+ private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
+ String traitName, TraitType traitType) throws AtlasException {
+ ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
+
+ mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(), traitName, traitType, traitInstance);
+ }
+
+ private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName, String traitName,
+ TraitType traitType, ITypedStruct traitInstance) throws AtlasException {
+ String relationshipLabel = GraphHelper.getTraitLabel(typedInstanceTypeName, traitName);
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
+ if (traitInstanceVertex != null) {
+ LOG.debug("Found trait instance vertex {}, mapping to instance {} ", traitInstanceVertex,
+ traitInstance.getTypeName());
+ mapVertexToInstance(traitInstanceVertex, traitInstance, traitType.fieldMapping().fields);
+ break;
+ }
+ }
+ }
+
+ private void mapVertexToPrimitive(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws AtlasException {
+ LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
+ final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ if (instanceVertex.getProperty(vertexPropertyName) == null) {
+ return;
+ }
+
+ if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
+ typedInstance.setString(attributeInfo.name, instanceVertex.<String>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
+ typedInstance.setShort(attributeInfo.name, instanceVertex.<Short>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
+ typedInstance.setInt(attributeInfo.name, instanceVertex.<Integer>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ typedInstance.setBigInt(attributeInfo.name, instanceVertex.<BigInteger>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
+ typedInstance.setBoolean(attributeInfo.name, instanceVertex.<Boolean>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
+ typedInstance.setByte(attributeInfo.name, instanceVertex.<Byte>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
+ typedInstance.setLong(attributeInfo.name, instanceVertex.<Long>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
+ typedInstance.setFloat(attributeInfo.name, instanceVertex.<Float>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
+ typedInstance.setDouble(attributeInfo.name, instanceVertex.<Double>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ typedInstance
+ .setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+ final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName);
+ typedInstance.setDate(attributeInfo.name, new Date(dateVal));
+ }
+ }
+
+ public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
+ final Edge edge = titanGraph.getEdge(edgeId);
+ if (edge != null) {
+ final Vertex referredVertex = edge.getVertex(Direction.IN);
+ if (referredVertex != null) {
+ switch (referredType.getTypeCategory()) {
+ case STRUCT:
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ", referredVertex,
+ referredType.getName());
+ StructType structType = (StructType) referredType;
+ ITypedStruct instance = structType.createInstance();
+ Map<String, AttributeInfo> fields = structType.fieldMapping().fields;
+ mapVertexToInstance(referredVertex, instance, fields);
+ return instance;
+ case CLASS:
+ //TODO isComposite handling for class loads
+ final String guid = referredVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ Id referenceId =
+ new Id(guid, referredVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ referredType.getName());
+ return referenceId;
+ default:
+ throw new UnsupportedOperationException("Loading " + referredType.getTypeCategory() + " is not supported");
+ }
+ }
+ }
+ return null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
new file mode 100644
index 0000000..7ef5c50
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
@@ -0,0 +1,633 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import com.thinkaurelius.titan.core.SchemaViolationException;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.exception.EntityExistsException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.utils.MD5Utils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public final class TypedInstanceToGraphMapper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class);
+ private final Map<Id, Vertex> idToVertexMap = new HashMap<>();
+ private final TypeSystem typeSystem = TypeSystem.getInstance();
+
+ private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
+
+ private static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+ private final String SIGNATURE_HASH_PROPERTY_KEY = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "signature";
+
+ public enum Operation {
+ CREATE,
+ UPDATE_PARTIAL,
+ UPDATE_FULL,
+ DELETE
+ }
+
+ public TypedInstanceToGraphMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
+ this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
+ }
+
+ String[] mapTypedInstanceToGraph(Operation operation, ITypedReferenceableInstance... typedInstances)
+ throws AtlasException {
+ List<String> guids = new ArrayList<>();
+ for (ITypedReferenceableInstance typedInstance : typedInstances) {
+ Collection<IReferenceableInstance> newInstances = walkClassInstances(typedInstance);
+ Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> instancesPair =
+ createVerticesAndDiscoverInstances(newInstances);
+
+ switch (operation) {
+ case CREATE:
+ addOrUpdateAttributesAndTraits(operation, instancesPair.getLeft());
+ addFullTextProperty(instancesPair.getLeft());
+ break;
+
+ case UPDATE_FULL:
+ case UPDATE_PARTIAL:
+ List<ITypedReferenceableInstance> instancesForUpdate = instancesPair.getLeft();
+ instancesForUpdate.addAll(instancesPair.getRight());
+ addOrUpdateAttributesAndTraits(operation, instancesForUpdate);
+ addFullTextProperty(instancesForUpdate);
+ break;
+
+ case DELETE:
+ throw new UnsupportedOperationException("Not handled - " + operation);
+ }
+
+ //Return guid for
+ addToGuids(typedInstance, guids);
+ }
+ return guids.toArray(new String[guids.size()]);
+ }
+
+ private Collection<IReferenceableInstance> walkClassInstances(ITypedReferenceableInstance typedInstance)
+ throws RepositoryException {
+
+ EntityProcessor entityProcessor = new EntityProcessor();
+ try {
+ LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
+ new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
+ } catch (AtlasException me) {
+ throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
+ }
+
+ entityProcessor.addInstanceIfNotExists(typedInstance);
+ return entityProcessor.getInstances();
+ }
+
+ private void addOrUpdateAttributesAndTraits(Operation operation, List<ITypedReferenceableInstance> instances) throws AtlasException {
+ for (ITypedReferenceableInstance instance : instances) {
+ try {
+ //new vertex, set all the properties
+ addOrUpdateAttributesAndTraits(operation, instance);
+ } catch (SchemaViolationException e) {
+ throw new EntityExistsException(instance, e);
+ }
+ }
+ }
+
+ private void addOrUpdateAttributesAndTraits(Operation operation, ITypedReferenceableInstance typedInstance)
+ throws AtlasException {
+ LOG.debug("Adding/Updating typed instance {}", typedInstance.getTypeName());
+
+ Id id = typedInstance.getId();
+ if (id == null) { // oops
+ throw new RepositoryException("id cannot be null");
+ }
+
+ Vertex instanceVertex = idToVertexMap.get(id);
+
+ // add the attributes for the instance
+ ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
+ final Map<String, AttributeInfo> fields = classType.fieldMapping().fields;
+
+ mapInstanceToVertex(typedInstance, instanceVertex, fields, false, operation);
+
+ if (Operation.CREATE.equals(operation)) {
+ //TODO - Handle Trait updates
+ addTraits(typedInstance, instanceVertex, classType);
+ }
+ }
+
+ private void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation)
+ throws AtlasException {
+ LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(),
+ instanceVertex);
+ for (AttributeInfo attributeInfo : fields.values()) {
+ if (mapOnlyUniqueAttributes && !attributeInfo.isUnique) {
+ continue;
+ }
+ mapAttributesToVertex(typedInstance, instanceVertex, attributeInfo, operation);
+ }
+ }
+
+ void mapAttributesToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo, Operation operation) throws AtlasException {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
+ final String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
+
+ if (attrValue != null || operation == Operation.UPDATE_FULL) {
+ switch (attributeInfo.dataType().getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ mapPrimitiveOrEnumToVertex(typedInstance, instanceVertex, attributeInfo);
+ break;
+
+ case ARRAY:
+ mapArrayCollectionToVertex(typedInstance, instanceVertex, attributeInfo, operation);
+ break;
+
+ case MAP:
+ mapMapCollectionToVertex(typedInstance, instanceVertex, attributeInfo, operation);
+ break;
+
+ case STRUCT:
+ case CLASS:
+ Iterator<Edge> outGoingEdgesIterator =
+ GraphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel).iterator();
+ String currentEntry =
+ outGoingEdgesIterator.hasNext() ? outGoingEdgesIterator.next().getId().toString() : null;
+ addOrUpdateCollectionEntry(instanceVertex, attributeInfo, attributeInfo.dataType(), attrValue,
+ currentEntry, propertyName, operation);
+ break;
+
+ case TRAIT:
+ // do NOTHING - this is taken care of earlier
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown type category: " + attributeInfo.dataType().getTypeCategory());
+ }
+ }
+ }
+
+ private Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> createVerticesAndDiscoverInstances(
+ Collection<IReferenceableInstance> instances) throws AtlasException {
+
+ List<ITypedReferenceableInstance> instancesToCreate = new ArrayList<>();
+ List<ITypedReferenceableInstance> instancesToUpdate = new ArrayList<>();
+
+ for (IReferenceableInstance instance : instances) {
+ Id id = instance.getId();
+ if (!idToVertexMap.containsKey(id)) {
+ Vertex instanceVertex;
+ if (id.isAssigned()) { // has a GUID
+ instanceVertex = graphHelper.getVertexForGUID(id.id);
+ if (!(instance instanceof ReferenceableInstance)) {
+ throw new IllegalStateException(
+ String.format("%s is not of type ITypedReferenceableInstance", instance));
+ }
+ instancesToUpdate.add((ITypedReferenceableInstance) instance);
+ } else {
+ //Check if there is already an instance with the same unique attribute value
+ ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
+ instanceVertex = graphHelper.getVertexForInstanceByUniqueAttribute(classType, instance);
+
+ //no entity with the given unique attribute, create new
+ if (instanceVertex == null) {
+ ITypedReferenceableInstance newInstance = classType.convert(instance, Multiplicity.REQUIRED);
+ instanceVertex = graphHelper.createVertexWithIdentity(newInstance, classType.getAllSuperTypeNames());
+ instancesToCreate.add(newInstance);
+
+ //Map only unique attributes for cases of circular references
+ mapInstanceToVertex(newInstance, instanceVertex, classType.fieldMapping().fields, true, Operation.CREATE);
+ } else {
+ if (!(instance instanceof ReferenceableInstance)) {
+ throw new IllegalStateException(
+ String.format("%s is not of type ITypedReferenceableInstance", instance));
+ }
+ instancesToUpdate.add((ITypedReferenceableInstance) instance);
+ }
+ }
+
+ idToVertexMap.put(id, instanceVertex);
+ }
+ }
+ return Pair.of(instancesToCreate, instancesToUpdate);
+ }
+
+ private void addToGuids(ITypedReferenceableInstance typedInstance, List<String> guids) {
+ Vertex instanceVertex = idToVertexMap.get(typedInstance.getId());
+ String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ guids.add(guid);
+ }
+
+ private void addFullTextProperty(List<ITypedReferenceableInstance> instances) throws AtlasException {
+ FullTextMapper fulltextMapper = new FullTextMapper(graphToTypedInstanceMapper);
+ for (ITypedReferenceableInstance typedInstance : instances) { // Traverse
+ Vertex instanceVertex = getClassVertex(typedInstance);
+ String fullText = fulltextMapper.mapRecursive(instanceVertex, true);
+ GraphHelper.setProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
+ }
+ }
+
+ private void addTraits(ITypedReferenceableInstance typedInstance, Vertex instanceVertex, ClassType classType) throws AtlasException {
+ for (String traitName : typedInstance.getTraits()) {
+ LOG.debug("mapping trait {}", traitName);
+ GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
+ ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
+
+ // add the attributes for the trait instance
+ mapTraitInstanceToVertex(traitInstance, classType, instanceVertex);
+ }
+ }
+
+ /******************************************** STRUCT **************************************************/
+
+ private Pair<Vertex, Edge> updateStructVertex(ITypedStruct structInstance, Edge relEdge, Operation operation) throws AtlasException {
+ //Already existing vertex. Update
+ Vertex structInstanceVertex = relEdge.getVertex(Direction.IN);
+
+ // Update attributes
+ final MessageDigest digester = MD5Utils.getDigester();
+ String newSignature = structInstance.getSignatureHash(digester);
+ String curSignature = structInstanceVertex.getProperty(SIGNATURE_HASH_PROPERTY_KEY);
+
+ if (!newSignature.equals(curSignature)) {
+ //Update struct vertex instance only if there is a change
+ LOG.debug("Updating struct {} since signature has changed {} {} ", structInstance, curSignature, newSignature);
+ mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false, operation);
+ GraphHelper.setProperty(structInstanceVertex, SIGNATURE_HASH_PROPERTY_KEY, String.valueOf(newSignature));
+ }
+ return Pair.of(structInstanceVertex, relEdge);
+ }
+
+ private Pair<Vertex, Edge> addStructVertex(ITypedStruct structInstance, Vertex instanceVertex, AttributeInfo attributeInfo, String edgeLabel) throws AtlasException {
+ // add a new vertex for the struct or trait instance
+ Vertex structInstanceVertex = graphHelper.createVertexWithoutIdentity(structInstance.getTypeName(), null,
+ Collections.<String>emptySet()); // no super types for struct type
+ LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name, structInstance);
+
+ // map all the attributes to this new vertex
+ mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false, Operation.CREATE);
+ // add an edge to the newly created vertex from the parent
+ Edge relEdge = graphHelper.addEdge(instanceVertex, structInstanceVertex, edgeLabel);
+
+ return Pair.of(structInstanceVertex, relEdge);
+ }
+
+ /******************************************** ARRAY **************************************************/
+
+ private void mapArrayCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo, Operation operation) throws AtlasException {
+ LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
+ attributeInfo.name);
+ List newElements = (List) typedInstance.get(attributeInfo.name);
+ boolean empty = (newElements == null || newElements.isEmpty());
+ if (!empty || operation == Operation.UPDATE_FULL) {
+ String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ List<String> currentEntries = instanceVertex.getProperty(propertyName);
+
+ IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
+ List<String> newEntries = new ArrayList<>();
+
+ if (newElements != null && !newElements.isEmpty()) {
+ int index = 0;
+ for (; index < newElements.size(); index++) {
+ String currentEntry =
+ (currentEntries != null && index < currentEntries.size()) ? currentEntries.get(index) : null;
+ String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
+ newElements.get(index), currentEntry, propertyName, operation);
+ newEntries.add(newEntry);
+ }
+
+ //Remove extra entries in the list
+ if (currentEntries != null) {
+ if (index < currentEntries.size()) {
+ for (; index < currentEntries.size(); index++) {
+ removeUnusedReference(currentEntries.get(index), attributeInfo, elementType);
+ }
+ }
+ }
+ }
+
+ // for dereference on way out
+ GraphHelper.setProperty(instanceVertex, propertyName, newEntries);
+ }
+ }
+
+ /******************************************** MAP **************************************************/
+
+ private void mapMapCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo, Operation operation) throws AtlasException {
+ LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
+ attributeInfo.name);
+ @SuppressWarnings("unchecked") Map<Object, Object> collection =
+ (Map<Object, Object>) typedInstance.get(attributeInfo.name);
+ boolean empty = (collection == null || collection.isEmpty());
+ if (!empty || operation == Operation.UPDATE_FULL) {
+
+ String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
+
+ if (!empty) {
+ for (Map.Entry entry : collection.entrySet()) {
+ String myPropertyName = propertyName + "." + entry.getKey().toString();
+
+ String currentEntry = instanceVertex.getProperty(myPropertyName);
+ String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
+ entry.getValue(), currentEntry, myPropertyName, operation);
+
+ //Add/Update/Remove property value
+ GraphHelper.setProperty(instanceVertex, myPropertyName, newEntry);
+ }
+
+ //Remove unused key references
+ List<Object> origKeys = instanceVertex.getProperty(propertyName);
+ if (origKeys != null) {
+ if (collection != null) {
+ origKeys.removeAll(collection.keySet());
+ }
+ for (Object unusedKey : origKeys) {
+ String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo) + "." + unusedKey;
+ if (instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().hasNext()) {
+ Edge edge = instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().next();
+ removeUnusedReference(edge.getId().toString(), attributeInfo,
+ ((DataTypes.MapType) attributeInfo.dataType()).getValueType());
+ }
+ }
+ }
+
+ }
+
+ // for dereference on way out
+ GraphHelper.setProperty(instanceVertex, propertyName, collection == null ? null : new ArrayList(collection.keySet()));
+ }
+ }
+
+ /******************************************** ARRAY & MAP **************************************************/
+
+ private String addOrUpdateCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
+ IDataType elementType, Object newVal, String curVal, String propertyName,
+ Operation operation)
+ throws AtlasException {
+
+ final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
+ switch (elementType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ return newVal != null ? newVal.toString() : null;
+
+ case ARRAY:
+ case MAP:
+ case TRAIT:
+ // do nothing
+ return null;
+
+ case STRUCT:
+ return addOrUpdateStruct(instanceVertex, attributeInfo, elementType, (ITypedStruct) newVal, curVal, edgeLabel, operation);
+
+ case CLASS:
+ return addOrUpdateClassVertex(instanceVertex, attributeInfo, elementType,
+ (ITypedReferenceableInstance) newVal, curVal, edgeLabel, operation);
+
+ default:
+ throw new IllegalArgumentException("Unknown type category: " + elementType.getTypeCategory());
+ }
+ }
+
+ private String addOrUpdateStruct(Vertex instanceVertex, AttributeInfo attributeInfo, IDataType elementType,
+ ITypedStruct structAttr, String curVal,
+ String edgeLabel, Operation operation) throws AtlasException {
+ Pair<Vertex, Edge> vertexEdgePair = null;
+ if (curVal != null && structAttr == null) {
+ //remove edge
+ removeUnusedReference(curVal, attributeInfo, elementType);
+ } else if (curVal != null && structAttr != null) {
+ //update
+ Edge edge = graphHelper.getOutGoingEdgeById(curVal);
+ vertexEdgePair = updateStructVertex(structAttr, edge, operation);
+ } else if (structAttr != null) {
+ //add
+ vertexEdgePair = addStructVertex(structAttr, instanceVertex, attributeInfo, edgeLabel);
+ }
+
+ return (vertexEdgePair != null) ? vertexEdgePair.getRight().getId().toString() : null;
+ }
+
+ private String addOrUpdateClassVertex(Vertex instanceVertex, AttributeInfo attributeInfo, IDataType elementType,
+ ITypedReferenceableInstance newVal, String curVal,
+ String edgeLabel, Operation operation) throws AtlasException {
+ Vertex toVertex = getClassVertex(newVal);
+ if(toVertex == null && newVal != null) {
+ LOG.error("Could not find vertex for Class Reference " + newVal);
+ throw new EntityNotFoundException("Could not find vertex for Class Reference " + newVal);
+ }
+
+ Pair<Vertex, Edge> vertexEdgePair = null;
+ if (curVal != null && newVal == null) {
+ //remove edge
+ removeUnusedReference(curVal, attributeInfo, elementType);
+ } else if (curVal != null && newVal != null) {
+ Edge edge = graphHelper.getOutGoingEdgeById(curVal);
+ Id classRefId = getId(newVal);
+ vertexEdgePair = updateClassEdge(classRefId, newVal, instanceVertex, edge, toVertex, attributeInfo, elementType, edgeLabel, operation);
+ } else if (newVal != null){
+ vertexEdgePair = addClassEdge(instanceVertex, toVertex, edgeLabel);
+ }
+
+ return (vertexEdgePair != null) ? vertexEdgePair.getRight().getId().toString() : null;
+ }
+
+ /******************************************** CLASS **************************************************/
+
+ private Pair<Vertex, Edge> addClassEdge(Vertex instanceVertex, Vertex toVertex, String edgeLabel) throws AtlasException {
+ // add an edge to the class vertex from the instance
+ Edge edge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel);
+ return Pair.of(toVertex, edge);
+ }
+
+ private Vertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
+ Vertex referenceVertex = null;
+ Id id = null;
+ if (typedReference != null) {
+ id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
+ if (id.isAssigned()) {
+ referenceVertex = graphHelper.getVertexForGUID(id.id);
+ } else {
+ referenceVertex = idToVertexMap.get(id);
+ }
+ }
+
+ return referenceVertex;
+ }
+
+ private Id getId(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
+ Id id = null;
+ if (typedReference != null) {
+ id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
+ }
+
+ if (id.isUnassigned()) {
+ Vertex classVertex = idToVertexMap.get(id);
+ String guid = classVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ id = new Id(guid, 0, typedReference.getTypeName());
+ }
+ return id;
+ }
+
+
+ private Pair<Vertex, Edge> updateClassEdge(Id id, final ITypedReferenceableInstance typedInstance,
+ Vertex instanceVertex, Edge edge, Vertex toVertex,
+ AttributeInfo attributeInfo, IDataType dataType,
+ String edgeLabel, Operation operation) throws AtlasException {
+ Pair<Vertex, Edge> result = Pair.of(toVertex, edge);
+ Edge newEdge = edge;
+ // Update edge if it exists
+ Vertex invertex = edge.getVertex(Direction.IN);
+ String currentGUID = invertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ Id currentId = new Id(currentGUID, 0, (String) invertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
+ if (!currentId.equals(id)) {
+ // add an edge to the class vertex from the instance
+ if(toVertex != null) {
+ newEdge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel);
+ result = Pair.of(toVertex, newEdge);
+ }
+ removeUnusedReference(edge.getId().toString(), attributeInfo, dataType);
+ }
+
+ if (attributeInfo.isComposite) {
+ //Update the attributes also if composite
+ if (typedInstance.fieldMapping() != null) {
+ //In case of Id instance, fieldMapping is null
+ mapInstanceToVertex(typedInstance, toVertex, typedInstance.fieldMapping().fields , false, operation);
+ //Update full text for the updated composite vertex
+ addFullTextProperty(new ArrayList<ITypedReferenceableInstance>() {{ add(typedInstance); }});
+ }
+ }
+
+ return result;
+ }
+
+ /******************************************** TRAITS ****************************************************/
+
+ void mapTraitInstanceToVertex(ITypedStruct traitInstance, IDataType entityType, Vertex parentInstanceVertex)
+ throws AtlasException {
+ // add a new vertex for the struct or trait instance
+ final String traitName = traitInstance.getTypeName();
+ Vertex traitInstanceVertex = graphHelper.createVertexWithoutIdentity(traitInstance.getTypeName(), null,
+ typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
+ LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
+
+ // map all the attributes to this newly created vertex
+ mapInstanceToVertex(traitInstance, traitInstanceVertex, traitInstance.fieldMapping().fields, false, Operation.CREATE);
+
+ // add an edge to the newly created vertex from the parent
+ String relationshipLabel = GraphHelper.getTraitLabel(entityType.getName(), traitName);
+ graphHelper.addEdge(parentInstanceVertex, traitInstanceVertex, relationshipLabel);
+ }
+
+ /******************************************** PRIMITIVES **************************************************/
+
+ private void mapPrimitiveOrEnumToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo) throws AtlasException {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+
+ final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ Object propertyValue = null;
+
+ if (attrValue == null ) {
+ propertyValue = null;
+ } else if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
+ propertyValue = typedInstance.getString(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
+ propertyValue = typedInstance.getShort(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
+ propertyValue = typedInstance.getInt(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ propertyValue = typedInstance.getBigInt(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
+ propertyValue = typedInstance.getBoolean(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
+ propertyValue = typedInstance.getByte(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
+ propertyValue = typedInstance.getLong(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
+ propertyValue = typedInstance.getFloat(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
+ propertyValue = typedInstance.getDouble(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+ final Date dateVal = typedInstance.getDate(attributeInfo.name);
+ //Convert Property value to Long while persisting
+ propertyValue = dateVal.getTime();
+ } else if (attributeInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
+ if (attrValue != null) {
+ propertyValue = ((EnumValue)attrValue).value;
+ }
+ }
+
+
+ GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
+ }
+
+ private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) {
+ //Remove edges for property values which do not exist any more
+ Edge removedRelation = null;
+ switch (elementType.getTypeCategory()) {
+ case STRUCT:
+ removedRelation = graphHelper.removeRelation(edgeId, true);
+ //Remove the vertex from state so that further processing no longer uses this
+ break;
+ case CLASS:
+ removedRelation = graphHelper.removeRelation(edgeId, attributeInfo.isComposite);
+ break;
+ }
+ return removedRelation;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java b/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
index 86141e2..9351be9 100755
--- a/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
@@ -55,7 +55,6 @@ public class ReplaceIdWithInstance implements ObjectGraphWalker.NodeProcessor {
} else if (!nd.aInfo.isComposite || nd.value == null) {
// do nothing
} else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
-
if (nd.value != null && nd.value instanceof Id) {
Id id = (Id) nd.value;
ITypedReferenceableInstance r = getInstance(id);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 0c90709..fb782a2 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -23,8 +23,11 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Provider;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.ParamChecker;
-import org.apache.atlas.TypeNotFoundException;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.exception.TypeNotFoundException;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
@@ -39,6 +42,7 @@ import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
@@ -61,6 +65,7 @@ import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
import javax.inject.Singleton;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -282,7 +287,7 @@ public class DefaultMetadataService implements MetadataService {
entitites.add(repository.getEntityDefinition(guid));
}
- onEntitiesAddedToRepo(entitites);
+ onEntitiesAdded(entitites);
return new JSONArray(Arrays.asList(guids)).toString();
}
@@ -325,12 +330,17 @@ public class DefaultMetadataService implements MetadataService {
return InstanceSerialization.toJson(instance, true);
}
- @Override
- public String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException {
+ private ITypedReferenceableInstance getEntityDefinitionReference(String entityType, String attribute, String value)
+ throws AtlasException {
validateTypeExists(entityType);
validateUniqueAttribute(entityType, attribute);
- final ITypedReferenceableInstance instance = repository.getEntityDefinition(entityType, attribute, value);
+ return repository.getEntityDefinition(entityType, attribute, value);
+ }
+
+ @Override
+ public String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException {
+ final ITypedReferenceableInstance instance = getEntityDefinitionReference(entityType, attribute, value);
return InstanceSerialization.toJson(instance, true);
}
@@ -361,15 +371,146 @@ public class DefaultMetadataService implements MetadataService {
return repository.getEntityList(entityType);
}
+ /**
+ * Updates an entity, instance of the type based on the guid set.
+ *
+ * @param entityInstanceDefinition json array of entity definitions
+ * @return guids - json array of guids
+ */
@Override
- public void updateEntity(String guid, String property, String value) throws AtlasException {
+ public String updateEntities(String entityInstanceDefinition) throws AtlasException {
+
+ ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition cannot be empty");
+ ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition);
+
+ String[] guids = repository.updateEntities(typedInstances);
+ onEntitiesAdded(Arrays.asList(typedInstances));
+
+ return new JSONArray(Arrays.asList(guids)).toString();
+ }
+
+ @Override
+ public void updateEntityAttributeByGuid(final String guid, String attributeName, String value) throws AtlasException {
ParamChecker.notEmpty(guid, "guid cannot be null");
- ParamChecker.notEmpty(property, "property cannot be null");
+ ParamChecker.notEmpty(attributeName, "property cannot be null");
ParamChecker.notEmpty(value, "property value cannot be null");
- repository.updateEntity(guid, property, value);
+ ITypedReferenceableInstance existInstance = validateEntityExists(guid);
+ ClassType type = typeSystem.getDataType(ClassType.class, existInstance.getTypeName());
+ ITypedReferenceableInstance newInstance = type.createInstance();
+
+ AttributeInfo attributeInfo = type.fieldMapping.fields.get(attributeName);
+ if (attributeInfo == null) {
+ throw new AtlasException("Invalid property " + attributeName + " for entity " + existInstance.getTypeName());
+ }
+
+ DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
+
+ switch(attrTypeCategory) {
+ case PRIMITIVE:
+ newInstance.set(attributeName, value);
+ break;
+ case CLASS:
+ Id id = new Id(value, 0, attributeInfo.dataType().getName());
+ newInstance.set(attributeName, id);
+ break;
+ default:
+ throw new AtlasException("Update of " + attrTypeCategory + " is not supported");
+ }
+
+ ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
+ repository.updatePartial(newInstance);
+ onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
+ add(repository.getEntityDefinition(guid));
+ }});
+ }
+
+ private ITypedReferenceableInstance validateEntityExists(String guid)
+ throws EntityNotFoundException, RepositoryException {
+ final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
+ if (instance == null) {
+ throw new EntityNotFoundException(String.format("Entity with guid %s not found ", guid));
+ }
+ return instance;
+ }
+
+ @Override
+ public void updateEntityPartialByGuid(final String guid, Referenceable newEntity) throws AtlasException {
+ ParamChecker.notEmpty(guid, "guid cannot be null");
+ ParamChecker.notNull(newEntity, "updatedEntity cannot be null");
+ ITypedReferenceableInstance existInstance = validateEntityExists(guid);
- onEntityUpdated(repository.getEntityDefinition(guid));
+ ITypedReferenceableInstance newInstance = convertToTypedInstance(newEntity, existInstance.getTypeName());
+ ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
+
+ repository.updatePartial(newInstance);
+ onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
+ add(repository.getEntityDefinition(guid));
+ }});
+ }
+
+ private ITypedReferenceableInstance convertToTypedInstance(Referenceable updatedEntity, String typeName) throws AtlasException {
+ ClassType type = typeSystem.getDataType(ClassType.class, typeName);
+ ITypedReferenceableInstance newInstance = type.createInstance();
+
+ for (String attributeName : updatedEntity.getValuesMap().keySet()) {
+ AttributeInfo attributeInfo = type.fieldMapping.fields.get(attributeName);
+ if (attributeInfo == null) {
+ throw new AtlasException("Invalid property " + attributeName + " for entity " + updatedEntity);
+ }
+
+ DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
+ Object value = updatedEntity.get(attributeName);
+ if (value != null) {
+ switch (attrTypeCategory) {
+ case CLASS:
+ if (value instanceof Referenceable) {
+ newInstance.set(attributeName, value);
+ } else {
+ Id id = new Id((String) value, 0, attributeInfo.dataType().getName());
+ newInstance.set(attributeName, id);
+ }
+ break;
+
+ case ENUM:
+ case PRIMITIVE:
+ case ARRAY:
+ case STRUCT:
+ case MAP:
+ newInstance.set(attributeName, value);
+ break;
+
+ case TRAIT:
+ //TODO - handle trait updates as well?
+ default:
+ throw new AtlasException("Update of " + attrTypeCategory + " is not supported");
+ }
+ }
+ }
+
+ return newInstance;
+ }
+
+ @Override
+ public String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue,
+ Referenceable updatedEntity) throws AtlasException {
+ ParamChecker.notEmpty(typeName, "typeName cannot be null");
+ ParamChecker.notEmpty(uniqueAttributeName, "uniqueAttributeName cannot be null");
+ ParamChecker.notNull(attrValue, "value cannot be null");
+ ParamChecker.notNull(updatedEntity, "updatedEntity cannot be null");
+
+ ITypedReferenceableInstance oldInstance = getEntityDefinitionReference(typeName, uniqueAttributeName, attrValue);
+
+ final ITypedReferenceableInstance newInstance = convertToTypedInstance(updatedEntity, typeName);
+ ((ReferenceableInstance)newInstance).replaceWithNewId(oldInstance.getId());
+
+ repository.updatePartial(newInstance);
+
+ onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
+ add(newInstance);
+ }});
+
+ return newInstance.getId()._getId();
}
private void validateTypeExists(String entityType) throws AtlasException {
@@ -485,8 +626,7 @@ public class DefaultMetadataService implements MetadataService {
}
}
- private void onEntitiesAddedToRepo(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
-
+ private void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntitiesAdded(entities);
}
@@ -509,10 +649,10 @@ public class DefaultMetadataService implements MetadataService {
}
}
- private void onEntityUpdated(ITypedReferenceableInstance entity)
+ private void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities)
throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
- listener.onEntityUpdated(entity);
+ listener.onEntitiesUpdated(entities);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/services/MetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/MetadataService.java b/repository/src/main/java/org/apache/atlas/services/MetadataService.java
deleted file mode 100755
index f027b79..0000000
--- a/repository/src/main/java/org/apache/atlas/services/MetadataService.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.listener.EntityChangeListener;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.util.List;
-
-/**
- * Metadata service.
- */
-public interface MetadataService {
-
- /**
- * Creates a new type based on the type system to enable adding
- * entities (instances for types).
- *
- * @param typeDefinition definition as json
- * @return a unique id for this type
- */
- JSONObject createType(String typeDefinition) throws AtlasException;
-
- /**
- * Updates the given types in the type definition
- * @param typeDefinition
- * @return
- * @throws AtlasException
- */
- JSONObject updateType(String typeDefinition) throws AtlasException;
-
- /**
- * Return the definition for the given type.
- *
- * @param typeName name for this type, must be unique
- * @return type definition as JSON
- */
- String getTypeDefinition(String typeName) throws AtlasException;
-
- /**
- * Return the list of types in the type system.
- *
- * @return list of type names in the type system
- */
- List<String> getTypeNamesList() throws AtlasException;
-
- /**
- * Return the list of trait type names in the type system.
- *
- * @return list of trait type names in the type system
- */
- List<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) throws AtlasException;
-
- /**
- * Creates an entity, instance of the type.
- *
- * @param entityDefinition definition
- * @return guid
- */
- String createEntities(String entityDefinition) throws AtlasException;
-
- /**
- * Return the definition for the given guid.
- *
- * @param guid guid
- * @return entity definition as JSON
- */
- String getEntityDefinition(String guid) throws AtlasException;
-
- /**
- * Return the definition given type and attribute. The attribute has to be unique attribute for the type
- * @param entityType - type name
- * @param attribute - attribute name
- * @param value - attribute value
- * @return
- * @throws AtlasException
- */
- String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
-
- /**
- * Return the list of entity names for the given type in the repository.
- *
- * @param entityType type
- * @return list of entity names for the given type in the repository
- */
- List<String> getEntityList(String entityType) throws AtlasException;
-
- /**
- * Adds the property to the given entity id(guid).
- *
- * @param guid entity id
- * @param property property name
- * @param value property value
- */
- void updateEntity(String guid, String property, String value) throws AtlasException;
-
- // Trait management functions
-
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- * @throws AtlasException
- */
- List<String> getTraitNames(String guid) throws AtlasException;
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitInstanceDefinition trait instance that needs to be added to entity
- * @throws AtlasException
- */
- void addTrait(String guid, String traitInstanceDefinition) throws AtlasException;
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitNameToBeDeleted name of the trait
- * @throws AtlasException
- */
- void deleteTrait(String guid, String traitNameToBeDeleted) throws AtlasException;
-
- /**
- * Register a listener for entity change.
- *
- * @param listener the listener to register
- */
- void registerListener(EntityChangeListener listener);
-
- /**
- * Unregister an entity change listener.
- *
- * @param listener the listener to unregister
- */
- void unregisterListener(EntityChangeListener listener);
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
index d857a66..34d101a 100755
--- a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
@@ -26,7 +26,7 @@ import com.tinkerpop.blueprints.{Vertex, Direction}
import org.apache.atlas.AtlasException
import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
import org.apache.atlas.query.TypeUtils.FieldInfo
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository
+import org.apache.atlas.repository.graph.{GraphHelper, GraphBackedMetadataRepository}
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types.DataTypes._
import org.apache.atlas.typesystem.types._
@@ -199,7 +199,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName"
- def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphBackedMetadataRepository.getQualifiedName(dataType, aInfo.name)
+ def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphHelper.getQualifiedFieldName(dataType, aInfo.name)
def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id =
new Id(v.getId.toString, 0, dataTypeNm)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
index f9378e4..1075d85 100644
--- a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
@@ -21,10 +21,10 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
-import org.apache.atlas.services.DefaultMetadataService;
+import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
@@ -55,10 +55,10 @@ import java.util.List;
public class BaseHiveRepositoryTest {
@Inject
- protected DefaultMetadataService metadataService;
+ protected MetadataService metadataService;
@Inject
- protected GraphBackedMetadataRepository repository;
+ protected MetadataRepository repository;
@Inject
protected GraphProvider<TitanGraph> graphProvider;
@@ -67,7 +67,7 @@ public class BaseHiveRepositoryTest {
setUpTypes();
new GraphBackedSearchIndexer(graphProvider);
setupInstances();
- // TestUtils.dumpGraph(graphProvider.get());
+ TestUtils.dumpGraph(graphProvider.get());
}
protected void tearDown() throws Exception {
@@ -190,17 +190,20 @@ public class BaseHiveRepositoryTest {
Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
Referenceable sd =
- storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(
+ column("time_id", "int", "time id")));
List<Referenceable> salesFactColumns = ImmutableList
- .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"),
+ .of(column("time_id", "int", "time id"),
+ column("product_id", "int", "product id"),
column("customer_id", "int", "customer id", "PII"),
column("sales", "double", "product id", "Metric"));
Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
List<Referenceable> timeDimColumns = ImmutableList
- .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
+ .of(column("time_id", "int", "time id"),
+ column("dayOfYear", "int", "day Of Year"),
column("weekDay", "int", "week Day"));
Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
@@ -217,7 +220,8 @@ public class BaseHiveRepositoryTest {
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
List<Referenceable> productDimColumns = ImmutableList
- .of(column("product_id", "int", "product id"), column("product_name", "string", "product name"),
+ .of(column("product_id", "int", "product id"),
+ column("product_name", "string", "product name"),
column("brand_name", "int", "brand name"));
Id productDim =
@@ -226,7 +230,8 @@ public class BaseHiveRepositoryTest {
view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess");
- List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"),
+ List<Referenceable> customerDimColumns = ImmutableList.of(
+ column("customer_id", "int", "customer id", "PII"),
column("name", "string", "customer name", "PII"),
column("address", "string", "customer address", "PII"));
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index 01a8158..12c47d4 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -187,6 +187,8 @@ public final class TestUtils {
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
+ public static final String PARTITION_TYPE = "partition_type";
+ public static final String SERDE_TYPE = "serdeType";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String PII = "PII";
@@ -208,7 +210,8 @@ public final class TestUtils {
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType",
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- createRequiredAttrDef("serde", DataTypes.STRING_TYPE)});
+ createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
+ createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),};
@@ -244,21 +247,23 @@ public final class TestUtils {
new AttributeDefinition("parametersMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, true, null),
- // map of classes - todo - enable this
- // new AttributeDefinition("columnsMap",
- // DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
- // "column_type"),
- // Multiplicity.COLLECTION, true, null),
- // map of structs todo - enable this
- // new AttributeDefinition("partitionsMap",
- // DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
- // "partition_type"),
- // Multiplicity.COLLECTION, true, null),
+ //map of classes -
+ new AttributeDefinition("columnsMap",
+ DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
+ "column_type"),
+ Multiplicity.COLLECTION, true, null),
+ //map of structs
+ new AttributeDefinition("partitionsMap",
+ DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
+ "partition_type"),
+ Multiplicity.COLLECTION, true, null),
// struct reference
new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null),
// class reference
- new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
+ new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
+ //class reference as composite
+ new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null));
HierarchicalTypeDefinition<TraitType> piiTypeDefinition =
createTraitTypeDef(PII, ImmutableList.<String>of());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index 36b207a..ea8718d 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -24,7 +24,7 @@ import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
@@ -55,7 +55,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
private GraphProvider<TitanGraph> graphProvider;
@Inject
- private GraphBackedMetadataRepository repositoryService;
+ private MetadataRepository repositoryService;
@Inject
private GraphBackedDiscoveryService discoveryService;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
index 0e6913d..6d5a15a 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
@@ -20,8 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
-import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
@@ -32,7 +31,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
-import java.util.List;
/**
* Unit tests for Hive LineageService.
@@ -41,7 +39,7 @@ import java.util.List;
public class HiveLineageServiceTest extends BaseHiveRepositoryTest {
@Inject
- private GraphBackedDiscoveryService discoveryService;
+ private DiscoveryService discoveryService;
@Inject
private HiveLineageService hiveLineageService;
[2/5] incubator-atlas git commit: ATLAS-47 Entity mutations for
complex types (sumasai via shwethags)
Posted by sh...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
index b6e62aa..78af443 100755
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
@@ -29,13 +29,13 @@ import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
@@ -131,7 +131,7 @@ public class GraphBackedMetadataRepositoryTest {
Assert.assertNotNull(entity);
}
- @Test(expectedExceptions = RepositoryException.class)
+ @Test(expectedExceptions = EntityNotFoundException.class)
public void testGetEntityDefinitionNonExistent() throws Exception {
repositoryService.getEntityDefinition("blah");
Assert.fail();
@@ -342,13 +342,13 @@ public class GraphBackedMetadataRepositoryTest {
}
Id expected = new Id(guid, tableVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), TestUtils.TABLE_TYPE);
- Assert.assertEquals(repositoryService.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected);
+ Assert.assertEquals(GraphHelper.getIdFromVertex(TestUtils.TABLE_TYPE, tableVertex), expected);
}
@Test(dependsOnMethods = "testCreateEntity")
public void testGetTypeName() throws Exception {
Vertex tableVertex = getTableEntityVertex();
- Assert.assertEquals(repositoryService.getTypeName(tableVertex), TestUtils.TABLE_TYPE);
+ Assert.assertEquals(GraphHelper.getTypeName(tableVertex), TestUtils.TABLE_TYPE);
}
@Test(dependsOnMethods = "testCreateEntity")
@@ -415,6 +415,9 @@ public class GraphBackedMetadataRepositoryTest {
public void testBug37860() throws Exception {
String dslQuery = "hive_table as t where name = 'bar' "
+ "database where name = 'foo' and description = 'foo database' select t";
+
+ TestUtils.dumpGraph(graphProvider.get());
+
System.out.println("Executing dslQuery = " + dslQuery);
String jsonResults = discoveryService.searchByDSL(dslQuery);
Assert.assertNotNull(jsonResults);
@@ -446,6 +449,8 @@ public class GraphBackedMetadataRepositoryTest {
//but with elasticsearch, doesn't work without sleep. why??
long sleepInterval = 1000;
+ TestUtils.dumpGraph(graphProvider.get());
+
//person in hr department whose name is john
Thread.sleep(sleepInterval);
String response = discoveryService.searchByFullText("john");
@@ -475,31 +480,36 @@ public class GraphBackedMetadataRepositoryTest {
@Test(dependsOnMethods = "testSubmitEntity")
public void testUpdateEntity_MultiplicityOneNonCompositeReference() throws Exception {
ITypedReferenceableInstance john = repositoryService.getEntityDefinition("Person", "name", "John");
- String johnGuid = john.getId()._getId();
+ Id johnGuid = john.getId();
ITypedReferenceableInstance max = repositoryService.getEntityDefinition("Person", "name", "Max");
String maxGuid = max.getId()._getId();
ITypedReferenceableInstance jane = repositoryService.getEntityDefinition("Person", "name", "Jane");
- String janeGuid = jane.getId()._getId();
+ Id janeGuid = jane.getId();
// Update max's mentor reference to john.
- repositoryService.updateEntity(maxGuid, "mentor", johnGuid);
+ ClassType personType = typeSystem.getDataType(ClassType.class, "Person");
+ ITypedReferenceableInstance instance = personType.createInstance(max.getId());
+ instance.set("mentor", johnGuid);
+ repositoryService.updatePartial(instance);
// Verify the update was applied correctly - john should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
Object object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
ITypedReferenceableInstance refTarget = (ITypedReferenceableInstance) object;
- Assert.assertEquals(refTarget.getId()._getId(), johnGuid);
+ Assert.assertEquals(refTarget.getId()._getId(), johnGuid._getId());
// Update max's mentor reference to jane.
- repositoryService.updateEntity(maxGuid, "mentor", janeGuid);
-
+ instance = personType.createInstance(max.getId());
+ instance.set("mentor", janeGuid);
+ repositoryService.updatePartial(instance);
+
// Verify the update was applied correctly - jane should now be max's mentor.
max = repositoryService.getEntityDefinition(maxGuid);
object = max.get("mentor");
Assert.assertTrue(object instanceof ITypedReferenceableInstance);
refTarget = (ITypedReferenceableInstance) object;
- Assert.assertEquals(refTarget.getId()._getId(), janeGuid);
+ Assert.assertEquals(refTarget.getId()._getId(), janeGuid._getId());
}
private ITypedReferenceableInstance createHiveTableInstance(Referenceable databaseInstance) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
index 11e8219..c25ccf8 100755
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphRepoMapperScaleTest.java
@@ -18,15 +18,9 @@
package org.apache.atlas.repository.graph;
-import com.google.inject.Inject;
-import com.thinkaurelius.titan.core.TitanFactory;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.TitanIndexQuery;
import com.thinkaurelius.titan.core.util.TitanCleanup;
-import com.thinkaurelius.titan.diskstorage.BackendException;
-import com.thinkaurelius.titan.diskstorage.configuration.ReadConfiguration;
-import com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration;
-import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration;
import com.tinkerpop.blueprints.Compare;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Predicate;
@@ -42,19 +36,16 @@ import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.commons.io.FileUtils;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
-import java.io.File;
-import java.io.IOException;
+import javax.inject.Inject;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
-import java.util.Random;
@Test
@Guice(modules = RepositoryMetadataModule.class)
@@ -63,15 +54,13 @@ public class GraphRepoMapperScaleTest {
private static final String DATABASE_NAME = "foo";
private static final String TABLE_NAME = "bar";
- private static final String INDEX_DIR =
- System.getProperty("java.io.tmpdir", "/tmp") + "/atlas-test" + new Random().nextLong();
-
@Inject
GraphProvider<TitanGraph> graphProvider;
@Inject
private GraphBackedMetadataRepository repositoryService;
+ @Inject
private GraphBackedSearchIndexer searchIndexer;
private TypeSystem typeSystem = TypeSystem.getInstance();
@@ -81,7 +70,7 @@ public class GraphRepoMapperScaleTest {
@BeforeClass
@GraphTransaction
public void setUp() throws Exception {
- searchIndexer = new GraphBackedSearchIndexer(graphProvider);
+ //Make sure we can cleanup the index directory
Collection<IDataType> typesAdded = TestUtils.createHiveTypes(typeSystem);
searchIndexer.onAdd(typesAdded);
}
@@ -127,7 +116,6 @@ public class GraphRepoMapperScaleTest {
//Elasticsearch requires some time before index is updated
Thread.sleep(5000);
-
searchWithOutIndex(Constants.GUID_PROPERTY_KEY, dbGUID);
searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, "column_type");
searchWithOutIndex(Constants.ENTITY_TYPE_PROPERTY_KEY, TestUtils.TABLE_TYPE);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
index 9d32332..0352ef3 100644
--- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
@@ -18,29 +18,40 @@
package org.apache.atlas.service;
+import com.google.common.collect.ImmutableList;
import com.google.inject.Inject;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
+import org.apache.atlas.typesystem.exception.TypeNotFoundException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
-import org.apache.atlas.TypeNotFoundException;
-import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
@Guice(modules = RepositoryMetadataModule.class)
public class DefaultMetadataServiceTest {
@Inject
@@ -48,8 +59,16 @@ public class DefaultMetadataServiceTest {
@Inject
private GraphProvider<TitanGraph> graphProvider;
+ private Referenceable db = createDBEntity();
+
+ private Id dbId;
+
+ private Referenceable table;
+
+ private Id tableId;
+
- @BeforeClass
+ @BeforeTest
public void setUp() throws Exception {
TypesDef typesDef = TestUtils.defineHiveTypes();
try {
@@ -57,10 +76,21 @@ public class DefaultMetadataServiceTest {
} catch (TypeNotFoundException e) {
metadataService.createType(TypesSerialization.toJson(typesDef));
}
+
+ String dbGUid = createInstance(db);
+ dbId = new Id(dbGUid, 0, TestUtils.DATABASE_TYPE);
+
+ table = createTableEntity(dbId);
+ String tableGuid = createInstance(table);
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ table = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ tableId = new Id(tableGuid, 0, TestUtils.TABLE_TYPE);
}
- @AfterClass
- public void shudown() {
+ @AfterTest
+ public void shutdown() {
+ TypeSystem.getInstance().reset();
try {
//TODO - Fix failure during shutdown while using BDB
graphProvider.get().shutdown();
@@ -82,6 +112,16 @@ public class DefaultMetadataServiceTest {
return new JSONArray(response).getString(0);
}
+ private String updateInstance(Referenceable entity) throws Exception {
+ ParamChecker.notNull(entity, "Entity");
+ ParamChecker.notNull(entity.getId(), "Entity");
+ String entityjson = InstanceSerialization.toJson(entity, true);
+ JSONArray entitiesJson = new JSONArray();
+ entitiesJson.put(entityjson);
+ String response = metadataService.updateEntities(entitiesJson.toString());
+ return new JSONArray(response).getString(0);
+ }
+
private Referenceable createDBEntity() {
Referenceable entity = new Referenceable(TestUtils.DATABASE_TYPE);
String dbName = RandomStringUtils.randomAlphanumeric(10);
@@ -90,6 +130,18 @@ public class DefaultMetadataServiceTest {
return entity;
}
+ private Referenceable createTableEntity(Id dbId) {
+ Referenceable entity = new Referenceable(TestUtils.TABLE_TYPE);
+ String tableName = RandomStringUtils.randomAlphanumeric(10);
+ entity.set("name", tableName);
+ entity.set("description", "random table");
+ entity.set("type", "type");
+ entity.set("tableType", "MANAGED");
+ entity.set("database", dbId);
+ entity.set("created", new Date());
+ return entity;
+ }
+
@Test
public void testCreateEntityWithUniqueAttribute() throws Exception {
//name is the unique attribute
@@ -116,30 +168,447 @@ public class DefaultMetadataServiceTest {
table.set("description", "random table");
table.set("type", "type");
table.set("tableType", "MANAGED");
- table.set("database", db);
+ table.set("database", new Id(dbId, 0, TestUtils.DATABASE_TYPE));
+ table.set("databaseComposite", db);
createInstance(table);
//table create should re-use the db instance created earlier
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
- Referenceable actualDb = (Referenceable) tableDefinition.get("database");
+ Referenceable actualDb = (Referenceable) tableDefinition.get("databaseComposite");
Assert.assertEquals(actualDb.getId().id, dbId);
}
@Test
- public void testCreateEntityWithEnum() throws Exception {
- Referenceable dbEntity = createDBEntity();
- String db = createInstance(dbEntity);
+ public void testUpdateEntityByUniqueAttribute() throws Exception {
+ final List<String> colNameList = ImmutableList.of("col1", "col2");
+ Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
+ put("columnNames", colNameList);
+ }});
+ metadataService.updateEntityByUniqueAttribute(table.getTypeName(), "name", (String) table.get("name"), tableUpdated);
- Referenceable table = new Referenceable(TestUtils.TABLE_TYPE);
- table.set("name", TestUtils.randomString());
- table.set("description", "random table");
- table.set("type", "type");
- table.set("tableType", "MANAGED");
- table.set("database", dbEntity);
- createInstance(table);
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ List<String> actualColumns = (List) tableDefinition.get("columnNames");
+ Assert.assertEquals(actualColumns, colNameList);
+ }
+
+ @Test
+ public void testUpdateEntityWithMap() throws Exception {
+
+ final Map<String, Struct> partsMap = new HashMap<>();
+ partsMap.put("part0", new Struct("partition_type",
+ new HashMap<String, Object>() {{
+ put("name", "test");
+ }}));
+
+ table.set("partitionsMap", partsMap);
+
+ updateInstance(table);
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertTrue(partsMap.get("part0").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0")));
+
+ //update map - add a map key
+ partsMap.put("part1", new Struct("partition_type",
+ new HashMap<String, Object>() {{
+ put("name", "test1");
+ }}));
+ table.set("partitionsMap", partsMap);
+ updateInstance(table);
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
+ Assert.assertTrue(partsMap.get("part1").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part1")));
+
+ //update map - remove a key and add another key
+ partsMap.remove("part0");
+ partsMap.put("part2", new Struct("partition_type",
+ new HashMap<String, Object>() {{
+ put("name", "test2");
+ }}));
+ table.set("partitionsMap", partsMap);
+
+ updateInstance(table);
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
+ Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
+ Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
+
+
+ //update struct value for existing map key
+ Struct partition2 = (Struct)partsMap.get("part2");
+ partition2.set("name", "test2Updated");
+ updateInstance(table);
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertEquals(((Map<String, Struct>)tableDefinition.get("partitionsMap")).size(), 2);
+ Assert.assertNull(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part0"));
+ Assert.assertTrue(partsMap.get("part2").equalsContents(((Map<String, Struct>)tableDefinition.get("partitionsMap")).get("part2")));
+ }
+
+ @Test
+ public void testUpdateEntityAddAndUpdateArrayAttr() throws Exception {
+ //Update entity, add new array attribute
+ //add array of primitives
+ final List<String> colNameList = ImmutableList.of("col1", "col2");
+ Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
+ put("columnNames", colNameList);
+ }});
+ metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
+
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ List<String> actualColumns = (List) tableDefinition.get("columnNames");
+ Assert.assertEquals(actualColumns, colNameList);
+
+ //update array of primitives
+ final List<String> updatedColNameList = ImmutableList.of("col2", "col3");
+ tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
+ put("columnNames", updatedColNameList);
+ }});
+ metadataService.updateEntityPartialByGuid(tableId.getId()._getId(), tableUpdated);
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ actualColumns = (List) tableDefinition.get("columnNames");
+ Assert.assertEquals(actualColumns, updatedColNameList);
+ }
+
+ @Test
+ public void testUpdateEntityArrayOfClass() throws Exception {
+ //test array of class with id
+ final List<Referenceable> columns = new ArrayList<>();
+ Map<String, Object> values = new HashMap<>();
+ values.put("name", "col1");
+ values.put("type", "type");
+ Referenceable ref = new Referenceable("column_type", values);
+ columns.add(ref);
+ Referenceable tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
+ put("columns", columns);
+ }});
+ metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
+
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ final List<Referenceable> arrClsColumns = (List) tableDefinition.get("columns");
+ Assert.assertTrue(arrClsColumns.get(0).equalsContents(columns.get(0)));
+
+ //Partial update. Add col5 But also update col1
+ Map<String, Object> valuesCol5 = new HashMap<>();
+ valuesCol5.put("name", "col5");
+ valuesCol5.put("type", "type");
+ ref = new Referenceable("column_type", valuesCol5);
+ //update col1
+ arrClsColumns.get(0).set("type", "type1");
+
+ //add col5
+ final List<Referenceable> updateColumns = new ArrayList<>(arrClsColumns);
+ updateColumns.add(ref);
+
+ tableUpdated = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
+ put("columns", updateColumns);
+ }});
+ metadataService.updateEntityPartialByGuid(tableId._getId(), tableUpdated);
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ List<Referenceable> arrColumnsList = (List) tableDefinition.get("columns");
+ Assert.assertEquals(arrColumnsList.size(), 2);
+ Assert.assertTrue(arrColumnsList.get(0).equalsContents(updateColumns.get(0)));
+ Assert.assertTrue(arrColumnsList.get(1).equalsContents(updateColumns.get(1)));
+
+ //Complete update. Add array elements - col3,4
+ Map<String, Object> values1 = new HashMap<>();
+ values1.put("name", "col3");
+ values1.put("type", "type");
+ Referenceable ref1 = new Referenceable("column_type", values1);
+ columns.add(ref1);
+
+ Map<String, Object> values2 = new HashMap<>();
+ values2.put("name", "col4");
+ values2.put("type", "type");
+ Referenceable ref2 = new Referenceable("column_type", values2);
+ columns.add(ref2);
+
+ table.set("columns", columns);
+ updateInstance(table);
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ arrColumnsList = (List) tableDefinition.get("columns");
+ Assert.assertEquals(arrColumnsList.size(), columns.size());
+ Assert.assertTrue(arrColumnsList.get(1).equalsContents(columns.get(1)));
+ Assert.assertTrue(arrColumnsList.get(2).equalsContents(columns.get(2)));
+
+
+ //Remove a class reference/Id and insert another reference
+ //Also covers isComposite case since columns is a composite
+ values.clear();
+ columns.clear();
+
+ values.put("name", "col2");
+ values.put("type", "type");
+ ref = new Referenceable("column_type", values);
+ columns.add(ref);
+ table.set("columns", columns);
+ updateInstance(table);
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ arrColumnsList = (List) tableDefinition.get("columns");
+ Assert.assertEquals(arrColumnsList.size(), columns.size());
+ Assert.assertTrue(arrColumnsList.get(0).equalsContents(columns.get(0)));
+
+ //Update array column to null
+ table.setNull("columns");
+ String newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ Assert.assertNull(tableDefinition.get("columns"));
+ }
+
+
+ @Test
+ public void testStructs() throws Exception {
+ Struct serdeInstance = new Struct(TestUtils.SERDE_TYPE);
+ serdeInstance.set("name", "serde1Name");
+ serdeInstance.set("serde", "test");
+ serdeInstance.set("description", "testDesc");
+ table.set("serde1", serdeInstance);
+
+ String newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ Assert.assertNotNull(tableDefinition.get("serde1"));
+ Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
+
+ //update struct attribute
+ serdeInstance.set("serde", "testUpdated");
+ updateInstance(table);
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertTrue(serdeInstance.equalsContents(tableDefinition.get("serde1")));
+
+ //set to null
+ serdeInstance.setNull("description");
+ updateInstance(table);
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(tableId._getId());
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ Assert.assertNull(((Struct)tableDefinition.get("serde1")).get("description"));
+ }
+
+ @Test
+ public void testClassUpdate() throws Exception {
+ //Create new db instance
+ final Referenceable databaseInstance = new Referenceable(TestUtils.DATABASE_TYPE);
+ databaseInstance.set("name", TestUtils.randomString());
+ databaseInstance.set("description", "new database");
+
+ String dbId = createInstance(databaseInstance);
+
+ /*Update reference property with Id */
+ metadataService.updateEntityAttributeByGuid(tableId._getId(), "database", dbId);
+
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(tableId._getId());
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertEquals(dbId, (((Id)tableDefinition.get("database"))._getId()));
+
+ /* Update with referenceable - TODO - Fails . Need to fix this */
+ /*final String dbName = TestUtils.randomString();
+ final Referenceable databaseInstance2 = new Referenceable(TestUtils.DATABASE_TYPE);
+ databaseInstance2.set("name", dbName);
+ databaseInstance2.set("description", "new database 2");
+
+ Referenceable updateTable = new Referenceable(TestUtils.TABLE_TYPE, new HashMap<String, Object>() {{
+ put("database", databaseInstance2);
+ }});
+ metadataService.updateEntityAttributeByGuid(tableId._getId(), updateTable);
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(tableId._getId());
+ Referenceable tableDefinitionActual = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ String dbDefJson = metadataService.getEntityDefinition(TestUtils.DATABASE_TYPE, "name", dbName);
+ Referenceable dbDef = InstanceSerialization.fromJsonReferenceable(dbDefJson, true);
+
+ Assert.assertNotEquals(dbId, (((Id) tableDefinitionActual.get("database"))._getId()));
+ Assert.assertEquals(dbDef.getId()._getId(), (((Id) tableDefinitionActual.get("database"))._getId())); */
+
+ }
+
+ @Test
+ public void testArrayOfStructs() throws Exception {
+ //Add array of structs
+ TestUtils.dumpGraph(graphProvider.get());
+
+ final Struct partition1 = new Struct(TestUtils.PARTITION_TYPE);
+ partition1.set("name", "part1");
+
+ final Struct partition2 = new Struct(TestUtils.PARTITION_TYPE);
+ partition2.set("name", "part2");
+
+ List<Struct> partitions = new ArrayList<Struct>(){{ add(partition1); add(partition2); }};
+ table.set("partitions", partitions);
+
+ String newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertNotNull(tableDefinition.get("partitions"));
+ List<Struct> partitionsActual = (List<Struct>) tableDefinition.get("partitions");
+ Assert.assertEquals(partitionsActual.size(), 2);
+ Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0)));
+
+ //add a new element to array of struct
+ final Struct partition3 = new Struct(TestUtils.PARTITION_TYPE);
+ partition3.set("name", "part3");
+ partitions.add(partition3);
+ table.set("partitions", partitions);
+ newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertNotNull(tableDefinition.get("partitions"));
+ partitionsActual = (List<Struct>) tableDefinition.get("partitions");
+ Assert.assertEquals(partitionsActual.size(), 3);
+ Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2)));
+
+ //remove one of the struct values
+ partitions.remove(1);
+ table.set("partitions", partitions);
+ newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertNotNull(tableDefinition.get("partitions"));
+ partitionsActual = (List<Struct>) tableDefinition.get("partitions");
+ Assert.assertEquals(partitionsActual.size(), 2);
+ Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0)));
+ Assert.assertTrue(partitions.get(1).equalsContents(partitionsActual.get(1)));
+
+ //Update struct value within array of struct
+ partition1.set("name", "part4");
+ newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertNotNull(tableDefinition.get("partitions"));
+ partitionsActual = (List<Struct>) tableDefinition.get("partitions");
+ Assert.assertEquals(partitionsActual.size(), 2);
+ Assert.assertTrue(partitions.get(0).equalsContents(partitionsActual.get(0)));
+
+ //add a repeated element to array of struct
+ final Struct partition4 = new Struct(TestUtils.PARTITION_TYPE);
+ partition4.set("name", "part4");
+ partitions.add(partition4);
+ table.set("partitions", partitions);
+ newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertNotNull(tableDefinition.get("partitions"));
+ partitionsActual = (List<Struct>) tableDefinition.get("partitions");
+ Assert.assertEquals(partitionsActual.size(), 3);
+ Assert.assertEquals(partitionsActual.get(2).get("name"), "part4");
+ Assert.assertEquals(partitionsActual.get(0).get("name"), "part4");
+ Assert.assertTrue(partitions.get(2).equalsContents(partitionsActual.get(2)));
+
+
+ // Remove all elements. Should set array attribute to null
+ partitions.clear();
+ newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertNull(tableDefinition.get("partitions"));
+ }
+
+
+ @Test(expectedExceptions = ValueConversionException.class)
+ public void testUpdateRequiredAttrToNull() throws Exception {
+ //Update required attribute
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ Assert.assertEquals(tableDefinition.get("description"), "random table");
+ table.setNull("description");
+
+ updateInstance(table);
+ Assert.fail("Expected exception while updating required attribute to null");
+ }
+
+ @Test
+ public void testUpdateOptionalAttrToNull() throws Exception {
+
+ String tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+
+ //Update optional Attribute
+ Assert.assertNotNull(tableDefinition.get("created"));
+ //Update optional attribute
+ table.setNull("created");
+
+ String newtableId = updateInstance(table);
+ Assert.assertEquals(newtableId, tableId._getId());
+
+ tableDefinitionJson =
+ metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
+ tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
+ Assert.assertNull(tableDefinition.get("created"));
+ }
+
+ @Test
+ public void testCreateEntityWithEnum() throws Exception {
String tableDefinitionJson =
metadataService.getEntityDefinition(TestUtils.TABLE_TYPE, "name", (String) table.get("name"));
Referenceable tableDefinition = InstanceSerialization.fromJsonReferenceable(tableDefinitionJson, true);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
----------------------------------------------------------------------
diff --git a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
index 1097828..0b57df3 100755
--- a/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
+++ b/repository/src/test/scala/org/apache/atlas/query/GremlinTest.scala
@@ -39,7 +39,6 @@ class GremlinTest extends BaseGremlinTest {
gProvider = new TitanGraphProvider();
gp = new DefaultGraphPersistenceStrategy(new GraphBackedMetadataRepository(gProvider))
g = QueryTestsUtils.setupTestGraph(gProvider)
-
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/pom.xml
----------------------------------------------------------------------
diff --git a/server-api/pom.xml b/server-api/pom.xml
new file mode 100644
index 0000000..88de030
--- /dev/null
+++ b/server-api/pom.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>apache-atlas</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <version>0.6-incubating-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>atlas-server-api</artifactId>
+
+ <name>Apache Atlas Server API</name>
+ <description>Apache Atlas Server related APIs</description>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-typesystem</artifactId>
+ </dependency>
+
+ </dependencies>
+
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryException.java b/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
new file mode 100644
index 0000000..ba69af7
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.AtlasException;
+
+import java.security.PrivilegedActionException;
+
+public class DiscoveryException extends AtlasException {
+
+ /**
+ * Constructs a new exception with the specified detail message. The
+ * cause is not initialized, and may subsequently be initialized by
+ * a call to {@link #initCause}.
+ *
+ * @param message the detail message. The detail message is saved for
+ * later retrieval by the {@link #getMessage()} method.
+ */
+ public DiscoveryException(String message) {
+ super(message);
+ }
+
+ /**
+ * Constructs a new exception with the specified detail message and
+ * cause. <p>Note that the detail message associated with
+ * {@code cause} is <i>not</i> automatically incorporated in
+ * this exception's detail message.
+ *
+ * @param message the detail message (which is saved for later retrieval
+ * by the {@link #getMessage()} method).
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A <tt>null</tt> value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public DiscoveryException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Constructs a new exception with the specified cause and a detail
+ * message of <tt>(cause==null ? null : cause.toString())</tt> (which
+ * typically contains the class and detail message of <tt>cause</tt>).
+ * This constructor is useful for exceptions that are little more than
+ * wrappers for other throwables (for example, {@link
+ * PrivilegedActionException}).
+ *
+ * @param cause the cause (which is saved for later retrieval by the
+ * {@link #getCause()} method). (A <tt>null</tt> value is
+ * permitted, and indicates that the cause is nonexistent or
+ * unknown.)
+ * @since 1.4
+ */
+ public DiscoveryException(Throwable cause) {
+ super(cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryService.java b/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
new file mode 100644
index 0000000..e347c2c
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Metadata discovery service.
+ */
+public interface DiscoveryService {
+
+ /**
+ * Full text search
+ */
+ String searchByFullText(String query) throws DiscoveryException;
+
+ /**
+ * Search using query DSL.
+ *
+ * @param dslQuery query in DSL format.
+ * @return JSON representing the type and results.
+ */
+ String searchByDSL(String dslQuery) throws DiscoveryException;
+
+ /**
+ * Assumes the User is familiar with the persistence structure of the Repository.
+ * The given query is run uninterpreted against the underlying Graph Store.
+ * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
+ *
+ * @param gremlinQuery query in gremlin dsl format
+ * @return List of Maps
+ * @throws org.apache.atlas.discovery.DiscoveryException
+ */
+ List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/discovery/LineageService.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/discovery/LineageService.java b/server-api/src/main/java/org/apache/atlas/discovery/LineageService.java
new file mode 100644
index 0000000..8dc36cd
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/discovery/LineageService.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.AtlasException;
+
+/**
+ * Lineage service interface.
+ */
+public interface LineageService {
+
+ /**
+ * Return the lineage outputs for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Outputs as JSON
+ */
+ String getOutputs(String tableName) throws AtlasException;
+
+ /**
+ * Return the lineage outputs graph for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Outputs Graph as JSON
+ */
+ String getOutputsGraph(String tableName) throws AtlasException;
+
+ /**
+ * Return the lineage inputs for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Inputs as JSON
+ */
+ String getInputs(String tableName) throws AtlasException;
+
+ /**
+ * Return the lineage inputs graph for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Inputs Graph as JSON
+ */
+ String getInputsGraph(String tableName) throws AtlasException;
+
+ /**
+ * Return the schema for the given tableName.
+ *
+ * @param tableName tableName
+ * @return Schema as JSON
+ */
+ String getSchema(String tableName) throws AtlasException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java b/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
new file mode 100644
index 0000000..619ed85
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.listener;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.IStruct;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+
+import java.util.Collection;
+
+/**
+ * Entity (a Typed instance) change notification listener.
+ */
+public interface EntityChangeListener {
+
+ /**
+ * This is upon adding new entities to the repository.
+ *
+ * @param entities the created entities
+ *
+ * @throws AtlasException if the listener notification fails
+ */
+ void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException;
+
+ /**
+ * This is upon updating an entity.
+ *
+ * @param entities the updated entities
+ *
+ * @throws AtlasException if the listener notification fails
+ */
+ void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException;
+
+ /**
+ * This is upon adding a new trait to a typed instance.
+ *
+ * @param entity the entity
+ * @param trait trait that needs to be added to entity
+ *
+ * @throws AtlasException if the listener notification fails
+ */
+ void onTraitAdded(ITypedReferenceableInstance entity, IStruct trait) throws AtlasException;
+
+ /**
+ * This is upon deleting a trait from a typed instance.
+ *
+ * @param entity the entity
+ * @param traitName trait name for the instance that needs to be deleted from entity
+ *
+ * @throws AtlasException if the listener notification fails
+ */
+ void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/listener/TypesChangeListener.java b/server-api/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
new file mode 100644
index 0000000..5ff6d4a
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.listener;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.IDataType;
+
+import java.util.Collection;
+
+/**
+ * Types change notification listener.
+ */
+public interface TypesChangeListener {
+
+ /**
+ * This is upon adding new type(s) to Store.
+ *
+ * @param dataTypes data type
+ * @throws AtlasException
+ */
+ void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException;
+
+ /**
+ * This is upon removing an existing type from the Store.
+ *
+ * @param typeName type name
+ * @throws AtlasException
+ */
+ // void onRemove(String typeName) throws MetadataException;
+
+ //This is upon updating an existing type to the store
+ void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/services/MetadataService.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/services/MetadataService.java b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java
new file mode 100644
index 0000000..0cfed2e
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/services/MetadataService.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.services;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.listener.EntityChangeListener;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+
+/**
+ * Metadata service.
+ */
+public interface MetadataService {
+
+ /**
+ * Creates a new type based on the type system to enable adding
+ * entities (instances for types).
+ *
+ * @param typeDefinition definition as json
+ * @return a unique id for this type
+ */
+ JSONObject createType(String typeDefinition) throws AtlasException;
+
+ /**
+ * Updates the given types in the type definition
+ * @param typeDefinition
+ * @return
+ * @throws AtlasException
+ */
+ JSONObject updateType(String typeDefinition) throws AtlasException;
+
+ /**
+ * Return the definition for the given type.
+ *
+ * @param typeName name for this type, must be unique
+ * @return type definition as JSON
+ */
+ String getTypeDefinition(String typeName) throws AtlasException;
+
+ /**
+ * Return the list of types in the type system.
+ *
+ * @return list of type names in the type system
+ */
+ List<String> getTypeNamesList() throws AtlasException;
+
+ /**
+ * Return the list of trait type names in the type system.
+ *
+ * @return list of trait type names in the type system
+ */
+ List<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) throws AtlasException;
+
+ /**
+ * Creates an entity, instance of the type.
+ *
+ * @param entityDefinition definition
+ * @return guid
+ */
+ String createEntities(String entityDefinition) throws AtlasException;
+
+ /**
+ * Return the definition for the given guid.
+ *
+ * @param guid guid
+ * @return entity definition as JSON
+ */
+ String getEntityDefinition(String guid) throws AtlasException;
+
+ /**
+ * Return the definition given type and attribute. The attribute has to be unique attribute for the type
+ * @param entityType - type name
+ * @param attribute - attribute name
+ * @param value - attribute value
+ * @return
+ * @throws AtlasException
+ */
+ String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
+
+ /**
+ * Return the list of entity names for the given type in the repository.
+ *
+ * @param entityType type
+ * @return list of entity names for the given type in the repository
+ */
+ List<String> getEntityList(String entityType) throws AtlasException;
+
+ /**
+ * Adds the property to the given entity id(guid).
+ * Currently supports updates only on PRIMITIVE, CLASS attribute types
+ *
+ * @param guid entity id
+ * @param attribute property name
+ * @param value property value
+ */
+ void updateEntityAttributeByGuid(String guid, String attribute, String value) throws AtlasException;
+
+ /**
+ * Supports Partial updates of an entity. Users can update a subset of attributes for an entity identified by its guid
+ * Note however that it cannot be used to set attribute values to null or delete attrbute values
+ *
+ */
+ void updateEntityPartialByGuid(String guid, Referenceable entity) throws AtlasException;
+
+ /**
+ * Batch API - Adds/Updates the given entity id(guid).
+ *
+ * @param entityJson entity json
+ * @return List of guids which were updated and ones which were newly created as part of the updated entity
+ */
+ String updateEntities(String entityJson) throws AtlasException;
+
+ // Trait management functions
+
+ /**
+ * Updates entity identified by a qualified name
+ *
+ * @param typeName
+ * @param uniqueAttributeName
+ * @param attrValue
+ * @param updatedEntity
+ * @return Guid of updated entity
+ * @throws AtlasException
+ */
+ String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue,
+ Referenceable updatedEntity) throws AtlasException;
+
+ /**
+ * Gets the list of trait names for a given entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @return a list of trait names for the given entity guid
+ * @throws AtlasException
+ */
+ List<String> getTraitNames(String guid) throws AtlasException;
+
+ /**
+ * Adds a new trait to an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitInstanceDefinition trait instance that needs to be added to entity
+ * @throws AtlasException
+ */
+ void addTrait(String guid, String traitInstanceDefinition) throws AtlasException;
+
+ /**
+ * Deletes a given trait from an existing entity represented by a guid.
+ *
+ * @param guid globally unique identifier for the entity
+ * @param traitNameToBeDeleted name of the trait
+ * @throws AtlasException
+ */
+ void deleteTrait(String guid, String traitNameToBeDeleted) throws AtlasException;
+
+ /**
+ * Register a listener for entity change.
+ *
+ * @param listener the listener to register
+ */
+ void registerListener(EntityChangeListener listener);
+
+ /**
+ * Unregister an entity change listener.
+ *
+ * @param listener the listener to unregister
+ */
+ void unregisterListener(EntityChangeListener listener);
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityExistsException.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityExistsException.java b/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityExistsException.java
new file mode 100644
index 0000000..b16cfa9
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityExistsException.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.exception;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+
+public class EntityExistsException extends AtlasException {
+ public EntityExistsException(IReferenceableInstance typedInstance, Exception e) {
+ super("Model violation for type "+ typedInstance.getTypeName(), e);
+ }
+
+ public EntityExistsException(IReferenceableInstance typedInstance) {
+ super("Model violation for type "+ typedInstance.getTypeName());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityNotFoundException.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityNotFoundException.java b/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityNotFoundException.java
new file mode 100644
index 0000000..0fec895
--- /dev/null
+++ b/server-api/src/main/java/org/apache/atlas/typesystem/exception/EntityNotFoundException.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.exception;
+
+import org.apache.atlas.AtlasException;
+
+/**
+ * A simple wrapper for 404.
+ */
+public class EntityNotFoundException extends AtlasException {
+ public EntityNotFoundException() {
+ }
+
+ public EntityNotFoundException(String message) {
+ super(message);
+ }
+
+ public EntityNotFoundException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public EntityNotFoundException(Throwable cause) {
+ super(cause);
+ }
+
+ public EntityNotFoundException(String message, Throwable cause, boolean enableSuppression,
+ boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/pom.xml
----------------------------------------------------------------------
diff --git a/typesystem/pom.xml b/typesystem/pom.xml
index efb49ed..129de58 100755
--- a/typesystem/pom.xml
+++ b/typesystem/pom.xml
@@ -110,13 +110,8 @@
</dependency>
<dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-common</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/ApplicationProperties.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/ApplicationProperties.java b/typesystem/src/main/java/org/apache/atlas/ApplicationProperties.java
deleted file mode 100644
index 738ec53..0000000
--- a/typesystem/src/main/java/org/apache/atlas/ApplicationProperties.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-import org.apache.commons.configuration.CompositeConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.PropertiesConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.net.URL;
-import java.util.Arrays;
-import java.util.Iterator;
-
-public class ApplicationProperties extends PropertiesConfiguration {
- private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class);
-
- public static final String APPLICATION_PROPERTIES = "application.properties";
- public static final String CLIENT_PROPERTIES = "client.properties";
-
- private static Configuration INSTANCE = null;
-
- private ApplicationProperties(URL url) throws ConfigurationException {
- super(url);
- }
-
- public static Configuration get() throws AtlasException {
- if (INSTANCE == null) {
- synchronized (ApplicationProperties.class) {
- if (INSTANCE == null) {
- Configuration applicationProperties = get(APPLICATION_PROPERTIES);
- Configuration clientProperties = get(CLIENT_PROPERTIES);
- INSTANCE = new CompositeConfiguration(Arrays.asList(applicationProperties, clientProperties));
- }
- }
- }
- return INSTANCE;
- }
-
- public static Configuration get(String fileName) throws AtlasException {
- String confLocation = System.getProperty("atlas.conf");
- try {
- URL url = confLocation == null ? ApplicationProperties.class.getResource("/" + fileName)
- : new File(confLocation, fileName).toURI().toURL();
- LOG.info("Loading {} from {}", fileName, url);
-
- Configuration configuration = new ApplicationProperties(url).interpolatedConfiguration();
- logConfiguration(configuration);
- return configuration;
- } catch (Exception e) {
- throw new AtlasException("Failed to load application properties", e);
- }
- }
-
- private static void logConfiguration(Configuration configuration) {
- if (LOG.isDebugEnabled()) {
- Iterator<String> keys = configuration.getKeys();
- LOG.debug("Configuration loaded:");
- while (keys.hasNext()) {
- String key = keys.next();
- LOG.debug("{} = {}", key, configuration.getProperty(key));
- }
- }
- }
-
- public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) {
- return inConf.subset(prefix);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/AtlasException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/AtlasException.java b/typesystem/src/main/java/org/apache/atlas/AtlasException.java
deleted file mode 100755
index 2eb0658..0000000
--- a/typesystem/src/main/java/org/apache/atlas/AtlasException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-/**
- * Base Exception class for metadata API.
- */
-public class AtlasException extends Exception {
-
- public AtlasException() {
- }
-
- public AtlasException(String message) {
- super(message);
- }
-
- public AtlasException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public AtlasException(Throwable cause) {
- super(cause);
- }
-
- public AtlasException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/ParamChecker.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/ParamChecker.java b/typesystem/src/main/java/org/apache/atlas/ParamChecker.java
deleted file mode 100644
index e900ba3..0000000
--- a/typesystem/src/main/java/org/apache/atlas/ParamChecker.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-public class ParamChecker {
-
- /**
- * Check that a value is not null. If null throws an IllegalArgumentException.
- *
- * @param obj value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static <T> T notNull(T obj, String name) {
- if (obj == null) {
- throw new IllegalArgumentException(name + " cannot be null");
- }
- return obj;
- }
-
- /**
- * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
- * throws an IllegalArgumentException.
- * @param list the list of T.
- * @param name parameter name for the exception message.
- */
- public static <T> Collection<T> notNullElements(Collection<T> list, String name) {
- notEmpty(list, name);
- for (T ele : list) {
- notNull(ele, String.format("Collection %s element %s", name, ele));
- }
- return list;
- }
-
- /**
- * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
- * throws an IllegalArgumentException.
- * @param array the array of T.
- * @param name parameter name for the exception message.
- */
- public static <T> T[] notNullElements(T[] array, String name) {
- notEmpty(Arrays.asList(array), name);
- for (T ele : array) {
- notNull(ele, String.format("Collection %s element %s", name, ele));
- }
- return array;
- }
-
- /**
- * Check that a list is not null and not empty.
- * @param list the list of T.
- * @param name parameter name for the exception message.
- */
- public static <T> Collection<T> notEmpty(Collection<T> list, String name) {
- notNull(list, name);
- if (list.isEmpty()) {
- throw new IllegalArgumentException(String.format("Collection %s is empty", name));
- }
- return list;
- }
-
- /**
- * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static String notEmpty(String value, String name) {
- return notEmpty(value, name, null);
- }
-
- /**
- * Check that a string is not empty if its not null.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static String notEmptyIfNotNull(String value, String name) {
- return notEmptyIfNotNull(value, name, null);
- }
-
- /**
- * Check that a string is not empty if its not null.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @return the given value.
- */
- public static String notEmptyIfNotNull(String value, String name, String info) {
- if (value == null) {
- return value;
- }
-
- if (value.trim().length() == 0) {
- throw new IllegalArgumentException(name + " cannot be empty" + (info == null ? "" : ", " + info));
- }
- return value.trim();
- }
-
- /**
- * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
- *
- * @param value value.
- * @param name parameter name for the exception message.
- * @param info additional information to be printed with the exception message
- * @return the given value.
- */
- public static String notEmpty(String value, String name, String info) {
- if (value == null) {
- throw new IllegalArgumentException(name + " cannot be null" + (info == null ? "" : ", " + info));
- }
- return notEmptyIfNotNull(value, name, info);
- }
-
- /**
- * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
- * throws an IllegalArgumentException.
- * @param list the list of strings.
- * @param name parameter name for the exception message.
- */
- public static Collection<String> notEmptyElements(Collection<String> list, String name) {
- notEmpty(list, name);
- for (String ele : list) {
- notEmpty(ele, String.format("list %s element %s", name, ele));
- }
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/TypeExistsException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/TypeExistsException.java b/typesystem/src/main/java/org/apache/atlas/TypeExistsException.java
deleted file mode 100644
index 1a2cb7c..0000000
--- a/typesystem/src/main/java/org/apache/atlas/TypeExistsException.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-public class TypeExistsException extends AtlasException {
- public TypeExistsException(String message) {
- super(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java b/typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java
deleted file mode 100644
index 348dd57..0000000
--- a/typesystem/src/main/java/org/apache/atlas/TypeNotFoundException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas;
-
-/**
- * A simple wrapper for 404.
- */
-public class TypeNotFoundException extends AtlasException {
- public TypeNotFoundException() {
- }
-
- public TypeNotFoundException(String message) {
- super(message);
- }
-
- public TypeNotFoundException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public TypeNotFoundException(Throwable cause) {
- super(cause);
- }
-
- public TypeNotFoundException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
index c903301..ffe40a7 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/IInstance.java
@@ -33,6 +33,8 @@ public interface IInstance {
void set(String attrName, Object val) throws AtlasException;
+ void setNull(String attrName) throws AtlasException;
+
Map<String, Object> getValuesMap() throws AtlasException;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java b/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java
index c951172..d7f4cb7 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/ITypedInstance.java
@@ -23,6 +23,7 @@ import org.apache.atlas.typesystem.types.FieldMapping;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.security.MessageDigest;
import java.util.Date;
/**
@@ -37,8 +38,6 @@ public interface ITypedInstance extends IInstance {
FieldMapping fieldMapping();
- void setNull(String attrName) throws AtlasException;
-
boolean getBoolean(String attrName) throws AtlasException;
byte getByte(String attrName) throws AtlasException;
@@ -82,4 +81,6 @@ public interface ITypedInstance extends IInstance {
void setDate(String attrName, Date val) throws AtlasException;
void setString(String attrName, String val) throws AtlasException;
+
+ String getSignatureHash(MessageDigest digester) throws AtlasException;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java b/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
index aaf0aa4..b8dcc7e 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/Referenceable.java
@@ -24,7 +24,6 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.persistence.Id;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -95,8 +94,10 @@ public class Referenceable extends Struct implements IReferenceableInstance {
*/
@SuppressWarnings("unused")
private Referenceable() {
- this("", "", Collections.<String, Object>emptyMap(), Collections.<String>emptyList(),
- Collections.<String, IStruct>emptyMap());
+ super(null, null);
+ id = null;
+ traitNames = ImmutableList.of();
+ traits = ImmutableMap.of();
}
@Override
@@ -114,6 +115,42 @@ public class Referenceable extends Struct implements IReferenceableInstance {
return traits.get(typeName);
}
+ /**
+ * Matches traits, values associated with this Referenceable and skips the id match
+ * @param o The Referenceable which needs to be matched with
+ * @return
+ */
+ public boolean equalsContents(Object o) {
+ if(this == o) {
+ return true;
+ }
+ if(o == null) {
+ return false;
+ }
+ if (o.getClass() != getClass()) {
+ return false;
+ }
+
+ if(!super.equalsContents(o)) {
+ return false;
+ }
+
+ Referenceable obj = (Referenceable)o;
+ if (!traitNames.equals(obj.getTraits())) {
+ return false;
+ }
+
+ return true;
+ }
+
+ public String toString() {
+ return "{" +
+ "Id='" + id + '\'' +
+ ", traits=" + traitNames +
+ ", values=" + getValuesMap() +
+ '}';
+ }
+
private static Map<String, IStruct> getTraits(IReferenceableInstance instance) throws AtlasException {
Map<String, IStruct> traits = new HashMap<>();
for (String traitName : instance.getTraits() ) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java b/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
index d03e2c2..70deab2 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/Struct.java
@@ -18,6 +18,7 @@
package org.apache.atlas.typesystem;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import java.util.Collections;
@@ -67,7 +68,52 @@ public class Struct implements IStruct {
}
@Override
+ public void setNull(String attrName) throws AtlasException {
+ values.remove(attrName);
+ }
+
+ @Override
public Map<String, Object> getValuesMap() {
return values;
}
+
+ @Override
+ public int hashCode() {
+ int result = typeName.hashCode();
+ result = 31 * result + values.hashCode();
+ return result;
+ }
+
+ /**
+ * equalContents instead of equals since values is a mutable attribute and could lead
+ * to incorrect behaviour when added to collections and mutated after that
+ * i.e when the attribute is mutated collections.contains(struct) returns false
+ * due to hashcode having changed for the struct.
+ * @param o
+ * @return
+ */
+ public boolean equalsContents(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null) {
+ return false;
+ }
+
+ if (o.getClass() != getClass()) {
+ return false;
+ }
+
+ Struct obj = (Struct)o;
+ if(!typeName.equals(obj.getTypeName())) {
+ return false;
+ }
+
+ if(!values.equals(obj.getValuesMap())) {
+ return false;
+ }
+
+ return true;
+ }
}
[5/5] incubator-atlas git commit: ATLAS-47 Entity mutations for
complex types (sumasai via shwethags)
Posted by sh...@apache.org.
ATLAS-47 Entity mutations for complex types (sumasai via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/51656991
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/51656991
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/51656991
Branch: refs/heads/master
Commit: 51656991fc008e57936bfc12afa51b9e372ae6a5
Parents: 6c3f096
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Tue Dec 8 12:09:57 2015 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Tue Dec 8 12:09:57 2015 +0530
----------------------------------------------------------------------
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 9 +-
.../org/apache/atlas/hive/hook/HiveHookIT.java | 2 +-
.../main/java/org/apache/atlas/AtlasClient.java | 141 ++-
common/pom.xml | 14 +-
.../org/apache/atlas/ApplicationProperties.java | 86 ++
.../java/org/apache/atlas/AtlasException.java | 44 +
.../atlas/listener/EntityChangeListener.java | 69 --
.../atlas/listener/TypesChangeListener.java | 49 -
.../java/org/apache/atlas/utils/MD5Utils.java | 59 +
.../org/apache/atlas/utils/ParamChecker.java | 148 +++
docs/src/site/twiki/Configuration.twiki | 4 +-
docs/src/site/twiki/InstallationSteps.twiki | 2 +-
notification/pom.xml | 5 +
.../NotificationEntityChangeListener.java | 4 +-
pom.xml | 7 +
release-log.txt | 1 +
repository/pom.xml | 2 +-
.../apache/atlas/RepositoryMetadataModule.java | 4 +-
.../atlas/discovery/DiscoveryException.java | 74 --
.../atlas/discovery/DiscoveryService.java | 52 -
.../atlas/discovery/HiveLineageService.java | 4 +-
.../apache/atlas/discovery/LineageService.java | 67 --
.../graph/DefaultGraphPersistenceStrategy.java | 14 +-
.../atlas/repository/EntityExistsException.java | 32 -
.../repository/EntityNotFoundException.java | 44 -
.../atlas/repository/MetadataRepository.java | 38 +-
.../atlas/repository/graph/EntityProcessor.java | 81 ++
.../atlas/repository/graph/FullTextMapper.java | 146 +++
.../graph/GraphBackedMetadataRepository.java | 1094 +-----------------
.../atlas/repository/graph/GraphHelper.java | 180 ++-
.../graph/GraphToTypedInstanceMapper.java | 419 +++++++
.../graph/TypedInstanceToGraphMapper.java | 633 ++++++++++
.../memory/ReplaceIdWithInstance.java | 1 -
.../atlas/services/DefaultMetadataService.java | 168 ++-
.../apache/atlas/services/MetadataService.java | 157 ---
.../query/GraphPersistenceStrategies.scala | 4 +-
.../apache/atlas/BaseHiveRepositoryTest.java | 25 +-
.../test/java/org/apache/atlas/TestUtils.java | 29 +-
.../GraphBackedDiscoveryServiceTest.java | 4 +-
.../atlas/discovery/HiveLineageServiceTest.java | 6 +-
.../GraphBackedMetadataRepositoryTest.java | 32 +-
.../graph/GraphRepoMapperScaleTest.java | 18 +-
.../service/DefaultMetadataServiceTest.java | 509 +++++++-
.../org/apache/atlas/query/GremlinTest.scala | 1 -
server-api/pom.xml | 53 +
.../atlas/discovery/DiscoveryException.java | 74 ++
.../atlas/discovery/DiscoveryService.java | 52 +
.../apache/atlas/discovery/LineageService.java | 67 ++
.../atlas/listener/EntityChangeListener.java | 69 ++
.../atlas/listener/TypesChangeListener.java | 49 +
.../apache/atlas/services/MetadataService.java | 187 +++
.../exception/EntityExistsException.java | 32 +
.../exception/EntityNotFoundException.java | 46 +
typesystem/pom.xml | 9 +-
.../org/apache/atlas/ApplicationProperties.java | 86 --
.../java/org/apache/atlas/AtlasException.java | 44 -
.../java/org/apache/atlas/ParamChecker.java | 148 ---
.../org/apache/atlas/TypeExistsException.java | 25 -
.../org/apache/atlas/TypeNotFoundException.java | 44 -
.../org/apache/atlas/typesystem/IInstance.java | 2 +
.../apache/atlas/typesystem/ITypedInstance.java | 5 +-
.../apache/atlas/typesystem/Referenceable.java | 43 +-
.../org/apache/atlas/typesystem/Struct.java | 46 +
.../exception/TypeExistsException.java | 27 +
.../exception/TypeNotFoundException.java | 46 +
.../persistence/DownCastStructInstance.java | 5 +
.../apache/atlas/typesystem/persistence/Id.java | 14 +
.../persistence/ReferenceableInstance.java | 14 +-
.../typesystem/persistence/StructInstance.java | 35 +
.../typesystem/types/AttributeDefinition.java | 2 +-
.../atlas/typesystem/types/ClassType.java | 28 +-
.../atlas/typesystem/types/DataTypes.java | 44 +-
.../apache/atlas/typesystem/types/EnumType.java | 11 +-
.../typesystem/types/EnumTypeDefinition.java | 2 +-
.../atlas/typesystem/types/EnumValue.java | 2 +-
.../atlas/typesystem/types/IDataType.java | 4 +
.../atlas/typesystem/types/Multiplicity.java | 4 +-
.../typesystem/types/ObjectGraphWalker.java | 6 +
.../atlas/typesystem/types/StructType.java | 20 +
.../typesystem/types/StructTypeDefinition.java | 2 +-
.../atlas/typesystem/types/TraitType.java | 20 +
.../atlas/typesystem/types/TypeSystem.java | 4 +-
.../atlas/typesystem/json/Serialization.scala | 41 +-
.../apache/atlas/web/filters/AuditFilter.java | 6 +-
.../atlas/web/resources/EntityResource.java | 213 +++-
.../web/resources/HiveLineageResource.java | 4 +-
.../resources/MetadataDiscoveryResource.java | 2 +-
.../atlas/web/resources/TypesResource.java | 2 +-
.../org/apache/atlas/web/util/Servlets.java | 2 +-
.../notification/EntityNotificationIT.java | 2 +-
.../NotificationHookConsumerIT.java | 2 +-
.../atlas/web/resources/BaseResourceIT.java | 7 +-
.../web/resources/EntityJerseyResourceIT.java | 116 +-
93 files changed, 4016 insertions(+), 2234 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 90f3d96..f367317 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -479,12 +479,13 @@ public class HiveMetaStoreBridge {
HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
AtlasClient dgiClient = getAtlasClient();
- //Register hive data model if its not already registered
- if (dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName()) == null) {
+ try {
+ dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
+ LOG.info("Hive data model is already registered!");
+ } catch(AtlasServiceException ase) {
+ //Expected in case types do not exist
LOG.info("Registering Hive data model");
dgiClient.createType(dataModelGenerator.getModelAsJson());
- } else {
- LOG.info("Hive data model is already registered!");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 643a29a..5447de5 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -20,7 +20,7 @@ package org.apache.atlas.hive.hook;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/client/src/main/java/org/apache/atlas/AtlasClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java
index 2e8a268..114548c 100755
--- a/client/src/main/java/org/apache/atlas/AtlasClient.java
+++ b/client/src/main/java/org/apache/atlas/AtlasClient.java
@@ -133,40 +133,43 @@ public class AtlasClient {
enum API {
//Type operations
- CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST),
- UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT),
- GET_TYPE(BASE_URI + TYPES, HttpMethod.GET),
- LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET),
- LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET),
+ CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST, Response.Status.CREATED),
+ UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT, Response.Status.OK),
+ GET_TYPE(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK),
+ LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET, Response.Status.OK),
+ LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET, Response.Status.OK),
//Entity operations
- CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST),
- GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET),
- UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT),
- LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET),
+ CREATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED),
+ GET_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
+ UPDATE_ENTITY(BASE_URI + URI_ENTITY, HttpMethod.PUT, Response.Status.OK),
+ UPDATE_ENTITY_PARTIAL(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.OK),
+ LIST_ENTITIES(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
//Trait operations
- ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST),
- DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE),
- LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET),
+ ADD_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.POST, Response.Status.CREATED),
+ DELETE_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.DELETE, Response.Status.OK),
+ LIST_TRAITS(BASE_URI + URI_ENTITY, HttpMethod.GET, Response.Status.OK),
//Search operations
- SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET),
- SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET),
- SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET),
- SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET),
+ SEARCH(BASE_URI + URI_SEARCH, HttpMethod.GET, Response.Status.OK),
+ SEARCH_DSL(BASE_URI + URI_SEARCH + "/dsl", HttpMethod.GET, Response.Status.OK),
+ SEARCH_GREMLIN(BASE_URI + URI_SEARCH + "/gremlin", HttpMethod.GET, Response.Status.OK),
+ SEARCH_FULL_TEXT(BASE_URI + URI_SEARCH + "/fulltext", HttpMethod.GET, Response.Status.OK),
//Lineage operations
- LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
- LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET),
- LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET);
+ LINEAGE_INPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK),
+ LINEAGE_OUTPUTS_GRAPH(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK),
+ LINEAGE_SCHEMA(BASE_URI + URI_LINEAGE, HttpMethod.GET, Response.Status.OK);
private final String method;
private final String path;
+ private final Response.Status status;
- API(String path, String method) {
+ API(String path, String method, Response.Status status) {
this.path = path;
this.method = method;
+ this.status = status;
}
public String getMethod() {
@@ -176,6 +179,8 @@ public class AtlasClient {
public String getPath() {
return path;
}
+
+ public Response.Status getExpectedStatus() { return status; }
}
/**
@@ -231,7 +236,7 @@ public class AtlasClient {
JSONObject response = callAPIWithResource(API.GET_TYPE, resource);
return response.getString(DEFINITION);
} catch (AtlasServiceException e) {
- if (e.getStatus() == ClientResponse.Status.NOT_FOUND) {
+ if (Response.Status.NOT_FOUND.equals(e.getStatus())) {
return null;
}
throw e;
@@ -266,11 +271,82 @@ public class AtlasClient {
}
public JSONArray createEntity(Referenceable... entities) throws AtlasServiceException {
+ JSONArray entityArray = getEntitiesArray(entities);
+ return createEntity(entityArray);
+ }
+
+ private JSONArray getEntitiesArray(Referenceable[] entities) {
JSONArray entityArray = new JSONArray(entities.length);
for (Referenceable entity : entities) {
entityArray.put(InstanceSerialization.toJson(entity, true));
}
- return createEntity(entityArray);
+ return entityArray;
+ }
+
+ /**
+ * Replaces entity definitions identified by their guid or unique attribute
+ * Updates properties set in the definition for the entity corresponding to guid
+ * @param entities entities to be updated
+ * @return json array of guids which were updated/created
+ * @throws AtlasServiceException
+ */
+ public JSONArray updateEntities(Referenceable... entities) throws AtlasServiceException {
+ JSONArray entitiesArray = getEntitiesArray(entities);
+ JSONObject response = callAPI(API.UPDATE_ENTITY, entitiesArray.toString());
+ try {
+ return response.getJSONArray(GUID);
+ } catch (JSONException e) {
+ throw new AtlasServiceException(API.UPDATE_ENTITY, e);
+ }
+ }
+
+ /**
+ * Supports Partial updates
+ * Updates property for the entity corresponding to guid
+ * @param guid guid
+ * @param attribute property key
+ * @param value property value
+ */
+ public void updateEntityAttribute(String guid, String attribute, String value) throws AtlasServiceException {
+ API api = API.UPDATE_ENTITY_PARTIAL;
+ WebResource resource = getResource(api, guid);
+ resource = resource.queryParam(ATTRIBUTE_NAME, attribute);
+ callAPIWithResource(api, resource, value);
+ }
+
+ /**
+ * Supports Partial updates
+ * Updates properties set in the definition for the entity corresponding to guid
+ * @param guid guid
+ * @param entity entity definition
+ */
+ public void updateEntity(String guid, Referenceable entity) throws AtlasServiceException {
+ String entityJson = InstanceSerialization.toJson(entity, true);
+ callAPI(API.UPDATE_ENTITY_PARTIAL, entityJson, guid);
+ }
+
+ /**
+ * Supports Partial updates
+ * Updates properties set in the definition for the entity corresponding to guid
+ * @param entityType Type of the entity being updated
+ * @param uniqueAttributeName Attribute Name that uniquely identifies the entity
+ * @param uniqueAttributeValue Attribute Value that uniquely identifies the entity
+ * @param entity entity definition
+ */
+ public String updateEntity(String entityType, String uniqueAttributeName, String uniqueAttributeValue,
+ Referenceable entity) throws AtlasServiceException {
+ API api = API.UPDATE_ENTITY_PARTIAL;
+ WebResource resource = getResource(api, "qualifiedName");
+ resource = resource.queryParam(TYPE, entityType);
+ resource = resource.queryParam(ATTRIBUTE_NAME, uniqueAttributeName);
+ resource = resource.queryParam(ATTRIBUTE_VALUE, uniqueAttributeValue);
+ String entityJson = InstanceSerialization.toJson(entity, true);
+ JSONObject response = callAPIWithResource(api, resource, entityJson);
+ try {
+ return response.getString(GUID);
+ } catch (JSONException e) {
+ throw new AtlasServiceException(api, e);
+ }
}
/**
@@ -351,19 +427,6 @@ public class AtlasClient {
}
/**
- * Updates property for the entity corresponding to guid
- * @param guid guid
- * @param property property key
- * @param value property value
- */
- public JSONObject updateEntity(String guid, String property, String value) throws AtlasServiceException {
- WebResource resource = getResource(API.UPDATE_ENTITY, guid);
- resource = resource.queryParam(ATTRIBUTE_NAME, property);
- resource = resource.queryParam(ATTRIBUTE_VALUE, value);
- return callAPIWithResource(API.UPDATE_ENTITY, resource);
- }
-
- /**
* Search using gremlin/dsl/full text
* @param searchQuery
* @return
@@ -488,13 +551,11 @@ public class AtlasClient {
}
private JSONObject callAPIWithResource(API api, WebResource resource, Object requestObject)
- throws AtlasServiceException {
+ throws AtlasServiceException {
ClientResponse clientResponse = resource.accept(JSON_MEDIA_TYPE).type(JSON_MEDIA_TYPE)
- .method(api.getMethod(), ClientResponse.class, requestObject);
+ .method(api.getMethod(), ClientResponse.class, requestObject);
- Response.Status expectedStatus =
- HttpMethod.POST.equals(api.getMethod()) ? Response.Status.CREATED : Response.Status.OK;
- if (clientResponse.getStatus() == expectedStatus.getStatusCode()) {
+ if (clientResponse.getStatus() == api.getExpectedStatus().getStatusCode()) {
String responseAsString = clientResponse.getEntity(String.class);
try {
return new JSONObject(responseAsString);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 5498105..a7a5544 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -33,13 +33,19 @@
<dependencies>
<dependency>
- <groupId>org.apache.atlas</groupId>
- <artifactId>atlas-typesystem</artifactId>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
</dependency>
<dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
</dependency>
+
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/ApplicationProperties.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/ApplicationProperties.java b/common/src/main/java/org/apache/atlas/ApplicationProperties.java
new file mode 100644
index 0000000..738ec53
--- /dev/null
+++ b/common/src/main/java/org/apache/atlas/ApplicationProperties.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Iterator;
+
+public class ApplicationProperties extends PropertiesConfiguration {
+ private static final Logger LOG = LoggerFactory.getLogger(ApplicationProperties.class);
+
+ public static final String APPLICATION_PROPERTIES = "application.properties";
+ public static final String CLIENT_PROPERTIES = "client.properties";
+
+ private static Configuration INSTANCE = null;
+
+ private ApplicationProperties(URL url) throws ConfigurationException {
+ super(url);
+ }
+
+ public static Configuration get() throws AtlasException {
+ if (INSTANCE == null) {
+ synchronized (ApplicationProperties.class) {
+ if (INSTANCE == null) {
+ Configuration applicationProperties = get(APPLICATION_PROPERTIES);
+ Configuration clientProperties = get(CLIENT_PROPERTIES);
+ INSTANCE = new CompositeConfiguration(Arrays.asList(applicationProperties, clientProperties));
+ }
+ }
+ }
+ return INSTANCE;
+ }
+
+ public static Configuration get(String fileName) throws AtlasException {
+ String confLocation = System.getProperty("atlas.conf");
+ try {
+ URL url = confLocation == null ? ApplicationProperties.class.getResource("/" + fileName)
+ : new File(confLocation, fileName).toURI().toURL();
+ LOG.info("Loading {} from {}", fileName, url);
+
+ Configuration configuration = new ApplicationProperties(url).interpolatedConfiguration();
+ logConfiguration(configuration);
+ return configuration;
+ } catch (Exception e) {
+ throw new AtlasException("Failed to load application properties", e);
+ }
+ }
+
+ private static void logConfiguration(Configuration configuration) {
+ if (LOG.isDebugEnabled()) {
+ Iterator<String> keys = configuration.getKeys();
+ LOG.debug("Configuration loaded:");
+ while (keys.hasNext()) {
+ String key = keys.next();
+ LOG.debug("{} = {}", key, configuration.getProperty(key));
+ }
+ }
+ }
+
+ public static final Configuration getSubsetConfiguration(Configuration inConf, String prefix) {
+ return inConf.subset(prefix);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/AtlasException.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/AtlasException.java b/common/src/main/java/org/apache/atlas/AtlasException.java
new file mode 100644
index 0000000..2eb0658
--- /dev/null
+++ b/common/src/main/java/org/apache/atlas/AtlasException.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+/**
+ * Base Exception class for metadata API.
+ */
+public class AtlasException extends Exception {
+
+ public AtlasException() {
+ }
+
+ public AtlasException(String message) {
+ super(message);
+ }
+
+ public AtlasException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public AtlasException(Throwable cause) {
+ super(cause);
+ }
+
+ public AtlasException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+ super(message, cause, enableSuppression, writableStackTrace);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java b/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
deleted file mode 100644
index 08ed0d3..0000000
--- a/common/src/main/java/org/apache/atlas/listener/EntityChangeListener.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.listener;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.IStruct;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-
-import java.util.Collection;
-
-/**
- * Entity (a Typed instance) change notification listener.
- */
-public interface EntityChangeListener {
-
- /**
- * This is upon adding new entities to the repository.
- *
- * @param entities the created entities
- *
- * @throws AtlasException if the listener notification fails
- */
- void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException;
-
- /**
- * This is upon updating an entity.
- *
- * @param entity the updated entity
- *
- * @throws AtlasException if the listener notification fails
- */
- void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException;
-
- /**
- * This is upon adding a new trait to a typed instance.
- *
- * @param entity the entity
- * @param trait trait that needs to be added to entity
- *
- * @throws AtlasException if the listener notification fails
- */
- void onTraitAdded(ITypedReferenceableInstance entity, IStruct trait) throws AtlasException;
-
- /**
- * This is upon deleting a trait from a typed instance.
- *
- * @param entity the entity
- * @param traitName trait name for the instance that needs to be deleted from entity
- *
- * @throws AtlasException if the listener notification fails
- */
- void onTraitDeleted(ITypedReferenceableInstance entity, String traitName) throws AtlasException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java b/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
deleted file mode 100644
index 5ff6d4a..0000000
--- a/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.listener;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.types.IDataType;
-
-import java.util.Collection;
-
-/**
- * Types change notification listener.
- */
-public interface TypesChangeListener {
-
- /**
- * This is upon adding new type(s) to Store.
- *
- * @param dataTypes data type
- * @throws AtlasException
- */
- void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException;
-
- /**
- * This is upon removing an existing type from the Store.
- *
- * @param typeName type name
- * @throws AtlasException
- */
- // void onRemove(String typeName) throws MetadataException;
-
- //This is upon updating an existing type to the store
- void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/utils/MD5Utils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/utils/MD5Utils.java b/common/src/main/java/org/apache/atlas/utils/MD5Utils.java
new file mode 100644
index 0000000..35e4744
--- /dev/null
+++ b/common/src/main/java/org/apache/atlas/utils/MD5Utils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.utils;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+public class MD5Utils {
+
+ private static final ThreadLocal<MessageDigest> DIGESTER_FACTORY =
+ new ThreadLocal<MessageDigest>() {
+ @Override
+ protected MessageDigest initialValue() {
+ try {
+ return MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ /**
+ * Create a thread local MD5 digester
+ */
+ public static MessageDigest getDigester() {
+ MessageDigest digester = DIGESTER_FACTORY.get();
+ digester.reset();
+ return digester;
+ }
+
+ private static final char[] HEX_DIGITS =
+ {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
+
+ public static String toString(byte[] digest) {
+ StringBuilder buf = new StringBuilder(MD5_LEN*2);
+ for (int i = 0; i < MD5_LEN; i++) {
+ int b = digest[i];
+ buf.append(HEX_DIGITS[(b >> 4) & 0xf]);
+ buf.append(HEX_DIGITS[b & 0xf]);
+ }
+ return buf.toString();
+ }
+
+ public static final int MD5_LEN = 16;
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/common/src/main/java/org/apache/atlas/utils/ParamChecker.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/utils/ParamChecker.java b/common/src/main/java/org/apache/atlas/utils/ParamChecker.java
new file mode 100644
index 0000000..ab543e6
--- /dev/null
+++ b/common/src/main/java/org/apache/atlas/utils/ParamChecker.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.utils;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+public class ParamChecker {
+
+ /**
+ * Check that a value is not null. If null throws an IllegalArgumentException.
+ *
+ * @param obj value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static <T> T notNull(T obj, String name) {
+ if (obj == null) {
+ throw new IllegalArgumentException(name + " cannot be null");
+ }
+ return obj;
+ }
+
+ /**
+ * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
+ * throws an IllegalArgumentException.
+ * @param list the list of T.
+ * @param name parameter name for the exception message.
+ */
+ public static <T> Collection<T> notNullElements(Collection<T> list, String name) {
+ notEmpty(list, name);
+ for (T ele : list) {
+ notNull(ele, String.format("Collection %s element %s", name, ele));
+ }
+ return list;
+ }
+
+ /**
+ * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
+ * throws an IllegalArgumentException.
+ * @param array the array of T.
+ * @param name parameter name for the exception message.
+ */
+ public static <T> T[] notNullElements(T[] array, String name) {
+ notEmpty(Arrays.asList(array), name);
+ for (T ele : array) {
+ notNull(ele, String.format("Collection %s element %s", name, ele));
+ }
+ return array;
+ }
+
+ /**
+ * Check that a list is not null and not empty.
+ * @param list the list of T.
+ * @param name parameter name for the exception message.
+ */
+ public static <T> Collection<T> notEmpty(Collection<T> list, String name) {
+ notNull(list, name);
+ if (list.isEmpty()) {
+ throw new IllegalArgumentException(String.format("Collection %s is empty", name));
+ }
+ return list;
+ }
+
+ /**
+ * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static String notEmpty(String value, String name) {
+ return notEmpty(value, name, null);
+ }
+
+ /**
+ * Check that a string is not empty if its not null.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static String notEmptyIfNotNull(String value, String name) {
+ return notEmptyIfNotNull(value, name, null);
+ }
+
+ /**
+ * Check that a string is not empty if its not null.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @return the given value.
+ */
+ public static String notEmptyIfNotNull(String value, String name, String info) {
+ if (value == null) {
+ return value;
+ }
+
+ if (value.trim().length() == 0) {
+ throw new IllegalArgumentException(name + " cannot be empty" + (info == null ? "" : ", " + info));
+ }
+ return value.trim();
+ }
+
+ /**
+ * Check that a string is not null and not empty. If null or emtpy throws an IllegalArgumentException.
+ *
+ * @param value value.
+ * @param name parameter name for the exception message.
+ * @param info additional information to be printed with the exception message
+ * @return the given value.
+ */
+ public static String notEmpty(String value, String name, String info) {
+ if (value == null) {
+ throw new IllegalArgumentException(name + " cannot be null" + (info == null ? "" : ", " + info));
+ }
+ return notEmptyIfNotNull(value, name, info);
+ }
+
+ /**
+ * Check that a list is not null and that none of its elements is null. If null or if the list has emtpy elements
+ * throws an IllegalArgumentException.
+ * @param list the list of strings.
+ * @param name parameter name for the exception message.
+ */
+ public static Collection<String> notEmptyElements(Collection<String> list, String name) {
+ notEmpty(list, name);
+ for (String ele : list) {
+ notEmpty(ele, String.format("list %s element %s", name, ele));
+ }
+ return list;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/docs/src/site/twiki/Configuration.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Configuration.twiki b/docs/src/site/twiki/Configuration.twiki
index 21d1c0e..526b300 100644
--- a/docs/src/site/twiki/Configuration.twiki
+++ b/docs/src/site/twiki/Configuration.twiki
@@ -60,8 +60,9 @@ Without Ranger, HBase shell can be used to set the permissions.
</verbatim>
---++++ Graph Search Index
+
This section sets up the graph db - titan - to use an search indexing system. The example
-configuration below setsup to use an embedded Elastic search indexing system.
+configuration below sets up to use an embedded Elastic search indexing system.
<verbatim>
atlas.graph.index.search.backend=elasticsearch
@@ -72,6 +73,7 @@ atlas.graph.index.search.elasticsearch.create.sleep=2000
</verbatim>
---++++ Graph Search Index - Solr
+Please note that Solr installation in Cloud mode is a prerequisite before configuring Solr as the search indexing backend. Refer InstallationSteps section for Solr installation/configuration.
<verbatim>
atlas.graph.index.search.backend=solr5
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/docs/src/site/twiki/InstallationSteps.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/InstallationSteps.twiki b/docs/src/site/twiki/InstallationSteps.twiki
index 8d1f3d0..0eefb20 100644
--- a/docs/src/site/twiki/InstallationSteps.twiki
+++ b/docs/src/site/twiki/InstallationSteps.twiki
@@ -124,7 +124,7 @@ export METADATA_SERVER_OPTS="-Djava.awt.headless=true -Djava.security.krb5.realm
* Hbase as the Storage Backend for the Graph Repository
By default, Atlas uses Titan as the graph repository and is the only graph repository implementation available currently.
-The HBase versions currently supported are 0.98.x, 1.0.x, 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section
+The HBase versions currently supported are 1.1.x. For configuring ATLAS graph persistence on HBase, please go through the "Configuration - Graph persistence engine - HBase" section
for more details.
Pre-requisites for running HBase as a distributed cluster
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/notification/pom.xml
----------------------------------------------------------------------
diff --git a/notification/pom.xml b/notification/pom.xml
index 796ea17..125ef75 100644
--- a/notification/pom.xml
+++ b/notification/pom.xml
@@ -48,6 +48,11 @@
</dependency>
<dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-server-api</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java
----------------------------------------------------------------------
diff --git a/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java b/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java
index 23a6d69..e2d16df 100644
--- a/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java
+++ b/notification/src/main/java/org/apache/atlas/notification/entity/NotificationEntityChangeListener.java
@@ -73,8 +73,8 @@ public class NotificationEntityChangeListener implements EntityChangeListener {
}
@Override
- public void onEntityUpdated(ITypedReferenceableInstance entity) throws AtlasException {
- notifyOfEntityEvent(Collections.singleton(entity), EntityNotification.OperationType.ENTITY_UPDATE);
+ public void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
+ notifyOfEntityEvent(entities, EntityNotification.OperationType.ENTITY_UPDATE);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2f7ee82..929d255 100755
--- a/pom.xml
+++ b/pom.xml
@@ -407,6 +407,7 @@
<modules>
<module>common</module>
<module>typesystem</module>
+ <module>server-api</module>
<module>notification</module>
<module>client</module>
<module>titan</module>
@@ -932,6 +933,12 @@
<dependency>
<groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-server-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 4ebba32..1dae6fb 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -14,6 +14,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
+ATLAS-47 Entity mutations for complex types (sumasai via shwethags)
ATLAS-345 UI: Should allow tag addition on any search result that returns a reference-able entity (darshankumar89 via shwethags)
ATLAS-279 UI not displaying results for certain successful "select" search queries (anilsg via shwethags)
ATLAS-242 The qualified name for hive entities should be backward compatible (shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/pom.xml
----------------------------------------------------------------------
diff --git a/repository/pom.xml b/repository/pom.xml
index 6e1baee..5810956 100755
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -41,7 +41,7 @@
<dependency>
<groupId>org.apache.atlas</groupId>
- <artifactId>atlas-common</artifactId>
+ <artifactId>atlas-server-api</artifactId>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
index b12c9a8..f77c237 100755
--- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
+++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
@@ -49,11 +49,13 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
@Override
protected void configure() {
// special wiring for Titan Graph
+
+
+
ThrowingProviderBinder.create(binder()).bind(GraphProvider.class, TitanGraph.class).to(TitanGraphProvider.class)
.asEagerSingleton();
// allow for dynamic binding of the metadata repo & graph service
-
// bind the MetadataRepositoryService interface to an implementation
bind(MetadataRepository.class).to(GraphBackedMetadataRepository.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
deleted file mode 100755
index ba69af7..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryException.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.AtlasException;
-
-import java.security.PrivilegedActionException;
-
-public class DiscoveryException extends AtlasException {
-
- /**
- * Constructs a new exception with the specified detail message. The
- * cause is not initialized, and may subsequently be initialized by
- * a call to {@link #initCause}.
- *
- * @param message the detail message. The detail message is saved for
- * later retrieval by the {@link #getMessage()} method.
- */
- public DiscoveryException(String message) {
- super(message);
- }
-
- /**
- * Constructs a new exception with the specified detail message and
- * cause. <p>Note that the detail message associated with
- * {@code cause} is <i>not</i> automatically incorporated in
- * this exception's detail message.
- *
- * @param message the detail message (which is saved for later retrieval
- * by the {@link #getMessage()} method).
- * @param cause the cause (which is saved for later retrieval by the
- * {@link #getCause()} method). (A <tt>null</tt> value is
- * permitted, and indicates that the cause is nonexistent or
- * unknown.)
- * @since 1.4
- */
- public DiscoveryException(String message, Throwable cause) {
- super(message, cause);
- }
-
- /**
- * Constructs a new exception with the specified cause and a detail
- * message of <tt>(cause==null ? null : cause.toString())</tt> (which
- * typically contains the class and detail message of <tt>cause</tt>).
- * This constructor is useful for exceptions that are little more than
- * wrappers for other throwables (for example, {@link
- * PrivilegedActionException}).
- *
- * @param cause the cause (which is saved for later retrieval by the
- * {@link #getCause()} method). (A <tt>null</tt> value is
- * permitted, and indicates that the cause is nonexistent or
- * unknown.)
- * @since 1.4
- */
- public DiscoveryException(Throwable cause) {
- super(cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
deleted file mode 100755
index e347c2c..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/DiscoveryService.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.discovery;
-
-import java.util.List;
-import java.util.Map;
-
-/**
- * Metadata discovery service.
- */
-public interface DiscoveryService {
-
- /**
- * Full text search
- */
- String searchByFullText(String query) throws DiscoveryException;
-
- /**
- * Search using query DSL.
- *
- * @param dslQuery query in DSL format.
- * @return JSON representing the type and results.
- */
- String searchByDSL(String dslQuery) throws DiscoveryException;
-
- /**
- * Assumes the User is familiar with the persistence structure of the Repository.
- * The given query is run uninterpreted against the underlying Graph Store.
- * The results are returned as a List of Rows. each row is a Map of Key,Value pairs.
- *
- * @param gremlinQuery query in gremlin dsl format
- * @return List of Maps
- * @throws org.apache.atlas.discovery.DiscoveryException
- */
- List<Map<String, String>> searchByGremlin(String gremlinQuery) throws DiscoveryException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
index 7a6ff55..00905d7 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/HiveLineageService.java
@@ -22,14 +22,14 @@ import com.thinkaurelius.titan.core.TitanGraph;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
-import org.apache.atlas.ParamChecker;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.discovery.graph.DefaultGraphPersistenceStrategy;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.Expressions;
import org.apache.atlas.query.GremlinQueryResult;
import org.apache.atlas.query.HiveLineageQuery;
import org.apache.atlas.query.HiveWhereUsedQuery;
-import org.apache.atlas.repository.EntityNotFoundException;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/LineageService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/LineageService.java b/repository/src/main/java/org/apache/atlas/discovery/LineageService.java
deleted file mode 100644
index 8dc36cd..0000000
--- a/repository/src/main/java/org/apache/atlas/discovery/LineageService.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.discovery;
-
-import org.apache.atlas.AtlasException;
-
-/**
- * Lineage service interface.
- */
-public interface LineageService {
-
- /**
- * Return the lineage outputs for the given tableName.
- *
- * @param tableName tableName
- * @return Outputs as JSON
- */
- String getOutputs(String tableName) throws AtlasException;
-
- /**
- * Return the lineage outputs graph for the given tableName.
- *
- * @param tableName tableName
- * @return Outputs Graph as JSON
- */
- String getOutputsGraph(String tableName) throws AtlasException;
-
- /**
- * Return the lineage inputs for the given tableName.
- *
- * @param tableName tableName
- * @return Inputs as JSON
- */
- String getInputs(String tableName) throws AtlasException;
-
- /**
- * Return the lineage inputs graph for the given tableName.
- *
- * @param tableName tableName
- * @return Inputs Graph as JSON
- */
- String getInputsGraph(String tableName) throws AtlasException;
-
- /**
- * Return the schema for the given tableName.
- *
- * @param tableName tableName
- * @return Schema as JSON
- */
- String getSchema(String tableName) throws AtlasException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
index 65a46a4..90718ed 100755
--- a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
@@ -30,6 +30,7 @@ import org.apache.atlas.query.TypeUtils;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.persistence.Id;
@@ -43,7 +44,7 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
+import javax.inject.Inject;
import java.util.List;
/**
@@ -55,6 +56,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
private final GraphBackedMetadataRepository metadataRepository;
+ @Inject
public DefaultGraphPersistenceStrategy(MetadataRepository metadataRepository) {
this.metadataRepository = (GraphBackedMetadataRepository) metadataRepository;
}
@@ -71,7 +73,11 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public String edgeLabel(IDataType<?> dataType, AttributeInfo aInfo) {
- return metadataRepository.getEdgeLabel(dataType, aInfo);
+ try {
+ return metadataRepository.getEdgeLabel(dataType, aInfo);
+ } catch (AtlasException e) {
+ throw new RuntimeException(e);
+ }
}
@Override
@@ -90,7 +96,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public List<String> traitNames(TitanVertex vertex) {
- return metadataRepository.getTraitNames(vertex);
+ return GraphHelper.getTraitNames(vertex);
}
@Override
@@ -100,7 +106,7 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
@Override
public Id getIdFromVertex(String dataTypeName, TitanVertex vertex) {
- return metadataRepository.getIdFromVertex(dataTypeName, vertex);
+ return GraphHelper.getIdFromVertex(dataTypeName, vertex);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java b/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java
deleted file mode 100644
index 7ea7e41..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/EntityExistsException.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.IReferenceableInstance;
-
-public class EntityExistsException extends AtlasException {
- public EntityExistsException(IReferenceableInstance typedInstance, Exception e) {
- super("Model violation for type "+ typedInstance.getTypeName(), e);
- }
-
- public EntityExistsException(IReferenceableInstance typedInstance) {
- super("Model violation for type "+ typedInstance.getTypeName());
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java b/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java
deleted file mode 100644
index db21bc7..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/EntityNotFoundException.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository;
-
-/**
- * A simple wrapper for 404.
- */
-public class EntityNotFoundException extends RepositoryException {
- public EntityNotFoundException() {
- }
-
- public EntityNotFoundException(String message) {
- super(message);
- }
-
- public EntityNotFoundException(String message, Throwable cause) {
- super(message, cause);
- }
-
- public EntityNotFoundException(Throwable cause) {
- super(cause);
- }
-
- public EntityNotFoundException(String message, Throwable cause, boolean enableSuppression,
- boolean writableStackTrace) {
- super(message, cause, enableSuppression, writableStackTrace);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
index c11d9a0..2091e71 100755
--- a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
@@ -19,6 +19,8 @@
package org.apache.atlas.repository;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.exception.EntityExistsException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.types.AttributeInfo;
@@ -70,7 +72,7 @@ public interface MetadataRepository {
* @param aInfo attribute info
* @return edge label for a given attribute
*/
- String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo);
+ String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException;
/**
* Creates an entity definition (instance) corresponding to a given type.
@@ -89,7 +91,7 @@ public interface MetadataRepository {
* @return entity (typed instance) definition
* @throws RepositoryException
*/
- ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException;
+ ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException;
/**
* Gets the list of entities for a given entity type.
@@ -108,20 +110,6 @@ public interface MetadataRepository {
* @throws RepositoryException
*/
// boolean deleteEntity(String guid) throws RepositoryException;
-
- /**
- * Updates an entity given its GUID with the attribute name and value.
- *
- * @param guid globally unique identifier for the entity
- * @param attributeName name of the attribute
- * @param attributeValue value of the attribute
- * @return an entity instance with updated state
- * @throws RepositoryException
- */
- //ITypedReferenceableInstance updateEntity(String guid, String attributeName,
- // String attributeValue) throws RepositoryException;
-
-
// Trait management functions
/**
@@ -149,15 +137,19 @@ public interface MetadataRepository {
* @param traitNameToBeDeleted name of the trait
* @throws RepositoryException
*/
- void deleteTrait(String guid, String traitNameToBeDeleted) throws RepositoryException;
+ void deleteTrait(String guid, String traitNameToBeDeleted) throws EntityNotFoundException, RepositoryException;
+
+ /**
+ * Adds/Updates the property to the entity that corresponds to the GUID
+ * Supports only primitive attribute/Class Id updations.
+ */
+ void updatePartial(ITypedReferenceableInstance entity) throws RepositoryException;
/**
- * Adds/Updates the property to/in the entity that corresponds to the GUID
- * @param guid entity id
- * @param property property name
- * @param value property value
+ * Adds the property to the entity that corresponds to the GUID
+ * @param entitiesToBeUpdated The entities to be updated
*/
- void updateEntity(String guid, String property, String value) throws RepositoryException;
+ String[] updateEntities(ITypedReferenceableInstance... entitiesToBeUpdated) throws RepositoryException;
/**
* Returns the entity for the given type and qualified name
@@ -166,5 +158,5 @@ public interface MetadataRepository {
* @param value
* @return entity instance
*/
- ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
+ ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value) throws AtlasException;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java b/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java
new file mode 100644
index 0000000..59472e4
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
+
+ private final Map<Id, IReferenceableInstance> idToInstanceMap;
+
+ public EntityProcessor() {
+ idToInstanceMap = new LinkedHashMap<>();
+ }
+
+ public Collection<IReferenceableInstance> getInstances() {
+ ArrayList<IReferenceableInstance> instances = new ArrayList<IReferenceableInstance>(idToInstanceMap.values());
+ Collections.reverse(instances);
+ return instances;
+ }
+
+ @Override
+ public void processNode(ObjectGraphWalker.Node nd) throws AtlasException {
+ IReferenceableInstance ref = null;
+ Id id = null;
+
+ if (nd.attributeName == null) {
+ ref = (IReferenceableInstance) nd.instance;
+ id = ref.getId();
+ } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
+ if (nd.value != null && (nd.value instanceof Id)) {
+ id = (Id) nd.value;
+ }
+ }
+
+ if (id != null) {
+ if (id.isUnassigned()) {
+ if (ref != null) {
+ if (idToInstanceMap.containsKey(id)) { // Oops
+ throw new RepositoryException(
+ String.format("Unexpected internal error: Id %s processed again", id));
+ }
+
+ idToInstanceMap.put(id, ref);
+ }
+ }
+ }
+ }
+
+ public void addInstanceIfNotExists(ITypedReferenceableInstance ref) {
+ if(!idToInstanceMap.containsKey(ref.getId())) {
+ idToInstanceMap.put(ref.getId(), ref);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
new file mode 100644
index 0000000..36d8034
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.List;
+import java.util.Map;
+
+public class FullTextMapper {
+
+ private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
+
+ private static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+ private static final String FULL_TEXT_DELIMITER = " ";
+
+ FullTextMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
+ this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
+ }
+
+ public String mapRecursive(Vertex instanceVertex, boolean followReferences) throws AtlasException {
+ String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ ITypedReferenceableInstance typedReference =
+ graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
+ String fullText = forInstance(typedReference, followReferences);
+ StringBuilder fullTextBuilder =
+ new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
+
+ List<String> traits = typedReference.getTraits();
+ for (String traitName : traits) {
+ String traitText = forInstance((ITypedInstance) typedReference.getTrait(traitName), false);
+ fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER)
+ .append(traitText);
+ }
+ return fullTextBuilder.toString();
+ }
+
+ private String forAttribute(IDataType type, Object value, boolean followReferences)
+ throws AtlasException {
+ if (value == null) {
+ return null;
+ }
+ switch (type.getTypeCategory()) {
+ case PRIMITIVE:
+ return String.valueOf(value);
+ case ENUM:
+
+ return ((EnumValue) value).value;
+
+ case ARRAY:
+ StringBuilder fullText = new StringBuilder();
+ IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
+ List list = (List) value;
+
+ for (Object element : list) {
+ String elemFullText = forAttribute(elemType, element, false);
+ if (StringUtils.isNotEmpty(elemFullText)) {
+ fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
+ }
+ }
+ return fullText.toString();
+
+ case MAP:
+ fullText = new StringBuilder();
+ IDataType keyType = ((DataTypes.MapType) type).getKeyType();
+ IDataType valueType = ((DataTypes.MapType) type).getValueType();
+ Map map = (Map) value;
+
+ for (Object entryObj : map.entrySet()) {
+ Map.Entry entry = (Map.Entry) entryObj;
+ String keyFullText = forAttribute(keyType, entry.getKey(), false);
+ if (StringUtils.isNotEmpty(keyFullText)) {
+ fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
+ }
+ String valueFullText = forAttribute(valueType, entry.getValue(), false);
+ if (StringUtils.isNotEmpty(valueFullText)) {
+ fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
+ }
+ }
+ return fullText.toString();
+
+ case CLASS:
+ if (followReferences) {
+ String refGuid = ((ITypedReferenceableInstance) value).getId()._getId();
+ Vertex refVertex = graphHelper.getVertexForGUID(refGuid);
+ return mapRecursive(refVertex, false);
+ }
+ break;
+
+ case STRUCT:
+ if (followReferences) {
+ return forInstance((ITypedInstance) value, true);
+ }
+ break;
+
+ default:
+ throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
+
+ }
+ return null;
+ }
+
+ private String forInstance(ITypedInstance typedInstance, boolean followReferences)
+ throws AtlasException {
+ StringBuilder fullText = new StringBuilder();
+ for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ if (attrValue == null) {
+ continue;
+ }
+
+ String attrFullText = forAttribute(attributeInfo.dataType(), attrValue, followReferences);
+ if (StringUtils.isNotEmpty(attrFullText)) {
+ fullText =
+ fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER)
+ .append(attrFullText);
+ }
+ }
+ return fullText.toString();
+ }
+}