You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/01/08 23:11:51 UTC
atlas git commit: ATLAS-2327: updated V1 to V2 conversion with
addition of validation of attribute values (#2)
Repository: atlas
Updated Branches:
refs/heads/master b5989c844 -> 7f7af565e
ATLAS-2327: updated V1 to V2 conversion with addition of validation of attribute values (#2)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/7f7af565
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/7f7af565
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/7f7af565
Branch: refs/heads/master
Commit: 7f7af565eb673486c1a2d0e7b1eb5fb1bb3923ab
Parents: b5989c8
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Mon Jan 8 13:07:19 2018 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Jan 8 14:51:41 2018 -0800
----------------------------------------------------------------------
.../AtlasAbstractFormatConverter.java | 17 +++++++
.../converters/AtlasArrayFormatConverter.java | 48 ++++++++++++++++++
.../converters/AtlasEntityFormatConverter.java | 11 +++++
.../converters/AtlasEnumFormatConverter.java | 45 +++++++++++++++++
.../converters/AtlasFormatConverter.java | 2 +
.../converters/AtlasMapFormatConverter.java | 51 ++++++++++++++++++++
.../converters/AtlasObjectIdConverter.java | 19 +++++++-
.../converters/AtlasStructFormatConverter.java | 13 ++++-
8 files changed, 203 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasAbstractFormatConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasAbstractFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasAbstractFormatConverter.java
index ab0d1fc..53c7282 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasAbstractFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasAbstractFormatConverter.java
@@ -19,9 +19,14 @@ package org.apache.atlas.repository.converters;
import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public abstract class AtlasAbstractFormatConverter implements AtlasFormatConverter {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasAbstractFormatConverter.class);
+
protected final AtlasFormatConverters converterRegistry;
protected final AtlasTypeRegistry typeRegistry;
@@ -34,6 +39,18 @@ public abstract class AtlasAbstractFormatConverter implements AtlasFormatConvert
}
@Override
+ public boolean isValidValueV1(Object v1Obj, AtlasType type) {
+ boolean ret = type.isValidValue(v1Obj);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasAbstractFormatConverter.isValidValueV1(type={}, value={}): {}", (v1Obj != null ? v1Obj.getClass().getCanonicalName() : null), v1Obj, ret);
+ }
+
+ return ret;
+ }
+
+
+ @Override
public TypeCategory getTypeCategory() {
return typeCategory;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasArrayFormatConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasArrayFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasArrayFormatConverter.java
index 882cd1b..c335f0a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasArrayFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasArrayFormatConverter.java
@@ -24,6 +24,10 @@ import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.v1.model.instance.Id;
+import org.apache.atlas.v1.model.instance.Referenceable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
@@ -32,12 +36,56 @@ import java.util.List;
import java.util.Set;
public class AtlasArrayFormatConverter extends AtlasAbstractFormatConverter {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasArrayFormatConverter.class);
public AtlasArrayFormatConverter(AtlasFormatConverters registry, AtlasTypeRegistry typeRegistry) {
super(registry, typeRegistry, TypeCategory.ARRAY);
}
@Override
+ public boolean isValidValueV1(Object v1Obj, AtlasType type) {
+ boolean ret = false;
+
+ if (v1Obj == null) {
+ return true;
+ } if (type instanceof AtlasArrayType) {
+ AtlasArrayType arrType = (AtlasArrayType) type;
+ AtlasType elemType = arrType.getElementType();
+ AtlasFormatConverter elemConverter = null;
+
+ try {
+ elemConverter = converterRegistry.getConverter(elemType.getTypeCategory());
+ } catch (AtlasBaseException excp) {
+ LOG.warn("failed to get element converter. type={}", type.getTypeName(), excp);
+
+ ret = false;
+ }
+
+ if (elemConverter != null) {
+ if (v1Obj instanceof Collection) {
+ ret = true; // for empty array
+
+ for (Object v1Elem : (Collection) v1Obj) {
+ ret = elemConverter.isValidValueV1(v1Elem, elemType);
+
+ if (!ret) {
+ break;
+ }
+ }
+ } else {
+ ret = elemConverter.isValidValueV1(v1Obj, elemType);
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasArrayFormatConverter.isValidValueV1(type={}, value={}): {}", (v1Obj != null ? v1Obj.getClass().getCanonicalName() : null), v1Obj, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
public Collection fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext ctx) throws AtlasBaseException {
Collection ret = null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
index 3d1d7ad..ab8c153 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
@@ -50,6 +50,17 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
}
@Override
+ public boolean isValidValueV1(Object v1Obj, AtlasType type) {
+ boolean ret = (v1Obj == null) || v1Obj instanceof Id || v1Obj instanceof Referenceable;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasEntityFormatConverter.isValidValueV1(type={}, value={}): {}", (v1Obj != null ? v1Obj.getClass().getCanonicalName() : null), v1Obj, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
public AtlasEntity fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException {
AtlasEntity entity = null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEnumFormatConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEnumFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEnumFormatConverter.java
index da76c5a..c2b0656 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEnumFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEnumFormatConverter.java
@@ -25,16 +25,61 @@ import org.apache.atlas.v1.model.typedef.EnumTypeDefinition.EnumValue;
import org.apache.atlas.type.AtlasEnumType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Map;
public class AtlasEnumFormatConverter extends AtlasAbstractFormatConverter {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasEnumFormatConverter.class);
+
public AtlasEnumFormatConverter(AtlasFormatConverters registry, AtlasTypeRegistry typeRegistry) {
super(registry, typeRegistry, TypeCategory.ENUM);
}
@Override
+ public boolean isValidValueV1(Object v1Obj, AtlasType type) {
+ boolean ret = false;
+
+ if (v1Obj == null) {
+ ret = true;
+ } else if (type instanceof AtlasEnumType) {
+ final AtlasEnumType enumType = (AtlasEnumType) type;
+
+ if (v1Obj instanceof EnumValue) {
+ Object enumValue = ((EnumValue)v1Obj).getValue();
+
+ if (enumValue != null) {
+ ret = enumType.getEnumDef().hasElement(enumValue.toString());
+ }
+ } else if (v1Obj instanceof Map) {
+ Object enumValue = ((Map)v1Obj).get("value");
+
+ if (enumValue != null) {
+ ret = enumType.getEnumDef().hasElement(enumValue.toString());
+ } else {
+ Object enumOrdinal = ((Map)v1Obj).get("ordinal");
+
+ if (enumOrdinal != null) {
+ ret = enumType.getEnumElementDef((Number) enumOrdinal) != null;
+ }
+ }
+ } else if (v1Obj instanceof Number) {
+ ret = enumType.getEnumElementDef((Number) v1Obj) != null;
+ } else {
+ ret = enumType.getEnumElementDef(v1Obj.toString()) != null;
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasEnumFormatConverter.isValidValueV1(type={}, value={}): {}", (v1Obj != null ? v1Obj.getClass().getCanonicalName() : null), v1Obj, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
public Object fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext ctx) throws AtlasBaseException {
String ret = null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverter.java
index a6d43da..1ec8e4a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasFormatConverter.java
@@ -26,6 +26,8 @@ import org.apache.atlas.type.AtlasType;
public interface AtlasFormatConverter {
+ boolean isValidValueV1(Object v1Ob, AtlasType typej);
+
Object fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext context) throws AtlasBaseException;
Object fromV2ToV1(Object v2Obj, AtlasType type, ConverterContext context) throws AtlasBaseException;
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasMapFormatConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasMapFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasMapFormatConverter.java
index bdfbf39..0eacd8e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasMapFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasMapFormatConverter.java
@@ -21,20 +21,71 @@ package org.apache.atlas.repository.converters;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
public class AtlasMapFormatConverter extends AtlasAbstractFormatConverter {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasMapFormatConverter.class);
+
public AtlasMapFormatConverter(AtlasFormatConverters registry, AtlasTypeRegistry typeRegistry) {
super(registry, typeRegistry, TypeCategory.MAP);
}
@Override
+ public boolean isValidValueV1(Object v1Obj, AtlasType type) {
+ boolean ret = false;
+
+ if (v1Obj == null) {
+ return true;
+ } if (type instanceof AtlasMapType && v1Obj instanceof Map) {
+ AtlasMapType mapType = (AtlasMapType) type;
+ AtlasType keyType = mapType.getKeyType();
+ AtlasType valueType = mapType.getValueType();
+ AtlasFormatConverter keyConverter = null;
+ AtlasFormatConverter valueConverter = null;
+ Map v1Map = (Map)v1Obj;
+
+ try {
+ keyConverter = converterRegistry.getConverter(keyType.getTypeCategory());
+ valueConverter = converterRegistry.getConverter(valueType.getTypeCategory());
+ } catch (AtlasBaseException excp) {
+ LOG.warn("failed to get key/value converter. type={}", type.getTypeName(), excp);
+
+ ret = false;
+ }
+
+ if (keyConverter != null && valueConverter != null) {
+ ret = true; // for empty map
+
+ for (Object key : v1Map.keySet()) {
+ Object value = v1Map.get(key);
+
+ ret = keyConverter.isValidValueV1(key, keyType) && valueConverter.isValidValueV1(value, valueType);
+
+ if (!ret) {
+ break;
+ }
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasArrayFormatConverter.isValidValueV1(type={}, value={}): {}", (v1Obj != null ? v1Obj.getClass().getCanonicalName() : null), v1Obj, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
public Map fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext ctx) throws AtlasBaseException {
Map ret = null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasObjectIdConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasObjectIdConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasObjectIdConverter.java
index a5b6d84..83e2ea2 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasObjectIdConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasObjectIdConverter.java
@@ -23,17 +23,21 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
-import org.apache.atlas.v1.model.instance.Id;
-import org.apache.atlas.v1.model.instance.Referenceable;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.v1.model.instance.Id;
+import org.apache.atlas.v1.model.instance.Referenceable;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Map;
public class AtlasObjectIdConverter extends AtlasAbstractFormatConverter {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasObjectIdConverter.class);
+
public AtlasObjectIdConverter(AtlasFormatConverters registry, AtlasTypeRegistry typeRegistry) {
this(registry, typeRegistry, TypeCategory.OBJECT_ID_TYPE);
@@ -44,6 +48,17 @@ public class AtlasObjectIdConverter extends AtlasAbstractFormatConverter {
}
@Override
+ public boolean isValidValueV1(Object v1Obj, AtlasType type) {
+ boolean ret = (v1Obj == null) || v1Obj instanceof Id || v1Obj instanceof Referenceable;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasObjectIdConverter.isValidValueV1(type={}, value={}): {}", (v1Obj != null ? v1Obj.getClass().getCanonicalName() : null), v1Obj, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
public Object fromV1ToV2(Object v1Obj, AtlasType type, AtlasFormatConverter.ConverterContext converterContext) throws AtlasBaseException {
Object ret = null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/7f7af565/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
index a5c9ded..539afdc 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
@@ -52,6 +52,17 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
}
@Override
+ public boolean isValidValueV1(Object v1Obj, AtlasType type) {
+ boolean ret = (v1Obj == null) || v1Obj instanceof Map || v1Obj instanceof Struct;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AtlasStructFormatConverter.isValidValueV1(type={}, value={}): {}", (v1Obj != null ? v1Obj.getClass().getCanonicalName() : null), v1Obj, ret);
+ }
+
+ return ret;
+ }
+
+ @Override
public Object fromV1ToV2(Object v1Obj, AtlasType type, ConverterContext converterContext) throws AtlasBaseException {
AtlasStruct ret = null;
@@ -248,7 +259,7 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
AtlasFormatConverter attrConverter = converterRegistry.getConverter(attrType.getTypeCategory());
Object v1Value = attributes.get(attrName);
- if (attrType.isValidValue(v1Value)) {
+ if (attrConverter.isValidValueV1(v1Value, attrType)) {
Object v2Value = attrConverter.fromV1ToV2(v1Value, attrType, context);
ret.put(attrName, v2Value);