You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/10/12 00:27:56 UTC
[02/17] atlas git commit: ATLAS-2875: Implement clear attribute value
transformer for Atlas Entity Transformer
ATLAS-2875: Implement clear attribute value transformer for Atlas Entity Transformer
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/afa314cb
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/afa314cb
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/afa314cb
Branch: refs/heads/master
Commit: afa314cb520774e6417339f3180d292a6b0c13d6
Parents: 4b3c078
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Thu Sep 20 11:20:39 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 15:40:21 2018 -0700
----------------------------------------------------------------------
.../apache/atlas/entitytransform/Action.java | 18 ++++
.../entitytransform/BaseEntityHandler.java | 8 ++
.../apache/atlas/entitytransform/Condition.java | 22 +++++
.../TransformationHandlerTest.java | 99 ++++++++++++++++++++
4 files changed, 147 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
index ca5f3a8..f01c6ce 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -31,6 +31,7 @@ public abstract class Action {
private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
private static final String ACTION_NAME_TO_UPPER = "TO_UPPER";
+ private static final String ACTION_NAME_CLEAR = "CLEAR";
protected final String attributeName;
@@ -80,6 +81,10 @@ public abstract class Action {
ret = new SetAction(key, actionValue);
break;
+ case ACTION_NAME_CLEAR:
+ ret = new ClearAction(key);
+ break;
+
default:
ret = new SetAction(key, value); // treat unspecified/unknown action as 'SET'
break;
@@ -196,4 +201,17 @@ public abstract class Action {
}
}
}
+
+ public static class ClearAction extends Action {
+ public ClearAction(String attributeName) {
+ super(attributeName);
+ }
+
+ @Override
+ public void apply(AtlasTransformableEntity entity) {
+ if (isValid() && entity.hasAttribute(attributeName)) {
+ entity.setAttribute(attributeName, null);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
index c1f2869..9d44043 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
@@ -95,6 +95,10 @@ public class BaseEntityHandler {
}
}
+ if (CollectionUtils.isEmpty(ret)) {
+ ret.add(new BaseEntityHandler(transformers));
+ }
+
if (LOG.isDebugEnabled()) {
LOG.debug("<== BaseEntityHandler.createEntityHandlers(transforms={}): ret.size={}", transforms, ret.size());
}
@@ -158,6 +162,10 @@ public class BaseEntityHandler {
}
}
+ public boolean hasAttribute(String attributeName) {
+ return getAttribute(attributeName) != null;
+ }
+
public void transformComplete() {
// implementations can override to set value of computed-attributes
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
index d44f575..bc63079 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
@@ -31,6 +31,7 @@ public abstract class Condition {
private static final String CONDITION_NAME_EQUALS_IGNORE_CASE = "EQUALS_IGNORE_CASE";
private static final String CONDITION_NAME_STARTS_WITH = "STARTS_WITH";
private static final String CONDITION_NAME_STARTS_WITH_IGNORE_CASE = "STARTS_WITH_IGNORE_CASE";
+ private static final String CONDITION_NAME_HAS_VALUE = "HAS_VALUE";
protected final String attributeName;
@@ -75,6 +76,10 @@ public abstract class Condition {
ret = new StartsWithIgnoreCaseCondition(key, conditionValue);
break;
+ case CONDITION_NAME_HAS_VALUE:
+ ret = new HasValueCondition(key, conditionValue);
+ break;
+
default:
ret = new EqualsCondition(key, value); // treat unspecified/unknown condition as 'EQUALS'
break;
@@ -158,4 +163,21 @@ public abstract class Condition {
return attributeValue != null && StringUtils.startsWithIgnoreCase(attributeValue.toString(), this.prefix);
}
}
+
+ public static class HasValueCondition extends Condition {
+ protected final String attributeValue;
+
+ public HasValueCondition(String attributeName, String attributeValue) {
+ super(attributeName);
+
+ this.attributeValue = attributeValue;
+ }
+
+ @Override
+ public boolean matches(AtlasTransformableEntity entity) {
+ Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null;
+
+ return attributeValue != null ? StringUtils.isNotEmpty(attributeValue.toString()) : false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
index 69fba1e..a0ebe59 100644
--- a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
+++ b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
@@ -25,9 +25,12 @@ import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE;
public class TransformationHandlerTest {
@Test
@@ -102,6 +105,100 @@ public class TransformationHandlerTest {
}
@Test
+ public void testHiveTableClearAttributeHandler() {
+ // clear replicatedTo attribute for hive_table entities
+ AttributeTransform p1 = new AttributeTransform(Collections.singletonMap("hive_table.replicatedTo", "HAS_VALUE:"),
+ Collections.singletonMap("hive_table.replicatedTo", "CLEAR:"));
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p1));
+
+ List<AtlasEntity> entities = getAllEntities();
+
+ for (AtlasEntity entity : entities) {
+ String replicatedTo = (String) entity.getAttribute("replicatedTo");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ }
+
+ applyTransforms(entity, handlers);
+
+ String transformedValue = (String) entity.getAttribute("replicatedTo");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isEmpty(transformedValue));
+ }
+ }
+ }
+
+ @Test
+ public void testEntityClearAttributesActionWithNoCondition() {
+ // clear replicatedFrom attribute for hive_table entities without any condition
+ Map<String, String> actions = new HashMap<String, String>() {{ put("__entity.replicatedTo", "CLEAR:");
+ put("__entity.replicatedFrom", "CLEAR:"); }};
+
+ AttributeTransform transform = new AttributeTransform(null, actions);
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(transform));
+
+
+ List<AtlasEntity> entities = getAllEntities();
+
+ for (AtlasEntity entity : entities) {
+ String replicatedTo = (String) entity.getAttribute("replicatedTo");
+ String replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom));
+ }
+
+ applyTransforms(entity, handlers);
+
+ replicatedTo = (String) entity.getAttribute("replicatedTo");
+ replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isEmpty(replicatedFrom));
+ }
+ }
+ }
+
+ @Test
+ public void testEntityClearAttributesActionWithNoTypeNameAndNoCondition() {
+ // clear replicatedFrom attribute for hive_table entities without any condition
+ Map<String, String> actions = new HashMap<String, String>() {{ put("replicatedTo", "CLEAR:");
+ put("replicatedFrom", "CLEAR:"); }};
+
+ AttributeTransform transform = new AttributeTransform(null, actions);
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(transform));
+
+ List<AtlasEntity> entities = getAllEntities();
+
+ for (AtlasEntity entity : entities) {
+ String replicatedTo = (String) entity.getAttribute("replicatedTo");
+ String replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom));
+ }
+
+ applyTransforms(entity, handlers);
+
+ replicatedTo = (String) entity.getAttribute("replicatedTo");
+ replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isEmpty(replicatedFrom));
+ }
+ }
+ }
+
+ @Test
public void testHdfsPathNameReplacePrefixHandler() {
// Prefix replace hdfs_path name from /aa/bb/ to /xx/yy/
AttributeTransform p1 = new AttributeTransform(Collections.singletonMap("hdfs_path.name", "STARTS_WITH: /aa/bb/"),
@@ -338,6 +435,8 @@ public class TransformationHandlerTest {
entity.setAttribute("tableType", "EXTERNAL_TABLE");
entity.setAttribute("createTime", "1535656355000");
entity.setAttribute("retention", 0);
+ entity.setAttribute("replicatedTo", "[{\"guid\":\"f378cfa5-c4aa-4699-a733-8f11d2f089cd\",\"typeName\":\"AtlasServer\"},{\"guid\":\"58e42789-ea3e-4eaa-a0c4-d38d8632e548\",\"typeName\":\"AtlasServer\"}]");
+ entity.setAttribute("replicatedFrom", "[{\"guid\":\"f378cfa5-c4aa-4699-a733-8f11d2f089cd\",\"typeName\":\"AtlasServer\"},{\"guid\":\"58e42789-ea3e-4eaa-a0c4-d38d8632e548\",\"typeName\":\"AtlasServer\"}]");
return entity;
}