You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2019/05/03 04:26:08 UTC
[atlas] branch master updated: ATLAS-3182: Import transforms
handling qualifiedName for hive_table entity.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 8b58a32 ATLAS-3182: Import transforms handling qualifiedName for hive_table entity.
8b58a32 is described below
commit 8b58a326197e897a937d28f53076f385bb6710a7
Author: Ashutosh Mestry <am...@hortonworks.com>
AuthorDate: Thu May 2 21:13:39 2019 -0700
ATLAS-3182: Import transforms handling qualifiedName for hive_table entity.
---
.../entitytransform/HiveTableEntityHandler.java | 19 +++-
.../entitytransform/TransformationHandlerTest.java | 104 +++++++++++++++------
intg/src/test/resources/json/entity1.json | 48 ++++++++++
intg/src/test/resources/json/transform1.json | 1 +
4 files changed, 139 insertions(+), 33 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
index 9eb44d7..caf08dd 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
@@ -43,18 +43,21 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
}
private static class HiveTableEntity extends AtlasTransformableEntity {
- private String databaseName;
- private String tableName;
- private String clusterName;
- private boolean isCustomAttributeUpdated = false;
+ private final String qualifiedName;
+ private String databaseName;
+ private String tableName;
+ private String clusterName;
+ private boolean isCustomAttributeUpdated = false;
+ private final boolean tableNameFromQualifiedNameDifferent;
+ private final String tableNameFromQualifiedName;
public HiveTableEntity(AtlasEntity entity) {
super(entity);
this.tableName = (String) entity.getAttribute(NAME_ATTRIBUTE);
- String qualifiedName = (String) entity.getAttribute(QUALIFIED_NAME_ATTRIBUTE);
+ this.qualifiedName = (String) entity.getAttribute(QUALIFIED_NAME_ATTRIBUTE);
if (qualifiedName != null) {
int databaseSeparatorIdx = qualifiedName.indexOf(DATABASE_DELIMITER);
@@ -62,9 +65,13 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
this.databaseName = databaseSeparatorIdx != -1 ? qualifiedName.substring(0, databaseSeparatorIdx) : "";
this.clusterName = clusterSeparatorIdx != -1 ? qualifiedName.substring(clusterSeparatorIdx + 1) : "";
+ this.tableNameFromQualifiedName = clusterSeparatorIdx != -1 ? this.qualifiedName.substring(databaseSeparatorIdx + 1, clusterSeparatorIdx) : "";
+ this.tableNameFromQualifiedNameDifferent = !this.tableNameFromQualifiedName.equals(this.tableName);
} else {
this.databaseName = "";
this.clusterName = "";
+ this.tableNameFromQualifiedName = "";
+ this.tableNameFromQualifiedNameDifferent = false;
}
}
@@ -121,6 +128,8 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
private String toQualifiedName() {
+ String tableName = tableNameFromQualifiedNameDifferent ? this.tableNameFromQualifiedName : this.tableName;
+
return String.format("%s.%s@%s", databaseName, tableName, clusterName);
}
}
diff --git a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
index d6b0ede..d3f4b57 100644
--- a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
+++ b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
@@ -24,13 +24,16 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
-import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
+import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -42,6 +45,7 @@ import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH
import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_COLUMN;
import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DATABASE;
import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_STORAGE_DESCRIPTOR;
+import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED_NAME_ATTRIBUTE;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
@@ -59,6 +63,7 @@ public class TransformationHandlerTest {
private static final String[] DATABASE_NAMES = new String[] { "hr", "sales", "engg" };
private static final String[] TABLE_NAMES = new String[] { "employees", "products", "invoice" };
private static final String[] COLUMN_NAMES = new String[] { "name", "age", "dob" };
+ private final String ATTR_NAME_QUALIFIED_NAME = "qualifiedName";
@Test
public void testHdfsClusterRenameHandler() {
@@ -69,12 +74,12 @@ public class TransformationHandlerTest {
List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p1));
for (AtlasEntity hdfsPath : getHdfsPathEntities()) {
- String qualifiedName = (String) hdfsPath.getAttribute("qualifiedName");
+ String qualifiedName = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME);
boolean endsWithCl1 = qualifiedName.endsWith("@cl1");
applyTransforms(hdfsPath, handlers);
- String transformedValue = (String) hdfsPath.getAttribute("qualifiedName");
+ String transformedValue = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME);
if (endsWithCl1) {
assertTrue(transformedValue.endsWith("@cl2"), transformedValue + ": expected to end with @cl2");
@@ -95,12 +100,12 @@ public class TransformationHandlerTest {
List<AtlasEntity> hdfsPaths = getHdfsPathEntities();
for (AtlasEntity hdfsPath : hdfsPaths) {
- String qualifiedName = (String) hdfsPath.getAttribute("qualifiedName");
+ String qualifiedName = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME);
boolean endsWithCl1 = qualifiedName.endsWith("@cl1");
applyTransforms(hdfsPath, handlers);
- String transformedValue = (String) hdfsPath.getAttribute("qualifiedName");
+ String transformedValue = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME);
if (endsWithCl1) {
assertTrue(transformedValue.endsWith("@CL1"), transformedValue + ": expected to end with @CL1");
@@ -116,12 +121,12 @@ public class TransformationHandlerTest {
handlers = initializeHandlers(Collections.singletonList(p2));
for (AtlasEntity hdfsPath : hdfsPaths) {
- String qualifiedName = (String) hdfsPath.getAttribute("qualifiedName");
+ String qualifiedName = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME);
boolean endsWithCL1 = qualifiedName.endsWith("@CL1");
applyTransforms(hdfsPath, handlers);
- String transformedValue = (String) hdfsPath.getAttribute("qualifiedName");
+ String transformedValue = (String) hdfsPath.getAttribute(ATTR_NAME_QUALIFIED_NAME);
if (endsWithCL1) {
assertTrue(transformedValue.endsWith("@cl1"), transformedValue + ": expected to end with @cl1");
@@ -258,14 +263,14 @@ public class TransformationHandlerTest {
List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p1));
for (AtlasEntity entity : getAllEntities()) {
- String qualifiedName = (String) entity.getAttribute("qualifiedName");
+ String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME);
boolean isHdfsPath = StringUtils.equals(entity.getTypeName(), HDFS_PATH);
boolean endsWithCl1 = qualifiedName.endsWith("@cl1");
boolean containsCl1 = qualifiedName.contains("@cl1"); // for stroage_desc
applyTransforms(entity, handlers);
- String transformedValue = (String) entity.getAttribute("qualifiedName");
+ String transformedValue = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME);
if (!isHdfsPath && endsWithCl1) {
assertTrue(transformedValue.endsWith("@cl1_backup"), transformedValue + ": expected to end with @cl1_backup");
@@ -286,18 +291,18 @@ public class TransformationHandlerTest {
List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p));
for (AtlasEntity entity : getAllEntities()) {
- String qualifiedName = (String) entity.getAttribute("qualifiedName");
+ String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME);
boolean startsWithHrDot = qualifiedName.startsWith("hr."); // for tables, columns
boolean startsWithHrAt = qualifiedName.startsWith("hr@"); // for databases
applyTransforms(entity, handlers);
if (startsWithHrDot) {
- assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup."));
+ assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr_backup."));
} else if (startsWithHrAt) {
- assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup@"));
+ assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr_backup@"));
} else {
- assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
+ assertEquals(qualifiedName, (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), "not expected to change");
}
}
}
@@ -313,18 +318,18 @@ public class TransformationHandlerTest {
List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p));
for (AtlasEntity entity : getAllEntities()) {
- String qualifiedName = (String) entity.getAttribute("qualifiedName");
+ String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME);
boolean startsWithHrEmployeesDot = qualifiedName.startsWith("hr.employees."); // for columns
boolean startsWithHrEmployeesAt = qualifiedName.startsWith("hr.employees@"); // for tables
applyTransforms(entity, handlers);
if (startsWithHrEmployeesDot) {
- assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup."));
+ assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr.employees_backup."));
} else if (startsWithHrEmployeesAt) {
- assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup@"));
+ assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr.employees_backup@"));
} else {
- assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
+ assertEquals(qualifiedName, (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), "not expected to change");
}
}
}
@@ -341,15 +346,15 @@ public class TransformationHandlerTest {
List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p));
for (AtlasEntity entity : getAllEntities()) {
- String qualifiedName = (String) entity.getAttribute("qualifiedName");
+ String qualifiedName = (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME);
boolean startsWithHrEmployeesAgeAt = qualifiedName.startsWith("hr.employees.age@");
applyTransforms(entity, handlers);
if (startsWithHrEmployeesAgeAt) {
- assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees.age_backup@"));
+ assertTrue(((String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME)).startsWith("hr.employees.age_backup@"));
} else {
- assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
+ assertEquals(qualifiedName, (String) entity.getAttribute(ATTR_NAME_QUALIFIED_NAME), "not expected to change");
}
}
}
@@ -369,7 +374,7 @@ public class TransformationHandlerTest {
public void verifyAddClassificationUsingScope() {
AtlasExportRequest exportRequest = new AtlasExportRequest();
- exportRequest.setItemsToExport(Collections.singletonList(new AtlasObjectId("hive_db", Collections.singletonMap("qualifiedName", "hr@cl1"))));
+ exportRequest.setItemsToExport(Collections.singletonList(new AtlasObjectId("hive_db", Collections.singletonMap(ATTR_NAME_QUALIFIED_NAME, "hr@cl1"))));
AtlasEntityTransformer transformer = new AtlasEntityTransformer(Collections.singletonMap("Referenceable.", "topLevel: "),
Collections.singletonMap("Referenceable", "ADD_CLASSIFICATION: replicated"),
@@ -420,11 +425,40 @@ public class TransformationHandlerTest {
assertEquals((String) nonAssetEntity.getAttribute("name"), "renamed", "name expected to be not updated for non-Asset type entity");
}
+ @Test
+ public void qualifiedNameDifferentFromName() throws IOException {
+ final String expectedQualifiedName = "test_partition_bootstrap_target.temptable_temp-5e93084a-f2d1-46f7-8f8f-30ffb859d2be@mycluster1";
+
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = readObjectFromJson("entity1", AtlasEntity.AtlasEntityWithExtInfo.class);
+ List<BaseEntityHandler> handlers = BaseEntityHandler.fromJson(getStringFromFile("transform1"), null);
+
+ assertNotNull(handlers);
+ assertEquals(handlers.size(), 4);
+ assertNotNull(entityWithExtInfo);
+
+ applyTransforms(entityWithExtInfo.getEntity(), handlers);
+ assertEquals(entityWithExtInfo.getEntity().getAttribute(ATTR_NAME_QUALIFIED_NAME), expectedQualifiedName);
+ }
+
+ @Test
+ public void emptyQualifiedNameHandling() throws IOException {
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = readObjectFromJson("entity1", AtlasEntity.AtlasEntityWithExtInfo.class);
+ entityWithExtInfo.getEntity().setAttribute(QUALIFIED_NAME_ATTRIBUTE, "");
+ List<BaseEntityHandler> handlers = BaseEntityHandler.fromJson("[{\"conditions\":{\"hive_db.clusterName\":\"EQUALS: mycluster0\"},\"action\":{\"hive_db.clusterName\":\"SET: mycluster1\"}}]", null);
+
+ assertNotNull(handlers);
+ assertEquals(handlers.size(), 4);
+ assertNotNull(entityWithExtInfo);
+
+ applyTransforms(entityWithExtInfo.getEntity(), handlers);
+ assertEquals(entityWithExtInfo.getEntity().getAttribute(ATTR_NAME_QUALIFIED_NAME), "");
+ }
+
private void assertApplyTransform(List<BaseEntityHandler> handlers) {
for (AtlasEntity entity : getAllEntities()) {
applyTransforms(entity, handlers);
- if(entity.getTypeName().equals("hive_db") && entity.getAttribute("qualifiedName").equals("hr@cl1")) {
+ if(entity.getTypeName().equals("hive_db") && entity.getAttribute(ATTR_NAME_QUALIFIED_NAME).equals("hr@cl1")) {
assertNotNull(entity.getClassifications());
} else{
assertNull(entity.getClassifications());
@@ -522,7 +556,7 @@ public class TransformationHandlerTest {
entity.setAttribute("name", "/aa/bb/employee");
entity.setAttribute("path", "hdfs://localhost.localdomain:8020/aa/bb/employee");
- entity.setAttribute("qualifiedName", "hdfs://localhost.localdomain:8020/aa/bb/employee@" + clusterName);
+ entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, "hdfs://localhost.localdomain:8020/aa/bb/employee@" + clusterName);
entity.setAttribute("clusterName", clusterName);
entity.setAttribute("isSymlink", false);
entity.setAttribute("modifiedTime", 0);
@@ -539,7 +573,7 @@ public class TransformationHandlerTest {
entity.setAttribute("name", "/cc/dd/employee");
entity.setAttribute("path", "hdfs://localhost.localdomain:8020/cc/dd/employee");
- entity.setAttribute("qualifiedName", "hdfs://localhost.localdomain:8020/cc/dd/employee@" + clusterName);
+ entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, "hdfs://localhost.localdomain:8020/cc/dd/employee@" + clusterName);
entity.setAttribute("clusterName", clusterName);
entity.setAttribute("isSymlink", false);
entity.setAttribute("modifiedTime", 0);
@@ -555,7 +589,7 @@ public class TransformationHandlerTest {
AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_DATABASE);
entity.setAttribute("name", dbName);
- entity.setAttribute("qualifiedName", dbName + "@" + clusterName);
+ entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, dbName + "@" + clusterName);
entity.setAttribute("location", "hdfs://localhost.localdomain:8020/warehouse/tablespace/managed/hive/" + dbName + ".db");
entity.setAttribute("clusterName", clusterName);
entity.setAttribute("owner", "hive");
@@ -570,7 +604,7 @@ public class TransformationHandlerTest {
AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_TABLE);
entity.setAttribute("name", tableName);
- entity.setAttribute("qualifiedName", qualifiedName);
+ entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, qualifiedName);
entity.setAttribute("owner", "hive");
entity.setAttribute("temporary", false);
entity.setAttribute("lastAccessTime", "1535656355000");
@@ -588,7 +622,7 @@ public class TransformationHandlerTest {
AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_STORAGE_DESCRIPTOR);
- entity.setAttribute("qualifiedName", dbName + "." + tableName + "@" + clusterName + "_storage");
+ entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, dbName + "." + tableName + "@" + clusterName + "_storage");
entity.setAttribute("storedAsSubDirectories", false);
entity.setAttribute("location", qualifiedName);
entity.setAttribute("compressed", false);
@@ -605,7 +639,7 @@ public class TransformationHandlerTest {
AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_COLUMN);
entity.setAttribute("owner", "hive");
- entity.setAttribute("qualifiedName", qualifiedName);
+ entity.setAttribute(ATTR_NAME_QUALIFIED_NAME, qualifiedName);
entity.setAttribute("name", columnName);
entity.setAttribute("position", 1);
entity.setAttribute("type", "string");
@@ -616,4 +650,18 @@ public class TransformationHandlerTest {
private AtlasEntity getNonAssetEntity() {
return new AtlasEntity(TYPENAME_NON_ASSET);
}
+
+ public static <T> T readObjectFromJson(String filename, Class<T> objectClass) throws IOException {
+ String json = getStringFromFile(filename);
+ return AtlasType.fromJson(json, objectClass);
+ }
+
+ private static String getStringFromFile(String filename) throws IOException {
+ final String userDir = System.getProperty("user.dir");
+ return FileUtils.readFileToString(new File(getTestJsonPath(userDir, filename)));
+ }
+
+ private static String getTestJsonPath(String startPath, String fileName) {
+ return startPath + "/src/test/resources/json/" + fileName + ".json";
+ }
}
\ No newline at end of file
diff --git a/intg/src/test/resources/json/entity1.json b/intg/src/test/resources/json/entity1.json
new file mode 100644
index 0000000..85aa62b
--- /dev/null
+++ b/intg/src/test/resources/json/entity1.json
@@ -0,0 +1,48 @@
+{
+ "entity": {
+ "attributes": {
+ "aliases": null,
+ "columns": [],
+ "comment": null,
+ "createTime": 1556559566000,
+ "db": {
+ "guid": "9ef83dcb-6cca-4e10-a19c-752d64227a94",
+ "typeName": "hive_db"
+ },
+ "description": null,
+ "lastAccessTime": 1556559566000,
+ "name": "temptable",
+ "owner": "hrt_qa",
+ "parameters": {
+ "bucketing_version": "2",
+ "numFiles": "1",
+ "numRows": "0",
+ "rawDataSize": "0",
+ "totalSize": "378890"
+ },
+ "partitionKeys": [],
+ "qualifiedName": "test_partition_bootstrap.temptable_temp-5e93084a-f2d1-46f7-8f8f-30ffb859d2be@mycluster0",
+ "replicatedFrom": null,
+ "replicatedTo": [
+ {
+ "guid": "2306c3e3-8b92-418e-b788-ade3d82d68db",
+ "typeName": "AtlasServer"
+ }
+ ],
+ "retention": 0,
+ "tableType": "MANAGED_TABLE",
+ "temporary": true,
+ "viewExpandedText": null,
+ "viewOriginalText": null
+ },
+ "createTime": 1556559566442,
+ "createdBy": "hrt_qa",
+ "guid": "e2d30fc3-026e-4597-b6a4-c706066bd480",
+ "status": "ACTIVE",
+ "typeName": "hive_table",
+ "updateTime": 1556559585192,
+ "updatedBy": "hrt_qa",
+ "version": 0
+ },
+ "referredEntities": {}
+}
diff --git a/intg/src/test/resources/json/transform1.json b/intg/src/test/resources/json/transform1.json
new file mode 100644
index 0000000..1093003
--- /dev/null
+++ b/intg/src/test/resources/json/transform1.json
@@ -0,0 +1 @@
+[{"conditions":{"hive_db.clusterName":"EQUALS: mycluster0"},"action":{"hive_db.clusterName":"SET: mycluster1"}},{"conditions":{"hive_db.name":"EQUALS: test_partition_bootstrap"},"action":{"hive_db.name":"SET: test_partition_bootstrap_target"}},{"conditions":{"hive_db.location":"STARTS_WITH_IGNORE_CASE: hdfs://ctr-e139-1542663976389-101273-01-000011.hwx.site:8020"},"action":{"hive_db.location":"REPLACE_PREFIX: = :hdfs://ctr-e139-1542663976389-101273-01-000011.hwx.site:8020=hdfs://ctr-e139 [...]
\ No newline at end of file