You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by su...@apache.org on 2016/04/09 18:54:05 UTC
incubator-atlas git commit: ATLAS-528 Support drop table,
view (sumasai)
Repository: incubator-atlas
Updated Branches:
refs/heads/master b8f4ffb68 -> 755e59c08
ATLAS-528 Support drop table,view (sumasai)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/755e59c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/755e59c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/755e59c0
Branch: refs/heads/master
Commit: 755e59c08dbbc3b03fc2f6fc4908396433b97e88
Parents: b8f4ffb
Author: Suma Shivaprasad <su...@gmail.com>
Authored: Sat Apr 9 09:53:58 2016 -0700
Committer: Suma Shivaprasad <su...@gmail.com>
Committed: Sat Apr 9 09:53:58 2016 -0700
----------------------------------------------------------------------
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 12 ++-
.../org/apache/atlas/hive/hook/HiveHook.java | 22 +++++
.../org/apache/atlas/hive/hook/HiveHookIT.java | 88 +++++++++++++++++++-
release-log.txt | 1 +
4 files changed, 115 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index eb5f1e6..6b348e2 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -335,8 +335,8 @@ public class HiveMetaStoreBridge {
tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableQualifiedName));
- // add reference to the StorageDescriptorx
- Referenceable sdReferenceable = fillStorageDescStruct(hiveTable.getSd(), tableQualifiedName, tableQualifiedName);
+ // add reference to the StorageDescriptor
+ Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName));
tableReference.set("sd", sdReferenceable);
// add reference to the Partition Keys
@@ -359,6 +359,10 @@ public class HiveMetaStoreBridge {
return tableReference;
}
+ private String getStorageDescQFName(String entityQualifiedName) {
+ return entityQualifiedName + "_storage";
+ }
+
private Referenceable registerTable(Referenceable dbReference, Table table) throws Exception {
String dbName = table.getDbName();
String tableName = table.getTableName();
@@ -410,7 +414,7 @@ public class HiveMetaStoreBridge {
return new Referenceable(sd.getId().id, sd.getTypeName(), null);
}
- public Referenceable fillStorageDescStruct(StorageDescriptor storageDesc, String tableQualifiedName,
+ public Referenceable fillStorageDesc(StorageDescriptor storageDesc, String tableQualifiedName,
String sdQualifiedName) throws Exception {
LOG.debug("Filling storage descriptor information for " + storageDesc);
@@ -466,7 +470,7 @@ public class HiveMetaStoreBridge {
ref.set("path", pathUri);
// Path path = new Path(pathUri);
// ref.set("name", path.getName());
-// TODO - Fix after ATLAS-542 to shorter Name
+ //TODO - Fix after ATLAS-542 to shorter Name
ref.set("name", pathUri);
ref.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, pathUri);
return ref;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index 4102263..505e5e7 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -308,6 +308,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
case EXPORT:
case IMPORT:
case QUERY:
+ case TRUNCATETABLE:
registerProcess(dgiBridge, event);
break;
@@ -326,6 +327,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
case ALTERTABLE_ADDCOLS:
case ALTERTABLE_REPLACECOLS:
case ALTERTABLE_RENAMECOL:
+ case ALTERTABLE_PARTCOLTYPE:
handleEventOutputs(dgiBridge, event, Type.TABLE);
break;
case ALTERTABLE_LOCATION:
@@ -334,17 +336,37 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
//Track altered lineage in case of external tables
handleExternalTables(dgiBridge, event, tablesUpdated.get(0).getLeft(), tablesUpdated.get(0).getRight());
}
+ break;
case ALTERDATABASE:
case ALTERDATABASE_OWNER:
handleEventOutputs(dgiBridge, event, Type.DATABASE);
break;
+ case DROPTABLE:
+ case DROPVIEW:
+ deleteTable(dgiBridge, event);
+ break;
+
default:
}
notifyEntities(messages);
}
+ private void deleteTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) {
+ for (Entity output : event.outputs) {
+ if (Type.TABLE.equals(output.getType())) {
+ final String tblQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(dgiBridge.getClusterName(), output.getTable().getDbName(), output.getTable().getTableName());
+ LOG.info("Deleting table {} ", tblQualifiedName);
+ messages.add(
+ new HookNotification.EntityDeleteRequest(event.getUser(),
+ HiveDataTypes.HIVE_TABLE.getName(),
+ HiveDataModelGenerator.NAME,
+ tblQualifiedName));
+ }
+ }
+ }
+
private void renameTable(HiveMetaStoreBridge dgiBridge, HiveEventContext event) throws Exception {
//crappy, no easy of getting new name
assert event.getInputs() != null && event.getInputs().size() == 1;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index 8ef8479..7a09b47 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -497,7 +497,6 @@ public class HiveHookIT {
runCommand(query);
Referenceable processReference = validateProcess(query, 1, 1);
validateHDFSPaths(processReference, filename, OUTPUTS);
-
validateInputTables(processReference, tableId);
//Import
@@ -510,7 +509,6 @@ public class HiveHookIT {
validateHDFSPaths(processReference, filename, INPUTS);
validateOutputTables(processReference, tableId);
-
}
@Test
@@ -541,8 +539,6 @@ public class HiveHookIT {
validateHDFSPaths(processReference, filename, INPUTS);
validateOutputTables(processReference, tableId);
-
-
}
@Test
@@ -684,6 +680,41 @@ public class HiveHookIT {
Assert.assertEquals(columns.get(0).get(HiveDataModelGenerator.NAME), "id");
}
+ @Test()
+ public void testTruncateTable() throws Exception {
+ String tableName = createTable(false);
+ String query = String.format("truncate table %s", tableName);
+ runCommand(query);
+
+ String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
+ validateProcess(query, 0, 1);
+
+ //Check lineage
+ String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
+ JSONObject response = dgiCLient.getInputGraph(datasetName);
+ JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
+ //Below should be assertTrue - Fix https://issues.apache.org/jira/browse/ATLAS-653
+ Assert.assertFalse(vertices.has(tableId));
+ }
+
+ @Test
+ public void testAlterTablePartitionColumnType() throws Exception {
+ String tableName = createTable(true, true, false);
+ final String newType = "int";
+ String query = String.format("ALTER TABLE %s PARTITION COLUMN (dt %s)", tableName, newType);
+ runCommand(query);
+
+ final String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
+ final String dtColId = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "dt"));
+ Referenceable table = dgiCLient.getEntity(tableId);
+ Referenceable column = dgiCLient.getEntity(dtColId);
+ Assert.assertEquals(column.get("type"), newType);
+
+ final List<Referenceable> partitionKeys = (List<Referenceable>) table.get("partitionKeys");
+ Assert.assertEquals(partitionKeys.size(), 1);
+ Assert.assertEquals(partitionKeys.get(0).getId()._getId(), dtColId);
+ }
+
@Test
public void testAlterViewRename() throws Exception {
String tableName = createTable();
@@ -834,7 +865,56 @@ public class HiveHookIT {
//Add another property
runSerdePropsQuery(tableName, expectedProps);
+ }
+
+ @Test
+ public void testDropTable() throws Exception {
+ //Test Deletion of tables and its corrresponding columns
+ String tableName = createTable(true, true, false);
+ assertTableIsRegistered(DEFAULT_DB, tableName);
+ assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "id"));
+ assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "name"));
+
+ final String query = String.format("drop table %s ", tableName);
+
+ runCommand(query);
+ assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "id"));
+ assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), "name"));
+ assertTableIsNotRegistered(DEFAULT_DB, tableName);
+ }
+
+ @Test
+ public void testDropNonExistingTable() throws Exception {
+ //Test Deletion of a non existing table
+ final String tableName = "nonexistingtable";
+ assertTableIsNotRegistered(DEFAULT_DB, tableName);
+ final String query = String.format("drop table if exists %s", tableName);
+ runCommand(query);
+
+ //Should have no effect
+ assertTableIsNotRegistered(DEFAULT_DB, tableName);
+ assertProcessIsNotRegistered(query);
+ }
+
+ @Test
+ public void testDropView() throws Exception {
+ //Test Deletion of tables and its corrresponding columns
+ String tableName = createTable(true, true, false);
+ String viewName = tableName();
+ String query = "create view " + viewName + " as select * from " + tableName;
+ runCommand(query);
+
+ assertTableIsRegistered(DEFAULT_DB, viewName);
+ assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "id"));
+ assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "name"));
+
+ query = String.format("drop view %s ", viewName);
+
+ runCommand(query);
+ assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "id"));
+ assertColumnIsNotRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, viewName), "name"));
+ assertTableIsNotRegistered(DEFAULT_DB, viewName);
}
private void runSerdePropsQuery(String tableName, Map<String, String> expectedProps) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/755e59c0/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a2fd4ea..2125f87 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -13,6 +13,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-528 Support drop table,view (sumasai)
ATLAS-603 Document High Availability of Atlas (yhemanth via sumasai)
ATLAS-498 Support Embedded HBase (tbeerbower via sumasai)
ATLAS-527 Support lineage for load table, import, export (sumasai via shwethags)