You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/09/18 21:40:12 UTC
[atlas] branch master updated: ATLAS-3414: Include metastore
catalog name in hive database name and qualifiedName
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new b6f35bf ATLAS-3414: Include metastore catalog name in hive database name and qualifiedName
b6f35bf is described below
commit b6f35bf9151bbd92c3c47679945e4d1628220af0
Author: Sarath Subramanian <sa...@apache.org>
AuthorDate: Tue Sep 17 22:39:36 2019 -0700
ATLAS-3414: Include metastore catalog name in hive database name and qualifiedName
---
.../apache/atlas/hive/bridge/HiveMetaStoreBridge.java | 15 ++++++++++++++-
.../apache/atlas/hive/hook/AtlasHiveHookContext.java | 3 ++-
.../apache/atlas/hive/hook/events/BaseHiveEvent.java | 17 ++++++++++-------
.../apache/atlas/hive/hook/events/CreateDatabase.java | 3 ++-
4 files changed, 28 insertions(+), 10 deletions(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 2f1bb0e..eb691dc 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -92,6 +92,7 @@ public class HiveMetaStoreBridge {
public static final String ATLAS_ENDPOINT = "atlas.rest.address";
public static final String SEP = ":".intern();
public static final String HDFS_PATH = "hdfs_path";
+ public static final String DEFAULT_METASTORE_CATALOG = "hive";
private static final int EXIT_CODE_SUCCESS = 0;
private static final int EXIT_CODE_FAILED = 1;
@@ -553,7 +554,7 @@ public class HiveMetaStoreBridge {
dbEntity = new AtlasEntity(HiveDataTypes.HIVE_DB.getName());
}
- String dbName = hiveDB.getName().toLowerCase();
+ String dbName = getDatabaseName(hiveDB);
dbEntity.setAttribute(ATTRIBUTE_QUALIFIED_NAME, getDBQualifiedName(metadataNamespace, dbName));
dbEntity.setAttribute(ATTRIBUTE_NAME, dbName);
@@ -570,6 +571,18 @@ public class HiveMetaStoreBridge {
return dbEntity;
}
+
+ public static String getDatabaseName(Database hiveDB) {
+ String dbName = hiveDB.getName().toLowerCase();
+ String catalogName = hiveDB.getCatalogName() != null ? hiveDB.getCatalogName().toLowerCase() : null;
+
+ if (StringUtils.isNotEmpty(catalogName) && !StringUtils.equals(catalogName, DEFAULT_METASTORE_CATALOG)) {
+ dbName = catalogName + SEP + dbName;
+ }
+
+ return dbName;
+ }
+
/**
* Create a new table instance in Atlas
* @param database AtlasEntity for Hive {@link AtlasEntity} to which this table belongs
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
index 0eee7c1..2e7a3c6 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/AtlasHiveHookContext.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import java.util.*;
+import static org.apache.atlas.hive.bridge.HiveMetaStoreBridge.getDatabaseName;
import static org.apache.atlas.hive.hook.events.BaseHiveEvent.toTable;
@@ -192,7 +193,7 @@ public class AtlasHiveHookContext {
}
public String getQualifiedName(Database db) {
- return (db.getName() + QNAME_SEP_METADATA_NAMESPACE).toLowerCase() + getMetadataNamespace();
+ return getDatabaseName(db) + QNAME_SEP_METADATA_NAMESPACE + getMetadataNamespace();
}
public String getQualifiedName(Table table) {
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index c6c0c66..42f78a0 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -62,6 +62,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import static org.apache.atlas.hive.bridge.HiveMetaStoreBridge.getDatabaseName;
import static org.apache.atlas.hive.hook.AtlasHiveHookContext.QNAME_SEP_METADATA_NAMESPACE;
import static org.apache.atlas.hive.hook.AtlasHiveHookContext.QNAME_SEP_ENTITY_NAME;
import static org.apache.atlas.hive.hook.AtlasHiveHookContext.QNAME_SEP_PROCESS;
@@ -257,8 +258,10 @@ public abstract class BaseHiveEvent {
switch (entity.getType()) {
case DATABASE: {
- if (!context.getIgnoreDummyDatabaseName().contains(entity.getDatabase().getName())) {
- Database db = getHive().getDatabase(entity.getDatabase().getName());
+ String dbName = getDatabaseName(entity.getDatabase());
+
+ if (!context.getIgnoreDummyDatabaseName().contains(dbName)) {
+ Database db = getHive().getDatabase(dbName);
ret = toDbEntity(db);
}
@@ -302,10 +305,10 @@ public abstract class BaseHiveEvent {
}
protected AtlasEntity toDbEntity(Database db) throws Exception {
- String dbQualifiedName = getQualifiedName(db);
- boolean isKnownDatabase = context.isKnownDatabase(dbQualifiedName);
-
- AtlasEntity ret = context.getEntity(dbQualifiedName);
+ String dbName = getDatabaseName(db);
+ String dbQualifiedName = getQualifiedName(db);
+ boolean isKnownDatabase = context.isKnownDatabase(dbQualifiedName);
+ AtlasEntity ret = context.getEntity(dbQualifiedName);
if (ret == null) {
ret = new AtlasEntity(HIVE_TYPE_DB);
@@ -318,7 +321,7 @@ public abstract class BaseHiveEvent {
}
ret.setAttribute(ATTRIBUTE_QUALIFIED_NAME, dbQualifiedName);
- ret.setAttribute(ATTRIBUTE_NAME, db.getName().toLowerCase());
+ ret.setAttribute(ATTRIBUTE_NAME, dbName);
ret.setAttribute(ATTRIBUTE_DESCRIPTION, db.getDescription());
ret.setAttribute(ATTRIBUTE_OWNER, db.getOwnerName());
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java
index f4f933e..7bea779 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/CreateDatabase.java
@@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
+import static org.apache.atlas.hive.bridge.HiveMetaStoreBridge.getDatabaseName;
import static org.apache.hadoop.hive.ql.hooks.Entity.Type.DATABASE;
public class CreateDatabase extends BaseHiveEvent {
@@ -80,7 +81,7 @@ public class CreateDatabase extends BaseHiveEvent {
Database db = entity.getDatabase();
if (db != null) {
- db = getHive().getDatabase(db.getName());
+ db = getHive().getDatabase(getDatabaseName(db));
}
if (db != null) {