You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2021/02/10 05:49:57 UTC
[atlas] branch master updated: ATLAS-4127: Optimize Hive table
calls in Atlas Hook
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new f848c97 ATLAS-4127: Optimize Hive table calls in Atlas Hook
f848c97 is described below
commit f848c9761bd340c9cbb790254420506d97c8e6b4
Author: Deep Singh <de...@gmail.com>
AuthorDate: Wed Feb 3 20:05:28 2021 -0600
ATLAS-4127: Optimize Hive table calls in Atlas Hook
Signed-off-by: Sarath Subramanian <sa...@apache.org>
---
.../java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index 3b2bbe0..7c269ce 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -1035,6 +1035,7 @@ public abstract class BaseHiveEvent {
Collections.sort(sortedEntities, entityComparator);
Set<String> dataSetsProcessed = new HashSet<>();
+ Map<String, Table> tableMap = new HashMap<>();
for (Entity entity : sortedEntities) {
if (ignoreHDFSPaths && (Entity.Type.DFS_DIR.equals(entity.getType()) || Entity.Type.LOCAL_DIR.equals(entity.getType()))) {
@@ -1046,8 +1047,14 @@ public abstract class BaseHiveEvent {
try {
if (entity.getType() == Entity.Type.PARTITION || entity.getType() == Entity.Type.TABLE) {
- Table table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName());
+ String tableKey = entity.getTable().getDbName() + "." + entity.getTable().getTableName();
+ Table table = tableMap.get(tableKey);
+ if (table == null) {
+ table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName());
+
+ tableMap.put(tableKey, table); //since there could be several partitions in a table, store it to avoid hive calls.
+ }
if (table != null) {
createTime = getTableCreateTime(table);
qualifiedName = getQualifiedName(table);