You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2021/02/10 05:50:33 UTC
[atlas] branch branch-2.0 updated: ATLAS-4127: Optimize Hive table
calls in Atlas Hook
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 5b3b0da ATLAS-4127: Optimize Hive table calls in Atlas Hook
5b3b0da is described below
commit 5b3b0da9e95830c8cf6330bb218993717de9e0ff
Author: Deep Singh <de...@gmail.com>
AuthorDate: Wed Feb 3 20:05:28 2021 -0600
ATLAS-4127: Optimize Hive table calls in Atlas Hook
Signed-off-by: Sarath Subramanian <sa...@apache.org>
(cherry picked from commit f848c9761bd340c9cbb790254420506d97c8e6b4)
---
.../java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
index dd1c41d..111b065 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/events/BaseHiveEvent.java
@@ -1038,6 +1038,7 @@ public abstract class BaseHiveEvent {
Collections.sort(sortedEntities, entityComparator);
Set<String> dataSetsProcessed = new HashSet<>();
+ Map<String, Table> tableMap = new HashMap<>();
for (Entity entity : sortedEntities) {
if (ignoreHDFSPaths && (Entity.Type.DFS_DIR.equals(entity.getType()) || Entity.Type.LOCAL_DIR.equals(entity.getType()))) {
@@ -1049,8 +1050,14 @@ public abstract class BaseHiveEvent {
try {
if (entity.getType() == Entity.Type.PARTITION || entity.getType() == Entity.Type.TABLE) {
- Table table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName());
+ String tableKey = entity.getTable().getDbName() + "." + entity.getTable().getTableName();
+ Table table = tableMap.get(tableKey);
+ if (table == null) {
+ table = getHive().getTable(entity.getTable().getDbName(), entity.getTable().getTableName());
+
+ tableMap.put(tableKey, table); //since there could be several partitions in a table, store it to avoid hive calls.
+ }
if (table != null) {
createTime = getTableCreateTime(table);
qualifiedName = getQualifiedName(table);