You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ga...@apache.org on 2024/09/10 18:50:22 UTC

(doris) 02/02: [enhance](Vault) Maintain VaultName to VaultId map in FE master (#37899)

This is an automated email from the ASF dual-hosted git repository.

gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git

commit a5b958d63c311293829565cdd4ecb34ce1f5892c
Author: AlexYue <yj...@gmail.com>
AuthorDate: Thu Jul 18 20:24:23 2024 +0800

    [enhance](Vault) Maintain VaultName to VaultId map in FE master (#37899)
    
    All the vault related sql stmt is processed in FE master, so FE master
    could serve as one cache for vault related map. For the create table
    stmt with vault name, it can try to get the vault id from the map first.
---
 .../main/java/org/apache/doris/catalog/StorageVaultMgr.java   |  3 ---
 .../doris/cloud/catalog/CloudInstanceStatusChecker.java       | 11 +++++++++++
 .../apache/doris/cloud/datasource/CloudInternalCatalog.java   |  2 +-
 .../java/org/apache/doris/datasource/InternalCatalog.java     |  3 ++-
 4 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
index c0942d916d1..2d1df457c05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java
@@ -58,9 +58,6 @@ public class StorageVaultMgr {
         this.systemInfoService = systemInfoService;
     }
 
-    // TODO(ByteYue): The CreateStorageVault should only be handled by master
-    // which indicates we can maintains one <VaultName, VaultId> map in FE master
-
     public void createStorageVaultResource(CreateStorageVaultStmt stmt) throws Exception {
         switch (stmt.getStorageVaultType()) {
             case HDFS:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
index 9342c705e9d..ec63e5b4d49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
@@ -27,6 +27,9 @@ import org.apache.doris.common.util.MasterDaemon;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import java.util.HashMap;
+import java.util.Map;
+
 public class CloudInstanceStatusChecker extends MasterDaemon {
     private static final Logger LOG = LogManager.getLogger(CloudInstanceStatusChecker.class);
     private CloudSystemInfoService cloudSystemInfoService;
@@ -49,6 +52,14 @@ public class CloudInstanceStatusChecker extends MasterDaemon {
                         + "cloud_unique_id={}, response={}", Config.cloud_unique_id, response);
             } else {
                 cloudSystemInfoService.setInstanceStatus(response.getInstance().getStatus());
+                Map<String, String> vaultMap = new HashMap<>();
+                int cnt = response.getInstance().getResourceIdsCount();
+                for (int i = 0; i < cnt; i++) {
+                    String name = response.getInstance().getStorageVaultNames(i);
+                    String id = response.getInstance().getResourceIds(i);
+                    vaultMap.put(name, id);
+                }
+                Env.getCurrentEnv().getStorageVaultMgr().refreshVaultMap(vaultMap);
                 Env.getCurrentEnv().getStorageVaultMgr().setDefaultStorageVault(
                         Pair.of(response.getInstance().getDefaultStorageVaultName(),
                                 response.getInstance().getDefaultStorageVaultId()));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index b7a932486ad..9eb1400ab52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -130,7 +130,7 @@ public class CloudInternalCatalog extends InternalCatalog {
         long version = partition.getVisibleVersion();
 
         final String storageVaultName = tbl.getStorageVaultName();
-        boolean storageVaultIdSet = false;
+        boolean storageVaultIdSet = tbl.getStorageVaultId().isEmpty();
 
         // short totalReplicaNum = replicaAlloc.getTotalReplicaNum();
         for (Map.Entry<Long, MaterializedIndex> entry : indexMap.entrySet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index d4b9d5eef14..2b7f6090bb9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2703,7 +2703,8 @@ public class InternalCatalog implements CatalogIf<Database> {
             }
 
             olapTable.setStorageVaultName(storageVaultName);
-            if (storageVaultId != null) {
+            storageVaultId = env.getStorageVaultMgr().getVaultIdByName(storageVaultName);
+            if (storageVaultId != null && !storageVaultId.isEmpty()) {
                 olapTable.setStorageVaultId(storageVaultId);
             }
         }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org