[incubator-doris] branch master updated: [Storage] Open data dirs parallelly (#3260)

[zh...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 162b1c5  [Storage] Open data dirs parallelly (#3260)
162b1c5 is described below

commit 162b1c5d8bfd8afb7b01951ea5c5a79d1ffe1679
Author: HuangWei <hu...@xiaomi.com>
AuthorDate: Tue Apr 7 20:59:56 2020 +0800

    [Storage] Open data dirs parallelly (#3260)
---
 be/src/olap/rowset/rowset.h    |  2 --
 be/src/olap/storage_engine.cpp | 45 ++++++++++++++++++++++++++++++++----------
 be/src/olap/storage_engine.h   |  2 ++
 3 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 30cf314..2c509eb 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -204,8 +204,6 @@ public:
     // return whether `path` is one of the files in this rowset
     virtual bool check_path(const std::string& path) = 0;
 
-    std::string rowset_path() const { return _rowset_path; }
-
     // return an unique identifier string for this rowset
     std::string unique_id() const {
         return _rowset_path + "/" + rowset_id().to_string();
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index d8be355..16b9025 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -148,16 +148,8 @@ void StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) {
 
 OLAPStatus StorageEngine::_open() {
     // init store_map
-    for (auto& path : _options.store_paths) {
-        DataDir* store = new DataDir(path.path, path.capacity_bytes, path.storage_medium,
-                                     _tablet_manager.get(), _txn_manager.get());
-        auto st = store->init();
-        if (!st.ok()) {
-            LOG(WARNING) << "Store load failed, path=" << path.path;
-            return OLAP_ERR_INVALID_ROOT_PATH;
-        }
-        _store_map.emplace(path.path, store);
-    }
+    RETURN_NOT_OK(_init_store_map());
+
     _effective_cluster_id = config::cluster_id;
     RETURN_NOT_OK_LOG(_check_all_root_path_cluster_id(), "fail to check cluster info.");
 
@@ -185,6 +177,39 @@ OLAPStatus StorageEngine::_open() {
     return OLAP_SUCCESS;
 }
 
+OLAPStatus StorageEngine::_init_store_map() {
+    std::vector<DataDir*> tmp_stores;
+    std::vector<std::thread> threads;
+    std::atomic<bool> init_error{false};
+    for (auto& path : _options.store_paths) {
+        DataDir* store = new DataDir(path.path, path.capacity_bytes, path.storage_medium,
+                                     _tablet_manager.get(), _txn_manager.get());
+        tmp_stores.emplace_back(store);
+        threads.emplace_back([store, &init_error]() {
+            auto st = store->init();
+            if (!st.ok()) {
+                init_error = true;
+                LOG(WARNING) << "Store load failed, status="<< st.to_string() << ", path=" << store->path();
+            }
+        });
+    }
+    for (auto& thread : threads) {
+        thread.join();
+    }
+
+    if (init_error) {
+        for (auto store : tmp_stores) {
+            delete store;
+        }
+        return OLAP_ERR_INVALID_ROOT_PATH;
+    }
+
+    for (auto store : tmp_stores) {
+        _store_map.emplace(store->path(), store);
+    }
+    return OLAP_SUCCESS;
+}
+
 void StorageEngine::_update_storage_medium_type_count() {
     set<TStorageMedium::type> available_storage_medium_types;
 
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 8d32d9b..7177968 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -200,6 +200,8 @@ private:
     // Clear status(tables, ...)
     void _clear();
 
+    OLAPStatus _init_store_map();
+
     void _update_storage_medium_type_count();
 
     // Some check methods


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org