You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2019/09/29 10:20:07 UTC
[incubator-doris] branch master updated: Improve unique id
performance (#1911)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new f852f50 Improve unique id performance (#1911)
f852f50 is described below
commit f852f50acb56deff19d1699bed2b89313ed3fe8d
Author: yiguolei <yi...@baidu.com>
AuthorDate: Sun Sep 29 18:20:02 2019 +0800
Improve unique id performance (#1911)
Remove the default constructor for UniqueID
Add a gen_uid method in UniqueId. If need to generate a new uid, users should call this api explicitly.
Reuse boost random generator not generate a new one every time.
---
be/src/olap/olap_snapshot_converter.cpp | 2 +-
be/src/olap/options.h | 2 +-
be/src/olap/rowset/rowset_writer_context.h | 5 ++-
be/src/olap/tablet_manager.cpp | 2 +-
be/src/olap/tablet_meta.cpp | 2 +-
be/src/runtime/routine_load/data_consumer.cpp | 2 +-
be/src/runtime/routine_load/data_consumer.h | 2 ++
be/src/runtime/routine_load/data_consumer_group.h | 4 ++-
be/src/runtime/stream_load/stream_load_context.h | 1 +
be/src/service/doris_main.cpp | 2 +-
be/src/util/uid_util.h | 21 +++++------
be/src/{olap/options.h => util/uuid_generator.h} | 41 ++++++++++++----------
be/test/olap/olap_snapshot_converter_test.cpp | 2 +-
be/test/olap/rowset/rowset_meta_manager_test.cpp | 4 +--
.../rowset/unique_rowset_id_generator_test.cpp | 4 +--
be/test/olap/tablet_mgr_test.cpp | 2 +-
be/test/olap/txn_manager_test.cpp | 4 +--
be/test/util/uid_util_test.cpp | 2 +-
18 files changed, 57 insertions(+), 47 deletions(-)
diff --git a/be/src/olap/olap_snapshot_converter.cpp b/be/src/olap/olap_snapshot_converter.cpp
index 361a26e..37ffc0b 100755
--- a/be/src/olap/olap_snapshot_converter.cpp
+++ b/be/src/olap/olap_snapshot_converter.cpp
@@ -157,7 +157,7 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola
tablet_meta_pb->set_in_restore_mode(olap_header.in_restore_mode());
}
tablet_meta_pb->set_tablet_state(TabletStatePB::PB_RUNNING);
- *(tablet_meta_pb->mutable_tablet_uid()) = TabletUid().to_proto();
+ *(tablet_meta_pb->mutable_tablet_uid()) = TabletUid::gen_uid().to_proto();
VLOG(3) << "convert tablet meta tablet id = " << olap_header.tablet_id()
<< " schema hash = " << olap_header.schema_hash() << " successfully.";
return OLAP_SUCCESS;
diff --git a/be/src/olap/options.h b/be/src/olap/options.h
index 0dd0d9a..489405f 100644
--- a/be/src/olap/options.h
+++ b/be/src/olap/options.h
@@ -38,7 +38,7 @@ OLAPStatus parse_conf_store_paths(const std::string& config_path, std::vector<St
struct EngineOptions {
// list paths that tablet will be put into.
std::vector<StorePath> store_paths;
- UniqueId backend_uid;
+ UniqueId backend_uid {0, 0};
};
}
diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h
index 2123045..e4737e4 100644
--- a/be/src/olap/rowset/rowset_writer_context.h
+++ b/be/src/olap/rowset/rowset_writer_context.h
@@ -39,11 +39,10 @@ struct RowsetWriterContext {
data_dir(nullptr),
version(Version(0, 0)),
version_hash(0),
- txn_id(0) {
+ txn_id(0),
+ tablet_uid(0, 0) {
load_id.set_hi(0);
load_id.set_lo(0);
- tablet_uid.hi = 0;
- tablet_uid.lo = 0;
}
RowsetId rowset_id;
int64_t tablet_id;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index cdc4f2b..8e7f3a7 100755
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1336,7 +1336,7 @@ OLAPStatus TabletManager::_create_tablet_meta(
LOG(INFO) << "next_unique_id:" << next_unique_id;
// it is a new tablet meta obviously, should generate a new tablet id
- TabletUid tablet_uid;
+ TabletUid tablet_uid = TabletUid::gen_uid();
res = TabletMeta::create(request.table_id, request.partition_id,
request.tablet_id, request.tablet_schema.schema_hash,
shard_id, request.tablet_schema,
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 086f48f..adee33e 100755
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -202,7 +202,7 @@ OLAPStatus TabletMeta::reset_tablet_uid(const std::string& file_path) {
<< " , meta_file=" << file_path;
return res;
}
- *(tmp_tablet_meta_pb.mutable_tablet_uid()) = TabletUid().to_proto();
+ *(tmp_tablet_meta_pb.mutable_tablet_uid()) = TabletUid::gen_uid().to_proto();
res = save(file_path, tmp_tablet_meta_pb);
if (res != OLAP_SUCCESS) {
LOG(FATAL) << "fail to save tablet meta pb to "
diff --git a/be/src/runtime/routine_load/data_consumer.cpp b/be/src/runtime/routine_load/data_consumer.cpp
index 46c94f3..f0a6b96 100644
--- a/be/src/runtime/routine_load/data_consumer.cpp
+++ b/be/src/runtime/routine_load/data_consumer.cpp
@@ -48,7 +48,7 @@ Status KafkaDataConsumer::init(StreamLoadContext* ctx) {
std::stringstream ss;
ss << BackendOptions::get_localhost() << "_";
- std::string group_id = ss.str() + UniqueId().to_string();
+ std::string group_id = ss.str() + UniqueId::gen_uid().to_string();
LOG(INFO) << "init kafka consumer with group id: " << group_id;
std::string errstr;
diff --git a/be/src/runtime/routine_load/data_consumer.h b/be/src/runtime/routine_load/data_consumer.h
index 3cbf515..5c0fb5d 100644
--- a/be/src/runtime/routine_load/data_consumer.h
+++ b/be/src/runtime/routine_load/data_consumer.h
@@ -36,6 +36,8 @@ class StreamLoadPipe;
class DataConsumer {
public:
DataConsumer(StreamLoadContext* ctx):
+ _id(UniqueId::gen_uid()),
+ _grp_id(UniqueId::gen_uid()),
_has_grp(false),
_init(false),
_cancelled(false),
diff --git a/be/src/runtime/routine_load/data_consumer_group.h b/be/src/runtime/routine_load/data_consumer_group.h
index b18ef53..52b756b 100644
--- a/be/src/runtime/routine_load/data_consumer_group.h
+++ b/be/src/runtime/routine_load/data_consumer_group.h
@@ -31,8 +31,10 @@ public:
typedef std::function<void (const Status&)> ConsumeFinishCallback;
DataConsumerGroup():
+ _grp_id(UniqueId::gen_uid()),
_thread_pool(3, 10),
- _counter(0) {}
+ _counter(0){
+ }
virtual ~DataConsumerGroup() {
_consumers.clear();
diff --git a/be/src/runtime/stream_load/stream_load_context.h b/be/src/runtime/stream_load/stream_load_context.h
index ec4ce5a..9f79c8d 100644
--- a/be/src/runtime/stream_load/stream_load_context.h
+++ b/be/src/runtime/stream_load/stream_load_context.h
@@ -81,6 +81,7 @@ class MessageBodySink;
class StreamLoadContext {
public:
StreamLoadContext(ExecEnv* exec_env) :
+ id(UniqueId::gen_uid()),
_exec_env(exec_env),
_refs(0) {
start_nanos = MonotonicNanos();
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index be396d8..1240ef3 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -154,7 +154,7 @@ int main(int argc, char** argv) {
// options
doris::EngineOptions options;
options.store_paths = paths;
- options.backend_uid = doris::UniqueId();
+ options.backend_uid = doris::UniqueId::gen_uid();
doris::StorageEngine* engine = nullptr;
auto st = doris::StorageEngine::open(options, &engine);
if (!st.ok()) {
diff --git a/be/src/util/uid_util.h b/be/src/util/uid_util.h
index a626aa3..eb4019f 100644
--- a/be/src/util/uid_util.h
+++ b/be/src/util/uid_util.h
@@ -22,14 +22,12 @@
#include <string>
#include <boost/functional/hash.hpp>
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_generators.hpp>
-#include <boost/uuid/uuid_io.hpp>
#include "gen_cpp/Types_types.h" // for TUniqueId
#include "gen_cpp/types.pb.h" // for PUniqueId
// #include "util/debug_util.h"
#include "util/hash_util.hpp"
+#include "util/uuid_generator.h"
namespace doris {
@@ -63,13 +61,6 @@ struct UniqueId {
int64_t hi;
int64_t lo;
- // !!!! Not modify this method, it is very important. it will generate a random uid
- // it need modify it contact yiguolei
- UniqueId() {
- auto uuid = boost::uuids::basic_random_generator<boost::mt19937>()();
- memcpy(&hi, uuid.data, sizeof(int64_t));
- memcpy(&lo, uuid.data + sizeof(int64_t), sizeof(int64_t));
- }
UniqueId(int64_t hi_, int64_t lo_) : hi(hi_), lo(lo_) { }
UniqueId(const TUniqueId& tuid) : hi(tuid.hi), lo(tuid.lo) { }
UniqueId(const PUniqueId& puid) : hi(puid.hi()), lo(puid.lo()) { }
@@ -77,6 +68,16 @@ struct UniqueId {
from_hex(&hi, hi_str);
from_hex(&lo, lo_str);
}
+
+ // currently, the implementation is uuid, but it may change in the future
+ static UniqueId gen_uid() {
+ UniqueId uid(0, 0);
+ auto uuid = UUIDGenerator::instance()->next_uuid();
+ memcpy(&uid.hi, uuid.data, sizeof(int64_t));
+ memcpy(&uid.lo, uuid.data + sizeof(int64_t), sizeof(int64_t));
+ return uid;
+ }
+
~UniqueId() noexcept { }
std::string to_string() const {
diff --git a/be/src/olap/options.h b/be/src/util/uuid_generator.h
similarity index 58%
copy from be/src/olap/options.h
copy to be/src/util/uuid_generator.h
index 0dd0d9a..0643a7e 100644
--- a/be/src/olap/options.h
+++ b/be/src/util/uuid_generator.h
@@ -17,28 +17,33 @@
#pragma once
+#include <mutex>
+#include <ostream>
#include <string>
-#include <vector>
-#include "olap/olap_define.h"
-#include "util/uid_util.h"
+#include <boost/functional/hash.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/uuid/uuid_io.hpp>
+#include "util/spinlock.h"
namespace doris {
-struct StorePath {
- StorePath() : capacity_bytes(-1) { }
- StorePath(const std::string& path_, int64_t capacity_bytes_)
- : path(path_), capacity_bytes(capacity_bytes_) { }
- std::string path;
- int64_t capacity_bytes;
+class UUIDGenerator {
+public:
+ boost::uuids::uuid next_uuid() {
+ std::lock_guard<SpinLock> lock(_uuid_gen_lock);
+ return _boost_uuid_generator();
+ }
+
+ static UUIDGenerator* instance() {
+ static UUIDGenerator generator;
+ return &generator;
+ }
+
+private:
+ boost::uuids::basic_random_generator<boost::mt19937> _boost_uuid_generator;
+ SpinLock _uuid_gen_lock;
};
-OLAPStatus parse_conf_store_paths(const std::string& config_path, std::vector<StorePath>* path);
-
-struct EngineOptions {
- // list paths that tablet will be put into.
- std::vector<StorePath> store_paths;
- UniqueId backend_uid;
-};
-
-}
+}
\ No newline at end of file
diff --git a/be/test/olap/olap_snapshot_converter_test.cpp b/be/test/olap/olap_snapshot_converter_test.cpp
index fef45db..87447ff 100755
--- a/be/test/olap/olap_snapshot_converter_test.cpp
+++ b/be/test/olap/olap_snapshot_converter_test.cpp
@@ -54,7 +54,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
- options.backend_uid = doris::UniqueId();
+ options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}
diff --git a/be/test/olap/rowset/rowset_meta_manager_test.cpp b/be/test/olap/rowset/rowset_meta_manager_test.cpp
index 22246ae..58a4ef5 100644
--- a/be/test/olap/rowset/rowset_meta_manager_test.cpp
+++ b/be/test/olap/rowset/rowset_meta_manager_test.cpp
@@ -51,7 +51,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
- options.backend_uid = doris::UniqueId();
+ options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}
@@ -83,7 +83,7 @@ public:
private:
OlapMeta* _meta;
std::string _json_rowset_meta;
- TabletUid _tablet_uid;
+ TabletUid _tablet_uid {0, 0};
};
TEST_F(RowsetMetaManagerTest, TestSaveAndGetAndRemove) {
diff --git a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp
index c2a9a04..5fc021e 100644
--- a/be/test/olap/rowset/unique_rowset_id_generator_test.cpp
+++ b/be/test/olap/rowset/unique_rowset_id_generator_test.cpp
@@ -61,8 +61,8 @@ TEST_F(UniqueRowsetIdGeneratorTest, RowsetIdFormatTest) {
TEST_F(UniqueRowsetIdGeneratorTest, GenerateIdTest) {
- UniqueId backend_uid;
- UniqueId backend_uid2;
+ UniqueId backend_uid = UniqueId::gen_uid();
+ UniqueId backend_uid2 = UniqueId::gen_uid();
ASSERT_TRUE(backend_uid != backend_uid2);
UniqueRowsetIdGenerator id_generator(backend_uid);
UniqueRowsetIdGenerator id_generator2(backend_uid2);
diff --git a/be/test/olap/tablet_mgr_test.cpp b/be/test/olap/tablet_mgr_test.cpp
index debcefb..8745688 100644
--- a/be/test/olap/tablet_mgr_test.cpp
+++ b/be/test/olap/tablet_mgr_test.cpp
@@ -59,7 +59,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
- options.backend_uid = doris::UniqueId();
+ options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}
diff --git a/be/test/olap/txn_manager_test.cpp b/be/test/olap/txn_manager_test.cpp
index 13c0bb5..e156839 100644
--- a/be/test/olap/txn_manager_test.cpp
+++ b/be/test/olap/txn_manager_test.cpp
@@ -98,7 +98,7 @@ public:
paths.emplace_back("_engine_data_path", -1);
EngineOptions options;
options.store_paths = paths;
- options.backend_uid = doris::UniqueId();
+ options.backend_uid = UniqueId::gen_uid();
if (k_engine == nullptr) {
k_engine = new StorageEngine(options);
}
@@ -166,7 +166,7 @@ private:
TTransactionId transaction_id = 111;
TTabletId tablet_id = 222;
SchemaHash schema_hash = 333;
- TabletUid _tablet_uid;
+ TabletUid _tablet_uid {0, 0};
PUniqueId load_id;
std::unique_ptr<TabletSchema> _schema;
RowsetSharedPtr _alpha_rowset;
diff --git a/be/test/util/uid_util_test.cpp b/be/test/util/uid_util_test.cpp
index 9aa9add..a9964b0 100644
--- a/be/test/util/uid_util_test.cpp
+++ b/be/test/util/uid_util_test.cpp
@@ -30,7 +30,7 @@ public:
TEST_F(UidUtilTest, UniqueId) {
{
- UniqueId id;
+ UniqueId id = UniqueId::gen_uid();
std::string hex_str = id.to_string();
ASSERT_STRNE("0000000000000000-0000000000000000", hex_str.c_str());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org