You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2019/10/12 15:30:21 UTC
[incubator-doris] branch master updated: Lost rowset during tablet
revise tablet meta (#1967)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8232261 Lost rowset during tablet revise tablet meta (#1967)
8232261 is described below
commit 8232261df1c0ddaf64388397e6209f4ad3f30a81
Author: yiguolei <yi...@baidu.com>
AuthorDate: Sat Oct 12 23:30:11 2019 +0800
Lost rowset during tablet revise tablet meta (#1967)
---
be/src/olap/snapshot_manager.cpp | 64 +---------------------
be/src/olap/snapshot_manager.h | 4 --
be/src/olap/tablet.cpp | 19 ++++++-
be/src/olap/tablet.h | 2 +
be/src/olap/tablet_meta_manager.cpp | 13 +----
be/src/olap/tablet_meta_manager.h | 3 -
be/src/olap/task/engine_storage_migration_task.cpp | 12 ++--
7 files changed, 31 insertions(+), 86 deletions(-)
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 4f3a9e3..df30fb8 100755
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -338,7 +338,6 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
path boost_path(snapshot_id_path);
string snapshot_id = canonical(boost_path).string();
do {
- DataDir* data_dir = ref_tablet->data_dir();
TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
if (new_tablet_meta == nullptr) {
LOG(WARNING) << "fail to malloc TabletMeta.";
@@ -365,8 +364,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
if (res != OLAP_SUCCESS) {
break;
}
- res = TabletMetaManager::get_meta(data_dir, ref_tablet->tablet_id(),
- ref_tablet->schema_hash(), new_tablet_meta);
+ res = ref_tablet->generate_tablet_meta_copy(new_tablet_meta);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to load header. res=" << res
<< " tablet_id=" << ref_tablet->tablet_id()
@@ -409,8 +407,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
break;
}
- res = TabletMetaManager::get_meta(data_dir, ref_tablet->tablet_id(),
- ref_tablet->schema_hash(), new_tablet_meta);
+ res = ref_tablet->generate_tablet_meta_copy(new_tablet_meta);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "fail to load header. res=" << res
<< " tablet_id=" << ref_tablet->tablet_id()
@@ -523,61 +520,4 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
return res;
}
-OLAPStatus SnapshotManager::_append_single_delta(
- const TSnapshotRequest& request, DataDir* store) {
- OLAPStatus res = OLAP_SUCCESS;
- string root_path = store->path();
- TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
- if (new_tablet_meta == nullptr) {
- LOG(WARNING) << "fail to malloc TabletMeta.";
- return OLAP_ERR_MALLOC_ERROR;
- }
-
- res = TabletMetaManager::get_meta(store, request.tablet_id, request.schema_hash, new_tablet_meta);
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to create tablet from header file. "
- << " tablet_id=" << request.tablet_id
- << " schema_hash=" << request.schema_hash;
- return res;
- }
- auto tablet = Tablet::create_tablet_from_meta(new_tablet_meta, store);
- if (tablet == nullptr) {
- LOG(WARNING) << "fail to load tablet. "
- << " res=" << res
- << " tablet_id=" << request.tablet_id
- << " schema_hash=" << request.schema_hash;
- return OLAP_ERR_INPUT_PARAMETER_ERROR;
- }
-
- res = tablet->init();
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to load tablet. [res=" << res
- << " header_path=" << store->path();
- return res;
- }
-
- const RowsetSharedPtr lastest_version = tablet->rowset_with_max_version();
- if (lastest_version->start_version() != request.version) {
- TPushReq empty_push;
- empty_push.tablet_id = request.tablet_id;
- empty_push.schema_hash = request.schema_hash;
- empty_push.version = request.version + 1;
- empty_push.version_hash = 0;
-
- PushHandler handler;
- // res = handler.process(tablet, empty_push, PUSH_NORMAL, NULL);
- // TODO (yiguolei) should create a empty version, call create new rowset meta and set version
- // just return success to skip push a empty rowset into the snapshot since has alreay removed
- // batch process code from push handler
- res = OLAP_SUCCESS;
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to push empty version. "
- << " res=" << res
- << " version=" << empty_push.version;
- return res;
- }
- }
- return res;
-}
-
} // namespace doris
diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h
index ec3c3f4..afcc624 100644
--- a/be/src/olap/snapshot_manager.h
+++ b/be/src/olap/snapshot_manager.h
@@ -93,10 +93,6 @@ private:
OLAPStatus _prepare_snapshot_dir(const TabletSharedPtr& ref_tablet,
std::string* snapshot_id_path);
-
- OLAPStatus _append_single_delta(
- const TSnapshotRequest& request,
- DataDir* store);
OLAPStatus _rename_rowset_id(const RowsetMetaPB& rs_meta_pb, const string& new_path,
DataDir& data_dir, TabletSchema& tablet_schema, const RowsetId& next_id, RowsetMetaPB* new_rs_meta_pb);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 58b9dd0..bdd6857 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -163,7 +163,7 @@ OLAPStatus Tablet::revise_tablet_meta(
do {
// load new local tablet_meta to operate on
TabletMetaSharedPtr new_tablet_meta(new (nothrow) TabletMeta());
- RETURN_NOT_OK(TabletMetaManager::get_meta(_data_dir, tablet_id(), schema_hash(), new_tablet_meta));
+ RETURN_NOT_OK(generate_tablet_meta_copy(new_tablet_meta));
// delete versions from new local tablet_meta
for (const Version& version : versions_to_delete) {
@@ -1031,6 +1031,12 @@ OLAPStatus Tablet::do_tablet_meta_checkpoint() {
}
// hold read lock not write lock, because it will not modify meta structure
ReadLock rdlock(&_meta_lock);
+ if (tablet_state() != TABLET_RUNNING) {
+ LOG(INFO) << "tablet is under state=" << tablet_state()
+ << ", not running, skip do checkpoint"
+ << ", tablet=" << full_name();
+ return OLAP_SUCCESS;
+ }
LOG(INFO) << "start to do tablet meta checkpoint, tablet=" << full_name();
RETURN_NOT_OK(save_meta());
// if save meta successfully, then should remove the rowset meta existing in tablet
@@ -1115,6 +1121,17 @@ void Tablet::build_tablet_report_info(TTabletInfo* tablet_info) {
tablet_info->__set_storage_medium(_data_dir->storage_medium());
tablet_info->__set_version_count(_tablet_meta->version_count());
tablet_info->__set_path_hash(_data_dir->path_hash());
+ return;
+}
+
+// should use this method to get a copy of current tablet meta
+// there are some rowset meta in local meta store and in in-memory tablet meta
+// but not in tablet meta in local meta store
+OLAPStatus Tablet::generate_tablet_meta_copy(TabletMetaSharedPtr new_tablet_meta) {
+ TabletMetaPB tablet_meta_pb;
+ RETURN_NOT_OK(_tablet_meta->to_meta_pb(&tablet_meta_pb));
+ RETURN_NOT_OK(new_tablet_meta->init_from_pb(tablet_meta_pb));
+ return OLAP_SUCCESS;
}
} // namespace doris
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 51f57c3..ff202d6 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -242,6 +242,8 @@ public:
void build_tablet_report_info(TTabletInfo* tablet_info);
+ OLAPStatus generate_tablet_meta_copy(TabletMetaSharedPtr new_tablet_meta);
+
private:
OLAPStatus _init_once_action();
void _print_missed_versions(const std::vector<Version>& missed_versions) const;
diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp
index 664580e..abd89d8 100755
--- a/be/src/olap/tablet_meta_manager.cpp
+++ b/be/src/olap/tablet_meta_manager.cpp
@@ -44,6 +44,9 @@ using rocksdb::kDefaultColumnFamilyName;
namespace doris {
+// should use tablet's generate tablet meta copy method to get a copy of current tablet meta
+// there are some rowset meta in local meta store and in in-memory tablet meta
+// but not in tablet meta in local meta store
OLAPStatus TabletMetaManager::get_meta(
DataDir* store, TTabletId tablet_id,
TSchemaHash schema_hash,
@@ -170,14 +173,4 @@ OLAPStatus TabletMetaManager::load_json_meta(DataDir* store, const std::string&
return save(store, tablet_id, schema_hash, meta_binary);
}
-OLAPStatus TabletMetaManager::dump_header(DataDir* store, TTabletId tablet_id,
- TSchemaHash schema_hash, const std::string& dump_path) {
- TabletMetaSharedPtr tablet_meta(new TabletMeta());
- OLAPStatus res = TabletMetaManager::get_meta(store, tablet_id, schema_hash, tablet_meta);
- if (res != OLAP_SUCCESS) {
- return res;
- }
- return tablet_meta->save(dump_path);
-}
-
}
diff --git a/be/src/olap/tablet_meta_manager.h b/be/src/olap/tablet_meta_manager.h
index 3898961..c2597de 100644
--- a/be/src/olap/tablet_meta_manager.h
+++ b/be/src/olap/tablet_meta_manager.h
@@ -51,9 +51,6 @@ public:
std::function<bool(long, long, const std::string&)> const& func, const string& header_prefix = "tabletmeta_");
static OLAPStatus load_json_meta(DataDir* store, const std::string& meta_path);
-
- static OLAPStatus dump_header(DataDir* store, TTabletId tablet_id,
- TSchemaHash schema_hash, const std::string& path);
};
}
diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp
index 0c2facc..d6da60d 100644
--- a/be/src/olap/task/engine_storage_migration_task.cpp
+++ b/be/src/olap/task/engine_storage_migration_task.cpp
@@ -162,12 +162,14 @@ OLAPStatus EngineStorageMigrationTask::_storage_medium_migrate(
LOG(WARNING) << "fail to copy index and data files when migrate. res=" << res;
break;
}
-
+ tablet->obtain_header_rdlock();
res = _generate_new_header(stores[0], shard, tablet, consistent_rowsets, new_tablet_meta);
if (res != OLAP_SUCCESS) {
+ tablet->release_header_lock();
LOG(WARNING) << "fail to generate new header file from the old. res=" << res;
break;
}
+ tablet->release_header_lock();
std::string new_meta_file = schema_hash_path + "/" + std::to_string(tablet_id) + ".hdr";
res = new_tablet_meta->save(new_meta_file);
if (res != OLAP_SUCCESS) {
@@ -233,11 +235,9 @@ OLAPStatus EngineStorageMigrationTask::_generate_new_header(
LOG(WARNING) << "fail to generate new header for store is null";
return OLAP_ERR_HEADER_INIT_FAILED;
}
- OLAPStatus res = OLAP_SUCCESS;
- res = TabletMetaManager::get_meta(tablet->data_dir(), tablet->tablet_id(), tablet->schema_hash(), new_tablet_meta);
- if (res == OLAP_ERR_META_KEY_NOT_FOUND) {
- LOG(WARNING) << "tablet_meta has already been dropped. "
- << "data_dir:" << tablet->data_dir()->path()
+ OLAPStatus res = tablet->generate_tablet_meta_copy(new_tablet_meta);
+ if (res != OLAP_SUCCESS) {
+ LOG(WARNING) << "could not generate new tablet meta. "
<< "tablet:" << tablet->full_name();
return res;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org