You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2019/11/28 16:03:33 UTC
[GitHub] [incubator-doris] kangpinghuang commented on a change in pull
request #2212: Add rowset convert
kangpinghuang commented on a change in pull request #2212: Add rowset convert
URL: https://github.com/apache/incubator-doris/pull/2212#discussion_r351856701
##########
File path: be/src/olap/snapshot_manager.cpp
##########
@@ -467,39 +468,114 @@ OLAPStatus SnapshotManager::_create_snapshot_files(
new_tablet_meta->revise_rs_metas(rs_metas);
}
if (snapshot_version < PREFERRED_SNAPSHOT_VERSION) {
- set<string> exist_old_files;
- Status ret = FileUtils::list_dirs_files(schema_full_path, nullptr, &exist_old_files,
- Env::Default());
- if (!ret.ok()) {
- res = OLAP_ERR_DISK_FAILURE;
- LOG(WARNING) << "failed to dir walk when convert old files. dir="
- << schema_full_path << ", error:" << ret.to_string();
- break;
- }
- OlapSnapshotConverter converter;
- TabletMetaPB tablet_meta_pb;
- OLAPHeaderMessage olap_header_msg;
- new_tablet_meta->to_meta_pb(&tablet_meta_pb);
- res = converter.to_old_snapshot(tablet_meta_pb, schema_full_path, schema_full_path, &olap_header_msg);
- if (res != OLAP_SUCCESS) {
- break;
- }
- // convert new version files to old version files successuflly, then should remove the old files
- vector<string> files_to_delete;
- for (auto file_name : exist_old_files) {
- string full_file_path = schema_full_path + "/" + file_name;
- files_to_delete.push_back(full_file_path);
- }
- // remove all files
- ret = FileUtils::remove_paths(files_to_delete);
- if (!ret.ok()) {
- res = OLAP_ERR_IO_ERROR;
- LOG(WARNING) << "remove paths failed. error: " << ret.to_string();
- break;
+ if (snapshot_version < ALPHA_ROWSET_VERSION) {
+ // convert old segment file name to new name
+ // from tabletid_schemahash_segmentgroupid_segmentid to rowsetid_segmentgroupid_segmentid
+ set<string> exist_old_files;
+ Status ret = FileUtils::list_dirs_files(schema_full_path, nullptr, &exist_old_files, Env::Default());
+ if (!ret.ok()) {
+ res = OLAP_ERR_DISK_FAILURE;
+ LOG(WARNING) << "failed to dir walk when convert old files. dir="
+ << schema_full_path << ", error:" << ret.to_string();
+ break;
+ }
+ OlapSnapshotConverter converter;
+ TabletMetaPB tablet_meta_pb;
+ OLAPHeaderMessage olap_header_msg;
+ new_tablet_meta->to_meta_pb(&tablet_meta_pb);
+ res = converter.to_old_snapshot(
+ tablet_meta_pb, schema_full_path, schema_full_path,
+ &olap_header_msg);
+ if (res != OLAP_SUCCESS) {
+ break;
+ }
+ // convert new version files to old version files successuflly, then should remove the old files
+ vector<string> files_to_delete;
+ for (auto file_name : exist_old_files) {
+ string full_file_path = schema_full_path + "/" + file_name;
+ files_to_delete.push_back(full_file_path);
+ }
+ // remove all files
+ ret = FileUtils::remove_paths(files_to_delete);
+ if (!ret.ok()) {
+ res = OLAP_ERR_IO_ERROR;
+ LOG(WARNING) << "remove paths failed. error: " << ret.to_string();
+ break;
+ }
+ // save new header to snapshot header path
+ res = converter.save(header_path, olap_header_msg);
+ LOG(INFO) << "finished convert new snapshot to old snapshot, res=" << res;
+ } else if (snapshot_version == ALPHA_ROWSET_VERSION) {
+ // convert beta rowset to alpha rowset
+ LOG(INFO) << "start to convert beta rowset to alpha rowset for snapshot";
+ RowsetConverter rowset_converter(new_tablet_meta);
+ std::vector<RowsetMetaSharedPtr> new_rowsets;
+ bool modified = false;
+ if (request.__isset.missing_version) {
+ for (auto& inc_meta : new_tablet_meta->all_inc_rs_metas()) {
+ if (inc_meta->rowset_type() == BETA_ROWSET) {
+ modified = true;
+ RowsetMetaPB rowset_meta_pb;
+ LOG(INFO) << "convert inc beta rowset:"
+ << inc_meta->rowset_id() << ", tablet_id:"
+ << new_tablet_meta->tablet_id() << " to alpha";
+ auto st = rowset_converter.convert_beta_to_alpha(inc_meta,
+ schema_full_path, &rowset_meta_pb);
+ if (st != OLAP_SUCCESS) {
+ res = st;
+ LOG(WARNING) << "convert beta to alpha failed. error: " << st;
+ break;
+ }
+ RowsetMetaSharedPtr new_rowset(new AlphaRowsetMeta());
+ bool ret = new_rowset->init_from_pb(rowset_meta_pb);
+ if (!ret) {
+ res = OLAP_ERR_INIT_FAILED;
+ break;
+ }
+ new_rowsets.push_back(new_rowset);
+ } else {
+ new_rowsets.push_back(inc_meta);
+ }
+ }
+ if (res == OLAP_SUCCESS && modified) {
+ res = new_tablet_meta->revise_inc_rs_metas(new_rowsets);
+ }
+ } else {
+ for (auto& rowset_meta : new_tablet_meta->all_rs_metas()) {
+ if (rowset_meta->rowset_type() == BETA_ROWSET) {
Review comment:
ok
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org