You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2020/07/07 08:57:49 UTC
[GitHub] [incubator-doris] wuyunfeng commented on a change in pull request #4039: Add delayed deletion of rowsets function, fix -230 error.
wuyunfeng commented on a change in pull request #4039:
URL: https://github.com/apache/incubator-doris/pull/4039#discussion_r450711406
##########
File path: be/src/olap/rowset_graph.cpp
##########
@@ -17,14 +17,247 @@
#include "olap/rowset_graph.h"
-#include <queue>
#include <memory>
+#include <queue>
#include "common/logging.h"
namespace doris {
-void RowsetGraph::construct_rowset_graph(const std::vector<RowsetMetaSharedPtr>& rs_metas) {
+void VersionedRowsetTracker::_construct_versioned_tracker(
+ const std::vector<RowsetMetaSharedPtr>& rs_metas,
+ const std::vector<RowsetMetaSharedPtr>& expired_snapshot_rs_metas) {
+ int64_t max_version = 0;
+
+ std::vector<RowsetMetaSharedPtr> all_rs_metas(rs_metas);
+ all_rs_metas.insert(all_rs_metas.end(), expired_snapshot_rs_metas.begin(),
+ expired_snapshot_rs_metas.end());
+ // construct the roset graph
+ _rowsetGraph.reconstruct_rowset_graph(all_rs_metas, max_version);
+
+ // fill main_path
+ std::unordered_map<Version, RowsetMetaSharedPtr, HashOfVersion> main_path;
+ for (auto& rs_meta : rs_metas) {
+ main_path[rs_meta->version()] = rs_meta;
+ }
+ // fill other_path
+ typedef std::shared_ptr<std::map<int64_t, RowsetMetaSharedPtr>> Edges;
+ std::map<int64_t, Edges> other_path;
+
+ for (auto& rs_meta : expired_snapshot_rs_metas) {
+ if (other_path.find(rs_meta->start_version()) == other_path.end()) {
+ other_path[rs_meta->start_version()] =
+ Edges(new std::map<int64_t, RowsetMetaSharedPtr>());
+ }
+ other_path[rs_meta->start_version()]->insert(
+ std::pair<int64_t, RowsetMetaSharedPtr>(rs_meta->end_version(), rs_meta));
+ }
+
+ auto iter = other_path.begin();
+ for (; iter != other_path.end(); iter++) {
+ Edges edges = iter->second;
+ auto min_begin_version = iter->first;
+
+ while (!edges->empty()) {
+ PathVersionListSharedPtr path_version_ptr(new std::vector<VersionTrackerSharedPtr>());
+ // 1. find a path
+ // begin from min_begin_version
Review comment:
merge this two comment into one line maybe better
##########
File path: be/src/common/config.h
##########
@@ -230,6 +230,8 @@ namespace config {
CONF_mInt32(pending_data_expire_time_sec, "1800");
// inc_rowset expired interval
CONF_mInt32(inc_rowset_expired_sec, "1800");
+ // inc_rowset snapshot rs sweep time interval
+ CONF_mInt32(tablet_rowset_expired_snapshot_sweep_time, "1800");
Review comment:
It is better to add `unit` to your config item
##########
File path: be/src/olap/tablet.cpp
##########
@@ -61,7 +61,10 @@ Tablet::Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir) :
_last_cumu_compaction_success_millis(0),
_last_base_compaction_success_millis(0),
_cumulative_point(kInvalidCumulativePoint) {
- _rs_graph.construct_rowset_graph(_tablet_meta->all_rs_metas());
+ // change _rs_graph to _versioned_rs_tracker
+ // _rs_graph.construct_rowset_graph(_tablet_meta->all_rs_metas());
+ _versioned_rs_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas(),
Review comment:
66 - 67 has nonstandard format
##########
File path: be/src/olap/rowset_graph.cpp
##########
@@ -17,14 +17,247 @@
#include "olap/rowset_graph.h"
-#include <queue>
#include <memory>
+#include <queue>
#include "common/logging.h"
namespace doris {
-void RowsetGraph::construct_rowset_graph(const std::vector<RowsetMetaSharedPtr>& rs_metas) {
+void VersionedRowsetTracker::_construct_versioned_tracker(
+ const std::vector<RowsetMetaSharedPtr>& rs_metas,
+ const std::vector<RowsetMetaSharedPtr>& expired_snapshot_rs_metas) {
+ int64_t max_version = 0;
+
+ std::vector<RowsetMetaSharedPtr> all_rs_metas(rs_metas);
+ all_rs_metas.insert(all_rs_metas.end(), expired_snapshot_rs_metas.begin(),
+ expired_snapshot_rs_metas.end());
+ // construct the roset graph
+ _rowsetGraph.reconstruct_rowset_graph(all_rs_metas, max_version);
+
+ // fill main_path
+ std::unordered_map<Version, RowsetMetaSharedPtr, HashOfVersion> main_path;
+ for (auto& rs_meta : rs_metas) {
+ main_path[rs_meta->version()] = rs_meta;
+ }
+ // fill other_path
+ typedef std::shared_ptr<std::map<int64_t, RowsetMetaSharedPtr>> Edges;
+ std::map<int64_t, Edges> other_path;
+
+ for (auto& rs_meta : expired_snapshot_rs_metas) {
+ if (other_path.find(rs_meta->start_version()) == other_path.end()) {
+ other_path[rs_meta->start_version()] =
+ Edges(new std::map<int64_t, RowsetMetaSharedPtr>());
+ }
+ other_path[rs_meta->start_version()]->insert(
+ std::pair<int64_t, RowsetMetaSharedPtr>(rs_meta->end_version(), rs_meta));
+ }
+
+ auto iter = other_path.begin();
+ for (; iter != other_path.end(); iter++) {
+ Edges edges = iter->second;
+ auto min_begin_version = iter->first;
+
+ while (!edges->empty()) {
+ PathVersionListSharedPtr path_version_ptr(new std::vector<VersionTrackerSharedPtr>());
+ // 1. find a path
+ // begin from min_begin_version
+ auto min_end_version = edges->begin()->first;
+ auto min_rs_meta = edges->begin()->second;
+ // tracker the first
Review comment:
```suggestion
// tracker the first
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org