You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/10/27 10:21:24 UTC
[incubator-doris] branch branch-0.15 updated (259cc7b -> 26a81f2)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git.
from 259cc7b [ARM64] Fix some problem when compiling on ARM64 platform (#6836) (#6872)
new b913aa1 Replace replica_allocation to replication_allocation (#6870)
new 56f01d6 [Bug][Memory Leak] Fix the issue of Catalog instance leakage (#6895)
new 582f417 [Flink][Bug] Fix potential NPE when cancel DorisSourceFunction (#6838)
new e83a867 [Bug] Fix bug to reject request with no SQL in TableQueryPlanAction (#6843)
new 6d35560 [Refactor] remove unused code (#6879)
new 2efb7f9 [Bug][Binlog] Fix the number of versions may exceed the limit during data synchronization (#6889)
new 0695671 [MetaTool]add error message when loading meta by meta tool (#6893)
new 959b7f1 [Docs] fix urls and format in routine load docs (#6896)
new a99f2b5 [Cache][Bug] Fix sql_key of getting and updating Cache is inconsistent (#6903)
new cef3d31 [Flink]Simplify the use of flink connector (#6892)
new 1241992 [Bug] Fix bug of decommission (#6826)
new 1c8b7df [Memory Usage] Implement segment lru cache to save memory of BE (#6829)
new b2b39f3 [Bug] Fix bug that truncate table may change the storage medium property (#6905)
new 20caf42 [Spark-Doris-Connector] fixed some spark-doris-connector doc typo
new 6e388a7 Fix String type column using zonemap to filter data maybe core dump (#6939)
new 4bb24d2 [Cache][Bug] Correct update cache timeout unit (#6888)
new 9cc9465 [Bug] Fix treat tuple_is_null_predicate is const expr cause core problem (#6919)
new 6664aae [Bug] fix Runtime filter can't find fragment-id when apply_filter called early (#6923)
new dd9dfba [Benchmark] Add star schema benchmark tools (#6925)
new c385536 [Enhance] Add BackendHbResponse info (#6929)
new 26a81f2 [Optimize] optimize some session variable and profile (#6920)
The 21 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/agent/cgroups_mgr.cpp | 4 +-
be/src/agent/task_worker_pool.cpp | 65 ++++--
be/src/agent/task_worker_pool.h | 3 +
be/src/common/config.h | 10 +-
be/src/exec/olap_scan_node.cpp | 8 +-
be/src/exec/tablet_sink.cpp | 32 +--
be/src/exec/tablet_sink.h | 12 +-
be/src/exprs/bloomfilter_predicate.cpp | 2 +-
be/src/exprs/runtime_filter_rpc.cpp | 4 +-
be/src/exprs/tuple_is_null_predicate.h | 3 +
be/src/olap/CMakeLists.txt | 1 +
be/src/olap/delta_writer.cpp | 3 +
be/src/olap/lru_cache.cpp | 119 ++++++++---
be/src/olap/lru_cache.h | 42 +++-
be/src/olap/olap_server.cpp | 4 +-
be/src/olap/rowset/alpha_rowset.cpp | 36 +---
be/src/olap/rowset/alpha_rowset.h | 6 +-
be/src/olap/rowset/alpha_rowset_reader.cpp | 2 +-
be/src/olap/rowset/beta_rowset.cpp | 14 +-
be/src/olap/rowset/beta_rowset.h | 5 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 10 +-
be/src/olap/rowset/beta_rowset_reader.h | 5 +
be/src/olap/rowset/rowset.cpp | 4 +-
be/src/olap/rowset/rowset.h | 6 +-
be/src/olap/segment_loader.cpp | 98 +++++++++
be/src/olap/segment_loader.h | 171 ++++++++++++++++
be/src/olap/snapshot_manager.cpp | 2 +-
be/src/olap/storage_engine.cpp | 6 +-
be/src/olap/storage_engine.h | 2 +-
be/src/olap/tablet.cpp | 16 +-
be/src/olap/tablet_manager.cpp | 5 +-
be/src/olap/tablet_meta_manager.cpp | 4 +-
be/src/olap/wrapper_field.cpp | 11 +-
be/src/olap/wrapper_field.h | 4 +
be/src/runtime/data_stream_sender.cpp | 2 +-
be/src/runtime/exec_env_init.cpp | 3 +
be/src/runtime/fragment_mgr.cpp | 14 +-
be/src/runtime/fragment_mgr.h | 2 +
be/src/runtime/runtime_filter_mgr.cpp | 2 +-
be/src/service/internal_service.cpp | 2 +-
be/test/olap/lru_cache_test.cpp | 82 ++++++--
be/test/olap/rowset/beta_rowset_test.cpp | 1 +
be/test/olap/rowset/rowset_converter_test.cpp | 1 +
docs/.vuepress/sidebar/en.js | 3 +-
docs/.vuepress/sidebar/zh-CN.js | 3 +-
docs/en/administrator-guide/config/be_config.md | 15 +-
.../load-data/binlog-load-manual.md | 33 +++-
.../load-data/broker-load-manual.md | 2 +-
.../load-data/routine-load-manual.md | 6 +-
docs/en/administrator-guide/multi-tenant.md | 4 +-
docs/en/administrator-guide/running-profile.md | 4 +-
docs/en/administrator-guide/runtime-filter.md | 2 +-
docs/en/administrator-guide/variables.md | 6 +-
docs/en/best-practices/star-schema-benchmark.md | 163 +++++++++++++++
docs/en/extending-doris/flink-doris-connector.md | 107 +++++++++-
.../Create/CREATE-TABLE.md | 12 +-
.../sql-statements/Data Definition/CREATE TABLE.md | 10 +-
docs/zh-CN/administrator-guide/config/be_config.md | 17 +-
.../load-data/binlog-load-manual.md | 34 +++-
.../load-data/broker-load-manual.md | 2 +-
.../load-data/routine-load-manual.md | 4 +-
docs/zh-CN/administrator-guide/multi-tenant.md | 4 +-
docs/zh-CN/administrator-guide/running-profile.md | 4 +-
docs/zh-CN/administrator-guide/runtime-filter.md | 2 +-
docs/zh-CN/administrator-guide/variables.md | 6 +-
docs/zh-CN/best-practices/star-schema-benchmark.md | 164 +++++++++++++++
.../zh-CN/extending-doris/flink-doris-connector.md | 106 +++++++++-
.../zh-CN/extending-doris/spark-doris-connector.md | 4 +-
.../Create/CREATE-TABLE.md | 12 +-
.../sql-statements/Data Definition/CREATE TABLE.md | 10 +-
.../doris/flink/cfg/DorisExecutionOptions.java | 20 +-
.../apache/doris/flink/cfg/DorisReadOptions.java | 5 +
.../java/org/apache/doris/flink/cfg/DorisSink.java | 95 +++++++++
.../doris/flink/cfg/GenericDorisSinkFunction.java | 53 +++++
.../flink/datastream/DorisSourceFunction.java | 19 +-
.../flink/table/DorisDynamicOutputFormat.java | 77 +++++---
.../doris/flink/datastream/ScalaValueReader.scala | 3 +-
.../doris/flink/DorisOutPutFormatExample.java | 84 ++++++++
.../apache/doris/flink/DorisStreamSinkExample.java | 219 +++++++++++++++++++++
.../apache/doris/alter/SchemaChangeHandler.java | 8 +-
.../doris/analysis/ModifyPartitionClause.java | 2 +-
.../java/org/apache/doris/analysis/SetVar.java | 13 +-
.../java/org/apache/doris/backup/BackupJob.java | 8 +-
.../java/org/apache/doris/catalog/Catalog.java | 3 +-
.../java/org/apache/doris/catalog/Database.java | 4 -
.../java/org/apache/doris/catalog/OlapTable.java | 47 ++---
.../main/java/org/apache/doris/catalog/Tablet.java | 15 ++
.../apache/doris/clone/ClusterLoadStatistic.java | 7 +
.../org/apache/doris/clone/TabletSchedCtx.java | 23 ++-
.../org/apache/doris/clone/TabletScheduler.java | 7 +-
.../main/java/org/apache/doris/common/Config.java | 36 +++-
.../org/apache/doris/common/ThreadPoolManager.java | 8 +-
.../doris/http/rest/TableQueryPlanAction.java | 2 +-
.../doris/httpv2/rest/TableQueryPlanAction.java | 2 +-
.../src/main/java/org/apache/doris/load/Load.java | 20 +-
.../apache/doris/load/loadv2/BrokerLoadJob.java | 10 +-
.../load/routineload/RoutineLoadTaskScheduler.java | 6 +-
.../apache/doris/load/sync/canal/CanalConfigs.java | 3 -
.../load/sync/canal/CanalSyncDataConsumer.java | 11 +-
.../apache/doris/load/sync/canal/CanalSyncJob.java | 5 +-
.../doris/load/sync/canal/SyncCanalClient.java | 15 +-
.../org/apache/doris/master/ReportHandler.java | 4 +-
.../apache/doris/metric/DorisMetricRegistry.java | 14 +-
.../java/org/apache/doris/metric/MetricRepo.java | 26 ++-
.../org/apache/doris/planner/OlapScanNode.java | 1 +
.../main/java/org/apache/doris/qe/Coordinator.java | 2 +-
.../java/org/apache/doris/qe/GlobalVariable.java | 1 -
.../java/org/apache/doris/qe/QeProcessorImpl.java | 2 +-
.../java/org/apache/doris/qe/SessionVariable.java | 14 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 2 +-
.../org/apache/doris/qe/cache/CacheBeProxy.java | 4 +-
.../org/apache/doris/qe/cache/PartitionCache.java | 4 +-
.../java/org/apache/doris/qe/cache/SqlCache.java | 3 +-
.../org/apache/doris/system/BackendHbResponse.java | 11 ++
.../main/java/org/apache/doris/task/PushTask.java | 2 +-
.../apache/doris/catalog/ModifyBackendTagTest.java | 2 +-
.../doris/clone/TabletRepairAndBalanceTest.java | 58 ++++++
.../doris/http/TableQueryPlanActionTest.java | 40 ++++
.../org/apache/doris/qe/SessionVariablesTest.java | 4 +-
.../java/org/apache/doris/qe/VariableMgrTest.java | 2 +-
.../doris/utframe/DemoMultiBackendsTest.java | 3 +-
.../doris/broker/hdfs/FileSystemManager.java | 4 +-
gensrc/thrift/PlanNodes.thrift | 1 +
.../ssb-tools/build-ssb-dbgen.sh | 29 ++-
tools/ssb-tools/create-tables.sql | 117 +++++++++++
.../ssb-tools/doris-cluster.conf | 17 +-
tools/ssb-tools/gen-ssb-data.sh | 112 +++++++++++
tools/ssb-tools/load-dimension-data.sh | 68 +++++++
tools/ssb-tools/load-fact-data.sh | 126 ++++++++++++
tools/ssb-tools/queries/q1.1.sql | 24 +++
tools/ssb-tools/queries/q1.2.sql | 24 +++
tools/ssb-tools/queries/q1.3.sql | 25 +++
tools/ssb-tools/queries/q2.1.sql | 26 +++
tools/ssb-tools/queries/q2.2.sql | 27 +++
tools/ssb-tools/queries/q2.3.sql | 26 +++
tools/ssb-tools/queries/q3.1.sql | 28 +++
tools/ssb-tools/queries/q3.2.sql | 28 +++
tools/ssb-tools/queries/q3.3.sql | 30 +++
tools/ssb-tools/queries/q3.4.sql | 30 +++
tools/ssb-tools/queries/q4.1.sql | 30 +++
tools/ssb-tools/queries/q4.2.sql | 31 +++
tools/ssb-tools/queries/q4.3.sql | 29 +++
142 files changed, 3006 insertions(+), 422 deletions(-)
create mode 100644 be/src/olap/segment_loader.cpp
create mode 100644 be/src/olap/segment_loader.h
create mode 100644 docs/en/best-practices/star-schema-benchmark.md
create mode 100644 docs/zh-CN/best-practices/star-schema-benchmark.md
create mode 100644 extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisSink.java
create mode 100644 extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/GenericDorisSinkFunction.java
create mode 100644 extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisOutPutFormatExample.java
create mode 100644 extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisStreamSinkExample.java
copy build-support/check-format.sh => tools/ssb-tools/build-ssb-dbgen.sh (57%)
create mode 100644 tools/ssb-tools/create-tables.sql
copy be/test/common/CMakeLists.txt => tools/ssb-tools/doris-cluster.conf (78%)
create mode 100755 tools/ssb-tools/gen-ssb-data.sh
create mode 100755 tools/ssb-tools/load-dimension-data.sh
create mode 100755 tools/ssb-tools/load-fact-data.sh
create mode 100644 tools/ssb-tools/queries/q1.1.sql
create mode 100644 tools/ssb-tools/queries/q1.2.sql
create mode 100644 tools/ssb-tools/queries/q1.3.sql
create mode 100644 tools/ssb-tools/queries/q2.1.sql
create mode 100644 tools/ssb-tools/queries/q2.2.sql
create mode 100644 tools/ssb-tools/queries/q2.3.sql
create mode 100644 tools/ssb-tools/queries/q3.1.sql
create mode 100644 tools/ssb-tools/queries/q3.2.sql
create mode 100644 tools/ssb-tools/queries/q3.3.sql
create mode 100644 tools/ssb-tools/queries/q3.4.sql
create mode 100644 tools/ssb-tools/queries/q4.1.sql
create mode 100644 tools/ssb-tools/queries/q4.2.sql
create mode 100644 tools/ssb-tools/queries/q4.3.sql
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 13/21: [Bug] Fix bug that truncate table may
change the storage medium property (#6905)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit b2b39f34e844b5a2ec95b4a9efc7afc28517fb70
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Mon Oct 25 10:07:27 2021 +0800
[Bug] Fix bug that truncate table may change the storage medium property (#6905)
---
be/src/exec/olap_scan_node.cpp | 2 -
be/src/exprs/bloomfilter_predicate.cpp | 2 +-
be/src/exprs/runtime_filter_rpc.cpp | 4 +-
be/src/olap/rowset/alpha_rowset.cpp | 4 +-
be/src/olap/tablet.cpp | 16 +++++---
be/src/olap/tablet_manager.cpp | 5 ++-
be/src/runtime/data_stream_sender.cpp | 2 +-
be/src/runtime/runtime_filter_mgr.cpp | 2 +-
be/src/service/internal_service.cpp | 2 +-
.../java/org/apache/doris/backup/BackupJob.java | 8 ++--
.../java/org/apache/doris/catalog/Catalog.java | 3 +-
.../java/org/apache/doris/catalog/OlapTable.java | 47 ++++++++++++----------
.../doris/broker/hdfs/FileSystemManager.java | 4 +-
13 files changed, 56 insertions(+), 45 deletions(-)
diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
index 3dc69024..b2d90e8 100644
--- a/be/src/exec/olap_scan_node.cpp
+++ b/be/src/exec/olap_scan_node.cpp
@@ -689,7 +689,6 @@ Status OlapScanNode::get_hints(const TPaloScanRange& scan_range, int block_row_c
res = table->split_range(key_range->begin_scan_range, key_range->end_scan_range,
block_row_count, &range);
if (res != OLAP_SUCCESS) {
- OLAP_LOG_WARNING("fail to show hints by split range. [res=%d]", res);
return Status::InternalError("fail to show hints");
}
ranges.emplace_back(std::move(range));
@@ -700,7 +699,6 @@ Status OlapScanNode::get_hints(const TPaloScanRange& scan_range, int block_row_c
std::vector<OlapTuple> range;
auto res = table->split_range({}, {}, block_row_count, &range);
if (res != OLAP_SUCCESS) {
- OLAP_LOG_WARNING("fail to show hints by split range. [res=%d]", res);
return Status::InternalError("fail to show hints");
}
ranges.emplace_back(std::move(range));
diff --git a/be/src/exprs/bloomfilter_predicate.cpp b/be/src/exprs/bloomfilter_predicate.cpp
index c3b199d..44906fd 100644
--- a/be/src/exprs/bloomfilter_predicate.cpp
+++ b/be/src/exprs/bloomfilter_predicate.cpp
@@ -73,7 +73,7 @@ BloomFilterPredicate::BloomFilterPredicate(const TExprNode& node)
_scan_rows(0) {}
BloomFilterPredicate::~BloomFilterPredicate() {
- LOG(INFO) << "bloom filter rows:" << _filtered_rows << ",scan_rows:" << _scan_rows
+ VLOG_NOTICE << "bloom filter rows:" << _filtered_rows << ",scan_rows:" << _scan_rows
<< ",rate:" << (double)_filtered_rows / _scan_rows;
}
diff --git a/be/src/exprs/runtime_filter_rpc.cpp b/be/src/exprs/runtime_filter_rpc.cpp
index 87bb579..f1e6e20 100644
--- a/be/src/exprs/runtime_filter_rpc.cpp
+++ b/be/src/exprs/runtime_filter_rpc.cpp
@@ -58,8 +58,8 @@ Status IRuntimeFilter::push_to_remote(RuntimeState* state, const TNetworkAddress
Status serialize_status = serialize(&_rpc_context->request, &data, &len);
if (serialize_status.ok()) {
- LOG(INFO) << "Producer:" << _rpc_context->request.ShortDebugString() << addr->hostname
- << ":" << addr->port;
+ VLOG_NOTICE << "Producer:" << _rpc_context->request.ShortDebugString() << addr->hostname
+ << ":" << addr->port;
if (len > 0) {
DCHECK(data != nullptr);
_rpc_context->cntl.request_attachment().append(data, len);
diff --git a/be/src/olap/rowset/alpha_rowset.cpp b/be/src/olap/rowset/alpha_rowset.cpp
index c6bec67..7a98d82 100644
--- a/be/src/olap/rowset/alpha_rowset.cpp
+++ b/be/src/olap/rowset/alpha_rowset.cpp
@@ -166,7 +166,9 @@ OLAPStatus AlphaRowset::split_range(const RowCursor& start_key, const RowCursor&
std::vector<OlapTuple>* ranges) {
if (key_num > _schema->num_short_key_columns()) {
// should not happen
- LOG(WARNING) << "key num " << key_num << " should less than or equal to short key column number: "
+ // But since aloha rowset is deprecated in future and it will not fail the query,
+ // just use VLOG to avoid too many warning logs.
+ VLOG_NOTICE << "key num " << key_num << " should less than or equal to short key column number: "
<< _schema->num_short_key_columns();
return OLAP_ERR_INVALID_SCHEMA;
}
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index b6dc448..7a77448 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -560,13 +560,19 @@ OLAPStatus Tablet::capture_consistent_versions(const Version& spec_version,
std::vector<Version> missed_versions;
calc_missed_versions_unlocked(spec_version.second, &missed_versions);
if (missed_versions.empty()) {
- LOG(WARNING) << "tablet:" << full_name()
- << ", version already has been merged. spec_version: " << spec_version;
+ // if version_path is null, it may be a compaction check logic.
+ // so to avoid print too many logs.
+ if (version_path != nullptr) {
+ LOG(WARNING) << "tablet:" << full_name()
+ << ", version already has been merged. spec_version: " << spec_version;
+ }
status = OLAP_ERR_VERSION_ALREADY_MERGED;
} else {
- LOG(WARNING) << "status:" << status << ", tablet:" << full_name()
- << ", missed version for version:" << spec_version;
- _print_missed_versions(missed_versions);
+ if (version_path != nullptr) {
+ LOG(WARNING) << "status:" << status << ", tablet:" << full_name()
+ << ", missed version for version:" << spec_version;
+ _print_missed_versions(missed_versions);
+ }
}
}
return status;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index cabd32d..9054cdb 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -45,6 +45,7 @@
#include "olap/tablet_meta.h"
#include "olap/tablet_meta_manager.h"
#include "olap/utils.h"
+#include "service/backend_options.h"
#include "util/doris_metrics.h"
#include "util/file_utils.h"
#include "util/histogram.h"
@@ -553,7 +554,7 @@ TabletSharedPtr TabletManager::_get_tablet_unlocked(TTabletId tablet_id, SchemaH
if (tablet == nullptr) {
if (err != nullptr) {
- *err = "tablet does not exist";
+ *err = "tablet does not exist. " + BackendOptions::get_localhost();
}
return nullptr;
}
@@ -561,7 +562,7 @@ TabletSharedPtr TabletManager::_get_tablet_unlocked(TTabletId tablet_id, SchemaH
if (!tablet->is_used()) {
LOG(WARNING) << "tablet cannot be used. tablet=" << tablet_id;
if (err != nullptr) {
- *err = "tablet cannot be used";
+ *err = "tablet cannot be used. " + BackendOptions::get_localhost();
}
return nullptr;
}
diff --git a/be/src/runtime/data_stream_sender.cpp b/be/src/runtime/data_stream_sender.cpp
index 6325c56..ceda653 100644
--- a/be/src/runtime/data_stream_sender.cpp
+++ b/be/src/runtime/data_stream_sender.cpp
@@ -71,7 +71,7 @@ DataStreamSender::Channel::Channel(DataStreamSender* parent, const RowDescriptor
std::string localhost = BackendOptions::get_localhost();
_is_local = _brpc_dest_addr.hostname == localhost && _brpc_dest_addr.port == config::brpc_port;
if (_is_local) {
- LOG(INFO) << "will use local exechange, dest_node_id:" << _dest_node_id;
+ VLOG_NOTICE << "will use local exechange, dest_node_id:" << _dest_node_id;
}
}
diff --git a/be/src/runtime/runtime_filter_mgr.cpp b/be/src/runtime/runtime_filter_mgr.cpp
index 58cec23..64f7501 100644
--- a/be/src/runtime/runtime_filter_mgr.cpp
+++ b/be/src/runtime/runtime_filter_mgr.cpp
@@ -231,7 +231,7 @@ Status RuntimeFilterMergeControllerEntity::merge(const PMergeFilterRequest* requ
PBackendService_Stub* stub = ExecEnv::GetInstance()->brpc_stub_cache()->get_stub(
targets[i].target_fragment_instance_addr);
- LOG(INFO) << "send filter " << rpc_contexts[i]->request.filter_id()
+ VLOG_NOTICE << "send filter " << rpc_contexts[i]->request.filter_id()
<< " to:" << targets[i].target_fragment_instance_addr.hostname << ":"
<< targets[i].target_fragment_instance_addr.port
<< rpc_contexts[i]->request.ShortDebugString();
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 621c692..35e94a9 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -308,7 +308,7 @@ void PInternalServiceImpl<T>::apply_filter(::google::protobuf::RpcController* co
auto attachment = static_cast<brpc::Controller*>(controller)->request_attachment();
UniqueId unique_id(request->query_id());
// TODO: avoid copy attachment copy
- LOG(INFO) << "rpc apply_filter recv";
+ VLOG_NOTICE << "rpc apply_filter recv";
Status st = _exec_env->fragment_mgr()->apply_filter(request, attachment.to_string().data());
if (!st.ok()) {
LOG(WARNING) << "apply filter meet error" << st.to_string();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index d75e1c7..55f676e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -49,9 +49,6 @@ import org.apache.doris.thrift.TFinishTaskRequest;
import org.apache.doris.thrift.TStatusCode;
import org.apache.doris.thrift.TTaskType;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicates;
@@ -61,6 +58,9 @@ import com.google.common.collect.Collections2;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
@@ -505,7 +505,7 @@ public class BackupJob extends AbstractJob {
// only copy visible indexes
List<String> reservedPartitions = tableRef.getPartitionNames() == null ? null
: tableRef.getPartitionNames().getPartitionNames();
- OlapTable copiedTbl = olapTable.selectiveCopy(reservedPartitions, true, IndexExtState.VISIBLE);
+ OlapTable copiedTbl = olapTable.selectiveCopy(reservedPartitions, IndexExtState.VISIBLE, true);
if (copiedTbl == null) {
status = new Status(ErrCode.COMMON_ERROR, "failed to copy table: " + tblName);
return;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index f82d09b..1c10990 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -777,6 +777,7 @@ public class Catalog {
// 1. check and create dirs and files
File meta = new File(metaDir);
if (!meta.exists()) {
+ LOG.warn("Doris' meta dir {} does not exist. You need to create it before starting FE", meta.getAbsolutePath());
throw new Exception(meta.getAbsolutePath() + " does not exist, will exit");
}
@@ -6676,7 +6677,7 @@ public class Catalog {
partitionsDistributionInfo.put(partition.getId(), partition.getDistributionInfo());
}
}
- copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), true, IndexExtState.VISIBLE);
+ copiedTbl = olapTable.selectiveCopy(origPartitions.keySet(), IndexExtState.VISIBLE, false);
} finally {
olapTable.readUnlock();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 1af2097..5fde029 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1278,34 +1278,37 @@ public class OlapTable extends Table {
return table instanceof OlapTable;
}
- public OlapTable selectiveCopy(Collection<String> reservedPartitions, boolean resetState, IndexExtState extState) {
+ public OlapTable selectiveCopy(Collection<String> reservedPartitions, IndexExtState extState, boolean isForBackup) {
OlapTable copied = new OlapTable();
if (!DeepCopy.copy(this, copied, OlapTable.class, FeConstants.meta_version)) {
LOG.warn("failed to copy olap table: " + getName());
return null;
}
-
- if (resetState) {
- // remove shadow index from copied table
- List<MaterializedIndex> shadowIndex = copied.getPartitions().stream().findFirst().get().getMaterializedIndices(IndexExtState.SHADOW);
+
+ // remove shadow index from copied table
+ List<MaterializedIndex> shadowIndex = copied.getPartitions().stream().findFirst().get().getMaterializedIndices(IndexExtState.SHADOW);
+ for (MaterializedIndex deleteIndex : shadowIndex) {
+ LOG.debug("copied table delete shadow index : {}", deleteIndex.getId());
+ copied.deleteIndexInfo(copied.getIndexNameById(deleteIndex.getId()));
+ }
+ copied.setState(OlapTableState.NORMAL);
+ for (Partition partition : copied.getPartitions()) {
+ // remove shadow index from partition
for (MaterializedIndex deleteIndex : shadowIndex) {
- LOG.debug("copied table delete shadow index : {}", deleteIndex.getId());
- copied.deleteIndexInfo(copied.getIndexNameById(deleteIndex.getId()));
+ partition.deleteRollupIndex(deleteIndex.getId());
}
- copied.setState(OlapTableState.NORMAL);
- for (Partition partition : copied.getPartitions()) {
- // remove shadow index from partition
- for (MaterializedIndex deleteIndex : shadowIndex) {
- partition.deleteRollupIndex(deleteIndex.getId());
- }
- partition.setState(PartitionState.NORMAL);
+ partition.setState(PartitionState.NORMAL);
+ if (isForBackup) {
+ // set storage medium to HDD for backup job, because we want that the backuped table
+ // can be able to restored to another Doris cluster without SSD disk.
+ // But for other operation such as truncate table, keep the origin storage medium.
copied.getPartitionInfo().setDataProperty(partition.getId(), new DataProperty(TStorageMedium.HDD));
- for (MaterializedIndex idx : partition.getMaterializedIndices(extState)) {
- idx.setState(IndexState.NORMAL);
- for (Tablet tablet : idx.getTablets()) {
- for (Replica replica : tablet.getReplicas()) {
- replica.setState(ReplicaState.NORMAL);
- }
+ }
+ for (MaterializedIndex idx : partition.getMaterializedIndices(extState)) {
+ idx.setState(IndexState.NORMAL);
+ for (Tablet tablet : idx.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ replica.setState(ReplicaState.NORMAL);
}
}
}
@@ -1315,10 +1318,10 @@ public class OlapTable extends Table {
// reserve all
return copied;
}
-
+
Set<String> partNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
partNames.addAll(copied.getPartitionNames());
-
+
for (String partName : partNames) {
if (!reservedPartitions.contains(partName)) {
copied.dropPartitionAndReserveTablet(partName);
diff --git a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
index 562e1ac..6cedd70 100644
--- a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
+++ b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
@@ -241,7 +241,7 @@ public class FileSystemManager {
return null;
}
if (fileSystem.getDFSFileSystem() == null) {
- logger.info("could not find file system for path " + path + " create a new one");
+ logger.info("create file system for new path: " + path);
UserGroupInformation ugi = null;
// create a new filesystem
@@ -402,7 +402,7 @@ public class FileSystemManager {
return null;
}
if (fileSystem.getDFSFileSystem() == null) {
- logger.info("could not find file system for path " + path + " create a new one");
+ logger.info("create file system for new path " + path);
// create a new filesystem
Configuration conf = new Configuration();
conf.set(FS_S3A_ACCESS_KEY, accessKey);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 06/21: [Bug][Binlog] Fix the number of versions
may exceed the limit during data synchronization (#6889)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 2efb7f90cc3a4783e8c29ce34a9715d0a4e480fe
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Sat Oct 23 16:47:32 2021 +0800
[Bug][Binlog] Fix the number of versions may exceed the limit during data synchronization (#6889)
Bug detail: #6887
To solve this problem, the commit of transaction must meet any of the following conditions to avoid commit too freqently:
1. The current accumulated event quantity is greater than the `min_sync_commit_size`.
2. The current accumulated data size is greater than the `min_bytes_sync_commit`.
In addition, when the accumulated data size exceeds `max_bytes_sync_commit`, the transaction needs to be committed immediately.
Before:
After:
---
.../load-data/binlog-load-manual.md | 33 ++++++++++++++++++++-
.../load-data/binlog-load-manual.md | 34 +++++++++++++++++++++-
.../main/java/org/apache/doris/common/Config.java | 34 ++++++++++++++++++++++
.../apache/doris/load/sync/canal/CanalConfigs.java | 3 --
.../load/sync/canal/CanalSyncDataConsumer.java | 11 +++++--
.../apache/doris/load/sync/canal/CanalSyncJob.java | 5 ++--
.../doris/load/sync/canal/SyncCanalClient.java | 15 +++++-----
7 files changed, 118 insertions(+), 17 deletions(-)
diff --git a/docs/en/administrator-guide/load-data/binlog-load-manual.md b/docs/en/administrator-guide/load-data/binlog-load-manual.md
index 90db6da..a13e520 100644
--- a/docs/en/administrator-guide/load-data/binlog-load-manual.md
+++ b/docs/en/administrator-guide/load-data/binlog-load-manual.md
@@ -104,7 +104,7 @@ A Send task is a request from Channel to Be, which contains the data of the same
Channel controls the begin, commit and abort of transaction of single table. In a transaction, the consumer may distribute multiple Batches of data to a channel, so multiple send tasks may be generated. These tasks will not actually take effect until the transaction is committed successfully.
-When certain conditions are met (for example, a certain period of time was passed, an empty batch is received), the Consumer will block and notify each channel to try commit the transaction.
+When certain conditions are met (for example, a certain period of time was passed, reach the maximun data size of commit), the Consumer will block and notify each channel to try commit the transaction.
If and only if all channels are committed successfully, Canal Server will be notified by the ACK request and Canal Client continue to get and consume data.
@@ -456,6 +456,25 @@ You can use `HELP STOP SYNC JOB;`, `HELP PAUSE SYNC JOB`; And `HELP RESUME SYNC
## Related Parameters
+### Canal configuration
+
+* `canal.ip`
+
+ canal server's ip address
+
+* `canal.port`
+
+ canal server's port
+
+* `canal.instance.memory.buffer.size`
+
+ The queue length of the store ring queue, must be set to the power of 2, the default length is 16384. This value is equal to the maximum number of events that can be cached on the canal side and directly determines the maximum number of events that can be accommodated in a transaction on the Doris side. It is recommended to make it large enough to prevent the upper limit of the amount of data that can be accommodated in a transaction on the Doris side from being too small, resulting in [...]
+
+* `canal.instance.memory.buffer.memunit`
+
+ The default space occupied by an event at the canal end, default value is 1024 bytes. This value multiplied by `canal.instance.memory.buffer.size` is equal to the maximum space of the store. For example, if the queue length of the store is 16384, the space of the store is 16MB. However, the actual size of an event is not actually equal to this value, but is determined by the number of rows of data in the event and the length of each row of data. For example, the insert event of a table [...]
+
+
### Fe configuration
The following configuration belongs to the system level configuration of SyncJob. The configuration value can be modified in configuration file fe.conf.
@@ -468,6 +487,18 @@ The following configuration belongs to the system level configuration of SyncJob
Maximum interval time between commit transactions. If there is still data in the channel that has not been committed after this time, the consumer will notify the channel to commit the transaction.
+* `min_sync_commit_size`
+
+ The minimum number of events required to commit a transaction. If the number of events received by Fe is less than it, Fe will continue to wait for the next batch of data until the time exceeds `sync_commit_interval_second`. The default value is 10000 events. If you want to modify this configuration, please ensure that this value is less than the `canal.instance.memory.buffer.size` configuration on the canal side (16384 by default). Otherwise, Fe will try to get more events than the len [...]
+
+* `min_bytes_sync_commit`
+
+ The minimum data size required to commit a transaction. If the data size received by Fe is smaller than it, it will continue to wait for the next batch of data until the time exceeds `sync_commit_interval_second`. The default value is 15MB. If you want to modify this configuration, please ensure that this value is less than the product `canal.instance.memory.buffer.size` and `canal.instance.memory.buffer.memunit` on the canal side (16MB by default). Otherwise, Fe will try to get data fr [...]
+
+* `max_bytes_sync_commit`
+
+ The maximum size of the data when the transaction is committed. If the data size received by Fe is larger than it, it will immediately commit the transaction and send the accumulated data. The default value is 64MB. If you want to modify this configuration, please ensure that this value is greater than the product of `canal.instance.memory.buffer.size` and `canal.instance.memory.buffer.mmemunit` on the canal side (16MB by default) and `min_bytes_sync_commit`。
+
* `max_sync_task_threads_num`
The maximum number of threads in the SyncJobs' thread pool. There is only one thread pool in the whole Fe for synchronization, which is used to process the tasks created by all SyncJobs in the Fe.
diff --git a/docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md b/docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md
index 3ed946b..8862a0a 100644
--- a/docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md
+++ b/docs/zh-CN/administrator-guide/load-data/binlog-load-manual.md
@@ -105,7 +105,7 @@ client中的receiver将负责通过Get命令接收数据，每获取到一个数
channel控制着单个表事务的开始、提交、终止。一个事务周期内，一般会从consumer获取到多个batch的数据，因此会产生多个向BE发送数据的子任务Task，在提交事务成功前，这些Task不会实际生效。
-满足一定条件时（比如超过一定时间、获取到了空的batch），consumer将会阻塞并通知各个channel提交事务。
+满足一定条件时（比如超过一定时间、达到提交最大数据大小），consumer将会阻塞并通知各个channel提交事务。
当且仅当所有channel都提交成功，才会通过Ack命令通知canal并继续获取并消费数据。
@@ -434,6 +434,26 @@ ALTER TABLE canal_test.test1 ENABLE FEATURE "BATCH_DELETE";
## 相关参数
+### CANAL配置
+
+下面配置属于canal端的配置，主要通过修改 conf 目录下的 canal.properties 调整配置值。
+
+* `canal.ip`
+
+ canal server的ip地址
+
+* `canal.port`
+
+ canal server的端口
+
+* `canal.instance.memory.buffer.size`
+
+ canal端的store环形队列的队列长度，必须设为2的幂次方，默认长度16384。此值等于canal端能缓存event数量的最大值，也直接决定了Doris端一个事务内所能容纳的最大event数量。建议将它改的足够大，防止Doris端一个事务内能容纳的数据量上限太小，导致提交事务太过频繁造成数据的版本堆积。
+
+* `canal.instance.memory.buffer.memunit`
+
+ canal端默认一个event所占的空间，默认空间为1024 bytes。此值乘上store环形队列的队列长度等于store的空间最大值，比如store队列长度为16384，则store的空间为16MB。但是，一个event的实际大小并不等于此值，而是由这个event内有多少行数据和每行数据的长度决定的，比如一张只有两列的表的insert event只有30字节，但delete event可能达到数千字节，这是因为通常delete event的行数比insert event多。
+
### FE配置
下面配置属于数据同步作业的系统级别配置，主要通过修改 fe.conf 来调整配置值。
@@ -446,6 +466,18 @@ ALTER TABLE canal_test.test1 ENABLE FEATURE "BATCH_DELETE";
提交事务的最大时间间隔。若超过了这个时间channel中还有数据没有提交，consumer会通知channel提交事务。
+* `min_sync_commit_size`
+
+ 提交事务需满足的最小event数量。若Fe接收到的event数量小于它，会继续等待下一批数据直到时间超过了`sync_commit_interval_second `为止。默认值是10000个events，如果你想修改此配置，请确保此值小于canal端的`canal.instance.memory.buffer.size`配置（默认16384），否则在ack前Fe会尝试获取比store队列长度更多的event，导致store队列阻塞至超时为止。
+
+* `min_bytes_sync_commit`
+
+ 提交事务需满足的最小数据大小。若Fe接收到的数据大小小于它，会继续等待下一批数据直到时间超过了`sync_commit_interval_second `为止。默认值是15MB，如果你想修改此配置，请确保此值小于canal端的`canal.instance.memory.buffer.size`和`canal.instance.memory.buffer.memunit`的乘积（默认16MB），否则在ack前Fe会尝试获取比store空间更大的数据，导致store队列阻塞至超时为止。
+
+* `max_bytes_sync_commit`
+
+ 提交事务时的数据大小的最大值。若Fe接收到的数据大小大于它，会立即提交事务并发送已积累的数据。默认值是64MB，如果你想修改此配置，请确保此值大于canal端的`canal.instance.memory.buffer.size`和`canal.instance.memory.buffer.memunit`的乘积（默认16MB）和`min_bytes_sync_commit`。
+
* `max_sync_task_threads_num`
数据同步作业线程池中的最大线程数量。此线程池整个FE中只有一个，用于处理FE中所有数据同步作业向BE发送数据的任务task，线程池的实现在`SyncTaskPool`类。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index d319d3c..aea8d98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -640,6 +640,40 @@ public class Config extends ConfigBase {
*/
@ConfField public static int max_sync_task_threads_num = 10;
+
+ /**
+ * Min event size that a sync job will commit.
+ * When receiving events less than it, SyncJob will continue
+ * to wait for the next batch of data until the time exceeds
+ * `sync_commit_interval_second`.
+ * The default value is 10000 (canal default event buffer size is 16384).
+ * You should set it smaller than canal buffer size.
+ */
+ @ConfField(mutable = true, masterOnly = true)
+ public static long min_sync_commit_size = 10000;
+
+ /**
+ * Min bytes that a sync job will commit.
+ * When receiving bytes less than it, SyncJob will continue
+ * to wait for the next batch of data until the time exceeds
+ * `sync_commit_interval_second`.
+ * The default value is 15 MB (canal default memory is 16 MB).
+ * You should set it slightly smaller than canal memory.
+ */
+ @ConfField(mutable = true, masterOnly = true)
+ public static long min_bytes_sync_commit = 15 * 1024 * 1024; // 15 MB
+
+ /**
+ * Max bytes that a sync job will commit.
+ * When receiving bytes less than it, SyncJob will commit
+ * all data immediately.
+ * The default value is 64 MB (canal default memory is 16 MB).
+ * You should set it larger than canal memory and
+ * `min_bytes_sync_commit`.
+ */
+ @ConfField(mutable = true, masterOnly = true)
+ public static long max_bytes_sync_commit = 64 * 1024 * 1024; // 64 MB
+
/**
* Default number of waiting jobs for routine load and version 2 of load
* This is a desired number.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalConfigs.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalConfigs.java
index 841acb6..394bb67 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalConfigs.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalConfigs.java
@@ -24,7 +24,4 @@ public class CanalConfigs {
// Maximal waiting time for consumer to poll one batch
public static long pollWaitingTimeoutMs = 80L;
-
- // Maximal waiting time for channel to poll one batch
- public static long channelWaitingTimeoutMs = 1000L;
}
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java
index bdfc732..16b897d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncDataConsumer.java
@@ -59,7 +59,9 @@ public class CanalSyncDataConsumer extends SyncDataConsumer {
private static Logger logger = LogManager.getLogger(CanalSyncDataConsumer.class);
private static final String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
- private static final long COMMIT_MEM_SIZE = 64 * 1024 * 1024; // 64mb;
+ private static final long MIN_COMMIT_EVENT_SIZE = Config.min_sync_commit_size;
+ private static final long MIN_COMMIT_MEM_SIZE = Config.min_bytes_sync_commit;
+ private static final long MAX_COMMIT_MEM_SIZE = Config.max_bytes_sync_commit;
private CanalSyncJob syncJob;
private CanalConnector connector;
@@ -198,7 +200,8 @@ public class CanalSyncDataConsumer extends SyncDataConsumer {
// do nothing
}
if (dataEvents == null) {
- if (totalSize > 0 || totalMemSize > 0) {
+ // If not, continue to wait for the next batch of data
+ if (totalSize >= MIN_COMMIT_EVENT_SIZE || totalMemSize >= MIN_COMMIT_MEM_SIZE) {
break;
}
try {
@@ -218,7 +221,8 @@ public class CanalSyncDataConsumer extends SyncDataConsumer {
executeOneBatch(dataEvents);
totalSize += size;
totalMemSize += dataEvents.getMemSize();
- if (totalMemSize >= COMMIT_MEM_SIZE) {
+ // size of bytes received so far is larger than max commit memory size.
+ if (totalMemSize >= MAX_COMMIT_MEM_SIZE) {
break;
}
}
@@ -228,6 +232,7 @@ public class CanalSyncDataConsumer extends SyncDataConsumer {
}
}
+ // wait all channels done
Status st = waitForTxn();
if (!running) {
abortForTxn("stopping client");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncJob.java
index 7c1b800..3235c2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/CanalSyncJob.java
@@ -60,7 +60,7 @@ public class CanalSyncJob extends SyncJob {
protected final static String CANAL_DEBUG = "canal.debug";
@SerializedName(value = "remote")
- private CanalDestination remote;
+ private final CanalDestination remote;
@SerializedName(value = "username")
private String username;
@SerializedName(value = "password")
@@ -139,7 +139,7 @@ public class CanalSyncJob extends SyncJob {
password = properties.get(CANAL_PASSWORD);
}
- // optional binlog properties
+ // optional
if (properties.containsKey(CANAL_BATCH_SIZE)) {
try {
batchSize = Integer.parseInt(properties.get(CANAL_BATCH_SIZE));
@@ -148,6 +148,7 @@ public class CanalSyncJob extends SyncJob {
}
}
+ // optional
if (properties.containsKey(CANAL_DEBUG)) {
debug = Boolean.parseBoolean(properties.get(CANAL_DEBUG));
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java
index 421c46d..e1c1eb7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/sync/canal/SyncCanalClient.java
@@ -34,13 +34,12 @@ import java.util.concurrent.locks.ReentrantLock;
public class SyncCanalClient {
protected static Logger logger = LogManager.getLogger(SyncCanalClient.class);
- private CanalConnector connector;
-
- private CanalSyncDataReceiver receiver;
- private CanalSyncDataConsumer consumer;
+ private final CanalConnector connector;
+ private final CanalSyncDataReceiver receiver;
+ private final CanalSyncDataConsumer consumer;
// channel id -> channel
- private Map<Long, CanalSyncChannel> idToChannels;
+ private final Map<Long, CanalSyncChannel> idToChannels;
protected ReentrantLock lock = new ReentrantLock(true);
protected ReentrantLock getLock = new ReentrantLock();
@@ -53,11 +52,13 @@ public class SyncCanalClient {
lock.unlock();
}
- public SyncCanalClient(CanalSyncJob syncJob, String destination, CanalConnector connector, int batchSize, boolean debug) {
+ public SyncCanalClient(CanalSyncJob syncJob, String destination, CanalConnector connector,
+ int batchSize, boolean debug) {
this(syncJob, destination, connector, batchSize, debug, ".*\\..*");
}
- public SyncCanalClient(CanalSyncJob syncJob, String destination, CanalConnector connector, int batchSize, boolean debug, String filter) {
+ public SyncCanalClient(CanalSyncJob syncJob, String destination, CanalConnector connector,
+ int batchSize, boolean debug, String filter) {
this.connector = connector;
this.consumer = new CanalSyncDataConsumer(syncJob, connector, getLock, debug);
this.receiver = new CanalSyncDataReceiver(syncJob, connector, destination, filter, consumer, batchSize, getLock);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 11/21: [Bug] Fix bug of decommission (#6826)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 1241992fbe5f620c6e97ff0752c3e773bea5042c
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Mon Oct 25 10:07:04 2021 +0800
[Bug] Fix bug of decommission (#6826)
---
.../main/java/org/apache/doris/catalog/Tablet.java | 15 ++++++
.../apache/doris/clone/ClusterLoadStatistic.java | 7 +++
.../org/apache/doris/clone/TabletSchedCtx.java | 23 ++++++---
.../org/apache/doris/clone/TabletScheduler.java | 7 +--
.../doris/clone/TabletRepairAndBalanceTest.java | 58 ++++++++++++++++++++++
.../doris/utframe/DemoMultiBackendsTest.java | 3 +-
6 files changed, 101 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
index cec8d8f..f40216c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -562,6 +562,8 @@ public class Tablet extends MetaObject implements Writable {
// in ColocateTableCheckerAndBalancer.
Short totalReplicaNum = replicaAlloc.getTotalReplicaNum();
// 1. check if replicas' backends are mismatch
+ // There is no REPLICA_MISSING status for colocate table's tablet.
+ // Because if the following check doesn't pass, the COLOCATE_MISMATCH will return.
Set<Long> replicaBackendIds = getBackendIds();
if (!replicaBackendIds.containsAll(backendsSet)) {
return TabletStatus.COLOCATE_MISMATCH;
@@ -574,6 +576,19 @@ public class Tablet extends MetaObject implements Writable {
// eg: replicaBackendIds=(1,2,3,4); backendsSet=(1,2,3), then replica 4 should be skipped here and then goto ```COLOCATE_REDUNDANT``` in step 3
continue;
}
+
+ if (!replica.isAlive()) {
+ if (replica.isBad()) {
+ // If this replica is bad but located on one of backendsSet,
+ // we have drop it first, or we can find any other BE for new replica.
+ return TabletStatus.COLOCATE_REDUNDANT;
+ } else {
+ // maybe in replica's DECOMMISSION state
+ // Here we return VERSION_INCOMPLETE, and the tablet scheduler will finally set it's state to NORMAL.
+ return TabletStatus.VERSION_INCOMPLETE;
+ }
+ }
+
if (replica.getLastFailedVersion() > 0 || replica.getVersion() < visibleVersion) {
// this replica is alive but version incomplete
return TabletStatus.VERSION_INCOMPLETE;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/ClusterLoadStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/clone/ClusterLoadStatistic.java
index 6dd18c1..14d7230 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/ClusterLoadStatistic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/ClusterLoadStatistic.java
@@ -83,6 +83,13 @@ public class ClusterLoadStatistic {
public void init() {
List<Backend> backends = infoService.getBackendsByTagInCluster(clusterName, tag);
for (Backend backend : backends) {
+ if (backend.isDecommissioned()) {
+ // Skip the decommission backend.
+ // Otherwise, before the decommission is done, these BE will always to treated
+ // as LOW Backends. But the balancer will not handle BE in decommission state.
+ // So balance will be blocked.
+ continue;
+ }
BackendLoadStatistic beStatistic = new BackendLoadStatistic(backend.getId(),
backend.getOwnerClusterName(), backend.getTag(), infoService, invertedIndex);
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
index b34af4e..d86c2c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
@@ -487,14 +487,19 @@ public class TabletSchedCtx implements Comparable<TabletSchedCtx> {
}
// database lock should be held.
- public void chooseSrcReplica(Map<Long, PathSlot> backendsWorkingSlots) throws SchedException {
+ // If exceptBeId != -1, should not choose src replica with same BE id as exceptBeId
+ public void chooseSrcReplica(Map<Long, PathSlot> backendsWorkingSlots, long exceptBeId) throws SchedException {
/*
* get all candidate source replicas
* 1. source replica should be healthy.
- * 2. slot of this source replica is available.
+ * 2. slot of this source replica is available.
*/
List<Replica> candidates = Lists.newArrayList();
for (Replica replica : tablet.getReplicas()) {
+ if (exceptBeId != -1 && replica.getBackendId() == exceptBeId) {
+ continue;
+ }
+
if (replica.isBad()) {
continue;
}
@@ -504,7 +509,7 @@ public class TabletSchedCtx implements Comparable<TabletSchedCtx> {
// backend which is in decommission can still be the source backend
continue;
}
-
+
if (replica.getLastFailedVersion() > 0) {
continue;
}
@@ -545,10 +550,9 @@ public class TabletSchedCtx implements Comparable<TabletSchedCtx> {
*/
public void chooseSrcReplicaForVersionIncomplete(Map<Long, PathSlot> backendsWorkingSlots)
throws SchedException {
- chooseSrcReplica(backendsWorkingSlots);
- if (srcReplica.getBackendId() == destBackendId) {
- throw new SchedException(Status.SCHEDULE_FAILED, "the chosen source replica is in dest backend");
- }
+ chooseSrcReplica(backendsWorkingSlots, destBackendId);
+ Preconditions.checkState(srcReplica.getBackendId() != destBackendId,
+ "wrong be id: " + destBackendId);
}
/*
@@ -573,9 +577,12 @@ public class TabletSchedCtx implements Comparable<TabletSchedCtx> {
continue;
}
+ // check version and replica state.
+ // if the replica's state is DECOMMISSION, it may be chose as dest replica,
+ // and its state will be set to NORMAL later.
if (replica.getLastFailedVersion() <= 0
&& ((replica.getVersion() == visibleVersion && replica.getVersionHash() == visibleVersionHash)
- || replica.getVersion() > visibleVersion)) {
+ || replica.getVersion() > visibleVersion) && replica.getState() != ReplicaState.DECOMMISSION) {
// skip healthy replica
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 39804ae..7aca236 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -625,7 +625,7 @@ public class TabletScheduler extends MasterDaemon {
tabletCtx.setDest(destPath.getBeId(), destPath.getPathHash());
// choose a source replica for cloning from
- tabletCtx.chooseSrcReplica(backendsWorkingSlots);
+ tabletCtx.chooseSrcReplica(backendsWorkingSlots, -1);
// create clone task
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
@@ -955,10 +955,11 @@ public class TabletScheduler extends MasterDaemon {
private boolean handleColocateRedundant(TabletSchedCtx tabletCtx) throws SchedException {
Preconditions.checkNotNull(tabletCtx.getColocateBackendsSet());
for (Replica replica : tabletCtx.getReplicas()) {
- if (tabletCtx.getColocateBackendsSet().contains(replica.getBackendId())) {
+ if (tabletCtx.getColocateBackendsSet().contains(replica.getBackendId()) && !replica.isBad()) {
continue;
}
+ // If the replica is not in ColocateBackendsSet or is bad, delete it.
deleteReplicaInternal(tabletCtx, replica, "colocate redundant", false);
throw new SchedException(Status.FINISHED, "colocate redundant replica is deleted");
}
@@ -1074,7 +1075,7 @@ public class TabletScheduler extends MasterDaemon {
tabletCtx.setDest(destPath.getBeId(), destPath.getPathHash());
// choose a source replica for cloning from
- tabletCtx.chooseSrcReplica(backendsWorkingSlots);
+ tabletCtx.chooseSrcReplica(backendsWorkingSlots, -1);
// create clone task
batchTask.addTask(tabletCtx.createCloneReplicaAndTask());
diff --git a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
index 5099696..65a9a24 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/clone/TabletRepairAndBalanceTest.java
@@ -27,11 +27,13 @@ import org.apache.doris.catalog.ColocateGroupSchema;
import org.apache.doris.catalog.ColocateTableIndex;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DiskInfo;
+import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.ReplicaAllocation;
+import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.AnalysisException;
@@ -456,6 +458,62 @@ public class TabletRepairAndBalanceTest {
// test colocate table index persist
ExceptionChecker.expectThrowsNoException(() -> testColocateTableIndexSerialization(colocateTableIndex));
+
+ // test colocate tablet repair
+ String createStr6 = "create table test.col_tbl3\n" +
+ "(k1 date, k2 int)\n" +
+ "distributed by hash(k2) buckets 1\n" +
+ "properties\n" +
+ "(\n" +
+ " \"replication_num\" = \"3\",\n" +
+ " \"colocate_with\" = \"g3\"\n" +
+ ")";
+ ExceptionChecker.expectThrowsNoException(() -> createTable(createStr6));
+
+ OlapTable tbl3 = db.getOlapTableOrDdlException("col_tbl3");
+ updateReplicaPathHash();
+ // Set one replica's state as DECOMMISSION, see if it can be changed to NORMAL
+ Tablet oneTablet = null;
+ Replica oneReplica = null;
+ for (Partition partition : tbl3.getPartitions()) {
+ for (MaterializedIndex mIndex : partition.getMaterializedIndices(MaterializedIndex.IndexExtState.ALL)) {
+ for (Tablet tablet : mIndex.getTablets()) {
+ oneTablet = tablet;
+ for (Replica replica : tablet.getReplicas()) {
+ oneReplica = replica;
+ oneReplica.setState(Replica.ReplicaState.DECOMMISSION);
+ break;
+ }
+ }
+ }
+ }
+ Assert.assertTrue(checkReplicaState(oneReplica));
+
+ // set one replica to bad, see if it can be repaired
+ oneReplica.setBad(true);
+ Assert.assertTrue(checkReplicaBad(oneTablet, oneReplica));
+ }
+
+ private static boolean checkReplicaState(Replica replica) throws Exception {
+ for (int i = 0; i < 10; i++) {
+ Thread.sleep(1000);
+ if (replica.getState() != Replica.ReplicaState.NORMAL) {
+ continue;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private static boolean checkReplicaBad(Tablet tablet, Replica replica) throws Exception {
+ for (int i = 0; i < 10; i++) {
+ Thread.sleep(1000);
+ if (tablet.getReplicaById(replica.getId()) != null) {
+ continue;
+ }
+ return true;
+ }
+ return false;
}
private void testColocateTableIndexSerialization(ColocateTableIndex colocateTableIndex) throws IOException {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java
index 460f49b..70578f4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/DemoMultiBackendsTest.java
@@ -132,7 +132,8 @@ public class DemoMultiBackendsTest {
Catalog.getCurrentCatalog().createDb(createDbStmt);
System.out.println(Catalog.getCurrentCatalog().getDbNames());
// 3. create table tbl1
- String createTblStmtStr = "create table db1.tbl1(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '3');";
+ String createTblStmtStr = "create table db1.tbl1(k1 int) distributed by hash(k1) buckets 3 properties('replication_num' = '3'," +
+ "'colocate_with' = 'g1');";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
// must set replicas' path hash, or the tablet scheduler won't work
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 03/21: [Flink][Bug] Fix potential NPE when cancel
DorisSourceFunction (#6838)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 582f417aaba5a999972aaaae38b0471e4e834826
Author: Yun Tang <my...@live.com>
AuthorDate: Sat Oct 23 16:45:24 2021 +0800
[Flink][Bug] Fix potential NPE when cancel DorisSourceFunction (#6838)
Fix potential NPE of `scalaValueReader` when cancelling DorisSourceFunction.
---
.../doris/flink/datastream/DorisSourceFunction.java | 19 +++++++++++++------
.../doris/flink/datastream/ScalaValueReader.scala | 3 +--
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
index 08ec5b0..85f5f6b 100644
--- a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
+++ b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/datastream/DorisSourceFunction.java
@@ -43,8 +43,8 @@ public class DorisSourceFunction extends RichSourceFunction<List<?>> implements
private final DorisDeserializationSchema<List<?>> deserializer;
private final DorisOptions options;
private final DorisReadOptions readOptions;
+ private transient volatile boolean isRunning;
private List<PartitionDefinition> dorisPartitions;
- private ScalaValueReader scalaValueReader;
public DorisSourceFunction(DorisStreamOptions streamOptions, DorisDeserializationSchema<List<?>> deserializer) {
this.deserializer = deserializer;
@@ -55,25 +55,32 @@ public class DorisSourceFunction extends RichSourceFunction<List<?>> implements
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
+ this.isRunning = true;
this.dorisPartitions = RestService.findPartitions(options, readOptions, logger);
}
@Override
public void run(SourceContext<List<?>> sourceContext) {
for (PartitionDefinition partitions : dorisPartitions) {
- scalaValueReader = new ScalaValueReader(partitions, options, readOptions);
- while (scalaValueReader.hasNext()) {
- List<?> next = scalaValueReader.next();
- sourceContext.collect(next);
+ try (ScalaValueReader scalaValueReader = new ScalaValueReader(partitions, options, readOptions)) {
+ while (isRunning && scalaValueReader.hasNext()) {
+ List<?> next = scalaValueReader.next();
+ sourceContext.collect(next);
+ }
}
}
}
@Override
public void cancel() {
- scalaValueReader.close();
+ isRunning = false;
}
+ @Override
+ public void close() throws Exception {
+ super.close();
+ isRunning = false;
+ }
@Override
public TypeInformation<List<?>> getProducedType() {
diff --git a/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala b/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
index 093390d..06df2ef 100644
--- a/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
+++ b/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
@@ -19,7 +19,6 @@ package org.apache.doris.flink.datastream
import java.util.concurrent._
import java.util.concurrent.atomic.AtomicBoolean
-
import org.apache.doris.flink.backend.BackendClient
import org.apache.doris.flink.cfg.ConfigurationOptions._
import org.apache.doris.flink.cfg.{DorisOptions, DorisReadOptions}
@@ -41,7 +40,7 @@ import scala.util.control.Breaks
* @param partition Doris RDD partition
* @param options request configuration
*/
-class ScalaValueReader(partition: PartitionDefinition, options: DorisOptions, readOptions: DorisReadOptions) {
+class ScalaValueReader(partition: PartitionDefinition, options: DorisOptions, readOptions: DorisReadOptions) extends AutoCloseable {
protected val logger = Logger.getLogger(classOf[ScalaValueReader])
protected val client = new BackendClient(new Routing(partition.getBeAddress), readOptions)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 07/21: [MetaTool]add error message when loading
meta by meta tool (#6893)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 0695671cc8430fe488388bdc89f786eec34a3f6e
Author: qiye <ji...@gmail.com>
AuthorDate: Sat Oct 23 16:51:58 2021 +0800
[MetaTool]add error message when loading meta by meta tool (#6893)
When loading meta by meta_tool goes wrong, we only get an error code from `json2pb`,
which is inconvenient for us to locate the problem.
This change is adding error message when loading meta goes wrong.
Log change is like below.
```
# before
./meta_tool --root_path=/home/disk1/qjl/mydoris/be/storage --operation=load_meta --json_meta_path=/home/disk1/qjl/data/meta-json.json
WARNING: Logging before InitGoogleLogging() is written to STDERR
I1020 11:41:56.564241 74937 data_dir.cpp:837] path: /home/disk1/qjl/mydoris/be/storage total capacity: 7750843404288, available capacity: 7583325925376
I1020 11:41:56.564415 74937 data_dir.cpp:275] path: /home/disk1/qjl/mydoris/be/storage, hash: 7528840506668047470
load meta failed, status:-1410
# after
./meta_tool --root_path=/home/disk1/qjl/mydoris/be/storage --operation=load_meta --json_meta_path=/home/disk1/qjl/data/meta-json.json
WARNING: Logging before InitGoogleLogging() is written to STDERR
I1020 14:41:40.084342 50727 data_dir.cpp:837] path: /home/disk1/qjl/mydoris/be/storage total capacity: 7750843404288, available capacity: 7584601022464
I1020 14:41:40.084496 50727 data_dir.cpp:275] path: /home/disk1/qjl/mydoris/be/storage, hash: 7528840506668047470
E1020 14:41:40.163007 50727 tablet_meta_manager.cpp:161] JSON to protobuf message failed: Fail to decode base64 string=0
load meta failed, status:-1410
```
---
be/src/olap/tablet_meta_manager.cpp | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp
index 27775f8..7844d44 100644
--- a/be/src/olap/tablet_meta_manager.cpp
+++ b/be/src/olap/tablet_meta_manager.cpp
@@ -155,8 +155,10 @@ OLAPStatus TabletMetaManager::load_json_meta(DataDir* store, const std::string&
}
boost::algorithm::trim(json_meta);
TabletMetaPB tablet_meta_pb;
- bool ret = json2pb::JsonToProtoMessage(json_meta, &tablet_meta_pb);
+ std::string error;
+ bool ret = json2pb::JsonToProtoMessage(json_meta, &tablet_meta_pb, &error);
if (!ret) {
+ LOG(ERROR) << "JSON to protobuf message failed: " << error;
return OLAP_ERR_HEADER_LOAD_JSON_HEADER;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 20/21: [Enhance] Add BackendHbResponse info
(#6929)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit c385536faf88641b199d7bf06f11978842eff17b
Author: dh-cloud <60...@users.noreply.github.com>
AuthorDate: Wed Oct 27 09:56:07 2021 +0800
[Enhance] Add BackendHbResponse info (#6929)
when be has excepiton, fe doesn't log the BackendHbResponse info, so we can't know which be has exception
the exception log is：
`WARN (heartbeat mgr|31) [HeartbeatMgr.runAfterCatalogReady():141] get bad heartbeat response: type: BACKEND, status: BAD, msg: java.net.ConnectException: Connection refused (Connection refused)
`
so need add toString(), then fe can log the BackendHbResponse info
---
.../main/java/org/apache/doris/system/BackendHbResponse.java | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java
index 9db8790..1818c01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/BackendHbResponse.java
@@ -104,5 +104,16 @@ public class BackendHbResponse extends HeartbeatResponse implements Writable {
httpPort = in.readInt();
brpcPort = in.readInt();
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(super.toString());
+ sb.append(", beId: ").append(beId);
+ sb.append(", bePort: ").append(bePort);
+ sb.append(", httpPort: ").append(httpPort);
+ sb.append(", brpcPort: ").append(brpcPort);
+ return sb.toString();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 21/21: [Optimize] optimize some session variable
and profile (#6920)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 26a81f2a9c01a9bf6ec0036230ea2d97c3e98bf9
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Wed Oct 27 18:03:12 2021 +0800
[Optimize] optimize some session variable and profile (#6920)
1. optimize error message when using batch delete
2. rename session variable is_report_success to enable_profile
3. add table name to OlapScanner profile
---
be/src/exec/olap_scan_node.cpp | 6 +++++-
.../load-data/broker-load-manual.md | 2 +-
docs/en/administrator-guide/running-profile.md | 4 ++--
docs/en/administrator-guide/runtime-filter.md | 2 +-
docs/en/administrator-guide/variables.md | 6 +++---
.../load-data/broker-load-manual.md | 2 +-
docs/zh-CN/administrator-guide/running-profile.md | 4 ++--
docs/zh-CN/administrator-guide/runtime-filter.md | 2 +-
docs/zh-CN/administrator-guide/variables.md | 6 +++---
.../org/apache/doris/alter/SchemaChangeHandler.java | 8 +++++++-
.../main/java/org/apache/doris/analysis/SetVar.java | 13 ++++++++-----
.../src/main/java/org/apache/doris/load/Load.java | 20 ++++++++++++++++----
.../org/apache/doris/load/loadv2/BrokerLoadJob.java | 10 +++++-----
.../java/org/apache/doris/planner/OlapScanNode.java | 1 +
.../main/java/org/apache/doris/qe/Coordinator.java | 2 +-
.../java/org/apache/doris/qe/GlobalVariable.java | 1 -
.../java/org/apache/doris/qe/QeProcessorImpl.java | 2 +-
.../java/org/apache/doris/qe/SessionVariable.java | 14 +++++++-------
.../main/java/org/apache/doris/qe/StmtExecutor.java | 2 +-
.../org/apache/doris/qe/SessionVariablesTest.java | 4 ++--
.../java/org/apache/doris/qe/VariableMgrTest.java | 2 +-
gensrc/thrift/PlanNodes.thrift | 1 +
22 files changed, 70 insertions(+), 44 deletions(-)
diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
index b2d90e8..50abc26 100644
--- a/be/src/exec/olap_scan_node.cpp
+++ b/be/src/exec/olap_scan_node.cpp
@@ -96,7 +96,11 @@ Status OlapScanNode::init(const TPlanNode& tnode, RuntimeState* state) {
}
void OlapScanNode::init_scan_profile() {
- _scanner_profile.reset(new RuntimeProfile("OlapScanner"));
+ std::string scanner_profile_name = "OlapScanner";
+ if (_olap_scan_node.__isset.table_name) {
+ scanner_profile_name = fmt::format("OlapScanner({0})", _olap_scan_node.table_name);
+ }
+ _scanner_profile.reset(new RuntimeProfile(scanner_profile_name));
runtime_profile()->add_child(_scanner_profile.get(), true, NULL);
_segment_profile.reset(new RuntimeProfile("SegmentIterator"));
diff --git a/docs/en/administrator-guide/load-data/broker-load-manual.md b/docs/en/administrator-guide/load-data/broker-load-manual.md
index 2b410b6..930b5e1 100644
--- a/docs/en/administrator-guide/load-data/broker-load-manual.md
+++ b/docs/en/administrator-guide/load-data/broker-load-manual.md
@@ -463,7 +463,7 @@ We will only discuss the case of a single BE. If the user cluster has more than
### Performance analysis
-You can execute `set is_report_success=true` to open the load job profile before submitting the import job. After the import job is completed, you can view the profile of the import job in the `Queris` tab of the FE web page.
+You can execute `set enable_profile=true` to open the load job profile before submitting the import job. After the import job is completed, you can view the profile of the import job in the `Queris` tab of the FE web page.
This profile can help analyze the running status of the import job.
diff --git a/docs/en/administrator-guide/running-profile.md b/docs/en/administrator-guide/running-profile.md
index e175271..736e73c 100644
--- a/docs/en/administrator-guide/running-profile.md
+++ b/docs/en/administrator-guide/running-profile.md
@@ -44,10 +44,10 @@ FE splits the query plan into fragments and distributes them to BE for task exec
Turn on the report switch on FE through MySQL command
```
-mysql> set is_report_success=true;
+mysql> set enable_profile=true;
```
-After executing the corresponding SQL statement, we can see the report information of the corresponding SQL statement on the FE web page like the picture below.
+After executing the corresponding SQL statement(`is_report_success` in old versions), we can see the report information of the corresponding SQL statement on the FE web page like the picture below.
The latest **100 statements** executed will be listed here. We can view detailed statistics of RunningProfile.
diff --git a/docs/en/administrator-guide/runtime-filter.md b/docs/en/administrator-guide/runtime-filter.md
index 297aec6..a008068 100644
--- a/docs/en/administrator-guide/runtime-filter.md
+++ b/docs/en/administrator-guide/runtime-filter.md
@@ -251,7 +251,7 @@ EXPLAIN SELECT t1 FROM test JOIN test2 where test.t1 = test2.t2;
SELECT t1 FROM test JOIN test2 where test.t1 = test2.t2;
-- Return 2 rows of results [3, 4];
--- Through the query profile (set is_report_success=true;) you can view the detailed information of the internal work of the query,
+-- Through the query profile (set enable_profile=true;) you can view the detailed information of the internal work of the query,
-- Including whether each Runtime Filter is pushed down, waiting time,
-- and the total time from prepare to receiving Runtime Filter for OLAP_SCAN_NODE.
RuntimeFilter:in:
diff --git a/docs/en/administrator-guide/variables.md b/docs/en/administrator-guide/variables.md
index d5e1bf6..815ceef 100644
--- a/docs/en/administrator-guide/variables.md
+++ b/docs/en/administrator-guide/variables.md
@@ -67,7 +67,7 @@ Variables that support both session-level and global-level setting include:
* `time_zone`
* `wait_timeout`
* `sql_mode`
-* `is_report_success`
+* `enable_profile`
* `query_timeout`
* `exec_mem_limit`
* `batch_size`
@@ -229,7 +229,7 @@ Note that the comment must start with /*+ and can only follow the SELECT.
Used for compatibility with MySQL clients. No practical effect.
-* `is_report_success`
+* `enable_profile`
Used to set whether you need to view the profile of the query. The default is false, which means no profile is required.
@@ -239,7 +239,7 @@ Note that the comment must start with /*+ and can only follow the SELECT.
`fe_host:fe_http:port/query`
- It will display the most recent 100 queries which `is_report_success` is set to true.
+ It will display the most recent 100 queries which `enable_profile` is set to true.
* `language`
diff --git a/docs/zh-CN/administrator-guide/load-data/broker-load-manual.md b/docs/zh-CN/administrator-guide/load-data/broker-load-manual.md
index 2ba0b63..8d85457 100644
--- a/docs/zh-CN/administrator-guide/load-data/broker-load-manual.md
+++ b/docs/zh-CN/administrator-guide/load-data/broker-load-manual.md
@@ -472,7 +472,7 @@ LoadFinishTime: 2019-07-27 11:50:16
### 性能分析
-可以在提交 LOAD 作业前，先执行 `set is_report_success=true` 打开会话变量。然后提交导入作业。待导入作业完成后，可以在 FE 的 web 页面的 `Queris` 标签中查看到导入作业的 Profile。
+可以在提交 LOAD 作业前，先执行 `set enable_profile=true` 打开会话变量。然后提交导入作业。待导入作业完成后，可以在 FE 的 web 页面的 `Queris` 标签中查看到导入作业的 Profile。
这个 Profile 可以帮助分析导入作业的运行状态。
diff --git a/docs/zh-CN/administrator-guide/running-profile.md b/docs/zh-CN/administrator-guide/running-profile.md
index b5f2d93..e24cc3b 100644
--- a/docs/zh-CN/administrator-guide/running-profile.md
+++ b/docs/zh-CN/administrator-guide/running-profile.md
@@ -44,10 +44,10 @@ FE将查询计划拆分成为Fragment下发到BE进行任务执行。BE在执行
通过Mysql命令，将FE上的Report的开关打开
```
-mysql> set is_report_success=true;
+mysql> set enable_profile=true;
```
-之后执行对应的SQL语句之后，在FE的Web页面就可以看到对应SQL语句执行的Report信息：
+之后执行对应的SQL语句之后（旧版本为`is_report_success`），在FE的Web页面就可以看到对应SQL语句执行的Report信息：
这里会列出最新执行完成的**100条语句**，我们可以通过Profile查看详细的统计信息。
diff --git a/docs/zh-CN/administrator-guide/runtime-filter.md b/docs/zh-CN/administrator-guide/runtime-filter.md
index 2f71cb3..6f8b251 100644
--- a/docs/zh-CN/administrator-guide/runtime-filter.md
+++ b/docs/zh-CN/administrator-guide/runtime-filter.md
@@ -251,7 +251,7 @@ EXPLAIN SELECT t1 FROM test JOIN test2 where test.t1 = test2.t2;
SELECT t1 FROM test JOIN test2 where test.t1 = test2.t2;
-- 返回2行结果[3, 4];
--- 通过query的profile（set is_report_success=true;）可以查看查询内部工作的详细信息，
+-- 通过query的profile（set enable_profile=true;）可以查看查询内部工作的详细信息，
-- 包括每个Runtime Filter是否下推、等待耗时、以及OLAP_SCAN_NODE从prepare到接收到Runtime Filter的总时长。
RuntimeFilter:in:
- HasPushDownToEngine: true
diff --git a/docs/zh-CN/administrator-guide/variables.md b/docs/zh-CN/administrator-guide/variables.md
index a5457ce..72b6fb7 100644
--- a/docs/zh-CN/administrator-guide/variables.md
+++ b/docs/zh-CN/administrator-guide/variables.md
@@ -67,7 +67,7 @@ SET GLOBAL exec_mem_limit = 137438953472
* `time_zone`
* `wait_timeout`
* `sql_mode`
-* `is_report_success`
+* `enable_profile`
* `query_timeout`
* `exec_mem_limit`
* `batch_size`
@@ -226,7 +226,7 @@ SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3);
用于兼容 MySQL 客户端。无实际作用。
-* `is_report_success`
+* `enable_profile`
用于设置是否需要查看查询的 profile。默认为 false，即不需要 profile。
@@ -235,7 +235,7 @@ SELECT /*+ SET_VAR(query_timeout = 1, enable_partition_cache=true) */ sleep(3);
`fe_host:fe_http_port/query`
- 其中会显示最近100条，开启 `is_report_success` 的查询的 profile。
+ 其中会显示最近100条，开启 `enable_profile` 的查询的 profile。
* `language`
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 2e6ba17..ccaa773 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -810,7 +810,13 @@ public class SchemaChangeHandler extends AlterHandler {
}
}
if (found) {
- throw new DdlException("Can not add column which already exists in base table: " + newColName);
+ if (newColName.equalsIgnoreCase(Column.DELETE_SIGN)) {
+ throw new DdlException("Can not enable batch delete support, already supported batch delete.");
+ } else if (newColName.equalsIgnoreCase(Column.SEQUENCE_COL)) {
+ throw new DdlException("Can not enable sequence column support, already supported sequence column.");
+ } else {
+ throw new DdlException("Can not add column which already exists in base table: " + newColName);
+ }
}
/*
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
index cdb28d0..784489e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetVar.java
@@ -129,23 +129,26 @@ public class SetVar {
}
}
- if (getVariable().toLowerCase().equals("prefer_join_method")) {
+ if (getVariable().equalsIgnoreCase(SessionVariable.PREFER_JOIN_METHOD)) {
String value = getValue().getStringValue();
- if (!value.toLowerCase().equals("broadcast") && !value.toLowerCase().equals("shuffle")) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR, "prefer_join_method", value);
+ if (!value.equalsIgnoreCase("broadcast") && !value.equalsIgnoreCase("shuffle")) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_FOR_VAR, SessionVariable.PREFER_JOIN_METHOD, value);
}
}
// Check variable time_zone value is valid
- if (getVariable().toLowerCase().equals("time_zone")) {
+ if (getVariable().equalsIgnoreCase(SessionVariable.TIME_ZONE)) {
this.value = new StringLiteral(TimeUtils.checkTimeZoneValidAndStandardize(getValue().getStringValue()));
this.result = (LiteralExpr) this.value;
}
- if (getVariable().toLowerCase().equals("exec_mem_limit")) {
+ if (getVariable().equalsIgnoreCase(SessionVariable.EXEC_MEM_LIMIT)) {
this.value = new StringLiteral(Long.toString(ParseUtil.analyzeDataVolumn(getValue().getStringValue())));
this.result = (LiteralExpr) this.value;
}
+ if (getVariable().equalsIgnoreCase("is_report_success")) {
+ variable = SessionVariable.ENABLE_PROFILE;
+ }
}
public String toSql() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index 616d043..9469e4c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -1065,8 +1065,14 @@ public class Load {
for (SlotRef slot : slots) {
SlotDescriptor slotDesc = slotDescByName.get(slot.getColumnName());
if (slotDesc == null) {
- throw new UserException("unknown reference column, column=" + entry.getKey()
- + ", reference=" + slot.getColumnName());
+ if (entry.getKey().equalsIgnoreCase(Column.DELETE_SIGN)) {
+ throw new UserException("unknown reference column in DELETE ON clause:" + slot.getColumnName());
+ } else if (entry.getKey().equalsIgnoreCase(Column.SEQUENCE_COL)) {
+ throw new UserException("unknown reference column in ORDER BY clause:" + slot.getColumnName());
+ } else {
+ throw new UserException("unknown reference column, column=" + entry.getKey()
+ + ", reference=" + slot.getColumnName());
+ }
}
smap.getLhs().add(slot);
smap.getRhs().add(new SlotRef(slotDesc));
@@ -1099,8 +1105,14 @@ public class Load {
smap.getRhs().add(new CastExpr(tbl.getColumn(slot.getColumnName()).getType(),
exprsByName.get(slot.getColumnName())));
} else {
- throw new UserException("unknown reference column, column=" + entry.getKey()
- + ", reference=" + slot.getColumnName());
+ if (entry.getKey().equalsIgnoreCase(Column.DELETE_SIGN)) {
+ throw new UserException("unknown reference column in DELETE ON clause:" + slot.getColumnName());
+ } else if (entry.getKey().equalsIgnoreCase(Column.SEQUENCE_COL)) {
+ throw new UserException("unknown reference column in ORDER BY clause:" + slot.getColumnName());
+ } else {
+ throw new UserException("unknown reference column, column=" + entry.getKey()
+ + ", reference=" + slot.getColumnName());
+ }
}
}
Expr expr = entry.getValue().clone(smap);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index afd153d..b57a2de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -75,7 +75,7 @@ public class BrokerLoadJob extends BulkLoadJob {
// Profile of this load job, including all tasks' profiles
private RuntimeProfile jobProfile;
// If set to true, the profile of load job with be pushed to ProfileManager
- private boolean isReportSuccess = false;
+ private boolean enableProfile = false;
// for log replay and unit test
public BrokerLoadJob() {
@@ -87,8 +87,8 @@ public class BrokerLoadJob extends BulkLoadJob {
throws MetaNotFoundException {
super(EtlJobType.BROKER, dbId, label, originStmt, userInfo);
this.brokerDesc = brokerDesc;
- if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().isReportSucc()) {
- isReportSuccess = true;
+ if (ConnectContext.get() != null && ConnectContext.get().getSessionVariable().enableProfile()) {
+ enableProfile = true;
}
}
@@ -199,7 +199,7 @@ public class BrokerLoadJob extends BulkLoadJob {
LoadLoadingTask task = new LoadLoadingTask(db, table, brokerDesc,
brokerFileGroups, getDeadlineMs(), getExecMemLimit(),
isStrictMode(), transactionId, this, getTimeZone(), getTimeout(),
- getLoadParallelism(), getSendBatchParallelism(), isReportSuccess ? jobProfile : null);
+ getLoadParallelism(), getSendBatchParallelism(), enableProfile ? jobProfile : null);
UUID uuid = UUID.randomUUID();
TUniqueId loadId = new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
task.init(loadId, attachment.getFileStatusByTable(aggKey),
@@ -306,7 +306,7 @@ public class BrokerLoadJob extends BulkLoadJob {
}
private void writeProfile() {
- if (!isReportSuccess) {
+ if (!enableProfile) {
return;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
index 246788c..d980c28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java
@@ -759,6 +759,7 @@ public class OlapScanNode extends ScanNode {
msg.olap_scan_node.setSortColumn(sortColumn);
}
msg.olap_scan_node.setKeyType(olapTable.getKeysType().toThrift());
+ msg.olap_scan_node.setTableName(olapTable.getName());
}
// export some tablets
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 70c4624..98b4661 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -239,7 +239,7 @@ public class Coordinator {
}
this.tResourceInfo = new TResourceInfo(context.getQualifiedUser(),
context.getSessionVariable().getResourceGroup());
- this.needReport = context.getSessionVariable().isReportSucc();
+ this.needReport = context.getSessionVariable().enableProfile();
this.nextInstanceId = new TUniqueId();
nextInstanceId.setHi(queryId.hi);
nextInstanceId.setLo(queryId.lo + 1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
index 923a8b3..b6612a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
@@ -82,7 +82,6 @@ public final class GlobalVariable {
// Don't allow create instance.
private GlobalVariable() {
-
}
public static List<String> getPersistentGlobalVarNames() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
index c421b56..c05bf00 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QeProcessorImpl.java
@@ -162,7 +162,7 @@ public final class QeProcessorImpl implements QeProcessor {
.db(context.getDatabase())
.fragmentInstanceInfos(info.getCoord().getFragmentInstanceInfos())
.profile(info.getCoord().getQueryProfile())
- .isReportSucc(context.getSessionVariable().isReportSucc()).build();
+ .isReportSucc(context.getSessionVariable().enableProfile()).build();
querySet.put(queryIdStr, item);
}
return querySet;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 4e9e0e9..859508b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -45,7 +45,7 @@ public class SessionVariable implements Serializable, Writable {
public static final String EXEC_MEM_LIMIT = "exec_mem_limit";
public static final String QUERY_TIMEOUT = "query_timeout";
- public static final String IS_REPORT_SUCCESS = "is_report_success";
+ public static final String ENABLE_PROFILE = "enable_profile";
public static final String SQL_MODE = "sql_mode";
public static final String RESOURCE_VARIABLE = "resource_group";
public static final String AUTO_COMMIT = "autocommit";
@@ -184,8 +184,8 @@ public class SessionVariable implements Serializable, Writable {
public int queryTimeoutS = 300;
// if true, need report to coordinator when plan fragment execute successfully.
- @VariableMgr.VarAttr(name = IS_REPORT_SUCCESS, needForward = true)
- public boolean isReportSucc = false;
+ @VariableMgr.VarAttr(name = ENABLE_PROFILE, needForward = true)
+ public boolean enableProfile = false;
// Set sqlMode to empty string
@VariableMgr.VarAttr(name = SQL_MODE, needForward = true)
@@ -391,8 +391,8 @@ public class SessionVariable implements Serializable, Writable {
return queryTimeoutS;
}
- public boolean isReportSucc() {
- return isReportSucc;
+ public boolean enableProfile() {
+ return enableProfile;
}
public int getWaitTimeoutS() {
@@ -800,7 +800,7 @@ public class SessionVariable implements Serializable, Writable {
tResult.setBufferPoolLimit(maxExecMemByte);
tResult.setQueryTimeout(queryTimeoutS);
- tResult.setIsReportSuccess(isReportSucc);
+ tResult.setIsReportSuccess(enableProfile);
tResult.setCodegenLevel(codegenLevel);
tResult.setEnableVectorizedEngine(enableVectorizedEngine);
@@ -900,7 +900,7 @@ public class SessionVariable implements Serializable, Writable {
Text.readString(in);
sqlMode = 0L;
}
- isReportSucc = in.readBoolean();
+ enableProfile = in.readBoolean();
queryTimeoutS = in.readInt();
maxExecMemByte = in.readLong();
if (Catalog.getCurrentCatalogJournalVersion() >= FeMetaVersion.VERSION_37) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index f1e8e5b..f435399 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -465,7 +465,7 @@ public class StmtExecutor implements ProfileWriter {
@Override
public void writeProfile(boolean isLastWriteProfile) {
- if (!context.getSessionVariable().isReportSucc()) {
+ if (!context.getSessionVariable().enableProfile()) {
return;
}
synchronized (writeProfileLock) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
index 3313630..98d7fc0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/SessionVariablesTest.java
@@ -51,9 +51,9 @@ public class SessionVariablesTest {
Assert.assertTrue(numOfForwardVars >= 6);
Assert.assertEquals(numOfForwardVars, vars.size());
- vars.put(SessionVariable.IS_REPORT_SUCCESS, "true");
+ vars.put(SessionVariable.ENABLE_PROFILE, "true");
sessionVariable.setForwardedSessionVariables(vars);
- Assert.assertEquals(true, sessionVariable.isReportSucc);
+ Assert.assertEquals(true, sessionVariable.enableProfile);
}
@Test
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
index 6ed14df..243832c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java
@@ -85,7 +85,7 @@ public class VariableMgrTest {
SessionVariable var = VariableMgr.newSessionVariable();
Assert.assertEquals(2147483648L, var.getMaxExecMemByte());
Assert.assertEquals(300, var.getQueryTimeoutS());
- Assert.assertEquals(false, var.isReportSucc());
+ Assert.assertEquals(false, var.enableProfile());
Assert.assertEquals(0L, var.getSqlMode());
List<List<String>> rows = VariableMgr.dump(SetType.SESSION, var, null);
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 0a90997..716002d 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -342,6 +342,7 @@ struct TOlapScanNode {
4: required bool is_preaggregation
5: optional string sort_column
6: optional Types.TKeysType keyType
+ 7: optional string table_name
}
struct TEqJoinCondition {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 12/21: [Memory Usage] Implement segment lru cache
to save memory of BE (#6829)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 1c8b7df710fb217a8a6aea5bf926df6df6f0e14c
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Mon Oct 25 10:07:15 2021 +0800
[Memory Usage] Implement segment lru cache to save memory of BE (#6829)
---
be/src/agent/task_worker_pool.cpp | 65 ++++++---
be/src/agent/task_worker_pool.h | 3 +
be/src/common/config.h | 8 +-
be/src/olap/CMakeLists.txt | 1 +
be/src/olap/lru_cache.cpp | 119 ++++++++++++----
be/src/olap/lru_cache.h | 42 ++++--
be/src/olap/olap_server.cpp | 4 +-
be/src/olap/rowset/alpha_rowset.cpp | 32 +----
be/src/olap/rowset/alpha_rowset.h | 6 +-
be/src/olap/rowset/alpha_rowset_reader.cpp | 2 +-
be/src/olap/rowset/beta_rowset.cpp | 14 +-
be/src/olap/rowset/beta_rowset.h | 5 +-
be/src/olap/rowset/beta_rowset_reader.cpp | 9 +-
be/src/olap/rowset/beta_rowset_reader.h | 5 +
be/src/olap/rowset/rowset.cpp | 4 +-
be/src/olap/rowset/rowset.h | 6 +-
be/src/olap/segment_loader.cpp | 94 +++++++++++++
be/src/olap/segment_loader.h | 152 +++++++++++++++++++++
be/src/olap/snapshot_manager.cpp | 2 +-
be/src/olap/storage_engine.cpp | 6 +-
be/src/olap/storage_engine.h | 2 +-
be/src/runtime/exec_env_init.cpp | 3 +
be/test/olap/lru_cache_test.cpp | 82 +++++++++--
be/test/olap/rowset/beta_rowset_test.cpp | 1 +
be/test/olap/rowset/rowset_converter_test.cpp | 1 +
docs/en/administrator-guide/config/be_config.md | 13 +-
docs/zh-CN/administrator-guide/config/be_config.md | 15 +-
.../load/routineload/RoutineLoadTaskScheduler.java | 6 +-
.../main/java/org/apache/doris/task/PushTask.java | 2 +-
29 files changed, 565 insertions(+), 139 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index a4b9463..1fb5bb3 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -51,6 +51,7 @@
#include "util/doris_metrics.h"
#include "util/file_utils.h"
#include "util/monotime.h"
+#include "util/random.h"
#include "util/scoped_cleanup.h"
#include "util/stopwatch.hpp"
#include "util/threadpool.h"
@@ -1098,19 +1099,36 @@ void TaskWorkerPool::_check_consistency_worker_thread_callback() {
}
void TaskWorkerPool::_report_task_worker_thread_callback() {
+ StorageEngine::instance()->register_report_listener(this);
TReportRequest request;
request.__set_backend(_backend);
- do {
+ while (_is_work) {
+ _is_doing_work = false;
+ // wait at most report_task_interval_seconds, or being notified
+ _worker_thread_condition_variable.wait_for(
+ MonoDelta::FromSeconds(config::report_task_interval_seconds));
+ if (!_is_work) {
+ break;
+ }
+
+ if (_master_info.network_address.port == 0) {
+ // port == 0 means not received heartbeat yet
+ // sleep a short time and try again
+ LOG(INFO) << "waiting to receive first heartbeat from frontend before doing task report";
+ continue;
+ }
+
_is_doing_work = true;
+ // See _random_sleep() comment in _report_disk_state_worker_thread_callback
+ _random_sleep(5);
{
lock_guard<Mutex> task_signatures_lock(_s_task_signatures_lock);
request.__set_tasks(_s_task_signatures);
}
_handle_report(request, ReportType::TASK);
- _is_doing_work = false;
- } while (!_stop_background_threads_latch.wait_for(
- MonoDelta::FromSeconds(config::report_task_interval_seconds)));
+ }
+ StorageEngine::instance()->deregister_report_listener(this);
}
/// disk state report thread will report disk state at a configurable fix interval.
@@ -1122,14 +1140,6 @@ void TaskWorkerPool::_report_disk_state_worker_thread_callback() {
while (_is_work) {
_is_doing_work = false;
- if (_master_info.network_address.port == 0) {
- // port == 0 means not received heartbeat yet
- // sleep a short time and try again
- LOG(INFO) << "waiting to receive first heartbeat from frontend";
- sleep(config::sleep_one_second);
- continue;
- }
-
// wait at most report_disk_state_interval_seconds, or being notified
_worker_thread_condition_variable.wait_for(
MonoDelta::FromSeconds(config::report_disk_state_interval_seconds));
@@ -1137,7 +1147,18 @@ void TaskWorkerPool::_report_disk_state_worker_thread_callback() {
break;
}
+ if (_master_info.network_address.port == 0) {
+ // port == 0 means not received heartbeat yet
+ LOG(INFO) << "waiting to receive first heartbeat from frontend before doing disk report";
+ continue;
+ }
+
_is_doing_work = true;
+ // Random sleep 1~5 seconds before doing report.
+ // In order to avoid the problem that the FE receives many report requests at the same time
+ // and can not be processed.
+ _random_sleep(5);
+
std::vector<DataDirInfo> data_dir_infos;
_env->storage_engine()->get_all_data_dir_info(&data_dir_infos, true /* update */);
@@ -1168,13 +1189,6 @@ void TaskWorkerPool::_report_tablet_worker_thread_callback() {
while (_is_work) {
_is_doing_work = false;
- if (_master_info.network_address.port == 0) {
- // port == 0 means not received heartbeat yet
- // sleep a short time and try again
- LOG(INFO) << "waiting to receive first heartbeat from frontend";
- sleep(config::sleep_one_second);
- continue;
- }
// wait at most report_tablet_interval_seconds, or being notified
_worker_thread_condition_variable.wait_for(
@@ -1183,7 +1197,15 @@ void TaskWorkerPool::_report_tablet_worker_thread_callback() {
break;
}
+ if (_master_info.network_address.port == 0) {
+ // port == 0 means not received heartbeat yet
+ LOG(INFO) << "waiting to receive first heartbeat from frontend before doing tablet report";
+ continue;
+ }
+
_is_doing_work = true;
+ // See _random_sleep() comment in _report_disk_state_worker_thread_callback
+ _random_sleep(5);
request.tablets.clear();
uint64_t report_version = _s_report_version;
OLAPStatus build_all_report_tablets_info_status =
@@ -1620,4 +1642,9 @@ void TaskWorkerPool::_handle_report(TReportRequest& request, ReportType type) {
}
}
+void TaskWorkerPool::_random_sleep(int second) {
+ Random rnd(UnixMillis());
+ sleep(rnd.Uniform(second) + 1);
+}
+
} // namespace doris
diff --git a/be/src/agent/task_worker_pool.h b/be/src/agent/task_worker_pool.h
index 4662a48..c451944 100644
--- a/be/src/agent/task_worker_pool.h
+++ b/be/src/agent/task_worker_pool.h
@@ -208,6 +208,9 @@ private:
OLAPStatus _check_migrate_requset(const TStorageMediumMigrateReq& req, TabletSharedPtr& tablet,
DataDir** dest_store);
+ // random sleep 1~second seconds
+ void _random_sleep(int second);
+
private:
std::string _name;
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 04ccb35..37e71d9 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -182,7 +182,8 @@ CONF_mInt64(column_dictionary_key_size_threshold, "0");
// memory_limitation_per_thread_for_schema_change unit GB
CONF_mInt32(memory_limitation_per_thread_for_schema_change, "2");
-CONF_mInt32(file_descriptor_cache_clean_interval, "3600");
+// the clean interval of file descriptor cache and segment cache
+CONF_mInt32(cache_clean_interval, "1800");
CONF_mInt32(disk_stat_monitor_interval, "5");
CONF_mInt32(unused_rowset_monitor_interval, "30");
CONF_String(storage_root_path, "${DORIS_HOME}/storage");
@@ -633,6 +634,11 @@ CONF_mInt32(max_segment_num_per_rowset, "100");
// The connection timeout when connecting to external table such as odbc table.
CONF_mInt32(external_table_connect_timeout_sec, "5");
+// The capacity of lur cache in segment loader.
+// Althought it is called "segment cache", but it caches segments in rowset granularity.
+// So the value of this config should corresponding to the number of rowsets on this BE.
+CONF_mInt32(segment_cache_capacity, "1000000");
+
} // namespace config
} // namespace doris
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index 7da0983..0302ff8 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -121,4 +121,5 @@ add_library(Olap STATIC
task/engine_alter_tablet_task.cpp
olap_snapshot_converter.cpp
column_vector.cpp
+ segment_loader.cpp
)
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index c860d42..0a2ce0f 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -175,7 +175,7 @@ void HandleTable::_resize() {
_length = new_length;
}
-LRUCache::LRUCache() {
+LRUCache::LRUCache(LRUCacheType type) : _type(type) {
// Make empty circular linked list
_lru_normal.next = &_lru_normal;
_lru_normal.prev = &_lru_normal;
@@ -234,7 +234,7 @@ void LRUCache::release(Cache::Handle* handle) {
MutexLock l(&_mutex);
last_ref = _unref(e);
if (last_ref) {
- _usage -= e->charge;
+ _usage -= e->total_size;
} else if (e->in_cache && e->refs == 1) {
// only exists in cache
if (_usage > _capacity) {
@@ -242,7 +242,7 @@ void LRUCache::release(Cache::Handle* handle) {
_table.remove(e);
e->in_cache = false;
_unref(e);
- _usage -= e->charge;
+ _usage -= e->total_size;
last_ref = true;
} else {
// put it to LRU free list
@@ -261,9 +261,9 @@ void LRUCache::release(Cache::Handle* handle) {
}
}
-void LRUCache::_evict_from_lru(size_t charge, LRUHandle** to_remove_head) {
+void LRUCache::_evict_from_lru(size_t total_size, LRUHandle** to_remove_head) {
// 1. evict normal cache entries
- while (_usage + charge > _capacity && _lru_normal.next != &_lru_normal) {
+ while (_usage + total_size > _capacity && _lru_normal.next != &_lru_normal) {
LRUHandle* old = _lru_normal.next;
DCHECK(old->priority == CachePriority::NORMAL);
_evict_one_entry(old);
@@ -271,7 +271,7 @@ void LRUCache::_evict_from_lru(size_t charge, LRUHandle** to_remove_head) {
*to_remove_head = old;
}
// 2. evict durable cache entries if need
- while (_usage + charge > _capacity && _lru_durable.next != &_lru_durable) {
+ while (_usage + total_size > _capacity && _lru_durable.next != &_lru_durable) {
LRUHandle* old = _lru_durable.next;
DCHECK(old->priority == CachePriority::DURABLE);
_evict_one_entry(old);
@@ -287,17 +287,19 @@ void LRUCache::_evict_one_entry(LRUHandle* e) {
_table.remove(e);
e->in_cache = false;
_unref(e);
- _usage -= e->charge;
+ _usage -= e->total_size;
}
Cache::Handle* LRUCache::insert(const CacheKey& key, uint32_t hash, void* value, size_t charge,
void (*deleter)(const CacheKey& key, void* value),
CachePriority priority) {
- LRUHandle* e = reinterpret_cast<LRUHandle*>(malloc(sizeof(LRUHandle) - 1 + key.size()));
+ size_t handle_size = sizeof(LRUHandle) - 1 + key.size();
+ LRUHandle* e = reinterpret_cast<LRUHandle*>(malloc(handle_size));
e->value = value;
e->deleter = deleter;
e->charge = charge;
e->key_length = key.size();
+ e->total_size = (_type == LRUCacheType::SIZE ? handle_size + charge : 1);
e->hash = hash;
e->refs = 2; // one for the returned handle, one for LRUCache.
e->next = e->prev = nullptr;
@@ -310,17 +312,17 @@ Cache::Handle* LRUCache::insert(const CacheKey& key, uint32_t hash, void* value,
// Free the space following strict LRU policy until enough space
// is freed or the lru list is empty
- _evict_from_lru(charge, &to_remove_head);
+ _evict_from_lru(e->total_size, &to_remove_head);
// insert into the cache
// note that the cache might get larger than its capacity if not enough
// space was freed
auto old = _table.insert(e);
- _usage += charge;
+ _usage += e->total_size;
if (old != nullptr) {
old->in_cache = false;
if (_unref(old)) {
- _usage -= old->charge;
+ _usage -= old->total_size;
// old is on LRU because it's in cache and its reference count
// was just 1 (Unref returned 0)
_lru_remove(old);
@@ -350,7 +352,7 @@ void LRUCache::erase(const CacheKey& key, uint32_t hash) {
if (e != nullptr) {
last_ref = _unref(e);
if (last_ref) {
- _usage -= e->charge;
+ _usage -= e->total_size;
if (e->in_cache) {
// locate in free list
_lru_remove(e);
@@ -372,10 +374,10 @@ void LRUCache::_prune_one(LRUHandle* old) {
_table.remove(old);
old->in_cache = false;
_unref(old);
- _usage -= old->charge;
+ _usage -= old->total_size;
}
-int LRUCache::prune() {
+int64_t LRUCache::prune() {
LRUHandle* to_remove_head = nullptr;
{
MutexLock l(&_mutex);
@@ -392,7 +394,47 @@ int LRUCache::prune() {
to_remove_head = old;
}
}
- int pruned_count = 0;
+ int64_t pruned_count = 0;
+ while (to_remove_head != nullptr) {
+ ++pruned_count;
+ LRUHandle* next = to_remove_head->next;
+ to_remove_head->free();
+ to_remove_head = next;
+ }
+ return pruned_count;
+}
+
+int64_t LRUCache::prune_if(bool (*pred)(const void* value)) {
+ LRUHandle* to_remove_head = nullptr;
+ {
+ MutexLock l(&_mutex);
+ LRUHandle* p = _lru_normal.next;
+ while (p != &_lru_normal) {
+ LRUHandle* old = _lru_normal.next;
+ if (pred(old->value)) {
+ _prune_one(old);
+ old->next = to_remove_head;
+ to_remove_head = old;
+ p = _lru_normal.next;
+ } else {
+ p = p->next;
+ }
+ }
+
+ p = _lru_durable.next;
+ while (p != &_lru_durable) {
+ LRUHandle* old = _lru_durable.next;
+ if (pred(old->value)) {
+ _prune_one(old);
+ old->next = to_remove_head;
+ to_remove_head = old;
+ p = _lru_durable.next;
+ } else {
+ p = p->next;
+ }
+ }
+ }
+ int64_t pruned_count = 0;
while (to_remove_head != nullptr) {
++pruned_count;
LRUHandle* next = to_remove_head->next;
@@ -410,12 +452,14 @@ uint32_t ShardedLRUCache::_shard(uint32_t hash) {
return hash >> (32 - kNumShardBits);
}
-ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity, std::shared_ptr<MemTracker> parent)
+ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity,
+ LRUCacheType type, std::shared_ptr<MemTracker> parent)
: _name(name), _last_id(1),
_mem_tracker(MemTracker::CreateTracker(-1, name, parent, true, false, MemTrackerLevel::OVERVIEW)) {
const size_t per_shard = (total_capacity + (kNumShards - 1)) / kNumShards;
for (int s = 0; s < kNumShards; s++) {
- _shards[s].set_capacity(per_shard);
+ _shards[s] = new LRUCache(type);
+ _shards[s]->set_capacity(per_shard);
}
_entity = DorisMetrics::instance()->metric_registry()->register_entity(
@@ -430,6 +474,9 @@ ShardedLRUCache::ShardedLRUCache(const std::string& name, size_t total_capacity,
}
ShardedLRUCache::~ShardedLRUCache() {
+ for (int s = 0; s < kNumShards; s++) {
+ delete _shards[s];
+ }
_entity->deregister_hook(_name);
DorisMetrics::instance()->metric_registry()->deregister_entity(_entity);
_mem_tracker->Release(_mem_tracker->consumption());
@@ -439,22 +486,22 @@ Cache::Handle* ShardedLRUCache::insert(const CacheKey& key, void* value, size_t
void (*deleter)(const CacheKey& key, void* value),
CachePriority priority) {
const uint32_t hash = _hash_slice(key);
- return _shards[_shard(hash)].insert(key, hash, value, charge, deleter, priority);
+ return _shards[_shard(hash)]->insert(key, hash, value, charge, deleter, priority);
}
Cache::Handle* ShardedLRUCache::lookup(const CacheKey& key) {
const uint32_t hash = _hash_slice(key);
- return _shards[_shard(hash)].lookup(key, hash);
+ return _shards[_shard(hash)]->lookup(key, hash);
}
void ShardedLRUCache::release(Handle* handle) {
LRUHandle* h = reinterpret_cast<LRUHandle*>(handle);
- _shards[_shard(h->hash)].release(handle);
+ _shards[_shard(h->hash)]->release(handle);
}
void ShardedLRUCache::erase(const CacheKey& key) {
const uint32_t hash = _hash_slice(key);
- _shards[_shard(hash)].erase(key, hash);
+ _shards[_shard(hash)]->erase(key, hash);
}
void* ShardedLRUCache::value(Handle* handle) {
@@ -470,12 +517,20 @@ uint64_t ShardedLRUCache::new_id() {
return _last_id.fetch_add(1, std::memory_order_relaxed);
}
-void ShardedLRUCache::prune() {
- int num_prune = 0;
+int64_t ShardedLRUCache::prune() {
+ int64_t num_prune = 0;
+ for (int s = 0; s < kNumShards; s++) {
+ num_prune += _shards[s]->prune();
+ }
+ return num_prune;
+}
+
+int64_t ShardedLRUCache::prune_if(bool (*pred)(const void* value)) {
+ int64_t num_prune = 0;
for (int s = 0; s < kNumShards; s++) {
- num_prune += _shards[s].prune();
+ num_prune += _shards[s]->prune_if(pred);
}
- VLOG_DEBUG << "Successfully prune cache, clean " << num_prune << " entries.";
+ return num_prune;
}
void ShardedLRUCache::update_cache_metrics() const {
@@ -484,10 +539,10 @@ void ShardedLRUCache::update_cache_metrics() const {
size_t total_lookup_count = 0;
size_t total_hit_count = 0;
for (int i = 0; i < kNumShards; i++) {
- total_capacity += _shards[i].get_capacity();
- total_usage += _shards[i].get_usage();
- total_lookup_count += _shards[i].get_lookup_count();
- total_hit_count += _shards[i].get_hit_count();
+ total_capacity += _shards[i]->get_capacity();
+ total_usage += _shards[i]->get_usage();
+ total_lookup_count += _shards[i]->get_lookup_count();
+ total_hit_count += _shards[i]->get_hit_count();
}
capacity->set_value(total_capacity);
@@ -501,7 +556,11 @@ void ShardedLRUCache::update_cache_metrics() const {
}
Cache* new_lru_cache(const std::string& name, size_t capacity, std::shared_ptr<MemTracker> parent_tracker) {
- return new ShardedLRUCache(name, capacity, parent_tracker);
+ return new ShardedLRUCache(name, capacity, LRUCacheType::SIZE, parent_tracker);
+}
+
+Cache* new_typed_lru_cache(const std::string& name, size_t capacity, LRUCacheType type, std::shared_ptr<MemTracker> parent_tracker) {
+ return new ShardedLRUCache(name, capacity, type, parent_tracker);
}
} // namespace doris
diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h
index 99bb7ef..94e262d 100644
--- a/be/src/olap/lru_cache.h
+++ b/be/src/olap/lru_cache.h
@@ -48,9 +48,18 @@ namespace doris {
class Cache;
class CacheKey;
-// Create a new cache with a specified name and a fixed size capacity. This implementation
-// of Cache uses a least-recently-used eviction policy.
-extern Cache* new_lru_cache(const std::string& name, size_t capacity, std::shared_ptr<MemTracker> parent_tracekr = nullptr);
+enum LRUCacheType {
+ SIZE, // The capacity of cache is based on the size of cache entry.
+ NUMBER // The capacity of cache is based on the number of cache entry.
+};
+
+// Create a new cache with a specified name and a fixed SIZE capacity.
+// This implementation of Cache uses a least-recently-used eviction policy.
+extern Cache* new_lru_cache(const std::string& name, size_t capacity,
+ std::shared_ptr<MemTracker> parent_tracekr = nullptr);
+
+extern Cache* new_typed_lru_cache(const std::string& name, size_t capacity, LRUCacheType type,
+ std::shared_ptr<MemTracker> parent_tracekr = nullptr);
class CacheKey {
public:
@@ -201,7 +210,13 @@ public:
// Default implementation of Prune() does nothing. Subclasses are strongly
// encouraged to override the default implementation. A future release of
// leveldb may change prune() to a pure abstract method.
- virtual void prune() {}
+ // return num of entries being pruned.
+ virtual int64_t prune() { return 0; }
+
+ // Same as prune(), but the entry will only be pruned if the predicate matched.
+ // NOTICE: the predicate should be simple enough, or the prune_if() function
+ // may hold lock for a long time to execute predicate.
+ virtual int64_t prune_if(bool (*pred)(const void* value)) { return 0; }
private:
DISALLOW_COPY_AND_ASSIGN(Cache);
@@ -218,6 +233,7 @@ typedef struct LRUHandle {
LRUHandle* prev = nullptr; // previous entry in lru list
size_t charge;
size_t key_length;
+ size_t total_size; // including key length
bool in_cache; // Whether entry is in the cache.
uint32_t refs;
uint32_t hash; // Hash of key(); used for fast sharding and comparisons
@@ -287,7 +303,7 @@ private:
// A single shard of sharded cache.
class LRUCache {
public:
- LRUCache();
+ LRUCache(LRUCacheType type);
~LRUCache();
// Separate from constructor so caller can easily make an array of LRUCache
@@ -300,7 +316,8 @@ public:
Cache::Handle* lookup(const CacheKey& key, uint32_t hash);
void release(Cache::Handle* handle);
void erase(const CacheKey& key, uint32_t hash);
- int prune();
+ int64_t prune();
+ int64_t prune_if(bool (*pred)(const void* value));
uint64_t get_lookup_count() const { return _lookup_count; }
uint64_t get_hit_count() const { return _hit_count; }
@@ -311,10 +328,13 @@ private:
void _lru_remove(LRUHandle* e);
void _lru_append(LRUHandle* list, LRUHandle* e);
bool _unref(LRUHandle* e);
- void _evict_from_lru(size_t charge, LRUHandle** to_remove_head);
+ void _evict_from_lru(size_t total_size, LRUHandle** to_remove_head);
void _evict_one_entry(LRUHandle* e);
void _prune_one(LRUHandle* old);
+private:
+ LRUCacheType _type;
+
// Initialized before use.
size_t _capacity = 0;
@@ -340,7 +360,8 @@ static const int kNumShards = 1 << kNumShardBits;
class ShardedLRUCache : public Cache {
public:
- explicit ShardedLRUCache(const std::string& name, size_t total_capacity, std::shared_ptr<MemTracker> parent);
+ explicit ShardedLRUCache(const std::string& name, size_t total_capacity,
+ LRUCacheType type, std::shared_ptr<MemTracker> parent);
// TODO(fdy): 析构时清除所有cache元素
virtual ~ShardedLRUCache();
virtual Handle* insert(const CacheKey& key, void* value, size_t charge,
@@ -352,7 +373,8 @@ public:
virtual void* value(Handle* handle);
Slice value_slice(Handle* handle) override;
virtual uint64_t new_id();
- virtual void prune();
+ virtual int64_t prune();
+ virtual int64_t prune_if(bool (*pred)(const void* value));
private:
void update_cache_metrics() const;
@@ -361,7 +383,7 @@ private:
static uint32_t _shard(uint32_t hash);
std::string _name;
- LRUCache _shards[kNumShards];
+ LRUCache* _shards[kNumShards];
std::atomic<uint64_t> _last_id;
std::shared_ptr<MemTracker> _mem_tracker;
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 851e807..a5a18e2 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -127,7 +127,7 @@ void StorageEngine::_fd_cache_clean_callback() {
#endif
int32_t interval = 600;
while (!_stop_background_threads_latch.wait_for(MonoDelta::FromSeconds(interval))) {
- interval = config::file_descriptor_cache_clean_interval;
+ interval = config::cache_clean_interval;
if (interval <= 0) {
OLAP_LOG_WARNING(
"config of file descriptor clean interval is illegal: [%d], "
@@ -136,7 +136,7 @@ void StorageEngine::_fd_cache_clean_callback() {
interval = 3600;
}
- _start_clean_fd_cache();
+ _start_clean_cache();
}
}
diff --git a/be/src/olap/rowset/alpha_rowset.cpp b/be/src/olap/rowset/alpha_rowset.cpp
index ae8d339..c6bec67 100644
--- a/be/src/olap/rowset/alpha_rowset.cpp
+++ b/be/src/olap/rowset/alpha_rowset.cpp
@@ -30,7 +30,7 @@ AlphaRowset::AlphaRowset(const TabletSchema* schema, std::string rowset_path,
RowsetMetaSharedPtr rowset_meta)
: Rowset(schema, std::move(rowset_path), std::move(rowset_meta)) {}
-OLAPStatus AlphaRowset::do_load(bool use_cache, std::shared_ptr<MemTracker>) {
+OLAPStatus AlphaRowset::do_load(bool use_cache) {
for (auto& segment_group : _segment_groups) {
// validate segment group
if (segment_group->validate() != OLAP_SUCCESS) {
@@ -407,34 +407,4 @@ std::shared_ptr<SegmentGroup> AlphaRowset::_segment_group_with_largest_size() {
return largest_segment_group;
}
-OLAPStatus AlphaRowset::reset_sizeinfo() {
- RETURN_NOT_OK(load());
- std::vector<SegmentGroupPB> segment_group_metas;
- AlphaRowsetMetaSharedPtr alpha_rowset_meta =
- std::dynamic_pointer_cast<AlphaRowsetMeta>(_rowset_meta);
- alpha_rowset_meta->get_segment_groups(&segment_group_metas);
- int32_t segment_group_idx = 0;
- int64_t data_disk_size = 0;
- int64_t index_disk_size = 0;
- int64_t num_rows = 0;
- for (auto segment_group : _segment_groups) {
- segment_group_metas.at(segment_group_idx).set_data_size(segment_group->data_size());
- segment_group_metas.at(segment_group_idx).set_index_size(segment_group->index_size());
- segment_group_metas.at(segment_group_idx).set_num_rows(segment_group->num_rows());
- data_disk_size += segment_group->data_size();
- index_disk_size += segment_group->index_size();
- num_rows += segment_group->num_rows();
- ++segment_group_idx;
- }
- alpha_rowset_meta->clear_segment_group();
- alpha_rowset_meta->set_num_rows(num_rows);
- alpha_rowset_meta->set_data_disk_size(data_disk_size);
- alpha_rowset_meta->set_index_disk_size(index_disk_size);
- alpha_rowset_meta->set_total_disk_size(data_disk_size + index_disk_size);
- for (auto& segment_group_meta : segment_group_metas) {
- alpha_rowset_meta->add_segment_group(segment_group_meta);
- }
- return OLAP_SUCCESS;
-}
-
} // namespace doris
diff --git a/be/src/olap/rowset/alpha_rowset.h b/be/src/olap/rowset/alpha_rowset.h
index 0a2186e..0fc3f71 100644
--- a/be/src/olap/rowset/alpha_rowset.h
+++ b/be/src/olap/rowset/alpha_rowset.h
@@ -67,10 +67,6 @@ public:
bool check_file_exist() override;
- // when convert from old be, should set row num, index size, data size
- // info by using segment's info
- OLAPStatus reset_sizeinfo();
-
protected:
friend class RowsetFactory;
@@ -80,7 +76,7 @@ protected:
// init segment groups
OLAPStatus init() override;
- OLAPStatus do_load(bool use_cache, std::shared_ptr<MemTracker>) override;
+ OLAPStatus do_load(bool use_cache) override;
void do_close() override {}
diff --git a/be/src/olap/rowset/alpha_rowset_reader.cpp b/be/src/olap/rowset/alpha_rowset_reader.cpp
index 9a72dae..8534ad7 100644
--- a/be/src/olap/rowset/alpha_rowset_reader.cpp
+++ b/be/src/olap/rowset/alpha_rowset_reader.cpp
@@ -31,7 +31,7 @@ AlphaRowsetReader::AlphaRowsetReader(int num_rows_per_row_block, AlphaRowsetShar
std::static_pointer_cast<AlphaRowsetMeta>(_rowset->rowset_meta()).get()),
_segment_groups(_rowset->_segment_groups),
_key_range_size(0) {
- _rowset->aquire();
+ _rowset->acquire();
}
AlphaRowsetReader::~AlphaRowsetReader() {
diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp
index 1b26a37..8e228b4 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -44,9 +44,13 @@ OLAPStatus BetaRowset::init() {
return OLAP_SUCCESS; // no op
}
-// `use_cache` is ignored because beta rowset doesn't support fd cache now
-OLAPStatus BetaRowset::do_load(bool /*use_cache*/, std::shared_ptr<MemTracker> parent) {
- // Open all segments under the current rowset
+OLAPStatus BetaRowset::do_load(bool /*use_cache*/) {
+ // do nothing.
+ // the segments in this rowset will be loaded by calling load_segments() explicitly.
+ return OLAP_SUCCESS;
+}
+
+OLAPStatus BetaRowset::load_segments(std::vector<segment_v2::SegmentSharedPtr>* segments) {
for (int seg_id = 0; seg_id < num_segments(); ++seg_id) {
std::string seg_path = segment_file_path(_rowset_path, rowset_id(), seg_id);
std::shared_ptr<segment_v2::Segment> segment;
@@ -56,7 +60,7 @@ OLAPStatus BetaRowset::do_load(bool /*use_cache*/, std::shared_ptr<MemTracker> p
<< " : " << s.to_string();
return OLAP_ERR_ROWSET_LOAD_FAILED;
}
- _segments.push_back(std::move(segment));
+ segments->push_back(std::move(segment));
}
return OLAP_SUCCESS;
}
@@ -108,7 +112,7 @@ OLAPStatus BetaRowset::remove() {
}
void BetaRowset::do_close() {
- _segments.clear();
+ // do nothing.
}
OLAPStatus BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id) {
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index 7300e7f..9db69bf 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -64,6 +64,8 @@ public:
bool check_file_exist() override;
+ OLAPStatus load_segments(std::vector<segment_v2::SegmentSharedPtr>* segments);
+
protected:
BetaRowset(const TabletSchema* schema, std::string rowset_path,
RowsetMetaSharedPtr rowset_meta);
@@ -71,14 +73,13 @@ protected:
// init segment groups
OLAPStatus init() override;
- OLAPStatus do_load(bool use_cache, std::shared_ptr<MemTracker> parent) override;
+ OLAPStatus do_load(bool use_cache) override;
void do_close() override;
private:
friend class RowsetFactory;
friend class BetaRowsetReader;
- std::vector<segment_v2::SegmentSharedPtr> _segments;
};
} // namespace doris
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index f50eb87..1c5e511 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -35,7 +35,7 @@ BetaRowsetReader::BetaRowsetReader(BetaRowsetSharedPtr rowset,
_rowset(std::move(rowset)),
_stats(&_owned_stats),
_parent_tracker(std::move(parent_tracker)) {
- _rowset->aquire();
+ _rowset->acquire();
}
OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
@@ -44,7 +44,7 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
_parent_tracker = read_context->runtime_state->instance_mem_tracker();
}
- RETURN_NOT_OK(_rowset->load(true, _parent_tracker));
+ RETURN_NOT_OK(_rowset->load());
_context = read_context;
if (_context->stats != nullptr) {
// schema change/compaction should use owned_stats
@@ -92,9 +92,12 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
}
read_options.use_page_cache = read_context->use_page_cache;
+ // load segments
+ RETURN_NOT_OK(SegmentLoader::instance()->load_segments(_rowset, &_segment_cache_handle));
+
// create iterator for each segment
std::vector<std::unique_ptr<RowwiseIterator>> seg_iterators;
- for (auto& seg_ptr : _rowset->_segments) {
+ for (auto& seg_ptr : _segment_cache_handle.value()->segments) {
std::unique_ptr<RowwiseIterator> iter;
auto s = seg_ptr->new_iterator(schema, read_options, _parent_tracker, &iter);
if (!s.ok()) {
diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h
index f073a65..34e7d9c 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -24,6 +24,7 @@
#include "olap/row_cursor.h"
#include "olap/rowset/beta_rowset.h"
#include "olap/rowset/rowset_reader.h"
+#include "olap/segment_loader.h"
namespace doris {
@@ -69,6 +70,10 @@ private:
std::unique_ptr<RowBlockV2> _input_block;
std::unique_ptr<RowBlock> _output_block;
std::unique_ptr<RowCursor> _row;
+
+ // make sure this handle is initialized and valid before
+ // reading data.
+ SegmentCacheHandle _segment_cache_handle;
};
} // namespace doris
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index 5e1237d..385657b 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -36,7 +36,7 @@ Rowset::Rowset(const TabletSchema* schema, std::string rowset_path, RowsetMetaSh
}
}
-OLAPStatus Rowset::load(bool use_cache, std::shared_ptr<MemTracker> parent) {
+OLAPStatus Rowset::load(bool use_cache) {
// if the state is ROWSET_UNLOADING it means close() is called
// and the rowset is already loaded, and the resource is not closed yet.
if (_rowset_state_machine.rowset_state() == ROWSET_LOADED) {
@@ -48,7 +48,7 @@ OLAPStatus Rowset::load(bool use_cache, std::shared_ptr<MemTracker> parent) {
// after lock, if rowset state is ROWSET_UNLOADING, it is ok to return
if (_rowset_state_machine.rowset_state() == ROWSET_UNLOADED) {
// first do load, then change the state
- RETURN_NOT_OK(do_load(use_cache, parent));
+ RETURN_NOT_OK(do_load(use_cache));
RETURN_NOT_OK(_rowset_state_machine.on_load());
}
}
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 0f99efc..19bdf7b 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -113,7 +113,7 @@ public:
//
// May be called multiple times, subsequent calls will no-op.
// Derived class implements the load logic by overriding the `do_load_once()` method.
- OLAPStatus load(bool use_cache = true, std::shared_ptr<MemTracker> parent = nullptr);
+ OLAPStatus load(bool use_cache = true);
// returns OLAP_ERR_ROWSET_CREATE_READER when failed to create reader
virtual OLAPStatus create_reader(std::shared_ptr<RowsetReader>* result) = 0;
@@ -224,7 +224,7 @@ public:
}
// this function is called by reader to increase reference of rowset
- void aquire() { ++_refs_by_reader; }
+ void acquire() { ++_refs_by_reader; }
void release() {
// if the refs by reader is 0 and the rowset is closed, should release the resouce
@@ -260,7 +260,7 @@ protected:
virtual OLAPStatus init() = 0;
// The actual implementation of load(). Guaranteed by to called exactly once.
- virtual OLAPStatus do_load(bool use_cache, std::shared_ptr<MemTracker> parent = nullptr) = 0;
+ virtual OLAPStatus do_load(bool use_cache) = 0;
// release resources in this api
virtual void do_close() = 0;
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
new file mode 100644
index 0000000..3a50dc9
--- /dev/null
+++ b/be/src/olap/segment_loader.cpp
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/segment_loader.h"
+
+#include "olap/rowset/rowset.h"
+#include "util/stopwatch.hpp"
+
+namespace doris {
+
+SegmentLoader* SegmentLoader::_s_instance = nullptr;
+
+void SegmentLoader::create_global_instance(size_t capacity) {
+ DCHECK(_s_instance == nullptr);
+ static SegmentLoader instance(capacity);
+ _s_instance = &instance;
+}
+
+SegmentLoader::SegmentLoader(size_t capacity)
+ : _mem_tracker(MemTracker::CreateTracker(capacity, "SegmentLoader", nullptr, true, true, MemTrackerLevel::OVERVIEW)) {
+ _cache = std::unique_ptr<Cache>(new_typed_lru_cache("SegmentCache", capacity, LRUCacheType::NUMBER, _mem_tracker));
+}
+
+bool SegmentLoader::_lookup(const SegmentLoader::CacheKey& key, SegmentCacheHandle* handle) {
+ auto lru_handle = _cache->lookup(key.encode());
+ if (lru_handle == nullptr) {
+ return false;
+ }
+ *handle = SegmentCacheHandle(_cache.get(), lru_handle);
+ return true;
+}
+
+void SegmentLoader::_insert(const SegmentLoader::CacheKey& key, SegmentLoader::CacheValue& value, SegmentCacheHandle* handle) {
+ auto deleter = [](const doris::CacheKey& key, void* value) {
+ SegmentLoader::CacheValue* cache_value = (SegmentLoader::CacheValue*) value;
+ cache_value->segments.clear();
+ delete cache_value;
+ };
+
+ auto lru_handle = _cache->insert(key.encode(), &value, sizeof(SegmentLoader::CacheValue), deleter, CachePriority::NORMAL);
+ *handle = SegmentCacheHandle(_cache.get(), lru_handle);
+}
+
+OLAPStatus SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
+ SegmentCacheHandle* cache_handle) {
+ SegmentCacheHandle handle;
+ SegmentLoader::CacheKey cache_key(rowset->rowset_id());
+ if (_lookup(cache_key, &handle)) {
+ *cache_handle = std::move(handle);
+ return OLAP_SUCCESS;
+ }
+
+ std::vector<segment_v2::SegmentSharedPtr> segments;
+ RETURN_NOT_OK(rowset->load_segments(&segments));
+
+ // memory of SegmentLoader::CacheValue will be handled by SegmentLoader
+ SegmentLoader::CacheValue* cache_value = new SegmentLoader::CacheValue();
+ cache_value->segments = std::move(segments);
+ _insert(cache_key, *cache_value, &handle);
+ *cache_handle = std::move(handle);
+
+ return OLAP_SUCCESS;
+}
+
+OLAPStatus SegmentLoader::prune() {
+ bool (*pred)(const void* value) = [](const void* value) -> bool {
+ int64_t curtime = UnixMillis();
+ SegmentLoader::CacheValue* cache_value = (SegmentLoader::CacheValue*) value;
+ return curtime - cache_value->last_visit_time > config::tablet_rowset_stale_sweep_time_sec * 1000;
+ };
+
+ MonotonicStopWatch watch;
+ watch.start();
+ int64_t prune_num = _cache->prune_if(pred);
+ LOG(INFO) << "prune " << prune_num << " entries in segment cache. cost(ms): "
+ << watch.elapsed_time() / 1000 / 1000;
+ return OLAP_SUCCESS;
+}
+
+} // namespace doris
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
new file mode 100644
index 0000000..8a2bba4
--- /dev/null
+++ b/be/src/olap/segment_loader.h
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "gutil/macros.h" // for DISALLOW_COPY_AND_ASSIGN
+#include "olap/lru_cache.h"
+#include "olap/olap_common.h" // for rowset id
+#include "olap/rowset/beta_rowset.h"
+#include "runtime/mem_tracker.h"
+#include "util/time.h"
+
+namespace doris {
+
+class SegmentCacheHandle;
+
+// SegmentLoader is used to load the Segment of BetaRowset.
+// An LRUCache is encapsulated inside it, which is used to cache the opened segments.
+// The caller should use the following method to load and obtain
+// the segments of a specified rowset:
+//
+// SegmentCacheHandle cache_handle;
+// RETURN_NOT_OK(SegmentCache::instance()->load_segments(_rowset, &cache_handle));
+// for (auto& seg_ptr : cache_handle.value()->segments) {
+// ... visit segment ...
+// }
+//
+// Make sure that cache_handle is valid during the segment usage period.
+using BetaRowsetSharedPtr = std::shared_ptr<BetaRowset>;
+class SegmentLoader {
+public:
+
+ // The cache key or segment lru cache
+ struct CacheKey {
+ CacheKey(RowsetId rowset_id_) : rowset_id(rowset_id_) {}
+ RowsetId rowset_id;
+
+ // Encode to a flat binary which can be used as LRUCache's key
+ std::string encode() const {
+ return rowset_id.to_string();
+ }
+ };
+
+ // The cache value of segment lru cache.
+ // Holding all opened segments of a rowset.
+ struct CacheValue {
+ // Save the last visit time of this cache entry.
+ // Use atomic because it may be modified by multi threads.
+ std::atomic<int64_t> last_visit_time = 0;
+ std::vector<segment_v2::SegmentSharedPtr> segments;
+ };
+
+ // Create global instance of this class.
+ // "capacity" is the capacity of lru cache.
+ // TODO: Currently we use the number of rowset as the cache capacity.
+ // That is, the limit of cache is the number of rowset.
+ // This is because currently we cannot accurately estimate the memory occupied by a segment.
+ // After the estimation of segment memory usage is provided later, it is recommended
+ // to use Memory as the capacity limit of the cache.
+ static void create_global_instance(size_t capacity);
+
+ // Return global instance.
+ // Client should call create_global_cache before.
+ static SegmentLoader* instance() { return _s_instance; }
+
+ SegmentLoader(size_t capacity);
+
+ // Load segments of "rowset" from _cache, return the "cache_handle" which contains segments
+ OLAPStatus load_segments(const BetaRowsetSharedPtr& rowset, SegmentCacheHandle* cache_handle);
+
+ // Try to prune the segment cache if expired.
+ OLAPStatus prune();
+
+private:
+ SegmentLoader();
+
+ // Lookup the given rowset in the cache.
+ // If the rowset is found, the cache entry will be written into handle.
+ // Return true if entry is found, otherwise return false.
+ bool _lookup(const SegmentLoader::CacheKey& key, SegmentCacheHandle* handle);
+
+ // Insert a cache entry by key.
+ // And the cache entry will be returned in handle.
+ // This function is thread-safe.
+ void _insert(const SegmentLoader::CacheKey& key, CacheValue& value, SegmentCacheHandle* handle);
+
+private:
+ static SegmentLoader* _s_instance;
+ // A LRU cache to cache all opened segments
+ std::unique_ptr<Cache> _cache = nullptr;
+ std::shared_ptr<MemTracker> _mem_tracker = nullptr;
+};
+
+// A handle for a single rowset from segment lru cache.
+// The handle can ensure that the segment is valid
+// and will not be closed while the holder of the handle is accessing the segment.
+// The handle will automatically release the cache entry when it is destroyed.
+// So the caller need to make sure the handle is valid in lifecycle.
+class SegmentCacheHandle {
+public:
+ SegmentCacheHandle() {}
+ SegmentCacheHandle(Cache* cache, Cache::Handle* handle) : _cache(cache), _handle(handle) {}
+
+ ~SegmentCacheHandle() {
+ if (_handle != nullptr) {
+ // last_visit_time is set when release.
+ // because it only be needed when pruning.
+ value()->last_visit_time = UnixMillis();
+ _cache->release(_handle);
+ }
+ }
+
+ SegmentCacheHandle(SegmentCacheHandle&& other) noexcept {
+ std::swap(_cache, other._cache);
+ std::swap(_handle, other._handle);
+ }
+
+ SegmentCacheHandle& operator=(SegmentCacheHandle&& other) noexcept {
+ std::swap(_cache, other._cache);
+ std::swap(_handle, other._handle);
+ return *this;
+ }
+
+ SegmentLoader::CacheValue* value() { return (SegmentLoader::CacheValue*) _cache->value(_handle); }
+
+private:
+ Cache* _cache = nullptr;
+ Cache::Handle* _handle = nullptr;
+
+ // Don't allow copy and assign
+ DISALLOW_COPY_AND_ASSIGN(SegmentCacheHandle);
+};
+
+} // namespace doris
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 812725f..9db0fa7 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -241,7 +241,7 @@ OLAPStatus SnapshotManager::_rename_rowset_id(const RowsetMetaPB& rs_meta_pb,
LOG(WARNING) << "failed to build rowset when rename rowset id";
return OLAP_ERR_MALLOC_ERROR;
}
- RETURN_NOT_OK(new_rowset->load());
+ RETURN_NOT_OK(new_rowset->load(false));
new_rowset->rowset_meta()->to_rowset_pb(new_rs_meta_pb);
org_rowset->remove();
return OLAP_SUCCESS;
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 4983c78..8e6aab6 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -51,6 +51,7 @@
#include "olap/rowset/rowset_meta_manager.h"
#include "olap/rowset/unique_rowset_id_generator.h"
#include "olap/schema_change.h"
+#include "olap/segment_loader.h"
#include "olap/tablet_meta.h"
#include "olap/tablet_meta_manager.h"
#include "olap/utils.h"
@@ -629,10 +630,9 @@ void StorageEngine::clear_transaction_task(const TTransactionId transaction_id,
LOG(INFO) << "finish to clear transaction task. transaction_id=" << transaction_id;
}
-void StorageEngine::_start_clean_fd_cache() {
- VLOG_TRACE << "start clean file descritpor cache";
+void StorageEngine::_start_clean_cache() {
_file_cache->prune();
- VLOG_TRACE << "end clean file descritpor cache";
+ SegmentLoader::instance()->prune();
}
OLAPStatus StorageEngine::start_trash_sweep(double* usage, bool ignore_guard) {
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index cfe7e87..55d56a9 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -240,7 +240,7 @@ private:
// parse the default rowset type config to RowsetTypePB
void _parse_default_rowset_type();
- void _start_clean_fd_cache();
+ void _start_clean_cache();
// 磁盘状态监测。监测unused_flag路劲新的对应root_path unused标识位，
// 当检测到有unused标识时，从内存中删除对应表信息，磁盘数据不动。
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 69d70cd..e7947fa 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -26,6 +26,7 @@
#include "gen_cpp/TExtDataSourceService.h"
#include "gen_cpp/TPaloBrokerService.h"
#include "olap/page_cache.h"
+#include "olap/segment_loader.h"
#include "olap/storage_engine.h"
#include "plugin/plugin_mgr.h"
#include "runtime/broker_mgr.h"
@@ -229,6 +230,8 @@ Status ExecEnv::_init_mem_tracker() {
LOG(INFO) << "Storage page cache memory limit: " << PrettyPrinter::print(storage_cache_limit, TUnit::BYTES)
<< ", origin config value: " << config::storage_page_cache_limit;
+ SegmentLoader::create_global_instance(config::segment_cache_capacity);
+
// 4. init other managers
RETURN_IF_ERROR(_disk_io_mgr->init(_mem_tracker));
RETURN_IF_ERROR(_tmp_file_mgr->init());
diff --git a/be/test/olap/lru_cache_test.cpp b/be/test/olap/lru_cache_test.cpp
index b2b62ac..1456365 100644
--- a/be/test/olap/lru_cache_test.cpp
+++ b/be/test/olap/lru_cache_test.cpp
@@ -68,7 +68,10 @@ public:
_s_current->_deleted_values.push_back(DecodeValue(v));
}
- static const int kCacheSize = 1000;
+ // there is 16 shards in ShardedLRUCache
+ // And the LRUHandle size is about 100B. So the cache size should big enough
+ // to run the UT.
+ static const int kCacheSize = 1000 * 16;
std::vector<int> _deleted_keys;
std::vector<int> _deleted_values;
Cache* _cache;
@@ -92,7 +95,7 @@ public:
void Insert(int key, int value, int charge) {
std::string result;
_cache->release(_cache->insert(EncodeKey(&result, key), EncodeValue(value), charge,
- &CacheTest::Deleter));
+ &CacheTest::Deleter));
}
void InsertDurable(int key, int value, int charge) {
@@ -158,6 +161,9 @@ TEST_F(CacheTest, EntriesArePinned) {
Insert(100, 101, 1);
std::string result1;
Cache::Handle* h1 = _cache->lookup(EncodeKey(&result1, 100));
+ if (h1 == nullptr) {
+ std::cout << "h1 is null" << std::endl;
+ }
ASSERT_EQ(101, DecodeValue(_cache->value(h1)));
Insert(100, 102, 1);
@@ -224,36 +230,88 @@ static void insert_LRUCache(LRUCache& cache, const CacheKey& key, int value,
}
TEST_F(CacheTest, Usage) {
- LRUCache cache;
- cache.set_capacity(1000);
+ LRUCache cache(LRUCacheType::SIZE);
+ cache.set_capacity(1050);
+ // The lru usage is handle_size + charge = 96 - 1 = 95
+ // 95 + 3 means handle_size + key size
CacheKey key1("100");
insert_LRUCache(cache, key1, 100, CachePriority::NORMAL);
- ASSERT_EQ(100, cache.get_usage());
+ ASSERT_EQ(198, cache.get_usage()); // 100 + 95 + 3
CacheKey key2("200");
insert_LRUCache(cache, key2, 200, CachePriority::DURABLE);
- ASSERT_EQ(300, cache.get_usage());
+ ASSERT_EQ(496, cache.get_usage()); // 198 + 200 + 95 + 3
CacheKey key3("300");
insert_LRUCache(cache, key3, 300, CachePriority::NORMAL);
- ASSERT_EQ(600, cache.get_usage());
+ ASSERT_EQ(894, cache.get_usage()); // 496 + 300 + 95 + 3
CacheKey key4("400");
insert_LRUCache(cache, key4, 400, CachePriority::NORMAL);
- ASSERT_EQ(1000, cache.get_usage());
+ ASSERT_EQ(796, cache.get_usage()); // 894 + 400 + 95 + 3 - (300 + 100 + (95 + 3) * 2)
CacheKey key5("500");
insert_LRUCache(cache, key5, 500, CachePriority::NORMAL);
- ASSERT_EQ(700, cache.get_usage());
+ ASSERT_EQ(896, cache.get_usage()); // 796 + 500 + 95 + 3 - (400 + 95 +3)
CacheKey key6("600");
insert_LRUCache(cache, key6, 600, CachePriority::NORMAL);
- ASSERT_EQ(800, cache.get_usage());
+ ASSERT_EQ(996, cache.get_usage()); // 896 + 600 + 95 +3 - (500 + 95 + 3)
CacheKey key7("950");
insert_LRUCache(cache, key7, 950, CachePriority::DURABLE);
- ASSERT_EQ(950, cache.get_usage());
+ ASSERT_EQ(1048, cache.get_usage()); // 996 + 950 + 95 +3 - (200 + 600 + (95 + 3) * 2)
+}
+
+TEST_F(CacheTest, Prune) {
+ LRUCache cache(LRUCacheType::NUMBER);
+ cache.set_capacity(5);
+
+ // The lru usage is handle_size + charge = 96 - 1 = 95
+ // 95 + 3 means handle_size + key size
+ CacheKey key1("100");
+ insert_LRUCache(cache, key1, 100, CachePriority::NORMAL);
+ ASSERT_EQ(1, cache.get_usage());
+
+ CacheKey key2("200");
+ insert_LRUCache(cache, key2, 200, CachePriority::DURABLE);
+ ASSERT_EQ(2, cache.get_usage());
+
+ CacheKey key3("300");
+ insert_LRUCache(cache, key3, 300, CachePriority::NORMAL);
+ ASSERT_EQ(3, cache.get_usage());
+
+ CacheKey key4("400");
+ insert_LRUCache(cache, key4, 400, CachePriority::NORMAL);
+ ASSERT_EQ(4, cache.get_usage());
+
+ CacheKey key5("500");
+ insert_LRUCache(cache, key5, 500, CachePriority::NORMAL);
+ ASSERT_EQ(5, cache.get_usage());
+
+ CacheKey key6("600");
+ insert_LRUCache(cache, key6, 600, CachePriority::NORMAL);
+ ASSERT_EQ(5, cache.get_usage());
+
+ CacheKey key7("700");
+ insert_LRUCache(cache, key7, 700, CachePriority::DURABLE);
+ ASSERT_EQ(5, cache.get_usage());
+
+ auto pred = [](const void* value) -> bool {
+ return false;
+ };
+ cache.prune_if(pred);
+ ASSERT_EQ(5, cache.get_usage());
+
+ auto pred2 = [](const void* value) -> bool {
+ return true;
+ };
+ cache.prune_if(pred2);
+ ASSERT_EQ(0, cache.get_usage());
+
+ cache.prune();
+ ASSERT_EQ(0, cache.get_usage());
}
TEST_F(CacheTest, HeavyEntries) {
@@ -319,6 +377,7 @@ TEST(CacheHandleTest, HandleTableTest) {
h->value = nullptr;
h->deleter = nullptr;
h->charge = 1;
+ h->total_size = sizeof(LRUHandle) - 1 + key->size() + 1;
h->key_length = key->size();
h->hash = 1; // make them in a same hash table linked-list
h->refs = 0;
@@ -355,6 +414,7 @@ TEST(CacheHandleTest, HandleTableTest) {
h->value = nullptr;
h->deleter = nullptr;
h->charge = 1;
+ h->total_size = sizeof(LRUHandle) - 1 + key->size() + 1;
h->key_length = key->size();
h->hash = 1; // make them in a same hash table linked-list
h->refs = 0;
diff --git a/be/test/olap/rowset/beta_rowset_test.cpp b/be/test/olap/rowset/beta_rowset_test.cpp
index 6191e6f..54973c7 100644
--- a/be/test/olap/rowset/beta_rowset_test.cpp
+++ b/be/test/olap/rowset/beta_rowset_test.cpp
@@ -354,6 +354,7 @@ TEST_F(BetaRowsetTest, BasicFunctionTest) {
int main(int argc, char** argv) {
doris::StoragePageCache::create_global_cache(1 << 30, 0.1);
+ doris::SegmentLoader::create_global_instance(1000);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
diff --git a/be/test/olap/rowset/rowset_converter_test.cpp b/be/test/olap/rowset/rowset_converter_test.cpp
index fece436..bb3f193 100644
--- a/be/test/olap/rowset/rowset_converter_test.cpp
+++ b/be/test/olap/rowset/rowset_converter_test.cpp
@@ -298,6 +298,7 @@ TEST_F(RowsetConverterTest, TestConvertBetaRowsetToAlpha) {
int main(int argc, char** argv) {
doris::StoragePageCache::create_global_cache(1 << 30, 0.1);
+ doris::SegmentLoader::create_global_instance(1000);
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md
index 359be42..4672101 100644
--- a/docs/en/administrator-guide/config/be_config.md
+++ b/docs/en/administrator-guide/config/be_config.md
@@ -582,11 +582,12 @@ Default：32768
File handle cache capacity, 32768 file handles are cached by default.
-### `file_descriptor_cache_clean_interval`
+### `cache_clean_interval`
-Default：3600（s）
+Default：1800(s)
File handle cache cleaning interval, used to clean up file handles that have not been used for a long time.
+Also the clean interval of Segment Cache.
### `flush_thread_num_per_store`
@@ -1447,3 +1448,11 @@ Increasing this value can reduce the number of calls to read remote data, but it
* Type: int32
* Description: The timeout when establishing connection with external table such as ODBC table.
* Default value: 5 seconds
+
+### `segment_cache_capacity`
+
+* Type: int32
+* Description: The maximum number of Segments cached by Segment Cache.
+* Default value: 1000000
+
+The default value is currently only an empirical value, and may need to be modified according to actual scenarios. Increasing this value can cache more segments and avoid some IO. Decreasing this value will reduce memory usage.
diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md
index 31b9bb0..fc4d893 100644
--- a/docs/zh-CN/administrator-guide/config/be_config.md
+++ b/docs/zh-CN/administrator-guide/config/be_config.md
@@ -580,11 +580,12 @@ ETL线程池的大小
文件句柄缓存的容量，默认缓存32768个文件句柄
-### `file_descriptor_cache_clean_interval`
+### `cache_clean_interval`
-默认值：3600 (s)
+默认值：1800 (s)
-文件句柄缓存清理的间隔，用于清理长期不用的文件句柄
+文件句柄缓存清理的间隔，用于清理长期不用的文件句柄。
+同时也是Segment Cache的清理间隔时间。
### `flush_thread_num_per_store`
@@ -1468,3 +1469,11 @@ webserver默认工作线程数
* 类型: int32
* 描述: 和外部表建立连接的超时时间。
* 默认值: 5秒
+
+### `segment_cache_capacity`
+
+* 类型: int32
+* 描述: Segment Cache 缓存的 Segment 最大数量
+* 默认值: 1000000
+
+默认值目前只是一个经验值，可能需要根据实际场景修改。增大该值可以缓存更多的segment从而避免一些IO。减少该值则会降低内存使用。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
index b83fa13..f2f03a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java
@@ -151,7 +151,7 @@ public class RoutineLoadTaskScheduler extends MasterDaemon {
routineLoadManager.getJob(routineLoadTaskInfo.getJobId()).updateState(JobState.PAUSED,
new ErrorReason(InternalErrorCode.CREATE_TASKS_ERR, "failed to allocate task: " + e.getMessage()), false);
LOG.warn(new LogBuilder(LogKey.ROUTINE_LOAD_TASK, routineLoadTaskInfo.getId()).add("error_msg",
- "allocate task encounter exception: " + e.getMessage()).build());
+ "allocate task encounter exception: " + e.getMessage()).build(), e);
throw e;
}
@@ -170,9 +170,9 @@ public class RoutineLoadTaskScheduler extends MasterDaemon {
routineLoadTaskInfo.setBeId(-1);
routineLoadManager.getJob(routineLoadTaskInfo.getJobId()).updateState(JobState.PAUSED,
new ErrorReason(InternalErrorCode.CREATE_TASKS_ERR,
- "failed to allocate task for txn: " + e.getMessage()), false);
+ "failed to begin txn: " + e.getMessage()), false);
LOG.warn(new LogBuilder(LogKey.ROUTINE_LOAD_TASK, routineLoadTaskInfo.getId()).add("error_msg",
- "begin task txn encounter exception: " + e.getMessage()).build());
+ "begin task txn encounter exception: " + e.getMessage()).build(), e);
throw e;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java
index e78c515..cc2cb6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/PushTask.java
@@ -195,7 +195,7 @@ public class PushTask extends AgentTask {
public void countDownLatch(long backendId, long tabletId) {
if (this.latch != null) {
if (latch.markedCountDown(backendId, tabletId)) {
- LOG.info("pushTask current latch count: {}. backend: {}, tablet:{}",
+ LOG.debug("pushTask current latch count: {}. backend: {}, tablet:{}",
latch.getCount(), backendId, tabletId);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 19/21: [Benchmark] Add star schema benchmark
tools (#6925)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit dd9dfba7ab4aa082efdace58248e59a02b95252f
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Wed Oct 27 09:55:36 2021 +0800
[Benchmark] Add star schema benchmark tools (#6925)
This CL mainly changes:
1. Add star schema benchmark tools in `tools/ssb-tools`, for user to easy load and test with SSB data set.
2. Disable the segment cache for some read scenario such as compaction and alter operation.(Fix #6924 )
3. Fix a bug that `max_segment_num_per_rowset` won't work(Fix #6926)
4. Enable `enable_batch_delete_by_default` by default.
---
be/src/agent/cgroups_mgr.cpp | 4 +-
be/src/common/config.h | 2 +-
be/src/exec/tablet_sink.cpp | 32 ++--
be/src/exec/tablet_sink.h | 12 +-
be/src/olap/delta_writer.cpp | 3 +
be/src/olap/rowset/beta_rowset_reader.cpp | 5 +-
be/src/olap/segment_loader.cpp | 24 +--
be/src/olap/segment_loader.h | 27 +++-
docs/.vuepress/sidebar/en.js | 3 +-
docs/.vuepress/sidebar/zh-CN.js | 3 +-
docs/en/administrator-guide/config/be_config.md | 2 +-
docs/en/best-practices/star-schema-benchmark.md | 163 ++++++++++++++++++++
docs/zh-CN/administrator-guide/config/be_config.md | 2 +-
docs/zh-CN/best-practices/star-schema-benchmark.md | 164 +++++++++++++++++++++
.../main/java/org/apache/doris/common/Config.java | 2 +-
tools/ssb-tools/build-ssb-dbgen.sh | 53 +++++++
tools/ssb-tools/create-tables.sql | 117 +++++++++++++++
tools/ssb-tools/doris-cluster.conf | 27 ++++
tools/ssb-tools/gen-ssb-data.sh | 112 ++++++++++++++
tools/ssb-tools/load-dimension-data.sh | 68 +++++++++
tools/ssb-tools/load-fact-data.sh | 126 ++++++++++++++++
tools/ssb-tools/queries/q1.1.sql | 24 +++
tools/ssb-tools/queries/q1.2.sql | 24 +++
tools/ssb-tools/queries/q1.3.sql | 25 ++++
tools/ssb-tools/queries/q2.1.sql | 26 ++++
tools/ssb-tools/queries/q2.2.sql | 27 ++++
tools/ssb-tools/queries/q2.3.sql | 26 ++++
tools/ssb-tools/queries/q3.1.sql | 28 ++++
tools/ssb-tools/queries/q3.2.sql | 28 ++++
tools/ssb-tools/queries/q3.3.sql | 30 ++++
tools/ssb-tools/queries/q3.4.sql | 30 ++++
tools/ssb-tools/queries/q4.1.sql | 30 ++++
tools/ssb-tools/queries/q4.2.sql | 31 ++++
tools/ssb-tools/queries/q4.3.sql | 29 ++++
34 files changed, 1259 insertions(+), 50 deletions(-)
diff --git a/be/src/agent/cgroups_mgr.cpp b/be/src/agent/cgroups_mgr.cpp
index edb274f..ffcbe21 100644
--- a/be/src/agent/cgroups_mgr.cpp
+++ b/be/src/agent/cgroups_mgr.cpp
@@ -325,8 +325,8 @@ AgentStatus CgroupsMgr::init_cgroups() {
_is_cgroups_init_success = true;
return AgentStatus::DORIS_SUCCESS;
} else {
- LOG(WARNING) << "Could not find a valid cgroups path for resource isolation,"
- << "current value is " << _root_cgroups_path << ". ignore it.";
+ VLOG_NOTICE << "Could not find a valid cgroups path for resource isolation,"
+ << "current value is " << _root_cgroups_path << ". ignore it.";
_is_cgroups_init_success = false;
return AgentStatus::DORIS_ERROR;
}
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 37e71d9..42c43fb 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -629,7 +629,7 @@ CONF_Int32(send_batch_thread_pool_queue_size, "102400");
// so if there are too many segment in a rowset, the compaction process
// will run out of memory.
// When doing compaction, each segment may take at least 1MB buffer.
-CONF_mInt32(max_segment_num_per_rowset, "100");
+CONF_mInt32(max_segment_num_per_rowset, "200");
// The connection timeout when connecting to external table such as odbc table.
CONF_mInt32(external_table_connect_timeout_sec, "5");
diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp
index d282058..c5c02d2 100644
--- a/be/src/exec/tablet_sink.cpp
+++ b/be/src/exec/tablet_sink.cpp
@@ -163,10 +163,7 @@ Status NodeChannel::open_wait() {
// add batch closure
_add_batch_closure = ReusableClosure<PTabletWriterAddBatchResult>::create();
_add_batch_closure->addFailedHandler([this]() {
- std::stringstream ss;
- ss << name() << " add batch req rpc failed, " << print_load_info()
- << ", node=" << node_info()->id << ":" << node_info()->brpc_port;
- _cancel_with_msg(ss.str());
+ _cancel_with_msg(fmt::format("{}, err: {}", channel_info(), _add_batch_closure->cntl.ErrorText()));
});
_add_batch_closure->addSuccessHandler([this](const PTabletWriterAddBatchResult& result,
@@ -183,14 +180,7 @@ Status NodeChannel::open_wait() {
_add_batches_finished = true;
}
} else {
- std::stringstream ss;
- // FIXME(cmy): There is a problem that when calling node_info, the node_info seems not initialized.
- // But I don't know why. so here I print node_info()->id instead of node_info()->host
- // to avoid BE crash. It needs further observation.
- ss << name() << " add batch req success but status isn't ok, " << print_load_info()
- << ", backend id=" << node_info()->id << ":" << node_info()->brpc_port
- << ", errmsg=" << status.get_error_msg();
- _cancel_with_msg(ss.str());
+ _cancel_with_msg(fmt::format("{}, add batch req success but status isn't ok, err: {}", channel_info(), status.get_error_msg()));
}
if (result.has_execution_time_us()) {
@@ -317,10 +307,10 @@ Status NodeChannel::close_wait(RuntimeState* state) {
return Status::InternalError(ss.str());
}
-void NodeChannel::cancel() {
+void NodeChannel::cancel(const std::string& cancel_msg) {
// we don't need to wait last rpc finished, cause closure's release/reset will join.
// But do we need brpc::StartCancel(call_id)?
- _cancelled = true;
+ _cancel_with_msg(cancel_msg);
PTabletWriterCancelRequest request;
request.set_allocated_id(&_parent->_load_id);
@@ -384,7 +374,7 @@ void NodeChannel::try_send_batch() {
int remain_ms = _rpc_timeout_ms - _timeout_watch.elapsed_time() / NANOS_PER_MILLIS;
if (UNLIKELY(remain_ms < _min_rpc_timeout_ms)) {
if (remain_ms <= 0 && !request.eos()) {
- cancel();
+ cancel(fmt::format("{}, err: timeout", channel_info()));
} else {
remain_ms = _min_rpc_timeout_ms;
}
@@ -672,12 +662,10 @@ Status OlapTableSink::open(RuntimeState* state) {
auto st = ch->open_wait();
if (!st.ok()) {
std::stringstream err;
- err << ch->name() << ": tablet open failed, " << ch->print_load_info()
- << ", node=" << ch->node_info()->host << ":" << ch->node_info()->brpc_port
- << ", errmsg=" << st.get_error_msg();
+ err << ch->channel_info() << ", tablet open failed, err: " << st.get_error_msg();
LOG(WARNING) << err.str();
- index_channel->mark_as_failed(ch);
ss << err.str() << "; ";
+ index_channel->mark_as_failed(ch);
}
});
@@ -786,9 +774,7 @@ Status OlapTableSink::close(RuntimeState* state, Status close_status) {
if (!s.ok()) {
// 'status' will store the last non-ok status of all channels
status = s;
- LOG(WARNING) << ch->name() << ": close channel failed, "
- << ch->print_load_info()
- << ". error_msg=" << s.get_error_msg();
+ LOG(WARNING) << ch->channel_info() << ", close channel failed, err: " << s.get_error_msg();
}
ch->time_report(&node_add_batch_counter_map, &serialize_batch_ns,
&mem_exceeded_block_ns, &queue_push_lock_ns,
@@ -837,7 +823,7 @@ Status OlapTableSink::close(RuntimeState* state, Status close_status) {
LOG(INFO) << ss.str();
} else {
for (auto channel : _channels) {
- channel->for_each_node_channel([](NodeChannel* ch) { ch->cancel(); });
+ channel->for_each_node_channel([&status](NodeChannel* ch) { ch->cancel(status.get_error_msg()); });
}
}
diff --git a/be/src/exec/tablet_sink.h b/be/src/exec/tablet_sink.h
index 277ecef..9b58548 100644
--- a/be/src/exec/tablet_sink.h
+++ b/be/src/exec/tablet_sink.h
@@ -17,6 +17,7 @@
#pragma once
+#include <fmt/format.h>
#include <memory>
#include <queue>
#include <set>
@@ -164,7 +165,7 @@ public:
Status mark_close();
Status close_wait(RuntimeState* state);
- void cancel();
+ void cancel(const std::string& cancel_msg);
// return:
// 0: stopped, send finished(eos request has been sent), or any internal error;
@@ -191,8 +192,6 @@ public:
}
int64_t node_id() const { return _node_id; }
- const NodeInfo* node_info() const { return _node_info; }
- std::string print_load_info() const { return _load_info; }
std::string name() const { return _name; }
Status none_of(std::initializer_list<bool> vars);
@@ -200,6 +199,13 @@ public:
// TODO(HW): remove after mem tracker shared
void clear_all_batches();
+ std::string channel_info() const {
+ // FIXME(cmy): There is a problem that when calling node_info, the node_info seems not initialized.
+ // But I don't know why. so here I print node_info->id instead of node_info->host
+ // to avoid BE crash. It needs further observation.
+ return fmt::format("{}, {}, node={}:{}", _name, _load_info, _node_info->id, _node_info->brpc_port);
+ }
+
private:
void _cancel_with_msg(const std::string& msg);
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index a5cb6de..3868e88 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -180,6 +180,9 @@ OLAPStatus DeltaWriter::write(Tuple* tuple) {
}
OLAPStatus DeltaWriter::_flush_memtable_async() {
+ if (++_segment_counter > config::max_segment_num_per_rowset) {
+ return OLAP_ERR_TOO_MANY_SEGMENTS;
+ }
return _flush_token->submit(_mem_table);
}
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 1c5e511..f395227 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -93,11 +93,12 @@ OLAPStatus BetaRowsetReader::init(RowsetReaderContext* read_context) {
read_options.use_page_cache = read_context->use_page_cache;
// load segments
- RETURN_NOT_OK(SegmentLoader::instance()->load_segments(_rowset, &_segment_cache_handle));
+ RETURN_NOT_OK(SegmentLoader::instance()->load_segments(
+ _rowset, &_segment_cache_handle, read_context->reader_type == ReaderType::READER_QUERY));
// create iterator for each segment
std::vector<std::unique_ptr<RowwiseIterator>> seg_iterators;
- for (auto& seg_ptr : _segment_cache_handle.value()->segments) {
+ for (auto& seg_ptr : _segment_cache_handle.get_segments()) {
std::unique_ptr<RowwiseIterator> iter;
auto s = seg_ptr->new_iterator(schema, read_options, _parent_tracker, &iter);
if (!s.ok()) {
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index 3a50dc9..872279f 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -56,23 +56,27 @@ void SegmentLoader::_insert(const SegmentLoader::CacheKey& key, SegmentLoader::C
}
OLAPStatus SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
- SegmentCacheHandle* cache_handle) {
- SegmentCacheHandle handle;
+ SegmentCacheHandle* cache_handle,
+ bool use_cache) {
SegmentLoader::CacheKey cache_key(rowset->rowset_id());
- if (_lookup(cache_key, &handle)) {
- *cache_handle = std::move(handle);
+ if (_lookup(cache_key, cache_handle)) {
+ cache_handle->owned = false;
return OLAP_SUCCESS;
}
+ cache_handle->owned = !use_cache;
std::vector<segment_v2::SegmentSharedPtr> segments;
RETURN_NOT_OK(rowset->load_segments(&segments));
- // memory of SegmentLoader::CacheValue will be handled by SegmentLoader
- SegmentLoader::CacheValue* cache_value = new SegmentLoader::CacheValue();
- cache_value->segments = std::move(segments);
- _insert(cache_key, *cache_value, &handle);
- *cache_handle = std::move(handle);
-
+ if (use_cache) {
+ // memory of SegmentLoader::CacheValue will be handled by SegmentLoader
+ SegmentLoader::CacheValue* cache_value = new SegmentLoader::CacheValue();
+ cache_value->segments = std::move(segments);
+ _insert(cache_key, *cache_value, cache_handle);
+ } else {
+ cache_handle->segments = std::move(segments);
+ }
+
return OLAP_SUCCESS;
}
diff --git a/be/src/olap/segment_loader.h b/be/src/olap/segment_loader.h
index 8a2bba4..2a75efa 100644
--- a/be/src/olap/segment_loader.h
+++ b/be/src/olap/segment_loader.h
@@ -83,8 +83,9 @@ public:
SegmentLoader(size_t capacity);
- // Load segments of "rowset" from _cache, return the "cache_handle" which contains segments
- OLAPStatus load_segments(const BetaRowsetSharedPtr& rowset, SegmentCacheHandle* cache_handle);
+ // Load segments of "rowset", return the "cache_handle" which contains segments.
+ // If use_cache is true, it will be loaded from _cache.
+ OLAPStatus load_segments(const BetaRowsetSharedPtr& rowset, SegmentCacheHandle* cache_handle, bool use_cache = false);
// Try to prune the segment cache if expired.
OLAPStatus prune();
@@ -121,9 +122,12 @@ public:
~SegmentCacheHandle() {
if (_handle != nullptr) {
+ CHECK(_cache != nullptr);
+ CHECK(segments.empty()) << segments.size();
+ CHECK(!owned);
// last_visit_time is set when release.
// because it only be needed when pruning.
- value()->last_visit_time = UnixMillis();
+ ((SegmentLoader::CacheValue*) _cache->value(_handle))->last_visit_time = UnixMillis();
_cache->release(_handle);
}
}
@@ -131,15 +135,30 @@ public:
SegmentCacheHandle(SegmentCacheHandle&& other) noexcept {
std::swap(_cache, other._cache);
std::swap(_handle, other._handle);
+ this->owned = other.owned;
+ this->segments = std::move(other.segments);
}
SegmentCacheHandle& operator=(SegmentCacheHandle&& other) noexcept {
std::swap(_cache, other._cache);
std::swap(_handle, other._handle);
+ this->owned = other.owned;
+ this->segments = std::move(other.segments);
return *this;
}
- SegmentLoader::CacheValue* value() { return (SegmentLoader::CacheValue*) _cache->value(_handle); }
+ std::vector<segment_v2::SegmentSharedPtr>& get_segments() {
+ if (owned) {
+ return segments;
+ } else {
+ return ((SegmentLoader::CacheValue*) _cache->value(_handle))->segments;
+ }
+ }
+
+public:
+ // If set to true, the loaded segments will be saved in segments, not in lru cache;
+ bool owned = false;
+ std::vector<segment_v2::SegmentSharedPtr> segments;
private:
Cache* _cache = nullptr;
diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js
index fe27f66..dba2221 100644
--- a/docs/.vuepress/sidebar/en.js
+++ b/docs/.vuepress/sidebar/en.js
@@ -214,9 +214,10 @@ module.exports = [
sidebarDepth: 1,
},
{
- title: "Bast Practices",
+ title: "Benchmark & Bast Practices",
directoryPath: "best-practices/",
children: [
+ "star-schema-benchmark",
"fe-load-balance",
"systemd",
"samples"
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index 6256acc..b148cd7 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -215,9 +215,10 @@ module.exports = [
sidebarDepth: 1,
},
{
- title: "最佳实践",
+ title: "性能测试和最佳实践",
directoryPath: "best-practices/",
children: [
+ "star-schema-benchmark",
"fe-load-balance",
"systemd",
"samples"
diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md
index 4672101..cfbe04b 100644
--- a/docs/en/administrator-guide/config/be_config.md
+++ b/docs/en/administrator-guide/config/be_config.md
@@ -1433,7 +1433,7 @@ The size of the buffer before flashing
* Type: int32
* Description: Used to limit the number of segments in the newly generated rowset when importing. If the threshold is exceeded, the import will fail with error -238. Too many segments will cause compaction to take up a lot of memory and cause OOM errors.
-* Default value: 100
+* Default value: 200
### `remote_storage_read_buffer_mb`
diff --git a/docs/en/best-practices/star-schema-benchmark.md b/docs/en/best-practices/star-schema-benchmark.md
new file mode 100644
index 0000000..f0ea9e1
--- /dev/null
+++ b/docs/en/best-practices/star-schema-benchmark.md
@@ -0,0 +1,163 @@
+---
+{
+ "title": "Star-Schema-Benchmark",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Star Schema Benchmark
+
+[Star Schema Benchmark(SSB)](https://www.cs.umb.edu/~poneil/StarSchemaB.PDF) is a lightweight data warehouse scenario performance test set. Based on [TPC-H](http://www.tpc.org/tpch/), SSB provides a simplified version of the star model data set, which is mainly used to test the performance of multi-table association queries under the star model.
+
+This document mainly introduces how to pass the preliminary performance test of the SSB process in Doris.
+
+> Note 1: The standard test set including SSB is usually far from the actual business scenario, and some tests will perform parameter tuning for the test set. Therefore, the test results of the standard test set can only reflect the performance of the database in a specific scenario. It is recommended that users use actual business data for further testing.
+>
+> Note 2: The operations involved in this document are all performed in the CentOS 7 environment.
+
+## Environmental preparation
+
+Please refer to the [official document](http://doris.incubator.apache.org/master/en/installing/install-deploy.html) to install and deploy Doris to obtain a normal running Doris cluster ( Contain at least 1 FE, 1 BE).
+
+The scripts involved in the following documents are all stored under `tools/ssb-tools/` in the Doris code base.
+
+## data preparation
+
+### 1. Download and install the SSB data generation tool.
+
+Execute the following script to download and compile the [ssb-dbgen](https://github.com/electrum/ssb-dbgen.git) tool.
+
+```
+sh build-ssb-dbgen.sh
+```
+
+After the installation is successful, the `dbgen` binary file will be generated in the `ssb-dbgen/` directory.
+
+### 2. Generate SSB test set
+
+Execute the following script to generate the SSB data set:
+
+```
+sh gen-ssb-data.sh -s 100 -c 100
+```
+
+> Note 1: `sh gen-ssb-data.sh -h View help`
+>
+> Note 2: The data will be generated under the directory `ssb-data/` with a suffix of `.tbl`. The total file size is about 60GB. The generation time may vary from a few minutes to an hour.
+>
+> Note 3: `-s 100` means that the test set size factor is 100, `-c 100` means that 100 threads concurrently generate data in the lineorder table. The `-c` parameter also determines the number of files in the final lineorder table. The larger the parameter, the more files and the smaller each file.
+
+Under the `-s 100` parameter, the generated data set size is:
+
+|Table |Rows |Size | File Number |
+|---|---|---|---|
+|lineorder| 600 million (600037902) | 60GB | 100|
+|customer|30 million (3000000) |277M |1|
+|part|1.4 million (1400000) | 116M|1|
+|supplier|200,000 (200,000) |17M |1|
+|date| 2556|228K |1|
+
+3. Build a table
+
+ Copy the table creation statement in [create-tables.sql](https://github.com/apache/incubator-doris/tree/master/tools/ssb-tools/create-tables.sql) and execute it in Doris.
+
+4. Import data
+
+ 1. Import 4 dimension table data (customer, part, supplier and date)
+
+ Because the data volume of these 4 dimension tables is small, and the import is simpler, we use the following command to import the data of these 4 tables first:
+
+ `sh load-dimension-data.sh`
+
+ 2. Import the fact table lineorder.
+
+ Import the lineorder table data with the following command:
+
+ `sh load-fact-data.sh -c 3`
+
+ `-c 3` means to start 5 concurrent threads to import (the default is 3). In the case of a single BE node, the import time of lineorder data generated by `sh gen-ssb-data.sh -s 100 -c 100` using `sh load-fact-data.sh -c 3` is about 10 minutes. The memory overhead is about 5-6GB. If you turn on more threads, you can speed up the import speed, but it will increase additional memory overhead.
+
+5. Check the imported data
+
+ ```
+ select count(*) from part;
+ select count(*) from customer;
+ select count(*) from supplier;
+ select count(*) from date;
+ select count(*) from lineorder;
+ ```
+
+ The amount of data should be the same as the number of rows of generated data.
+
+## Query test
+
+There are 4 groups of 14 SQL in the SSB test set. The query statement is in the [queries/](https://github.com/apache/incubator-doris/tree/master/tools/ssb-tools/queries) directory.
+
+## testing report
+
+The following test report is based on Doris [branch-0.15](https://github.com/apache/incubator-doris/tree/branch-0.15) branch code test, for reference only. (Update time: October 25, 2021)
+
+1. Hardware environment
+
+ * 1 FE + 1-3 BE mixed
+ * CPU: 96core, Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz
+ * Memory: 384GB
+ * Hard disk: 1 HDD
+ * Network card: 10 Gigabit network card
+
+2. Data set
+
+ |Table |Rows |Origin Size | Compacted Size(1 Replica) |
+ |---|---|---|---|
+ |lineorder| 600 million (600037902) | 60 GB | 14.846 GB |
+ |customer|30 million (3000000) |277 MB | 414.741 MB |
+ |part|1.4 million (1.400000) | 116 MB | 38.277 MB |
+ |supplier|200,000 (200,000) |17 MB | 27.428 MB |
+ |date| 2556|228 KB | 275.804 KB |
+
+3. Test results
+
+ |Query |Time(ms) (1 BE) | Time(ms) (3 BE) | Parallelism | Runtime Filter Mode |
+ |---|---|---|---|---|
+ | q1.1 | 200 | 140 | 8 | IN |
+ | q1.2 | 90 | 80 | 8 | IN |
+ | q1.3 | 90 | 80 | 8 | IN |
+ | q2.1 | 1100 | 400 | 8 | BLOOM_FILTER |
+ | q2.2 | 900 | 330 | 8 | BLOOM_FILTER |
+ | q2.3 | 790 | 320 | 8 | BLOOM_FILTER |
+ | q3.1 | 3100 | 1280 | 8 | BLOOM_FILTER |
+ | q3.2 | 700 | 270 | 8 | BLOOM_FILTER |
+ | q3.3 | 540 | 270 | 8 | BLOOM_FILTER |
+ | q3.4 | 560 | 240 | 8 | BLOOM_FILTER |
+ | q4.1 | 2820 | 1150 | 8 | BLOOM_FILTER |
+ | q4.2 | 1430 | 670 | 8 | BLOOM_FILTER |
+ | q4.2 | 1750 | 1030 | 8 | BLOOM_FILTER |
+
+ > Note 1: "This test set is far from your generation environment, don't be superstitious!"
+ >
+ > Note 2: The test result is the average value of multiple executions (Page Cache will play a certain acceleration role). And the data has undergone sufficient compaction (if you test immediately after importing the data, the query delay may be higher than the test result)
+ >
+ > Note 3: Due to environmental constraints, the hardware specifications used in this test are relatively high, but so many hardware resources will not be consumed during the entire test. The memory consumption is within 10GB, and the CPU usage is within 10%.
+ >
+ > Note 4: Parallelism means query concurrency, which is set by `set parallel_fragment_exec_instance_num=8`.
+ >
+ > Note 5: Runtime Filter Mode is the type of Runtime Filter, set by `set runtime_filter_type="BLOOM_FILTER"`. ([Runtime Filter](http://doris.incubator.apache.org/master/en/administrator-guide/runtime-filter.html) function has a significant effect on the SSB test set. Because in this test level, Join is calculated The data in the sub-right table can filter the left table very well. You can try to turn off this function through `set runtime_filter_mode=off` to see the change in query l [...]
diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md
index fc4d893..5907a3e 100644
--- a/docs/zh-CN/administrator-guide/config/be_config.md
+++ b/docs/zh-CN/administrator-guide/config/be_config.md
@@ -1454,7 +1454,7 @@ webserver默认工作线程数
* 类型: int32
* 描述: 用于限制导入时，新产生的rowset中的segment数量。如果超过阈值，导入会失败并报错 -238。过多的 segment 会导致compaction占用大量内存引发 OOM 错误。
-* 默认值: 100
+* 默认值: 200
### `remote_storage_read_buffer_mb`
diff --git a/docs/zh-CN/best-practices/star-schema-benchmark.md b/docs/zh-CN/best-practices/star-schema-benchmark.md
new file mode 100644
index 0000000..30f43aa
--- /dev/null
+++ b/docs/zh-CN/best-practices/star-schema-benchmark.md
@@ -0,0 +1,164 @@
+---
+{
+ "title": "Star-Schema-Benchmark 测试",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Star Schema Benchmark
+
+[Star Schema Benchmark(SSB)](https://www.cs.umb.edu/~poneil/StarSchemaB.PDF) 是一个轻量级的数仓场景下的性能测试集。SSB基于 [TPC-H](http://www.tpc.org/tpch/) 提供了一个简化版的星形模型数据集，主要用于测试在星形模型下，多表关联查询的性能表现。
+
+本文档主要介绍如何在 Doris 中通过 SSB 进程初步的性能测试。
+
+> 注1：包括 SSB 在内标准测试集通常和实际业务场景差距较大，并且部分测试会针对测试集进行参数调优。所以标准测试集的测试结果仅能反映数据库在特定场景下的性能表现。建议用户使用实际业务数据进行进一步的测试。
+>
+> 注2：本文档涉及的操作都在 CentOS 7 环境进行。
+
+## 环境准备
+
+请先参照 [官方文档](http://doris.incubator.apache.org/master/zh-CN/installing/install-deploy.html) 进行 Doris 的安装部署，以获得一个正常运行中的 Doris 集群（至少包含 1 FE，1 BE）。
+
+以下文档中涉及的的脚本都存放在 Doris 代码库的 `tools/ssb-tools/` 下。
+
+## 数据准备
+
+### 1. 下载安装 SSB 数据生成工具。
+
+执行以下脚本下载并编译 [ssb-dbgen](https://github.com/electrum/ssb-dbgen.git) 工具。
+
+```
+sh build-ssb-dbgen.sh
+```
+
+安装成功后，将在 `ssb-dbgen/` 目录下生成 `dbgen` 二进制文件。
+
+### 2. 生成 SSB 测试集
+
+执行以下脚本生成 SSB 数据集：
+
+```
+sh gen-ssb-data.sh -s 100 -c 100
+```
+
+> 注1：`sh gen-ssb-data.sh -h 查看帮助`
+>
+> 注2：数据会以 `.tbl` 为后缀生成在 `ssb-data/` 目录下。文件总大小约60GB。生成时间可能在数分钟到1小时不等。
+>
+> 注3：`-s 100` 表示测试集大小系数为 100，`-c 100` 表示并发100个线程生成 lineorder 表的数据。`-c` 参数也决定了最终 lineorder 表的文件数量。参数越大，文件数越多，每个文件越小。
+
+在 `-s 100` 参数下，生成的数据集大小为：
+
+|Table |Rows |Size | File Number |
+|---|---|---|---|
+|lineorder| 6亿（600037902） | 60GB | 100|
+|customer|3千万（3000000） |277M |1|
+|part|140万（1400000） | 116M|1|
+|supplier|20万（200000） |17M |1|
+|date| 2556|228K |1|
+
+3. 建表
+
+ 复制 [create-tables.sql](https://github.com/apache/incubator-doris/tree/master/tools/ssb-tools/create-tables.sql) 中的建表语句，在 Doris 中执行。
+
+4. 导入数据
+
+ 1. 导入 4 张维度表数据（customer, part, supplier and date）
+
+ 因为这4张维表数据量较小，导入较简答，我们使用以下命令先导入这4表的数据：
+
+ `sh load-dimension-data.sh`
+
+ 2. 导入事实表 lineorder。
+
+ 通过以下命令导入 lineorder 表数据：
+
+ `sh load-fact-data.sh -c 3`
+
+ `-c 3` 表示启动 5 个并发线程导入（默认为3）。在单 BE 节点情况下，由 `sh gen-ssb-data.sh -s 100 -c 100` 生成的 lineorder 数据，使用 `sh load-fact-data.sh -c 3` 的导入时间约为 10min。内存开销约为 5-6GB。如果开启更多线程，可以加快导入速度，但会增加额外的内存开销。
+
+5. 检查导入数据
+
+ ```
+ select count(*) from part;
+ select count(*) from customer;
+ select count(*) from supplier;
+ select count(*) from date;
+ select count(*) from lineorder;
+ ```
+
+ 数据量应和生成数据的行数一致。
+
+## 查询测试
+
+SSB 测试集共 4 组 14 个 SQL。查询语句在 [queries/](https://github.com/apache/incubator-doris/tree/master/tools/ssb-tools/queries) 目录下。
+
+## 测试报告
+
+以下测试报告基于 Doris [branch-0.15](https://github.com/apache/incubator-doris/tree/branch-0.15) 分支代码测试，仅供参考。（更新时间：2021年10月25号）
+
+1. 硬件环境
+
+ * 1 FE + 1-3 BE 混部
+ * CPU：96core, Intel(R) Xeon(R) Gold 6271C CPU @ 2.60GHz
+ * 内存：384GB
+ * 硬盘：1块机械硬盘
+ * 网卡：万兆网卡
+
+2. 数据集
+
+ |Table |Rows |Origin Size | Compacted Size(1 Replica) |
+ |---|---|---|---|
+ |lineorder| 6亿（600037902） | 60 GB | 14.846 GB |
+ |customer|3千万（3000000） |277 MB | 414.741 MB |
+ |part|140万（1400000） | 116 MB | 38.277 MB |
+ |supplier|20万（200000） |17 MB | 27.428 MB |
+ |date| 2556|228 KB | 275.804 KB |
+
+3. 测试结果
+
+ |Query |Time(ms) (1 BE) | Time(ms) (3 BE) | Parallelism | Runtime Filter Mode |
+ |---|---|---|---|---|
+ | q1.1 | 200 | 140 | 8 | IN |
+ | q1.2 | 90 | 80 | 8 | IN |
+ | q1.3 | 90 | 80 | 8 | IN |
+ | q2.1 | 1100 | 400 | 8 | BLOOM_FILTER |
+ | q2.2 | 900 | 330 | 8 | BLOOM_FILTER |
+ | q2.3 | 790 | 320 | 8 | BLOOM_FILTER |
+ | q3.1 | 3100 | 1280 | 8 | BLOOM_FILTER |
+ | q3.2 | 700 | 270 | 8 | BLOOM_FILTER |
+ | q3.3 | 540 | 270 | 8 | BLOOM_FILTER |
+ | q3.4 | 560 | 240 | 8 | BLOOM_FILTER |
+ | q4.1 | 2820 | 1150 | 8 | BLOOM_FILTER |
+ | q4.2 | 1430 | 670 | 8 | BLOOM_FILTER |
+ | q4.2 | 1750 | 1030 | 8 | BLOOM_FILTER |
+
+ > 注1：“这个测试集和你的生成环境相去甚远，不要迷信他！”
+ >
+ > 注2：测试结果为多次执行取平均值（Page Cache 会起到一定加速作用）。并且数据经过充分的 compaction （如果在刚导入数据后立刻测试，则查询延迟可能高于本测试结果）
+ >
+ > 注3：因环境受限，本测试使用的硬件规格较高，但整个测试过程中不会消耗如此多的硬件资源。其中内存消耗在 10GB 以内，CPU使用率在 10% 以内。
+ >
+ > 注4：Parallelism 表示查询并发度，通过 `set parallel_fragment_exec_instance_num=8` 设置。
+ >
+ > 注5：Runtime Filter Mode 是 Runtime Filter 的类型，通过 `set runtime_filter_type="BLOOM_FILTER"` 设置。（[Runtime Filter](http://doris.incubator.apache.org/master/zh-CN/administrator-guide/runtime-filter.html) 功能对 SSB 测试集效果显著。因为该测试级中，Join 算子右表的数据可以对左表起到很好的过滤作用。你可以尝试通过 `set runtime_filter_mode=off` 关闭该功能，看看查询延迟的变化。）
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index aea8d98..cc224a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -1370,7 +1370,7 @@ public class Config extends ConfigBase {
* Whether to add a delete sign column when create unique table
*/
@ConfField(mutable = true, masterOnly = true)
- public static boolean enable_batch_delete_by_default = false;
+ public static boolean enable_batch_delete_by_default = true;
/**
* Used to set default db data quota bytes.
diff --git a/tools/ssb-tools/build-ssb-dbgen.sh b/tools/ssb-tools/build-ssb-dbgen.sh
new file mode 100755
index 0000000..59af467
--- /dev/null
+++ b/tools/ssb-tools/build-ssb-dbgen.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to build ssb-dbgen
+# sssb-dbgen's source code is from https://github.com/electrum/ssb-dbgen.git
+# Usage:
+# sh build-ssb-dbgen.sh
+##############################################################
+
+set -eo pipefail
+
+ROOT=`dirname "$0"`
+ROOT=`cd "$ROOT"; pwd`
+
+CURDIR=${ROOT}
+SSB_DBGEN_DIR=$CURDIR/ssb-dbgen/
+
+# download ssb-dbgen first
+if [[ -d $SSB_DBGEN_DIR ]]; then
+ echo "Dir $CURDIR/ssb-dbgen/ already exists. No need to download."
+ echo "If you want to download ssb-dbgen again, please delete this dir first."
+else
+ curl https://palo-cloud-repo-bd.bd.bcebos.com/baidu-doris-release/ssb-dbgen-linux.tar.gz | tar xz -C $CURDIR/
+fi
+
+# compile ssb-dbgen
+cd $SSB_DBGEN_DIR/ && make
+cd -
+
+# check
+if [[ -f $CURDIR/ssb-dbgen/dbgen ]]; then
+ echo "Build succeed! Run $CURDIR/ssb-dbgen/dbgen -h"
+ exit 0
+else
+ echo "Build failed!"
+ exit 1
+fi
diff --git a/tools/ssb-tools/create-tables.sql b/tools/ssb-tools/create-tables.sql
new file mode 100644
index 0000000..fc971b1
--- /dev/null
+++ b/tools/ssb-tools/create-tables.sql
@@ -0,0 +1,117 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+CREATE TABLE `lineorder` (
+ `lo_orderkey` bigint(20) NULL COMMENT "",
+ `lo_linenumber` bigint(20) NULL COMMENT "",
+ `lo_custkey` int(11) NULL COMMENT "",
+ `lo_partkey` int(11) NULL COMMENT "",
+ `lo_suppkey` int(11) NULL COMMENT "",
+ `lo_orderdate` int(11) NULL COMMENT "",
+ `lo_orderpriority` varchar(16) NULL COMMENT "",
+ `lo_shippriority` int(11) NULL COMMENT "",
+ `lo_quantity` bigint(20) NULL COMMENT "",
+ `lo_extendedprice` bigint(20) NULL COMMENT "",
+ `lo_ordtotalprice` bigint(20) NULL COMMENT "",
+ `lo_discount` bigint(20) NULL COMMENT "",
+ `lo_revenue` bigint(20) NULL COMMENT "",
+ `lo_supplycost` bigint(20) NULL COMMENT "",
+ `lo_tax` bigint(20) NULL COMMENT "",
+ `lo_commitdate` bigint(20) NULL COMMENT "",
+ `lo_shipmode` varchar(11) NULL COMMENT ""
+)
+PARTITION BY RANGE(`lo_orderdate`)
+(PARTITION p1992 VALUES [("-2147483648"), ("19930101")),
+PARTITION p1993 VALUES [("19930101"), ("19940101")),
+PARTITION p1994 VALUES [("19940101"), ("19950101")),
+PARTITION p1995 VALUES [("19950101"), ("19960101")),
+PARTITION p1996 VALUES [("19960101"), ("19970101")),
+PARTITION p1997 VALUES [("19970101"), ("19980101")),
+PARTITION p1998 VALUES [("19980101"), ("19990101")))
+DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 48
+PROPERTIES (
+"replication_num" = "1"
+);
+
+CREATE TABLE `customer` (
+ `c_custkey` int(11) NULL COMMENT "",
+ `c_name` varchar(26) NULL COMMENT "",
+ `c_address` varchar(41) NULL COMMENT "",
+ `c_city` varchar(11) NULL COMMENT "",
+ `c_nation` varchar(16) NULL COMMENT "",
+ `c_region` varchar(13) NULL COMMENT "",
+ `c_phone` varchar(16) NULL COMMENT "",
+ `c_mktsegment` varchar(11) NULL COMMENT ""
+)
+DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 10
+PROPERTIES (
+"replication_num" = "1"
+);
+
+CREATE TABLE `date` (
+ `d_datekey` int(11) NULL COMMENT "",
+ `d_date` varchar(20) NULL COMMENT "",
+ `d_dayofweek` varchar(10) NULL COMMENT "",
+ `d_month` varchar(11) NULL COMMENT "",
+ `d_year` int(11) NULL COMMENT "",
+ `d_yearmonthnum` int(11) NULL COMMENT "",
+ `d_yearmonth` varchar(9) NULL COMMENT "",
+ `d_daynuminweek` int(11) NULL COMMENT "",
+ `d_daynuminmonth` int(11) NULL COMMENT "",
+ `d_daynuminyear` int(11) NULL COMMENT "",
+ `d_monthnuminyear` int(11) NULL COMMENT "",
+ `d_weeknuminyear` int(11) NULL COMMENT "",
+ `d_sellingseason` varchar(14) NULL COMMENT "",
+ `d_lastdayinweekfl` int(11) NULL COMMENT "",
+ `d_lastdayinmonthfl` int(11) NULL COMMENT "",
+ `d_holidayfl` int(11) NULL COMMENT "",
+ `d_weekdayfl` int(11) NULL COMMENT ""
+)
+DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1
+PROPERTIES (
+"replication_num" = "1"
+);
+
+CREATE TABLE `part` (
+ `p_partkey` int(11) NULL COMMENT "",
+ `p_name` varchar(23) NULL COMMENT "",
+ `p_mfgr` varchar(7) NULL COMMENT "",
+ `p_category` varchar(8) NULL COMMENT "",
+ `p_brand` varchar(10) NULL COMMENT "",
+ `p_color` varchar(12) NULL COMMENT "",
+ `p_type` varchar(26) NULL COMMENT "",
+ `p_size` int(11) NULL COMMENT "",
+ `p_container` varchar(11) NULL COMMENT ""
+)
+DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 10
+PROPERTIES (
+"replication_num" = "1"
+);
+
+CREATE TABLE `supplier` (
+ `s_suppkey` int(11) NULL COMMENT "",
+ `s_name` varchar(26) NULL COMMENT "",
+ `s_address` varchar(26) NULL COMMENT "",
+ `s_city` varchar(11) NULL COMMENT "",
+ `s_nation` varchar(16) NULL COMMENT "",
+ `s_region` varchar(13) NULL COMMENT "",
+ `s_phone` varchar(16) NULL COMMENT ""
+)
+DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 10
+PROPERTIES (
+"replication_num" = "1"
+);
diff --git a/tools/ssb-tools/doris-cluster.conf b/tools/ssb-tools/doris-cluster.conf
new file mode 100644
index 0000000..bef6c7b
--- /dev/null
+++ b/tools/ssb-tools/doris-cluster.conf
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Any of FE host
+export FE_HOST='127.0.0.1'
+# http_port in fe.conf
+export FE_HTTP_PORT=8030
+# Doris username
+export USER='admin'
+# Doris password
+export PASSWORD=''
+# The database where SSB tables located
+export DB='db1'
diff --git a/tools/ssb-tools/gen-ssb-data.sh b/tools/ssb-tools/gen-ssb-data.sh
new file mode 100755
index 0000000..594f129
--- /dev/null
+++ b/tools/ssb-tools/gen-ssb-data.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to generate ssb data set
+##############################################################
+
+set -eo pipefail
+
+ROOT=`dirname "$0"`
+ROOT=`cd "$ROOT"; pwd`
+
+CURDIR=${ROOT}
+SSB_DBGEN_DIR=$CURDIR/ssb-dbgen/
+SSB_DATA_DIR=$CURDIR/ssb-data/
+
+usage() {
+ echo "
+Usage: $0 <options>
+ Optional options:
+ -s scale factor, default is 100
+ -c parallelism to generate data of lineorder table, default is 10
+
+ Eg.
+ $0 generate data using default value.
+ $0 -s 10 generate data with scale factor 10.
+ $0 -s 10 -c 5 generate data with scale factor 10. And using 5 threads to generate data concurrently.
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -o 'hs:c:' \
+ -- "$@")
+
+eval set -- "$OPTS"
+
+SCALE_FACTOR=100
+PARALLEL=10
+HELP=0
+
+if [ $# == 0 ] ; then
+ usage
+fi
+
+while true; do
+ case "$1" in
+ -h) HELP=1 ; shift ;;
+ -s) SCALE_FACTOR=$2 ; shift 2 ;;
+ -c) PARALLEL=$2 ; shift 2 ;;
+ --) shift ; break ;;
+ *) echo "Internal error" ; exit 1 ;;
+ esac
+done
+
+if [[ ${HELP} -eq 1 ]]; then
+ usage
+ exit
+fi
+
+echo "Scale Factor: $SCALE_FACTOR"
+echo "Parallelism: $PARALLEL"
+
+# check if dbgen exists
+if [[ ! -f $SSB_DBGEN_DIR/dbgen ]]; then
+ echo "$SSB_DBGEN_DIR/dbgen does not exist. Run build-ssb-dbgen.sh first to build it first."
+ exit 1
+fi
+
+if [[ -d $SSB_DATA_DIR/ ]]; then
+ echo "$SSB_DATA_DIR exists. Remove it before generating data"
+ exit 1
+fi
+
+mkdir $SSB_DATA_DIR/
+
+# gen data
+cd $SSB_DBGEN_DIR
+echo "Begin to generate data for table: customer"
+$SSB_DBGEN_DIR/dbgen -f -s $SCALE_FACTOR -T c
+echo "Begin to generate data for table: part"
+$SSB_DBGEN_DIR/dbgen -f -s $SCALE_FACTOR -T p
+echo "Begin to generate data for table: supplier"
+$SSB_DBGEN_DIR/dbgen -f -s $SCALE_FACTOR -T s
+echo "Begin to generate data for table: date"
+$SSB_DBGEN_DIR/dbgen -f -s $SCALE_FACTOR -T d
+echo "Begin to generate data for table: lineorder"
+$SSB_DBGEN_DIR/dbgen -f -s $SCALE_FACTOR -T l -C $PARALLEL
+cd -
+
+# move data to $SSB_DATA_DIR
+mv $SSB_DBGEN_DIR/*.tbl* $SSB_DATA_DIR/
+
+# check data
+du -sh $SSB_DATA_DIR/*.tbl*
diff --git a/tools/ssb-tools/load-dimension-data.sh b/tools/ssb-tools/load-dimension-data.sh
new file mode 100755
index 0000000..f24ad3a
--- /dev/null
+++ b/tools/ssb-tools/load-dimension-data.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to load generated ssb data set to Doris
+# Only for 4 dimension tables: customer, part, supplier and date.
+# Usage:
+# sh load-dimension-data.sh
+##############################################################
+
+set -eo pipefail
+
+ROOT=`dirname "$0"`
+ROOT=`cd "$ROOT"; pwd`
+
+CURDIR=${ROOT}
+SSB_DATA_DIR=$CURDIR/ssb-data/
+
+# check if ssb-data exists
+if [[ ! -d $SSB_DATA_DIR/ ]]; then
+ echo "$SSB_DATA_DIR does not exist. Run sh gen-ssb-data.sh first."
+ exit 1
+fi
+
+check_prerequest() {
+ local CMD=$1
+ local NAME=$2
+ if ! $CMD; then
+ echo "$NAME is missing. This script depends on cURL to load data to Doris."
+ exit 1
+ fi
+}
+
+check_prerequest "curl --version" "curl"
+
+# load 4 small dimension tables
+
+source $CURDIR/doris-cluster.conf
+
+echo "FE_HOST: $FE_HOST"
+echo "FE_HTTP_PORT: $FE_HTTP_PORT"
+echo "USER: $USER"
+echo "PASSWORD: $PASSWORD"
+echo "DB: $DB"
+
+echo 'Loading data for table: part'
+curl --location-trusted -u $USER:$PASSWORD -H "column_separator:|" -H "columns:p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy" -T $SSB_DATA_DIR/part.tbl http://$FE_HOST:$FE_HTTP_PORT/api/$DB/part/_stream_load
+echo 'Loading data for table: date'
+curl --location-trusted -u $USER:$PASSWORD -H "column_separator:|" -H "columns:d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth,d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear,d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy" -T $SSB_DATA_DIR/date.tbl http://$FE_HOST:$FE_HTTP_PORT/api/$DB/date/_stream_load
+echo 'Loading data for table: supplier'
+curl --location-trusted -u $USER:$PASSWORD -H "column_separator:|" -H "columns:s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy" -T $SSB_DATA_DIR/supplier.tbl http://$FE_HOST:$FE_HTTP_PORT/api/$DB/supplier/_stream_load
+echo 'Loading data for table: customer'
+curl --location-trusted -u $USER:$PASSWORD -H "column_separator:|" -H "columns:c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use" -T $SSB_DATA_DIR/customer.tbl http://$FE_HOST:$FE_HTTP_PORT/api/$DB/customer/_stream_load
diff --git a/tools/ssb-tools/load-fact-data.sh b/tools/ssb-tools/load-fact-data.sh
new file mode 100755
index 0000000..24bc7f3
--- /dev/null
+++ b/tools/ssb-tools/load-fact-data.sh
@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to load generated ssb data set to Doris
+# Only for 1 fact table: lineorder
+##############################################################
+
+set -eo pipefail
+
+ROOT=`dirname "$0"`
+ROOT=`cd "$ROOT"; pwd`
+
+CURDIR=${ROOT}
+SSB_DATA_DIR=$CURDIR/ssb-data/
+
+usage() {
+ echo "
+Usage: $0 <options>
+ Optional options:
+ -c parallelism to load data of lineorder table, default is 5.
+
+ Eg.
+ $0 load data using default value.
+ $0 -c 10 load lineorder table data using parallelism 10.
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -o 'c:' \
+ -- "$@")
+
+eval set -- "$OPTS"
+
+PARALLEL=3
+HELP=0
+
+if [ $# == 0 ] ; then
+ usage
+fi
+
+while true; do
+ case "$1" in
+ -h) HELP=1 ; shift ;;
+ -c) PARALLEL=$2 ; shift 2 ;;
+ --) shift ; break ;;
+ *) echo "Internal error" ; exit 1 ;;
+ esac
+done
+
+if [[ ${HELP} -eq 1 ]]; then
+ usage
+ exit
+fi
+
+echo "Parallelism: $PARALLEL"
+
+# check if ssb-data exists
+if [[ ! -d $SSB_DATA_DIR/ ]]; then
+ echo "$SSB_DATA_DIR does not exist. Run sh gen-ssb-data.sh first."
+ exit 1
+fi
+
+check_prerequest() {
+ local CMD=$1
+ local NAME=$2
+ if ! $CMD; then
+ echo "$NAME is missing. This script depends on cURL to load data to Doris."
+ exit 1
+ fi
+}
+
+check_prerequest "curl --version" "curl"
+
+# load lineorder
+source $CURDIR/doris-cluster.conf
+
+echo "FE_HOST: $FE_HOST"
+echo "FE_HTTP_PORT: $FE_HTTP_PORT"
+echo "USER: $USER"
+echo "PASSWORD: $PASSWORD"
+echo "DB: $DB"
+
+function load()
+{
+ echo $@
+ curl --location-trusted -u $USER:$PASSWORD -H "column_separator:|" -H "columns:lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority,lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount,lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy" -T $@ http://$FE_HOST:$FE_HTTP_PORT/api/$DB/lineorder/_stream_load
+}
+
+
+# set parallelism
+[ -e /tmp/fd1 ] || mkfifo /tmp/fd1
+exec 3<>/tmp/fd1
+rm -rf /tmp/fd1
+
+for ((i=1;i<=$PARALLEL;i++))
+do
+ echo >&3
+done
+
+for file in `ls $SSB_DATA_DIR/lineorder.tbl.*`
+do
+ read -u3
+ {
+ load $file
+ echo >&3
+ }&
+done
diff --git a/tools/ssb-tools/queries/q1.1.sql b/tools/ssb-tools/queries/q1.1.sql
new file mode 100644
index 0000000..4ef15e9
--- /dev/null
+++ b/tools/ssb-tools/queries/q1.1.sql
@@ -0,0 +1,24 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT SUM(lo_extendedprice*lo_discount) AS
+REVENUE
+FROM lineorder, date
+WHERE lo_orderdate = d_datekey
+AND d_year = 1993
+AND lo_discount BETWEEN 1 AND 3
+AND lo_quantity < 25;
diff --git a/tools/ssb-tools/queries/q1.2.sql b/tools/ssb-tools/queries/q1.2.sql
new file mode 100644
index 0000000..1b8442b
--- /dev/null
+++ b/tools/ssb-tools/queries/q1.2.sql
@@ -0,0 +1,24 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT SUM(lo_extendedprice*lo_discount) AS
+REVENUE
+FROM lineorder, date
+WHERE lo_orderdate = d_datekey
+AND d_yearmonth = 'Jan1994'
+AND lo_discount BETWEEN 4 AND 6
+AND lo_quantity BETWEEN 26 AND 35;
diff --git a/tools/ssb-tools/queries/q1.3.sql b/tools/ssb-tools/queries/q1.3.sql
new file mode 100644
index 0000000..ed6e51b
--- /dev/null
+++ b/tools/ssb-tools/queries/q1.3.sql
@@ -0,0 +1,25 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT SUM(lo_extendedprice*lo_discount) AS
+REVENUE
+FROM lineorder, date
+WHERE lo_orderdate = d_datekey
+AND d_weeknuminyear= 6
+AND d_year = 1994
+AND lo_discount BETWEEN 5 AND 7
+AND lo_quantity BETWEEN 26 AND 35;
diff --git a/tools/ssb-tools/queries/q2.1.sql b/tools/ssb-tools/queries/q2.1.sql
new file mode 100644
index 0000000..e1a1f52
--- /dev/null
+++ b/tools/ssb-tools/queries/q2.1.sql
@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT SUM(lo_revenue), d_year, p_brand
+FROM lineorder, date, part, supplier
+WHERE lo_orderdate = d_datekey
+AND lo_partkey = p_partkey
+AND lo_suppkey = s_suppkey
+AND p_category = 'MFGR#12'
+AND s_region = 'AMERICA'
+GROUP BY d_year, p_brand
+ORDER BY d_year, p_brand;
diff --git a/tools/ssb-tools/queries/q2.2.sql b/tools/ssb-tools/queries/q2.2.sql
new file mode 100644
index 0000000..3db6170
--- /dev/null
+++ b/tools/ssb-tools/queries/q2.2.sql
@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT SUM(lo_revenue), d_year, p_brand
+FROM lineorder, date, part, supplier
+WHERE lo_orderdate = d_datekey
+AND lo_partkey = p_partkey
+AND lo_suppkey = s_suppkey
+AND p_brand BETWEEN 'MFGR#2221'
+AND 'MFGR#2228'
+AND s_region = 'ASIA'
+GROUP BY d_year, p_brand
+ORDER BY d_year, p_brand;
diff --git a/tools/ssb-tools/queries/q2.3.sql b/tools/ssb-tools/queries/q2.3.sql
new file mode 100644
index 0000000..b70ca90
--- /dev/null
+++ b/tools/ssb-tools/queries/q2.3.sql
@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT SUM(lo_revenue), d_year, p_brand
+FROM lineorder, date, part, supplier
+WHERE lo_orderdate = d_datekey
+AND lo_partkey = p_partkey
+AND lo_suppkey = s_suppkey
+AND p_brand = 'MFGR#2239'
+AND s_region = 'EUROPE'
+GROUP BY d_year, p_brand
+ORDER BY d_year, p_brand;
diff --git a/tools/ssb-tools/queries/q3.1.sql b/tools/ssb-tools/queries/q3.1.sql
new file mode 100644
index 0000000..70f17d7
--- /dev/null
+++ b/tools/ssb-tools/queries/q3.1.sql
@@ -0,0 +1,28 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT c_nation, s_nation, d_year,
+SUM(lo_revenue) AS REVENUE
+FROM customer, lineorder, supplier, date
+WHERE lo_custkey = c_custkey
+AND lo_suppkey = s_suppkey
+AND lo_orderdate = d_datekey
+AND c_region = 'ASIA'
+AND s_region = 'ASIA'
+AND d_year >= 1992 AND d_year <= 1997
+GROUP BY c_nation, s_nation, d_year
+ORDER BY d_year ASC, REVENUE DESC;
diff --git a/tools/ssb-tools/queries/q3.2.sql b/tools/ssb-tools/queries/q3.2.sql
new file mode 100644
index 0000000..a416fbe
--- /dev/null
+++ b/tools/ssb-tools/queries/q3.2.sql
@@ -0,0 +1,28 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT c_city, s_city, d_year, sum(lo_revenue)
+AS REVENUE
+FROM customer, lineorder, supplier, date
+WHERE lo_custkey = c_custkey
+AND lo_suppkey = s_suppkey
+AND lo_orderdate = d_datekey
+AND c_nation = 'UNITED STATES'
+AND s_nation = 'UNITED STATES'
+AND d_year >= 1992 AND d_year <= 1997
+GROUP BY c_city, s_city, d_year
+ORDER BY d_year ASC, REVENUE DESC;
diff --git a/tools/ssb-tools/queries/q3.3.sql b/tools/ssb-tools/queries/q3.3.sql
new file mode 100644
index 0000000..98e29b7
--- /dev/null
+++ b/tools/ssb-tools/queries/q3.3.sql
@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT c_city, s_city, d_year, SUM(lo_revenue)
+AS REVENUE
+FROM customer, lineorder, supplier, date
+WHERE lo_custkey = c_custkey
+AND lo_suppkey = s_suppkey
+AND lo_orderdate = d_datekey
+AND (c_city='UNITED KI1'
+OR c_city='UNITED KI5')
+AND (s_city='UNITED KI1'
+OR s_city='UNITED KI5')
+AND d_year >= 1992 AND d_year <= 1997
+GROUP BY c_city, s_city, d_year
+ORDER BY d_year ASC, REVENUE DESC;
diff --git a/tools/ssb-tools/queries/q3.4.sql b/tools/ssb-tools/queries/q3.4.sql
new file mode 100644
index 0000000..65fe992
--- /dev/null
+++ b/tools/ssb-tools/queries/q3.4.sql
@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT c_city, s_city, d_year, SUM(lo_revenue)
+AS REVENUE
+FROM customer, lineorder, supplier, date
+WHERE lo_custkey = c_custkey
+AND lo_suppkey = s_suppkey
+AND lo_orderdate = d_datekey
+AND (c_city='UNITED KI1'
+OR c_city='UNITED KI5')
+AND (s_city='UNITED KI1'
+OR s_city='UNITED KI5')
+AND d_yearmonth = 'Dec1997'
+GROUP BY c_city, s_city, d_year
+ORDER BY d_year ASC, REVENUE DESC;
diff --git a/tools/ssb-tools/queries/q4.1.sql b/tools/ssb-tools/queries/q4.1.sql
new file mode 100644
index 0000000..bdcd730
--- /dev/null
+++ b/tools/ssb-tools/queries/q4.1.sql
@@ -0,0 +1,30 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT d_year, c_nation,
+SUM(lo_revenue - lo_supplycost) AS PROFIT
+FROM date, customer, supplier, part, lineorder
+WHERE lo_custkey = c_custkey
+AND lo_suppkey = s_suppkey
+AND lo_partkey = p_partkey
+AND lo_orderdate = d_datekey
+AND c_region = 'AMERICA'
+AND s_region = 'AMERICA'
+AND (p_mfgr = 'MFGR#1'
+OR p_mfgr = 'MFGR#2')
+GROUP BY d_year, c_nation
+ORDER BY d_year, c_nation;
diff --git a/tools/ssb-tools/queries/q4.2.sql b/tools/ssb-tools/queries/q4.2.sql
new file mode 100644
index 0000000..24c82cf
--- /dev/null
+++ b/tools/ssb-tools/queries/q4.2.sql
@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT d_year, s_nation, p_category,
+SUM(lo_revenue - lo_supplycost) AS PROFIT
+FROM date, customer, supplier, part, lineorder
+WHERE lo_custkey = c_custkey
+AND lo_suppkey = s_suppkey
+AND lo_partkey = p_partkey
+AND lo_orderdate = d_datekey
+AND c_region = 'AMERICA'
+AND s_region = 'AMERICA'
+AND (d_year = 1997 OR d_year = 1998)
+AND (p_mfgr = 'MFGR#1'
+OR p_mfgr = 'MFGR#2')
+GROUP BY d_year, s_nation, p_category
+ORDER BY d_year, s_nation, p_category;
diff --git a/tools/ssb-tools/queries/q4.3.sql b/tools/ssb-tools/queries/q4.3.sql
new file mode 100644
index 0000000..0dcc08b
--- /dev/null
+++ b/tools/ssb-tools/queries/q4.3.sql
@@ -0,0 +1,29 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+SELECT d_year, s_city, p_brand,
+SUM(lo_revenue - lo_supplycost) AS PROFIT
+FROM date, customer, supplier, part, lineorder
+WHERE lo_custkey = c_custkey
+AND lo_suppkey = s_suppkey
+AND lo_partkey = p_partkey
+AND lo_orderdate = d_datekey
+AND s_nation = 'UNITED STATES'
+AND (d_year = 1997 OR d_year = 1998)
+AND p_category = 'MFGR#14'
+GROUP BY d_year, s_city, p_brand
+ORDER BY d_year, s_city, p_brand;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 05/21: [Refactor] remove unused code (#6879)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 6d35560a503f420a860ca309514073407f88ff3a
Author: Wei <hs...@163.com>
AuthorDate: Sat Oct 23 16:47:10 2021 +0800
[Refactor] remove unused code (#6879)
---
fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java | 4 ----
1 file changed, 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 0f15ff9..cfc89f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -76,10 +76,6 @@ import java.util.stream.Collectors;
public class Database extends MetaObject implements Writable {
private static final Logger LOG = LogManager.getLogger(Database.class);
- // empirical value.
- // assume that the time a lock is held by thread is less than 100ms
- public static final long TRY_LOCK_TIMEOUT_MS = 100L;
-
private long id;
private volatile String fullQualifiedName;
private String clusterName;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 14/21: [Spark-Doris-Connector] fixed some
spark-doris-connector doc typo
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 20caf423d741112a586c0b61e37eccc81dd56daf
Author: luzhijing <82...@users.noreply.github.com>
AuthorDate: Tue Oct 26 18:23:53 2021 +0800
[Spark-Doris-Connector] fixed some spark-doris-connector doc typo
---
docs/zh-CN/extending-doris/spark-doris-connector.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/docs/zh-CN/extending-doris/spark-doris-connector.md b/docs/zh-CN/extending-doris/spark-doris-connector.md
index fa69f38..61be81a 100644
--- a/docs/zh-CN/extending-doris/spark-doris-connector.md
+++ b/docs/zh-CN/extending-doris/spark-doris-connector.md
@@ -52,7 +52,7 @@ Spark Doris Connector 可以支持通过 Spark 读取 Doris 中存储的数据
```bash
sh build.sh 3 ## spark 3.x版本，默认是3.1.2
-sh build.sh 2 ## soark 2.x版本，默认是2.3.4
+sh build.sh 2 ## spark 2.x版本，默认是2.3.4
```
编译成功后，会在 `output/` 目录下生成文件 `doris-spark-1.0.0-SNAPSHOT.jar`。将此文件复制到 `Spark` 的 `ClassPath` 中即可使用 `Spark-Doris-Connector`。例如，`Local` 模式运行的 `Spark`，将此文件放入 `jars/` 文件夹下。`Yarn`集群模式运行的`Spark`，则将此文件放入预部署包中。
@@ -204,4 +204,4 @@ kafkaSource.selectExpr("CAST(key AS STRING)", "CAST(value as STRING)")
| TIME | DataTypes.DoubleType |
| HLL | Unsupported datatype |
-* 注：Connector中，将`DATE`和`DATETIME`映射为`String`。由于`Doris`底层存储引擎处理逻辑，直接使用时间类型时，覆盖的时间范围无法满足需求。所以使用 `String` 类型直接返回对应的时间可读文本。
\ No newline at end of file
+* 注：Connector中，将`DATE`和`DATETIME`映射为`String`。由于`Doris`底层存储引擎处理逻辑，直接使用时间类型时，覆盖的时间范围无法满足需求。所以使用 `String` 类型直接返回对应的时间可读文本。
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 02/21: [Bug][Memory Leak] Fix the issue of
Catalog instance leakage (#6895)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 56f01d67217dba9b2c543f0c52db7d43431de753
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Sat Oct 23 16:44:51 2021 +0800
[Bug][Memory Leak] Fix the issue of Catalog instance leakage (#6895)
The Checkpoint Catalog instance may be incorrectly stored in MetricRepo, causing memory leaks
---
.../org/apache/doris/common/ThreadPoolManager.java | 8 ++++---
.../org/apache/doris/master/ReportHandler.java | 4 ++--
.../apache/doris/metric/DorisMetricRegistry.java | 14 ++++++++++--
.../java/org/apache/doris/metric/MetricRepo.java | 26 +++++++++-------------
4 files changed, 30 insertions(+), 22 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java
index c699065..20ee89a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ThreadPoolManager.java
@@ -17,12 +17,14 @@
package org.apache.doris.common;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.doris.metric.GaugeMetric;
import org.apache.doris.metric.Metric.MetricUnit;
import org.apache.doris.metric.MetricLabel;
import org.apache.doris.metric.MetricRepo;
+
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -91,7 +93,7 @@ public class ThreadPoolManager {
};
gauge.addLabel(new MetricLabel("name", poolName))
.addLabel(new MetricLabel("type", poolMetricType));
- MetricRepo.addMetric(gauge);
+ MetricRepo.PALO_METRIC_REGISTER.addPaloMetrics(gauge);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 8ee321e..3775afb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -105,14 +105,14 @@ public class ReportHandler extends Daemon {
}
public ReportHandler() {
- GaugeMetric<Long> gaugeQueueSize = new GaugeMetric<Long>(
+ GaugeMetric<Long> gauge = new GaugeMetric<Long>(
"report_queue_size", MetricUnit.NOUNIT, "report queue size") {
@Override
public Long getValue() {
return (long) reportQueue.size();
}
};
- MetricRepo.addMetric(gaugeQueueSize);
+ MetricRepo.PALO_METRIC_REGISTER.addPaloMetrics(gauge);
}
public TMasterResult handleReport(TReportRequest request) throws TException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java
index a0940da..9b07810 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/DorisMetricRegistry.java
@@ -17,6 +17,8 @@
package org.apache.doris.metric;
+import org.apache.doris.catalog.Catalog;
+
import com.google.common.collect.Lists;
import java.util.Collection;
@@ -34,7 +36,12 @@ public class DorisMetricRegistry {
}
public synchronized void addPaloMetrics(Metric paloMetric) {
- paloMetrics.add(paloMetric);
+ // No metric needs to be added to the Checkpoint thread.
+ // And if you add a metric in Checkpoint thread, it will cause the metric to be added repeatedly,
+ // and the Checkpoint Catalog may be saved incorrectly, resulting in FE memory leaks.
+ if (!Catalog.isCheckpointThread()) {
+ paloMetrics.add(paloMetric);
+ }
}
public synchronized List<Metric> getPaloMetrics() {
@@ -47,6 +54,9 @@ public class DorisMetricRegistry {
}
public synchronized void removeMetrics(String name) {
- paloMetrics = paloMetrics.stream().filter(m -> !(m.getName().equals(name))).collect(Collectors.toList());
+ // Same reason as comment in addPaloMetrics()
+ if (!Catalog.isCheckpointThread()) {
+ paloMetrics = paloMetrics.stream().filter(m -> !(m.getName().equals(name))).collect(Collectors.toList());
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index f72c5eb..90221e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -113,6 +113,7 @@ public final class MetricRepo {
private static ScheduledThreadPoolExecutor metricTimer = ThreadPoolManager.newDaemonScheduledThreadPool(1, "Metric-Timer-Pool", true);
private static MetricCalculator metricCalculator = new MetricCalculator();
+ // init() should only be called after catalog is contructed.
public static synchronized void init() {
if (isInit) {
return;
@@ -184,8 +185,8 @@ public final class MetricRepo {
}
};
gauge.addLabel(new MetricLabel("job", "alter"))
- .addLabel(new MetricLabel("type", jobType.name()))
- .addLabel(new MetricLabel("state", "running"));
+ .addLabel(new MetricLabel("type", jobType.name()))
+ .addLabel(new MetricLabel("state", "running"));
PALO_METRIC_REGISTER.addPaloMetrics(gauge);
}
@@ -265,17 +266,17 @@ public final class MetricRepo {
PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_CACHE_MODE_SQL);
COUNTER_CACHE_HIT_SQL = new LongCounterMetric("cache_hit_sql", MetricUnit.REQUESTS, "total hits query by sql model");
PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_CACHE_HIT_SQL);
- COUNTER_CACHE_MODE_PARTITION = new LongCounterMetric("query_mode_partition", MetricUnit.REQUESTS,
- "total query of partition mode");
+ COUNTER_CACHE_MODE_PARTITION = new LongCounterMetric("query_mode_partition", MetricUnit.REQUESTS,
+ "total query of partition mode");
PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_CACHE_MODE_PARTITION);
- COUNTER_CACHE_HIT_PARTITION = new LongCounterMetric("cache_hit_partition", MetricUnit.REQUESTS,
- "total hits query by partition model");
+ COUNTER_CACHE_HIT_PARTITION = new LongCounterMetric("cache_hit_partition", MetricUnit.REQUESTS,
+ "total hits query by partition model");
PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_CACHE_HIT_PARTITION);
- COUNTER_CACHE_PARTITION_ALL = new LongCounterMetric("partition_all", MetricUnit.REQUESTS,
- "scan partition of cache partition model");
+ COUNTER_CACHE_PARTITION_ALL = new LongCounterMetric("partition_all", MetricUnit.REQUESTS,
+ "scan partition of cache partition model");
PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_CACHE_PARTITION_ALL);
- COUNTER_CACHE_PARTITION_HIT = new LongCounterMetric("partition_hit", MetricUnit.REQUESTS,
- "hit partition of cache partition model");
+ COUNTER_CACHE_PARTITION_HIT = new LongCounterMetric("partition_hit", MetricUnit.REQUESTS,
+ "hit partition of cache partition model");
PALO_METRIC_REGISTER.addPaloMetrics(COUNTER_CACHE_PARTITION_HIT);
COUNTER_LOAD_FINISHED = new LongCounterMetric("load_finished", MetricUnit.REQUESTS, "total load finished");
@@ -554,10 +555,5 @@ public final class MetricRepo {
public static synchronized List<Metric> getMetricsByName(String name) {
return PALO_METRIC_REGISTER.getPaloMetricsByName(name);
}
-
- public static void addMetric(Metric<?> metric) {
- init();
- PALO_METRIC_REGISTER.addPaloMetrics(metric);
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 04/21: [Bug] Fix bug to reject request with no
SQL in TableQueryPlanAction (#6843)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit e83a86776b846651d14f7a70d92043e4845407e0
Author: dohongdayi <do...@126.com>
AuthorDate: Sat Oct 23 16:46:24 2021 +0800
[Bug] Fix bug to reject request with no SQL in TableQueryPlanAction (#6843)
String.valueOf() returns string "null" with null input, in which case requests with no SQL
will be accepted by TableQueryPlanAction unexpectedly with potential risk.
---
.../doris/http/rest/TableQueryPlanAction.java | 2 +-
.../doris/httpv2/rest/TableQueryPlanAction.java | 2 +-
.../doris/http/TableQueryPlanActionTest.java | 40 ++++++++++++++++++++++
3 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java
index 7634fab..92074ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableQueryPlanAction.java
@@ -117,7 +117,7 @@ public class TableQueryPlanAction extends RestBaseAction {
} catch (JSONException e) {
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "malformed json [ " + postContent + " ]");
}
- sql = String.valueOf(jsonObject.opt("sql"));
+ sql = jsonObject.optString("sql");
if (Strings.isNullOrEmpty(sql)) {
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST, "POST body must contains [sql] root object");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
index d64a788..cf107df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableQueryPlanAction.java
@@ -104,7 +104,7 @@ public class TableQueryPlanAction extends RestBaseController {
return ResponseEntityBuilder.badRequest("malformed json: " + e.getMessage());
}
- sql = String.valueOf(jsonObject.opt("sql"));
+ sql = jsonObject.optString("sql");
if (Strings.isNullOrEmpty(sql)) {
return ResponseEntityBuilder.badRequest("POST body must contains [sql] root object");
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
index 9f9c436..695c5f8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/http/TableQueryPlanActionTest.java
@@ -80,6 +80,46 @@ public class TableQueryPlanActionTest extends DorisHttpTestCase {
}
@Test
+ public void testNoSqlFailure() throws IOException {
+ RequestBody body = RequestBody.create(JSON, "{}");
+ Request request = new Request.Builder()
+ .post(body)
+ .addHeader("Authorization", rootAuth)
+ .url(URI + PATH_URI)
+ .build();
+ Response response = networkClient.newCall(request).execute();
+ String respStr = response.body().string();
+ System.out.println(respStr);
+ Assert.assertNotNull(respStr);
+ expectThrowsNoException(() -> new JSONObject(respStr));
+ JSONObject jsonObject = new JSONObject(respStr);
+ Assert.assertEquals(400, jsonObject.getInt("status"));
+ String exception = jsonObject.getString("exception");
+ Assert.assertNotNull(exception);
+ Assert.assertEquals("POST body must contains [sql] root object", exception);
+ }
+
+ @Test
+ public void testEmptySqlFailure() throws IOException {
+ RequestBody body = RequestBody.create(JSON, "{ \"sql\" : \"\" }");
+ Request request = new Request.Builder()
+ .post(body)
+ .addHeader("Authorization", rootAuth)
+ .url(URI + PATH_URI)
+ .build();
+ Response response = networkClient.newCall(request).execute();
+ String respStr = response.body().string();
+ System.out.println(respStr);
+ Assert.assertNotNull(respStr);
+ expectThrowsNoException(() -> new JSONObject(respStr));
+ JSONObject jsonObject = new JSONObject(respStr);
+ Assert.assertEquals(400, jsonObject.getInt("status"));
+ String exception = jsonObject.getString("exception");
+ Assert.assertNotNull(exception);
+ Assert.assertEquals("POST body must contains [sql] root object", exception);
+ }
+
+ @Test
public void testInconsistentResource() throws IOException {
RequestBody body = RequestBody.create(JSON, "{ \"sql\" : \" select k1,k2 from " + DB_NAME + "." + TABLE_NAME + 1 + " \" }");
Request request = new Request.Builder()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 09/21: [Cache][Bug] Fix sql_key of getting and
updating Cache is inconsistent (#6903)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit a99f2b5c5f6f387e1d61ab41dcbc83ff440ef21c
Author: Zeno Yang <co...@qq.com>
AuthorDate: Sat Oct 23 16:54:00 2021 +0800
[Cache][Bug] Fix sql_key of getting and updating Cache is inconsistent (#6903)
Fix #6735
---
.../src/main/java/org/apache/doris/qe/cache/PartitionCache.java | 4 +++-
fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
index 1ff1fdb..dd2c53b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
@@ -130,7 +130,9 @@ public class PartitionCache extends Cache {
return;
}
- InternalService.PUpdateCacheRequest updateRequest = rowBatchBuilder.buildPartitionUpdateRequest(nokeyStmt.toSql());
+ String nokeyStmtWithViewStmt = nokeyStmt.toSql() + allViewExpandStmtListStr;
+ InternalService.PUpdateCacheRequest updateRequest
+ = rowBatchBuilder.buildPartitionUpdateRequest(nokeyStmtWithViewStmt);
if (updateRequest.getValuesCount() > 0) {
CacheBeProxy proxy = new CacheBeProxy();
Status status = new Status();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
index 9485547..f64312e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
@@ -75,8 +75,9 @@ public class SqlCache extends Cache {
return;
}
+ String originStmtWithViewStmt = selectStmt.getOrigStmt().originStmt + allViewExpandStmtListStr;
InternalService.PUpdateCacheRequest updateRequest =
- rowBatchBuilder.buildSqlUpdateRequest(selectStmt.getOrigStmt().originStmt, latestTable.latestPartitionId,
+ rowBatchBuilder.buildSqlUpdateRequest(originStmtWithViewStmt, latestTable.latestPartitionId,
latestTable.latestVersion, latestTable.latestTime);
if (updateRequest.getValuesCount() > 0) {
CacheBeProxy proxy = new CacheBeProxy();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 10/21: [Flink]Simplify the use of flink connector
(#6892)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit cef3d3181e030de51641b20d3ff0fdefcbf92bde
Author: xiaokangguo <40...@users.noreply.github.com>
AuthorDate: Sat Oct 23 18:10:47 2021 +0800
[Flink]Simplify the use of flink connector (#6892)
1. Simplify the use of flink connector like other stream sink by GenericDorisSinkFunction.
2. Add the use cases of flink connector.
## Use case
```
env.fromElements("{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
.addSink(
DorisSink.sink(
DorisOptions.builder()
.setFenodes("FE_IP:8030")
.setTableIdentifier("db.table")
.setUsername("root")
.setPassword("").build()
));
```
---
docs/en/extending-doris/flink-doris-connector.md | 107 +++++++++-
.../zh-CN/extending-doris/flink-doris-connector.md | 106 +++++++++-
.../doris/flink/cfg/DorisExecutionOptions.java | 20 +-
.../apache/doris/flink/cfg/DorisReadOptions.java | 5 +
.../java/org/apache/doris/flink/cfg/DorisSink.java | 95 +++++++++
.../doris/flink/cfg/GenericDorisSinkFunction.java | 53 +++++
.../flink/table/DorisDynamicOutputFormat.java | 77 +++++---
.../doris/flink/DorisOutPutFormatExample.java | 84 ++++++++
.../apache/doris/flink/DorisStreamSinkExample.java | 219 +++++++++++++++++++++
9 files changed, 728 insertions(+), 38 deletions(-)
diff --git a/docs/en/extending-doris/flink-doris-connector.md b/docs/en/extending-doris/flink-doris-connector.md
index 3046169..c42d237 100644
--- a/docs/en/extending-doris/flink-doris-connector.md
+++ b/docs/en/extending-doris/flink-doris-connector.md
@@ -119,7 +119,7 @@ CREATE TABLE flink_doris_sink (
INSERT INTO flink_doris_sink select name,age,price,sale from flink_doris_source
```
-### DataStream
+### DataStreamSource
```scala
Properties properties = new Properties();
@@ -130,6 +130,111 @@ INSERT INTO flink_doris_sink select name,age,price,sale from flink_doris_source
env.addSource(new DorisSourceFunction(new DorisStreamOptions(properties),new SimpleListDeserializationSchema())).print();
```
+### DataStreamSink
+
+```java
+// -------- sink with raw json string stream --------
+Properties pro = new Properties();
+pro.setProperty("format", "json");
+pro.setProperty("strip_outer_array", "true");
+env.fromElements( "{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
+ .addSink(
+ DorisSink.sink(
+ DorisReadOptions.builder().build(),
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0l)
+ .setMaxRetries(3)
+ .setStreamLoadProp(pro).build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+
+OR
+env.fromElements("{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
+ .addSink(
+ DorisSink.sink(
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+
+
+// -------- sink with RowData stream --------
+DataStream<RowData> source = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+String[] fields = {"city", "longitude", "latitude"};
+LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+
+source.addSink(
+ DorisSink.sink(
+ fields,
+ types,
+ DorisReadOptions.builder().build(),
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0L)
+ .setMaxRetries(3)
+ .build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+```
+
+### DataSetSink
+
+```java
+MapOperator<String, RowData> data = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+DorisOptions dorisOptions = DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build();
+DorisReadOptions readOptions = DorisReadOptions.defaults();
+DorisExecutionOptions executionOptions = DorisExecutionOptions.defaults();
+
+LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+String[] fiels = {"city", "longitude", "latitude"};
+
+DorisDynamicOutputFormat outputFormat =
+ new DorisDynamicOutputFormat(dorisOptions, readOptions, executionOptions, types, fiels);
+
+outputFormat.open(0, 1);
+data.output(outputFormat);
+outputFormat.close();
+```
+
+
+
### General
| Key | Default Value | Comment |
diff --git a/docs/zh-CN/extending-doris/flink-doris-connector.md b/docs/zh-CN/extending-doris/flink-doris-connector.md
index f9dca64..9ea1eba 100644
--- a/docs/zh-CN/extending-doris/flink-doris-connector.md
+++ b/docs/zh-CN/extending-doris/flink-doris-connector.md
@@ -120,7 +120,7 @@ CREATE TABLE flink_doris_sink (
INSERT INTO flink_doris_sink select name,age,price,sale from flink_doris_source
```
-### DataStream
+### DataStreamSource
```java
Properties properties = new Properties();
@@ -131,6 +131,110 @@ INSERT INTO flink_doris_sink select name,age,price,sale from flink_doris_source
env.addSource(new DorisSourceFunction(new DorisStreamOptions(properties),new SimpleListDeserializationSchema())).print();
```
+### DataStreamSink
+
+```java
+// -------- sink with raw json string stream --------
+Properties pro = new Properties();
+pro.setProperty("format", "json");
+pro.setProperty("strip_outer_array", "true");
+env.fromElements( "{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
+ .addSink(
+ DorisSink.sink(
+ DorisReadOptions.builder().build(),
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0l)
+ .setMaxRetries(3)
+ .setStreamLoadProp(pro).build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+
+OR
+env.fromElements("{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
+ .addSink(
+ DorisSink.sink(
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+
+
+// -------- sink with RowData stream --------
+DataStream<RowData> source = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+String[] fields = {"city", "longitude", "latitude"};
+LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+
+source.addSink(
+ DorisSink.sink(
+ fields,
+ types,
+ DorisReadOptions.builder().build(),
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0L)
+ .setMaxRetries(3)
+ .build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+```
+
+### DataSetSink
+
+```java
+MapOperator<String, RowData> data = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+DorisOptions dorisOptions = DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build();
+DorisReadOptions readOptions = DorisReadOptions.defaults();
+DorisExecutionOptions executionOptions = DorisExecutionOptions.defaults();
+
+LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+String[] fiels = {"city", "longitude", "latitude"};
+
+DorisDynamicOutputFormat outputFormat =
+ new DorisDynamicOutputFormat(dorisOptions, readOptions, executionOptions, types, fiels);
+
+outputFormat.open(0, 1);
+data.output(outputFormat);
+outputFormat.close();
+```
+
+
## 配置
diff --git a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
index 3d035ab..587ab07 100644
--- a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
+++ b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisExecutionOptions.java
@@ -20,7 +20,6 @@ package org.apache.doris.flink.cfg;
import org.apache.flink.util.Preconditions;
import java.io.Serializable;
-import java.time.Duration;
import java.util.Properties;
/**
@@ -29,6 +28,10 @@ import java.util.Properties;
public class DorisExecutionOptions implements Serializable {
private static final long serialVersionUID = 1L;
+ public static final Integer DEFAULT_BATCH_SIZE = 1000;
+ public static final Integer DEFAULT_MAX_RETRY_TIMES = 3;
+ private static final Long DEFAULT_INTERVAL_MILLIS = 10000L;
+
private final Integer batchSize;
private final Integer maxRetries;
private final Long batchIntervalMs;
@@ -66,14 +69,21 @@ public class DorisExecutionOptions implements Serializable {
return new Builder();
}
+ public static DorisExecutionOptions defaults() {
+ Properties pro = new Properties();
+ pro.setProperty("format", "json");
+ pro.setProperty("strip_outer_array", "true");
+ return new Builder().setStreamLoadProp(pro).build();
+ }
+
/**
* Builder of {@link DorisExecutionOptions}.
*/
public static class Builder {
- private Integer batchSize;
- private Integer maxRetries;
- private Long batchIntervalMs;
- private Properties streamLoadProp;
+ private Integer batchSize = DEFAULT_BATCH_SIZE;
+ private Integer maxRetries = DEFAULT_MAX_RETRY_TIMES;
+ private Long batchIntervalMs = DEFAULT_INTERVAL_MILLIS;
+ private Properties streamLoadProp = new Properties();
public Builder setBatchSize(Integer batchSize) {
this.batchSize = batchSize;
diff --git a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
index 53cefaa..0beb18c 100644
--- a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
+++ b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisReadOptions.java
@@ -103,6 +103,10 @@ public class DorisReadOptions implements Serializable {
return new Builder();
}
+ public static DorisReadOptions defaults(){
+ return DorisReadOptions.builder().build();
+ }
+
/**
* Builder of {@link DorisReadOptions}.
*/
@@ -179,6 +183,7 @@ public class DorisReadOptions implements Serializable {
public DorisReadOptions build() {
return new DorisReadOptions(readFields, filterQuery, requestTabletSize, requestConnectTimeoutMs, requestReadTimeoutMs, requestQueryTimeoutS, requestRetries, requestBatchSize, execMemLimit, deserializeQueueSize, deserializeArrowAsync);
}
+
}
diff --git a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisSink.java b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisSink.java
new file mode 100644
index 0000000..f11c587
--- /dev/null
+++ b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/DorisSink.java
@@ -0,0 +1,95 @@
+package org.apache.doris.flink.cfg;
+
+import org.apache.doris.flink.table.DorisDynamicOutputFormat;
+import org.apache.flink.streaming.api.functions.sink.SinkFunction;
+import org.apache.flink.table.types.logical.LogicalType;
+
+/** Facade to create Doris {@link SinkFunction sinks}. */
+public class DorisSink {
+
+
+ private DorisSink() {
+ }
+
+
+ /**
+ * Create a Doris DataStream sink with the default {@link DorisReadOptions}
+ * stream elements could only be JsonString.
+ *
+ * @see #sink(String[], LogicalType[], DorisReadOptions, DorisExecutionOptions, DorisOptions)
+ */
+ public static <T> SinkFunction<T> sink(DorisExecutionOptions executionOptions, DorisOptions dorisOptions) {
+
+ return sink(new String[]{}, new LogicalType[]{}, DorisReadOptions.defaults(), executionOptions, dorisOptions);
+ }
+
+ /**
+ * Create a Doris DataStream sink with the default {@link DorisReadOptions}
+ * stream elements could only be RowData.
+ *
+ * @see #sink(String[], LogicalType[], DorisReadOptions, DorisExecutionOptions, DorisOptions)
+ */
+ public static <T> SinkFunction<T> sink(String[] fiels, LogicalType[] types,
+ DorisExecutionOptions executionOptions, DorisOptions dorisOptions) {
+
+ return sink(fiels, types, DorisReadOptions.defaults(), executionOptions, dorisOptions);
+ }
+
+ /**
+ * Create a Doris DataStream sink with the default {@link DorisExecutionOptions}
+ * stream elements could only be JsonString.
+ *
+ * @see #sink(String[], LogicalType[], DorisReadOptions, DorisExecutionOptions, DorisOptions)
+ */
+ public static <T> SinkFunction<T> sink(DorisOptions dorisOptions) {
+
+ return sink(new String[]{}, new LogicalType[]{}, DorisReadOptions.defaults(),
+ DorisExecutionOptions.defaults(), dorisOptions);
+ }
+
+ /**
+ * Create a Doris DataStream sink with the default {@link DorisExecutionOptions}
+ * stream elements could only be RowData.
+ *
+ * @see #sink(String[], LogicalType[], DorisReadOptions, DorisExecutionOptions, DorisOptions)
+ */
+ public static <T> SinkFunction<T> sink(String[] fiels, LogicalType[] types, DorisOptions dorisOptions) {
+ return sink(fiels, types, DorisReadOptions.defaults(), DorisExecutionOptions.defaults(), dorisOptions);
+ }
+
+
+ /**
+ * Create a Doris DataStream sink, stream elements could only be JsonString.
+ *
+ * @see #sink(String[], LogicalType[], DorisReadOptions, DorisExecutionOptions, DorisOptions)
+ */
+ public static <T> SinkFunction<T> sink(DorisReadOptions readOptions,
+ DorisExecutionOptions executionOptions, DorisOptions dorisOptions) {
+
+ return sink(new String[]{}, new LogicalType[]{}, readOptions, executionOptions, dorisOptions);
+ }
+
+
+ /**
+ * Create a Doris DataStream sink, stream elements could only be RowData.
+ *
+ * <p>Note: the objects passed to the return sink can be processed in batch and retried.
+ * Therefore, objects can not be {@link org.apache.flink.api.common.ExecutionConfig#enableObjectReuse() reused}.
+ * </p>
+ *
+ * @param field array of field
+ * @param types types of field
+ * @param readOptions parameters of read, such as readFields, filterQuery
+ * @param executionOptions parameters of execution, such as batch size and maximum retries
+ * @param dorisOptions parameters of options, such as fenodes, username, password, tableIdentifier
+ * @param <T> type of data in {@link org.apache.flink.streaming.runtime.streamrecord.StreamRecord
+ * StreamRecord}.
+ */
+ public static <T> SinkFunction<T> sink(String[] field, LogicalType[] types, DorisReadOptions readOptions,
+ DorisExecutionOptions executionOptions, DorisOptions dorisOptions) {
+
+ return new GenericDorisSinkFunction(new DorisDynamicOutputFormat(
+ dorisOptions, readOptions, executionOptions, types, field));
+ }
+
+}
diff --git a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/GenericDorisSinkFunction.java b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/GenericDorisSinkFunction.java
new file mode 100644
index 0000000..92dd300
--- /dev/null
+++ b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/cfg/GenericDorisSinkFunction.java
@@ -0,0 +1,53 @@
+package org.apache.doris.flink.cfg;
+
+import org.apache.doris.flink.table.DorisDynamicOutputFormat;
+import org.apache.flink.api.common.functions.RuntimeContext;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nonnull;
+
+public class GenericDorisSinkFunction<T> extends RichSinkFunction<T>
+ implements CheckpointedFunction {
+
+ private final DorisDynamicOutputFormat outputFormat;
+
+ public GenericDorisSinkFunction(@Nonnull DorisDynamicOutputFormat outputFormat) {
+ this.outputFormat = Preconditions.checkNotNull(outputFormat);
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+ RuntimeContext ctx = getRuntimeContext();
+ outputFormat.setRuntimeContext(ctx);
+ outputFormat.open(ctx.getIndexOfThisSubtask(), ctx.getNumberOfParallelSubtasks());
+ }
+
+
+ @Override
+ public void invoke(T value, Context context) throws Exception {
+ outputFormat.writeRecord(value);
+ }
+
+ @Override
+ public void initializeState(FunctionInitializationContext context) throws Exception {
+
+ }
+
+ @Override
+ public void snapshotState(FunctionSnapshotContext context) throws Exception {
+
+ }
+
+ @Override
+ public void close() throws Exception {
+ outputFormat.close();
+ super.close();
+ }
+
+}
\ No newline at end of file
diff --git a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicOutputFormat.java b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicOutputFormat.java
index 73c68b6..0fd154a 100644
--- a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicOutputFormat.java
+++ b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/table/DorisDynamicOutputFormat.java
@@ -34,10 +34,10 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Arrays;
import java.util.StringJoiner;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -50,7 +50,7 @@ import static org.apache.flink.table.data.RowData.createFieldGetter;
/**
* DorisDynamicOutputFormat
**/
-public class DorisDynamicOutputFormat extends RichOutputFormat<RowData> {
+public class DorisDynamicOutputFormat<T> extends RichOutputFormat<T> {
private static final Logger LOG = LoggerFactory.getLogger(DorisDynamicOutputFormat.class);
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@@ -88,8 +88,10 @@ public class DorisDynamicOutputFormat extends RichOutputFormat<RowData> {
this.options = option;
this.readOptions = readOptions;
this.executionOptions = executionOptions;
- this.fieldDelimiter = executionOptions.getStreamLoadProp().getProperty(FIELD_DELIMITER_KEY, FIELD_DELIMITER_DEFAULT);
- this.lineDelimiter = executionOptions.getStreamLoadProp().getProperty(LINE_DELIMITER_KEY, LINE_DELIMITER_DEFAULT);
+ this.fieldDelimiter = executionOptions.getStreamLoadProp().getProperty(FIELD_DELIMITER_KEY,
+ FIELD_DELIMITER_DEFAULT);
+ this.lineDelimiter = executionOptions.getStreamLoadProp().getProperty(LINE_DELIMITER_KEY,
+ LINE_DELIMITER_DEFAULT);
this.fieldNames = fieldNames;
this.jsonFormat = FORMAT_JSON_VALUE.equals(executionOptions.getStreamLoadProp().getProperty(FORMAT_KEY));
this.fieldGetters = new RowData.FieldGetter[logicalTypes.length];
@@ -98,6 +100,7 @@ public class DorisDynamicOutputFormat extends RichOutputFormat<RowData> {
}
}
+
@Override
public void configure(Configuration configuration) {
}
@@ -105,16 +108,17 @@ public class DorisDynamicOutputFormat extends RichOutputFormat<RowData> {
@Override
public void open(int taskNumber, int numTasks) throws IOException {
dorisStreamLoad = new DorisStreamLoad(
- getBackend(),
- options.getTableIdentifier().split("\\.")[0],
- options.getTableIdentifier().split("\\.")[1],
- options.getUsername(),
- options.getPassword(),
- executionOptions.getStreamLoadProp());
+ getBackend(),
+ options.getTableIdentifier().split("\\.")[0],
+ options.getTableIdentifier().split("\\.")[1],
+ options.getUsername(),
+ options.getPassword(),
+ executionOptions.getStreamLoadProp());
LOG.info("Streamload BE:{}", dorisStreamLoad.getLoadUrlStr());
if (executionOptions.getBatchIntervalMs() != 0 && executionOptions.getBatchSize() != 1) {
- this.scheduler = Executors.newScheduledThreadPool(1, new ExecutorThreadFactory("doris-streamload-output-format"));
+ this.scheduler = Executors.newScheduledThreadPool(1, new ExecutorThreadFactory("doris-streamload-output" +
+ "-format"));
this.scheduledFuture = this.scheduler.scheduleWithFixedDelay(() -> {
synchronized (DorisDynamicOutputFormat.this) {
if (!closed) {
@@ -136,30 +140,37 @@ public class DorisDynamicOutputFormat extends RichOutputFormat<RowData> {
}
@Override
- public synchronized void writeRecord(RowData row) throws IOException {
+ public synchronized void writeRecord(T row) throws IOException {
checkFlushException();
-
addBatch(row);
if (executionOptions.getBatchSize() > 0 && batch.size() >= executionOptions.getBatchSize()) {
flush();
}
}
- private void addBatch(RowData row) {
- Map<String, String> valueMap = new HashMap<>();
- StringJoiner value = new StringJoiner(this.fieldDelimiter);
- for (int i = 0; i < row.getArity() && i < fieldGetters.length; ++i) {
- Object field = fieldGetters[i].getFieldOrNull(row);
- if (jsonFormat) {
- String data = field != null ? field.toString() : null;
- valueMap.put(this.fieldNames[i], data);
- } else {
- String data = field != null ? field.toString() : NULL_VALUE;
- value.add(data);
+ private void addBatch(T row) {
+ if (row instanceof RowData) {
+ RowData rowData = (RowData) row;
+ Map<String, String> valueMap = new HashMap<>();
+ StringJoiner value = new StringJoiner(this.fieldDelimiter);
+ for (int i = 0; i < rowData.getArity() && i < fieldGetters.length; ++i) {
+ Object field = fieldGetters[i].getFieldOrNull(rowData);
+ if (jsonFormat) {
+ String data = field != null ? field.toString() : null;
+ valueMap.put(this.fieldNames[i], data);
+ } else {
+ String data = field != null ? field.toString() : NULL_VALUE;
+ value.add(data);
+ }
}
+ Object data = jsonFormat ? valueMap : value.toString();
+ batch.add(data);
+
+ } else if (row instanceof String) {
+ batch.add(row);
+ } else {
+ throw new RuntimeException("The type of element should be 'RowData' or 'String' only.");
}
- Object data = jsonFormat ? valueMap : value.toString();
- batch.add(data);
}
@Override
@@ -189,7 +200,11 @@ public class DorisDynamicOutputFormat extends RichOutputFormat<RowData> {
}
String result;
if (jsonFormat) {
- result = OBJECT_MAPPER.writeValueAsString(batch);
+ if (batch.get(0) instanceof String) {
+ result = batch.toString();
+ } else {
+ result = OBJECT_MAPPER.writeValueAsString(batch);
+ }
} else {
result = String.join(this.lineDelimiter, batch);
}
@@ -292,11 +307,11 @@ public class DorisDynamicOutputFormat extends RichOutputFormat<RowData> {
public DorisDynamicOutputFormat build() {
final LogicalType[] logicalTypes =
- Arrays.stream(fieldDataTypes)
- .map(DataType::getLogicalType)
- .toArray(LogicalType[]::new);
+ Arrays.stream(fieldDataTypes)
+ .map(DataType::getLogicalType)
+ .toArray(LogicalType[]::new);
return new DorisDynamicOutputFormat(
- optionsBuilder.build(), readOptions, executionOptions, logicalTypes, fieldNames
+ optionsBuilder.build(), readOptions, executionOptions, logicalTypes, fieldNames
);
}
}
diff --git a/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisOutPutFormatExample.java b/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisOutPutFormatExample.java
new file mode 100644
index 0000000..a64e3d9
--- /dev/null
+++ b/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisOutPutFormatExample.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.doris.flink;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.table.DorisDynamicOutputFormat;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.operators.MapOperator;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.types.logical.DoubleType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.VarCharType;
+
+import java.io.IOException;
+
+/**
+ * example using {@link DorisDynamicOutputFormat} for batching.
+ */
+public class DorisOutPutFormatExample {
+
+ public static void main(String[] args) throws Exception {
+
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ MapOperator<String, RowData> data = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+ DorisOptions dorisOptions = DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build();
+ DorisReadOptions readOptions = DorisReadOptions.defaults();
+ DorisExecutionOptions executionOptions = DorisExecutionOptions.defaults();
+
+ LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+ String[] fiels = {"city", "longitude", "latitude"};
+
+ DorisDynamicOutputFormat outputFormat =
+ new DorisDynamicOutputFormat(dorisOptions, readOptions, executionOptions, types, fiels);
+
+ try {
+ outputFormat.open(0, 1);
+ data.output(outputFormat);
+ outputFormat.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ env.execute("doris batch sink example");
+
+
+ }
+
+}
diff --git a/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisStreamSinkExample.java b/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisStreamSinkExample.java
new file mode 100644
index 0000000..d37fd0d
--- /dev/null
+++ b/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/DorisStreamSinkExample.java
@@ -0,0 +1,219 @@
+package org.apache.doris.flink;
+
+import org.apache.doris.flink.cfg.DorisExecutionOptions;
+import org.apache.doris.flink.cfg.DorisOptions;
+import org.apache.doris.flink.cfg.DorisReadOptions;
+import org.apache.doris.flink.cfg.DorisSink;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.types.logical.DoubleType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.VarCharType;
+
+import java.util.Properties;
+
+/**
+ * example using {@link DorisSink} for streaming.
+ */
+public class DorisStreamSinkExample {
+
+
+ public void testJsonString() throws Exception {
+ /*
+ * Example for JsonString element
+ */
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ Properties pro = new Properties();
+ pro.setProperty("format", "json");
+ pro.setProperty("strip_outer_array", "true");
+ env.fromElements("{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
+ .addSink(
+ DorisSink.sink(
+ DorisReadOptions.builder().build(),
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0l)
+ .setMaxRetries(3)
+ .setStreamLoadProp(pro).build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+ env.execute("doris stream sink example");
+ }
+
+
+ public void testJsonStringWithDefaultReadOptions() throws Exception {
+ /*
+ * Example for JsonString element with default ReadOptions
+ */
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ Properties pro = new Properties();
+ pro.setProperty("format", "json");
+ pro.setProperty("strip_outer_array", "true");
+ env.fromElements("{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
+ .addSink(
+ DorisSink.sink(
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0l)
+ .setMaxRetries(3)
+ .setStreamLoadProp(pro).build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+ env.execute("doris stream sink example");
+ }
+
+
+ public void testJsonStringWithDefaultReadOptionsAndExecutionOptions() throws Exception {
+ /*
+ * Example for JsonString element with default ReadOptions and ExecutionOptions
+ */
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ env.fromElements("{\"longitude\": \"116.405419\", \"city\": \"北京\", \"latitude\": \"39.916927\"}")
+ .addSink(
+ DorisSink.sink(
+ DorisOptions.builder()
+ .setFenodes("192.168.52.101:8030")
+ .setTableIdentifier("smarttrip_db.doris_output_format")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+ env.execute("doris stream sink example");
+ }
+
+
+ public void testRowData() throws Exception {
+ /*
+ * Example for RowData element
+ */
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ DataStream<RowData> source = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+ String[] fields = {"city", "longitude", "latitude"};
+ LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+
+ source.addSink(
+ DorisSink.sink(
+ fields,
+ types,
+ DorisReadOptions.builder().build(),
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0L)
+ .setMaxRetries(3)
+ .build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+ env.execute("doris stream sink example");
+ }
+
+
+ public void testRowDataWithDefaultReadOptions() throws Exception {
+ /*
+ * Example for RowData element with default ReadOptions
+ */
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ DataStream<RowData> source = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+ String[] fields = {"city", "longitude", "latitude"};
+ LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+
+ source.addSink(
+ DorisSink.sink(
+ fields,
+ types,
+ DorisExecutionOptions.builder()
+ .setBatchSize(3)
+ .setBatchIntervalMs(0L)
+ .setMaxRetries(3)
+ .build(),
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+ env.execute("doris stream sink example");
+ }
+
+
+ public void testRowDataWithDefaultReadOptionsAndExecutionOptions() throws Exception {
+ /*
+ * Example for RowData element with default ReadOptions and ExecutionOptions
+ */
+ StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
+ env.setParallelism(1);
+
+ DataStream<RowData> source = env.fromElements("")
+ .map(new MapFunction<String, RowData>() {
+ @Override
+ public RowData map(String value) throws Exception {
+ GenericRowData genericRowData = new GenericRowData(3);
+ genericRowData.setField(0, StringData.fromString("北京"));
+ genericRowData.setField(1, 116.405419);
+ genericRowData.setField(2, 39.916927);
+ return genericRowData;
+ }
+ });
+
+ String[] fields = {"city", "longitude", "latitude"};
+ LogicalType[] types = {new VarCharType(), new DoubleType(), new DoubleType()};
+
+ source.addSink(
+ DorisSink.sink(
+ fields,
+ types,
+ DorisOptions.builder()
+ .setFenodes("FE_IP:8030")
+ .setTableIdentifier("db.table")
+ .setUsername("root")
+ .setPassword("").build()
+ ));
+ env.execute("doris stream sink example");
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 17/21: [Bug] Fix treat tuple_is_null_predicate is
const expr cause core problem (#6919)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 9cc94650d7b83ae0f082b15e9697057d9d847f18
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Wed Oct 27 09:54:25 2021 +0800
[Bug] Fix treat tuple_is_null_predicate is const expr cause core problem (#6919)
Fix treat tuple_is_null_predicate is const expr cause core problem
---
be/src/exprs/tuple_is_null_predicate.h | 3 +++
1 file changed, 3 insertions(+)
diff --git a/be/src/exprs/tuple_is_null_predicate.h b/be/src/exprs/tuple_is_null_predicate.h
index 2125887..58069d5 100644
--- a/be/src/exprs/tuple_is_null_predicate.h
+++ b/be/src/exprs/tuple_is_null_predicate.h
@@ -31,6 +31,9 @@ public:
return pool->add(new TupleIsNullPredicate(*this));
}
+ bool is_constant() const override {
+ return false;
+ }
protected:
friend class Expr;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 01/21: Replace replica_allocation to
replication_allocation (#6870)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit b913aa13e4f470b90eeaf0efa714651f4c43164c
Author: zh0122 <zh...@gmail.com>
AuthorDate: Wed Oct 20 15:32:35 2021 +0800
Replace replica_allocation to replication_allocation (#6870)
Fix #6869
---
docs/en/administrator-guide/multi-tenant.md | 4 ++--
.../Data-Definition-Statements/Create/CREATE-TABLE.md | 12 ++++++------
.../sql-statements/Data Definition/CREATE TABLE.md | 10 +++++-----
docs/zh-CN/administrator-guide/multi-tenant.md | 4 ++--
.../Data-Definition-Statements/Create/CREATE-TABLE.md | 12 ++++++------
.../sql-statements/Data Definition/CREATE TABLE.md | 10 +++++-----
.../org/apache/doris/analysis/ModifyPartitionClause.java | 2 +-
.../java/org/apache/doris/catalog/ModifyBackendTagTest.java | 2 +-
8 files changed, 28 insertions(+), 28 deletions(-)
diff --git a/docs/en/administrator-guide/multi-tenant.md b/docs/en/administrator-guide/multi-tenant.md
index ac0a808..a758ac8 100644
--- a/docs/en/administrator-guide/multi-tenant.md
+++ b/docs/en/administrator-guide/multi-tenant.md
@@ -72,7 +72,7 @@ Node resource division refers to setting tags for BE nodes in a Doris cluster, a
(k1 int, k2 int)
distributed by hash(k1) buckets 1
properties(
- "replica_allocation"
+ "replication_allocation"
=
"tag.location.group_a:1, tag.location.group_b:1, tag.location.group_c:1"
)
@@ -201,7 +201,7 @@ Here we give an example of the steps to start using the resource division functi
```
alter system modify backend "host1:9050, 1212:9050" set ("tag.location" = "group_a");
- alter table my_table modify partition p1 set ("replica_allocation" = "tag.location.group_a:2");
+ alter table my_table modify partition p1 set ("replication_allocation" = "tag.location.group_a:2");
```
3. Turn on data repair and balance logic
diff --git a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md
index fd44cf0..a2a706b 100644
--- a/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md
+++ b/docs/en/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md
@@ -238,11 +238,11 @@ distribution_info
Number of copies. The default number of copies is 3. If the number of BE nodes is less than 3, you need to specify that the number of copies is less than or equal to the number of BE nodes.
- After version 0.15, this attribute will be automatically converted to the `replica_allocation` attribute, such as:
+ After version 0.15, this attribute will be automatically converted to the `replication_allocation` attribute, such as:
- `"replication_num" = "3"` will be automatically converted to `"replica_allocation" = "tag.location.default:3"`
+ `"replication_num" = "3"` will be automatically converted to `"replication_allocation" = "tag.location.default:3"`
- * `replica_allocation`
+ * `replication_allocation`
Set the copy distribution according to Tag. This attribute can completely cover the function of the `replication_num` attribute.
@@ -487,7 +487,7 @@ distribution_info
PROPERTIES("replication_num" = "3");
```
-10. Set the replica of the table through the `replica_allocation` property.
+10. Set the replica of the table through the `replication_allocation` property.
```sql
CREATE TABLE example_db.table_hash
@@ -497,7 +497,7 @@ distribution_info
)
DISTRIBUTED BY HASH(k1) BUCKETS 32
PROPERTIES (
- "replica_allocation"="tag.location.group_a:1, tag.location.group_b:2"
+ "replication_allocation"="tag.location.group_a:1, tag.location.group_b:2"
);
CREATE TABLE example_db.dynamic_partition
@@ -516,7 +516,7 @@ distribution_info
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32",
- "dynamic_partition."replica_allocation" = "tag.location.group_a:3"
+ "dynamic_partition."replication_allocation" = "tag.location.group_a:3"
);
```
### Keywords
diff --git a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
index 707f19d..92e4829 100644
--- a/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
+++ b/docs/en/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
@@ -264,7 +264,7 @@ Syntax:
"storage_medium" = "[SSD|HDD]",
["storage_cooldown_time" = "yyyy-MM-dd HH:mm:ss"],
["replication_num" = "3"],
- ["replica_allocation" = "xxx"]
+ ["replication_allocation" = "xxx"]
)
```
@@ -274,7 +274,7 @@ Syntax:
Default is 30 days.
Format: "yyyy-MM-dd HH:mm:ss"
replication_num: Replication number of a partition. Default is 3.
- replica_allocation: Specify the distribution of replicas according to the resource tag.
+ replication_allocation: Specify the distribution of replicas according to the resource tag.
If table is not range partitions. This property takes on Table level. Or it will takes on Partition level.
User can specify different properties for different partition by `ADD PARTITION` or `MODIFY PARTITION` statements.
@@ -754,7 +754,7 @@ Syntax:
);
```
-16. Specify the replica distribution of the table through replica_allocation
+16. Specify the replica distribution of the table through replication_allocation
```
CREATE TABLE example_db.table_hash
@@ -764,7 +764,7 @@ Syntax:
)
DISTRIBUTED BY HASH(k1) BUCKETS 32
PROPERTIES (
- "replica_allocation"="tag.location.group_a:1, tag.location.group_b:2"
+ "replication_allocation"="tag.location.group_a:1, tag.location.group_b:2"
);
CREATE TABLE example_db.dynamic_partition
@@ -783,7 +783,7 @@ Syntax:
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32",
- "dynamic_partition."replica_allocation" = "tag.location.group_a:3"
+ "dynamic_partition."replication_allocation" = "tag.location.group_a:3"
);
```
diff --git a/docs/zh-CN/administrator-guide/multi-tenant.md b/docs/zh-CN/administrator-guide/multi-tenant.md
index 37b754b..18f5a29 100644
--- a/docs/zh-CN/administrator-guide/multi-tenant.md
+++ b/docs/zh-CN/administrator-guide/multi-tenant.md
@@ -72,7 +72,7 @@ FE 不参与用户数据的处理计算等工作，因此是一个资源消耗
(k1 int, k2 int)
distributed by hash(k1) buckets 1
properties(
- "replica_allocation"
+ "replication_allocation"
=
"tag.location.group_a:1, tag.location.group_b:1, tag.location.group_c:1"
)
@@ -201,7 +201,7 @@ Tag 划分和 CPU 限制是 0.15 版本中的新功能。为了保证可以从
```
alter system modify backend "host1:9050, 1212:9050" set ("tag.location" = "group_a");
- alter table my_table modify partition p1 set ("replica_allocation" = "tag.location.group_a:2");
+ alter table my_table modify partition p1 set ("replication_allocation" = "tag.location.group_a:2");
```
3. 开启数据修复与均衡逻辑
diff --git a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md
index 7e393ed..7ef911b 100644
--- a/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md
+++ b/docs/zh-CN/sql-reference-v2/sql-statements/Data-Definition-Statements/Create/CREATE-TABLE.md
@@ -238,11 +238,11 @@ distribution_info
副本数。默认副本数为3。如果 BE 节点数量小于3，则需指定副本数小于等于 BE 节点数量。
- 在 0.15 版本后，该属性将自动转换成 `replica_allocation` 属性，如：
+ 在 0.15 版本后，该属性将自动转换成 `replication_allocation` 属性，如：
- `"replication_num" = "3"` 会自动转换成 `"replica_allocation" = "tag.location.default:3"`
+ `"replication_num" = "3"` 会自动转换成 `"replication_allocation" = "tag.location.default:3"`
- * `replica_allocation`
+ * `replication_allocation`
根据 Tag 设置副本分布情况。该属性可以完全覆盖 `replication_num` 属性的功能。
@@ -487,7 +487,7 @@ distribution_info
PROPERTIES("replication_num" = "3");
```
-10. 通过 `replica_allocation` 属性设置表的副本。
+10. 通过 `replication_allocation` 属性设置表的副本。
```sql
CREATE TABLE example_db.table_hash
@@ -497,7 +497,7 @@ distribution_info
)
DISTRIBUTED BY HASH(k1) BUCKETS 32
PROPERTIES (
- "replica_allocation"="tag.location.group_a:1, tag.location.group_b:2"
+ "replication_allocation"="tag.location.group_a:1, tag.location.group_b:2"
);
@@ -517,7 +517,7 @@ distribution_info
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32",
- "dynamic_partition."replica_allocation" = "tag.location.group_a:3"
+ "dynamic_partition."replication_allocation" = "tag.location.group_a:3"
);
```
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
index a99fd99..9b0dcb1 100644
--- a/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Definition/CREATE TABLE.md
@@ -284,7 +284,7 @@ under the License.
"storage_medium" = "[SSD|HDD]",
["storage_cooldown_time" = "yyyy-MM-dd HH:mm:ss"],
["replication_num" = "3"]
- ["replica_allocation" = "xxx"]
+ ["replication_allocation" = "xxx"]
)
```
@@ -294,7 +294,7 @@ under the License.
默认存放 30 天。
格式为："yyyy-MM-dd HH:mm:ss"
replication_num: 指定分区的副本数。默认为 3。
- replica_allocation: 按照资源标签来指定副本分布。
+ replication_allocation: 按照资源标签来指定副本分布。
当表为单分区表时，这些属性为表的属性。
当表为两级分区时，这些属性为附属于每一个分区。
@@ -795,7 +795,7 @@ under the License.
);
```
-16. 通过 replica_allocation 指定表的副本分布
+16. 通过 replication_allocation 指定表的副本分布
```
CREATE TABLE example_db.table_hash
@@ -805,7 +805,7 @@ under the License.
)
DISTRIBUTED BY HASH(k1) BUCKETS 32
PROPERTIES (
- "replica_allocation"="tag.location.group_a:1, tag.location.group_b:2"
+ "replication_allocation"="tag.location.group_a:1, tag.location.group_b:2"
);
@@ -825,7 +825,7 @@ under the License.
"dynamic_partition.end" = "3",
"dynamic_partition.prefix" = "p",
"dynamic_partition.buckets" = "32",
- "dynamic_partition."replica_allocation" = "tag.location.group_a:3"
+ "dynamic_partition."replication_allocation" = "tag.location.group_a:3"
);
```
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyPartitionClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyPartitionClause.java
index 5dd3c31..504f197 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyPartitionClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyPartitionClause.java
@@ -93,7 +93,7 @@ public class ModifyPartitionClause extends AlterTableClause {
}
// Check the following properties' legality before modifying partition.
- // 1. replication_num or replica_allocation
+ // 1. replication_num or replication_allocation
// 2. storage_medium && storage_cooldown_time
// 3. in_memory
// 4. tablet type
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTagTest.java
index 4c6b63d..9db9b8e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTagTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTagTest.java
@@ -150,7 +150,7 @@ public class ModifyBackendTagTest {
Assert.assertEquals(ReplicaAllocation.DEFAULT_ALLOCATION, defaultAlloc);
TableProperty tableProperty = tbl.getTableProperty();
Map<String, String> tblProperties = tableProperty.getProperties();
- // if replication_num or replica_allocation is not set, it will be set to the default one
+ // if replication_num or replication_allocation is not set, it will be set to the default one
Assert.assertTrue(tblProperties.containsKey("default.replication_allocation"));
Assert.assertEquals("tag.location.default: 3", tblProperties.get("default.replication_allocation"));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 08/21: [Docs] fix urls and format in routine load
docs (#6896)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 959b7f10aee96a6b6686a8214c23614ca29be2fb
Author: qiye <ji...@gmail.com>
AuthorDate: Sat Oct 23 16:52:33 2021 +0800
[Docs] fix urls and format in routine load docs (#6896)
fix urls and format in routine load docs
---
docs/en/administrator-guide/load-data/routine-load-manual.md | 6 +++---
docs/zh-CN/administrator-guide/load-data/routine-load-manual.md | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/docs/en/administrator-guide/load-data/routine-load-manual.md b/docs/en/administrator-guide/load-data/routine-load-manual.md
index 3db45cd..35b3cf8 100644
--- a/docs/en/administrator-guide/load-data/routine-load-manual.md
+++ b/docs/en/administrator-guide/load-data/routine-load-manual.md
@@ -207,7 +207,7 @@ Accessing the SSL-certified Kafka cluster requires the user to provide a certifi
CREATE FILE "ca.pem" PROPERTIES("url" = "https://example_url/kafka-key/ca.pem", "catalog" = "kafka");
CREATE FILE "client.key" PROPERTIES("url" = "https://example_urlkafka-key/client.key", "catalog" = "kafka");
CREATE FILE "client.pem" PROPERTIES("url" = "https://example_url/kafka-key/client.pem", "catalog" = "kafka");
-```
+ ```
2. Create a routine load job
@@ -231,7 +231,7 @@ Accessing the SSL-certified Kafka cluster requires the user to provide a certifi
> Doris accesses Kafka clusters via Kafka's C++ API `librdkafka`. The parameters supported by `librdkafka` can be found.
>
-> `https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md`
+> <https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md>
### Viewing the status of the load job
@@ -243,7 +243,7 @@ You can only view tasks that are currently running, and tasks that have ended an
### Alter job
-Users can modify jobs that have been created. Specific instructions can be viewed through the `HELP ALTER ROUTINE LOAD;` command. Or refer to [ALTER ROUTINE LOAD](../../sql-reference/sql-statements/Data Manipulation/alter-routine-load.md).
+Users can modify jobs that have been created. Specific instructions can be viewed through the `HELP ALTER ROUTINE LOAD;` command. Or refer to [ALTER ROUTINE LOAD](../../sql-reference/sql-statements/Data%20Manipulation/alter-routine-load.md).
### Job Control
diff --git a/docs/zh-CN/administrator-guide/load-data/routine-load-manual.md b/docs/zh-CN/administrator-guide/load-data/routine-load-manual.md
index f009740..65ab1d6 100644
--- a/docs/zh-CN/administrator-guide/load-data/routine-load-manual.md
+++ b/docs/zh-CN/administrator-guide/load-data/routine-load-manual.md
@@ -229,7 +229,7 @@ FE 中的 JobScheduler 根据汇报结果，继续生成后续新的 Task，或
> Doris 通过 Kafka 的 C++ API `librdkafka` 来访问 Kafka 集群。`librdkafka` 所支持的参数可以参阅
>
-> `https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md`
+> <https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md>
### 查看导入作业状态
@@ -242,7 +242,7 @@ FE 中的 JobScheduler 根据汇报结果，继续生成后续新的 Task，或
### 修改作业属性
-用户可以修改已经创建的作业。具体说明可以通过 `HELP ALTER ROUTINE LOAD;` 命令查看。或参阅 [ALTER ROUTINE LOAD](../../sql-reference/sql-statements/Data Manipulation/alter-routine-load.md)。
+用户可以修改已经创建的作业。具体说明可以通过 `HELP ALTER ROUTINE LOAD;` 命令查看。或参阅 [ALTER ROUTINE LOAD](../../sql-reference/sql-statements/Data%20Manipulation/alter-routine-load.md)。
### 作业控制
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 15/21: Fix String type column using zonemap to
filter data maybe core dump (#6939)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 6e388a7180b525bb593b44a2e9bddd184aff9d7c
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Wed Oct 27 09:25:38 2021 +0800
Fix String type column using zonemap to filter data maybe core dump (#6939)
Fix String type column using zonemap to filter data maybe core dump, because of not allocating memory before parsing string type zonemap
---
be/src/olap/wrapper_field.cpp | 11 +++++++++--
be/src/olap/wrapper_field.h | 4 ++++
2 files changed, 13 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/wrapper_field.cpp b/be/src/olap/wrapper_field.cpp
index 61f5f88..be908a2 100644
--- a/be/src/olap/wrapper_field.cpp
+++ b/be/src/olap/wrapper_field.cpp
@@ -17,6 +17,8 @@
#include "olap/wrapper_field.h"
+#include "olap/row_cursor.h"
+
namespace doris {
const size_t DEFAULT_STRING_LENGTH = 50;
@@ -25,8 +27,9 @@ WrapperField* WrapperField::create(const TabletColumn& column, uint32_t len) {
bool is_string_type =
(column.type() == OLAP_FIELD_TYPE_CHAR || column.type() == OLAP_FIELD_TYPE_VARCHAR ||
column.type() == OLAP_FIELD_TYPE_HLL || column.type() == OLAP_FIELD_TYPE_OBJECT ||
- column.type() == OLAP_FIELD_TYPE_STRING);
- size_t max_length = column.type() == OLAP_FIELD_TYPE_STRING ? OLAP_STRING_MAX_LENGTH : OLAP_VARCHAR_MAX_LENGTH;
+ column.type() == OLAP_FIELD_TYPE_STRING);
+ size_t max_length = column.type() == OLAP_FIELD_TYPE_STRING ? OLAP_STRING_MAX_LENGTH
+ : OLAP_VARCHAR_MAX_LENGTH;
if (is_string_type && len > max_length) {
OLAP_LOG_WARNING("length of string parameter is too long[len=%lu, max_len=%lu].", len,
max_length);
@@ -89,6 +92,10 @@ WrapperField::WrapperField(Field* rep, size_t variable_len, bool is_string_type)
_string_content.reset(new char[slice->size]);
slice->data = _string_content.get();
}
+ if (_rep->type() == OLAP_FIELD_TYPE_STRING) {
+ _long_text_buf = (char*)malloc(RowCursor::DEFAULT_TEXT_LENGTH * sizeof(char));
+ rep->set_long_text_buf(&_long_text_buf);
+ }
}
WrapperField::WrapperField(Field* rep, const RowCursorCell& row_cursor_cell)
diff --git a/be/src/olap/wrapper_field.h b/be/src/olap/wrapper_field.h
index c14c019..3d22a7b 100644
--- a/be/src/olap/wrapper_field.h
+++ b/be/src/olap/wrapper_field.h
@@ -41,6 +41,9 @@ public:
virtual ~WrapperField() {
delete _rep;
delete[] _owned_buf;
+ if (_long_text_buf) {
+ delete _long_text_buf;
+ }
}
// 将内部的value转成string输出
@@ -103,6 +106,7 @@ private:
bool _is_string_type;
char* _field_buf = nullptr;
char* _owned_buf = nullptr;
+ char* _long_text_buf = nullptr;
//include fixed and variable length and null bytes
size_t _length;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 16/21: [Cache][Bug] Correct update cache timeout
unit (#6888)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 4bb24d2488482e8b76c3d94b99ffb82b6007c43b
Author: Lijia Liu <li...@yeah.net>
AuthorDate: Wed Oct 27 09:53:58 2021 +0800
[Cache][Bug] Correct update cache timeout unit (#6888)
Now FE update cache use MICROSECONDS as TimeUnit.
Replace it by MILLISECONDS.
---
fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java
index d7e5eda..cb30cbb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheBeProxy.java
@@ -53,7 +53,7 @@ public class CacheBeProxy extends CacheProxy {
try {
Future<InternalService.PCacheResponse> future = BackendServiceProxy.getInstance()
.updateCache(address, request);
- InternalService.PCacheResponse response = future.get(timeoutMs, TimeUnit.MICROSECONDS);
+ InternalService.PCacheResponse response = future.get(timeoutMs, TimeUnit.MILLISECONDS);
if (response.getStatus() == InternalService.PCacheStatus.CACHE_OK) {
status.setStatus(new Status(TStatusCode.OK, "CACHE_OK"));
} else {
@@ -125,7 +125,7 @@ public class CacheBeProxy extends CacheProxy {
request = request.toBuilder().setClearType(InternalService.PClearType.CLEAR_ALL).build();
LOG.info("clear all backend cache, backendId {}", backend.getId());
Future<InternalService.PCacheResponse> future = BackendServiceProxy.getInstance().clearCache(address, request);
- InternalService.PCacheResponse response = future.get(timeoutMs, TimeUnit.MICROSECONDS);
+ InternalService.PCacheResponse response = future.get(timeoutMs, TimeUnit.MILLISECONDS);
if (response.getStatus() == InternalService.PCacheStatus.CACHE_OK) {
status.setStatus(new Status(TStatusCode.OK, "CACHE_OK"));
return true;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[incubator-doris] 18/21: [Bug] fix Runtime filter can't find
fragment-id when apply_filter called early (#6923)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 6664aaeeb33a2b0cf01861c189ba48580be3439b
Author: Pxl <95...@qq.com>
AuthorDate: Wed Oct 27 09:54:52 2021 +0800
[Bug] fix Runtime filter can't find fragment-id when apply_filter called early (#6923)
#6921
---
be/src/runtime/fragment_mgr.cpp | 14 +++++++++++---
be/src/runtime/fragment_mgr.h | 2 ++
2 files changed, 13 insertions(+), 3 deletions(-)
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index f980e7b..a8ff209 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -610,6 +610,7 @@ Status FragmentMgr::exec_plan_fragment(const TExecPlanFragmentParams& params, Fi
{
std::lock_guard<std::mutex> lock(_lock);
_fragment_map.insert(std::make_pair(params.params.fragment_instance_id, exec_state));
+ _cv.notify_all();
}
auto st = _thread_pool->submit_func(
@@ -747,7 +748,7 @@ Status FragmentMgr::exec_external_plan_fragment(const TScanOpenParams& params,
}
VLOG_QUERY << "BackendService execute open() TQueryPlanInfo: "
- << apache::thrift::ThriftDebugString(t_query_plan_info);
+ << apache::thrift::ThriftDebugString(t_query_plan_info);
// assign the param used to execute PlanFragment
TExecPlanFragmentParams exec_fragment_params;
exec_fragment_params.protocol_version = (PaloInternalServiceVersion::type)0;
@@ -811,14 +812,21 @@ Status FragmentMgr::apply_filter(const PPublishFilterRequest* request, const cha
std::shared_ptr<FragmentExecState> fragment_state;
{
- std::lock_guard<std::mutex> lock(_lock);
+ std::unique_lock<std::mutex> lock(_lock);
+ if (!_fragment_map.count(tfragment_instance_id)) {
+ VLOG_NOTICE << "wait for fragment start execute, fragment-id:" << fragment_instance_id;
+ _cv.wait_for(lock, std::chrono::milliseconds(1000),
+ [&] { return _fragment_map.count(tfragment_instance_id); });
+ }
+
auto iter = _fragment_map.find(tfragment_instance_id);
if (iter == _fragment_map.end()) {
- LOG(WARNING) << "unknown.... fragment-id:" << fragment_instance_id;
+ VLOG_CRITICAL << "unknown.... fragment-id:" << fragment_instance_id;
return Status::InvalidArgument("fragment-id");
}
fragment_state = iter->second;
}
+
DCHECK(fragment_state != nullptr);
RuntimeFilterMgr* runtime_filter_mgr =
fragment_state->executor()->runtime_state()->runtime_filter_mgr();
diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h
index 70233e1..ba56216 100644
--- a/be/src/runtime/fragment_mgr.h
+++ b/be/src/runtime/fragment_mgr.h
@@ -99,6 +99,8 @@ private:
std::mutex _lock;
+ std::condition_variable _cv;
+
// Make sure that remove this before no data reference FragmentExecState
std::unordered_map<TUniqueId, std::shared_ptr<FragmentExecState>> _fragment_map;
// query id -> QueryFragmentsCtx
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org