You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/30 05:44:20 UTC
[doris] branch branch-hive-test updated (0148b39de0 -> 39aef0c1a7)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-hive-test
in repository https://gitbox.apache.org/repos/asf/doris.git
from 0148b39de0 [fix](metric) fix be down when enable_system_metrics is false (#16140)
new 3832ba64b8 [fix](multi-catalog)Use -1 for column_statistics internal table idx_id default value instead of null, for external catalog (#16177)
new cb29ad5708 [improvement](filecache) add profile for file cache (#16223)
new 39aef0c1a7 [tmpfix] add table name to file scan node explain string
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/io/cache/block/block_file_cache_profile.h | 45 ++++++++++++++++++++++
be/src/vec/exec/scan/vfile_scanner.cpp | 6 +++
.../planner/external/ExternalFileScanNode.java | 1 +
.../apache/doris/statistics/HiveAnalysisTask.java | 2 +-
4 files changed, 53 insertions(+), 1 deletion(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/03: [tmpfix] add table name to file scan node explain string
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-hive-test
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 39aef0c1a73c9323bb779c8c2f47af3e2b4045e1
Author: morningman <mo...@163.com>
AuthorDate: Mon Jan 30 13:44:07 2023 +0800
[tmpfix] add table name to file scan node explain string
---
.../java/org/apache/doris/planner/external/ExternalFileScanNode.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
index 37d7fd58f3..de007632c9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java
@@ -597,6 +597,7 @@ public class ExternalFileScanNode extends ExternalScanNode {
@Override
public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
StringBuilder output = new StringBuilder();
+ output.append(prefix).append("table: ").append(desc.getTable().getName()).append("\n");
if (!conjuncts.isEmpty()) {
output.append(prefix).append("predicates: ").append(getExplainString(conjuncts)).append("\n");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/03: [fix](multi-catalog)Use -1 for column_statistics internal table idx_id default value instead of null, for external catalog (#16177)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-hive-test
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3832ba64b8bf987a8842f267f26a7299c227010b
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Sun Jan 29 11:29:25 2023 +0800
[fix](multi-catalog)Use -1 for column_statistics internal table idx_id default value instead of null, for external catalog (#16177)
The internal statistic table column_statistics has a non-null field idx_id, the insert sql for hive table set the default value to NULL, which will failed to insert the result. Change it to -1.
---
.../src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
index 44be5bf078..836e3c6ae7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HiveAnalysisTask.java
@@ -62,7 +62,7 @@ public class HiveAnalysisTask extends HMSAnalysisTask {
private static final String ANALYZE_TABLE_SQL_TEMPLATE = "INSERT INTO "
+ "${internalDB}.${columnStatTbl}"
- + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', NULL, '${colId}', NULL, "
+ + " values ('${id}','${catalogId}', '${dbId}', '${tblId}', '-1', '${colId}', NULL, "
+ "${numRows}, ${ndv}, ${nulls}, '${min}', '${max}', ${dataSize}, '${update_time}')";
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/03: [improvement](filecache) add profile for file cache (#16223)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-hive-test
in repository https://gitbox.apache.org/repos/asf/doris.git
commit cb29ad5708a7932e5b6e0760eb2d622a089cebe0
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Mon Jan 30 10:46:31 2023 +0800
[improvement](filecache) add profile for file cache (#16223)
---
be/src/io/cache/block/block_file_cache_profile.h | 45 ++++++++++++++++++++++++
be/src/vec/exec/scan/vfile_scanner.cpp | 6 ++++
2 files changed, 51 insertions(+)
diff --git a/be/src/io/cache/block/block_file_cache_profile.h b/be/src/io/cache/block/block_file_cache_profile.h
index 64ca4eee9b..dfbd026f6c 100644
--- a/be/src/io/cache/block/block_file_cache_profile.h
+++ b/be/src/io/cache/block/block_file_cache_profile.h
@@ -29,6 +29,7 @@
#include "olap/olap_common.h"
#include "util/doris_metrics.h"
#include "util/metrics.h"
+#include "util/runtime_profile.h"
namespace doris {
namespace io {
@@ -111,5 +112,49 @@ struct FileCacheProfile {
FileCacheStatistics report(int64_t table_id, int64_t partition_id);
};
+struct FileCacheProfileReporter {
+ RuntimeProfile::Counter* num_io_total;
+ RuntimeProfile::Counter* num_io_hit_cache;
+ RuntimeProfile::Counter* num_io_bytes_read_total;
+ RuntimeProfile::Counter* num_io_bytes_read_from_file_cache;
+ RuntimeProfile::Counter* num_io_bytes_read_from_write_cache;
+ RuntimeProfile::Counter* num_io_written_in_file_cache;
+ RuntimeProfile::Counter* num_io_bytes_written_in_file_cache;
+ RuntimeProfile::Counter* num_io_bytes_skip_cache;
+
+ FileCacheProfileReporter(RuntimeProfile* profile) {
+ static const char* cache_profile = "FileCache";
+ ADD_TIMER(profile, cache_profile);
+ num_io_total = ADD_CHILD_COUNTER(profile, "IOTotalNum", TUnit::UNIT, cache_profile);
+ num_io_hit_cache = ADD_CHILD_COUNTER(profile, "IOHitCacheNum", TUnit::UNIT, cache_profile);
+ num_io_bytes_read_total =
+ ADD_CHILD_COUNTER(profile, "ReadTotalBytes", TUnit::BYTES, cache_profile);
+ num_io_bytes_read_from_file_cache =
+ ADD_CHILD_COUNTER(profile, "ReadFromFileCacheBytes", TUnit::BYTES, cache_profile);
+ num_io_bytes_read_from_write_cache =
+ ADD_CHILD_COUNTER(profile, "ReadFromWriteCacheBytes", TUnit::BYTES, cache_profile);
+ num_io_written_in_file_cache =
+ ADD_CHILD_COUNTER(profile, "WriteInFileCacheNum", TUnit::UNIT, cache_profile);
+ num_io_bytes_written_in_file_cache =
+ ADD_CHILD_COUNTER(profile, "WriteInFileCacheBytes", TUnit::BYTES, cache_profile);
+ num_io_bytes_skip_cache =
+ ADD_CHILD_COUNTER(profile, "SkipCacheBytes", TUnit::BYTES, cache_profile);
+ }
+
+ void update(FileCacheStatistics* statistics) {
+ COUNTER_UPDATE(num_io_total, statistics->num_io_total);
+ COUNTER_UPDATE(num_io_hit_cache, statistics->num_io_hit_cache);
+ COUNTER_UPDATE(num_io_bytes_read_total, statistics->num_io_bytes_read_total);
+ COUNTER_UPDATE(num_io_bytes_read_from_file_cache,
+ statistics->num_io_bytes_read_from_file_cache);
+ COUNTER_UPDATE(num_io_bytes_read_from_write_cache,
+ statistics->num_io_bytes_read_from_write_cache);
+ COUNTER_UPDATE(num_io_written_in_file_cache, statistics->num_io_written_in_file_cache);
+ COUNTER_UPDATE(num_io_bytes_written_in_file_cache,
+ statistics->num_io_bytes_written_in_file_cache);
+ COUNTER_UPDATE(num_io_bytes_skip_cache, statistics->num_io_bytes_skip_cache);
+ }
+};
+
} // namespace io
} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
index f78892fa0d..abebb723ab 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -25,6 +25,7 @@
#include "common/logging.h"
#include "common/utils.h"
#include "exec/text_converter.hpp"
+#include "io/cache/block/block_file_cache_profile.h"
#include "olap/iterators.h"
#include "runtime/descriptors.h"
#include "runtime/raw_value.h"
@@ -752,6 +753,11 @@ Status VFileScanner::close(RuntimeState* state) {
_push_down_expr->close(state);
}
+ if (config::enable_file_cache) {
+ io::FileCacheProfileReporter cache_profile(_profile);
+ cache_profile.update(_file_cache_statistics.get());
+ }
+
RETURN_IF_ERROR(VScanner::close(state));
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org