You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/14 13:19:28 UTC
[doris] 04/04: [fix](info_db) avoid infodb query timeout when external catalog info is too large or is not reachable (#18662)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ca78c132d48bde99b940f2268d4e6fd94b1e31b7
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Fri Apr 14 14:40:31 2023 +0800
[fix](info_db) avoid infodb query timeout when external catalog info is too large or is not reachable (#18662)
When query tables in information_schema databases, it may timeout due to:
There are external catalog with too many tables.
The external catalog is unreachable
So I add a new FE config infodb_support_ext_catalog.
The default is false, which means that when select from tables in information_schema database,
the result will not contain the information of the table in external catalog.
Describe your changes.
---
docs/en/docs/admin-manual/config/fe-config.md | 14 ++++++++++++++
docs/zh-CN/docs/admin-manual/config/fe-config.md | 15 +++++++++++++++
.../src/main/java/org/apache/doris/common/Config.java | 9 +++++++++
.../java/org/apache/doris/planner/SchemaScanNode.java | 3 +++
4 files changed, 41 insertions(+)
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
index d661c32edc..3ca3c32c66 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2386,3 +2386,17 @@ MasterOnly：true
Maximum number of error tablet showed in broker load.
+#### `infodb_support_ext_catalog`
+
+<version since="1.2.4"></version>
+
+Default: false
+
+IsMutable: true
+
+MasterOnly: false
+
+If false, when select from tables in information_schema database,
+the result will not contain the information of the table in external catalog.
+This is to avoid query time when external catalog is not reachable.
+
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index 2888279e0a..01d6abbdc7 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2449,3 +2449,18 @@ hive partition 的最大缓存数量。
是否为 Master FE 节点独有的配置项：true
broker load job 保存的失败tablet 信息的最大数量
+
+#### `infodb_support_ext_catalog`
+
+<version since="1.2.4"></version>
+
+默认值：false
+
+是否可以动态配置：true
+
+是否为 Master FE 节点独有的配置项：false
+
+当设置为 false 时，查询 `information_schema` 中的表时，将不再返回 external catalog 中的表的信息。
+
+这个参数主要用于避免因 external catalog 无法访问、信息过多等原因导致的查询 `information_schema` 超时的问题。
+
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 1c0b0acc6a..145b9fb894 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1993,5 +1993,14 @@ public class Config extends ConfigBase {
*/
@ConfField(mutable = false, masterOnly = false)
public static String mysql_ssl_default_certificate_password = "doris";
+
+ /**
+ * If false, when select from tables in information_schema database,
+ * the result will not contain the information of the table in external catalog.
+ * This is to avoid query time when external catalog is not reachable.
+ * TODO: this is a temp solution, we should support external catalog in the future.
+ */
+ @ConfField(mutable = true)
+ public static boolean infodb_support_ext_catalog = false;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java
index f00c8683f3..2107cb6c8e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SchemaScanNode.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.service.FrontendOptions;
import org.apache.doris.statistics.StatisticalType;
@@ -96,6 +97,8 @@ public class SchemaScanNode extends ScanNode {
}
if (schemaCatalog != null) {
msg.schema_scan_node.setCatalog(schemaCatalog);
+ } else if (!Config.infodb_support_ext_catalog) {
+ msg.schema_scan_node.setCatalog(InternalCatalog.INTERNAL_CATALOG_NAME);
}
msg.schema_scan_node.show_hidden_cloumns = Util.showHiddenColumns();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org