You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by pa...@apache.org on 2022/12/06 09:39:00 UTC
[doris] branch master updated: [refactor](odbc) move param to config (#14596)
This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5292880310 [refactor](odbc) move param to config (#14596)
5292880310 is described below
commit 5292880310e1529103d9c8b5a2aab48a2e50226a
Author: lsy3993 <11...@users.noreply.github.com>
AuthorDate: Tue Dec 6 17:38:52 2022 +0800
[refactor](odbc) move param to config (#14596)
move param to config
---
be/src/common/config.h | 2 ++
be/src/exec/odbc_connector.cpp | 8 ++------
be/src/exec/odbc_connector.h | 4 ++++
docs/en/docs/admin-manual/config/be-config.md | 12 ++++++++++++
docs/zh-CN/docs/admin-manual/config/be-config.md | 10 ++++++++++
5 files changed, 30 insertions(+), 6 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index a5e0c4826c..d46d7abd3d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -778,6 +778,8 @@ CONF_mInt32(parquet_column_max_buffer_mb, "8");
// OrcReader
CONF_mInt32(orc_natural_read_size_mb, "8");
+CONF_mInt64(big_column_size_buffer, "65535");
+CONF_mInt64(small_column_size_buffer, "100");
// When the rows number reached this limit, will check the filter rate the of bloomfilter
// if it is lower than a specific threshold, the predicate will be disabled.
diff --git a/be/src/exec/odbc_connector.cpp b/be/src/exec/odbc_connector.cpp
index 826d8b8bb9..255eb1715c 100644
--- a/be/src/exec/odbc_connector.cpp
+++ b/be/src/exec/odbc_connector.cpp
@@ -36,10 +36,6 @@
} \
}
-static constexpr uint32_t SMALL_COLUMN_SIZE_BUFFER = 100;
-// Now we only treat HLL, CHAR, VARCHAR as big column
-static constexpr uint32_t BIG_COLUMN_SIZE_BUFFER = 65535;
-
namespace doris {
ODBCConnector::ODBCConnector(const ODBCConnectorParam& param)
@@ -139,8 +135,8 @@ Status ODBCConnector::query() {
auto type = _tuple_desc->slots()[i]->type().type;
column_data->buffer_length = (type == TYPE_HLL || type == TYPE_CHAR ||
type == TYPE_VARCHAR || type == TYPE_STRING)
- ? BIG_COLUMN_SIZE_BUFFER
- : SMALL_COLUMN_SIZE_BUFFER;
+ ? big_column_size_buffer
+ : small_column_size_buffer;
column_data->target_value_ptr = malloc(sizeof(char) * column_data->buffer_length);
_columns_data.emplace_back(column_data);
}
diff --git a/be/src/exec/odbc_connector.h b/be/src/exec/odbc_connector.h
index 2aabcb61b6..5a62ca3d3b 100644
--- a/be/src/exec/odbc_connector.h
+++ b/be/src/exec/odbc_connector.h
@@ -71,6 +71,10 @@ public:
const DataBinding& get_column_data(int i) const { return *_columns_data.at(i).get(); }
Status init_to_write(RuntimeProfile* profile);
+ // Now we only treat HLL, CHAR, VARCHAR as big column
+ uint32_t big_column_size_buffer = config::big_column_size_buffer;
+ uint32_t small_column_size_buffer = config::small_column_size_buffer;
+
private:
static Status error_status(const std::string& prefix, const std::string& error_msg);
diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md
index 83596c9c4d..80de6b7d49 100644
--- a/docs/en/docs/admin-manual/config/be-config.md
+++ b/docs/en/docs/admin-manual/config/be-config.md
@@ -1588,3 +1588,15 @@ Translated with www.DeepL.com/Translator (free version)
* Type: int32
* Description: The segment whose row number above the threshold will be compacted during segcompaction
* Default value: 1048576
+
+### `big_column_size_buffer`
+* Type: int64
+* Description: When using the odbc external table, if a column type of the odbc source table is HLL, CHAR or VARCHAR, and the length of the column value exceeds this value, the query will report an error 'column value length longer than buffer length'. You can increase this value
+* Default value: 65535
+
+### `small_column_size_buffer`
+
+* Type: int64
+* Description: When using the odbc external table, if a column type of the odbc source table is not HLL, CHAR or VARCHAR, and the length of the column value exceeds this value, the query will report an error 'column value length longer than buffer length'. You can increase this value
+* Default value: 100
+
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md
index ef51e44e9d..df0119b30d 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -1611,3 +1611,13 @@ webserver默认工作线程数
* 类型：int32
* 描述：当 segment 文件超过此大小时则会在 segment compaction 时被 compact，否则跳过
* 默认值：1048576
+
+### `big_column_size_buffer`
+* 类型：int64
+* 描述：当使用odbc外表时，如果odbc源表的某一列类型为HLL, CHAR或者VARCHAR，并且列值长度超过该值，则查询报错'column value length longer than buffer length'. 可增大该值
+* 默认值：65535
+
+### `small_column_size_buffer`
+* 类型：int64
+* 描述：当使用odbc外表时，如果odbc源表的某一列类型不是HLL, CHAR或者VARCHAR，并且列值长度超过该值，则查询报错'column value length longer than buffer length'. 可增大该值
+* 默认值：100
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org