You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ya...@apache.org on 2021/04/08 01:17:06 UTC
[incubator-doris] branch master updated: Add fmt library to speed
up mysql text result serialization (#5554)
This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2b5e4dc Add fmt library to speed up mysql text result serialization (#5554)
2b5e4dc is described below
commit 2b5e4dc95103240a34ff989a1664f3d209fb8db0
Author: Dayue Gao <ga...@meituan.com>
AuthorDate: Thu Apr 8 09:16:58 2021 +0800
Add fmt library to speed up mysql text result serialization (#5554)
* Add fmt library to speed up mysql text result serialization
* use BUILD_SYSTEM instead of make
Co-authored-by: gaodayue <ga...@bytedance.com>
---
be/CMakeLists.txt | 4 +++
be/src/runtime/mysql_result_writer.cpp | 21 ++++++++------
be/src/util/mysql_row_buffer.cpp | 52 +++++++++++-----------------------
thirdparty/build-thirdparty.sh | 12 ++++++++
thirdparty/vars.sh | 8 ++++++
5 files changed, 52 insertions(+), 45 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index a7b726c..192ee3f 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -258,6 +258,9 @@ set_target_properties(bitshuffle PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/
add_library(roaring STATIC IMPORTED)
set_target_properties(roaring PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libroaring.a)
+add_library(fmt STATIC IMPORTED)
+set_target_properties(fmt PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libfmt.a)
+
add_library(orc STATIC IMPORTED)
set_target_properties(orc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/liborc.a)
@@ -478,6 +481,7 @@ set(DORIS_DEPENDENCIES
leveldb
bitshuffle
roaring
+ fmt
double-conversion
jemalloc
brotlicommon
diff --git a/be/src/runtime/mysql_result_writer.cpp b/be/src/runtime/mysql_result_writer.cpp
index 86e4675..d56e8ac 100644
--- a/be/src/runtime/mysql_result_writer.cpp
+++ b/be/src/runtime/mysql_result_writer.cpp
@@ -66,7 +66,6 @@ void MysqlResultWriter::_init_profile() {
}
Status MysqlResultWriter::_add_one_row(TupleRow* row) {
- SCOPED_TIMER(_convert_tuple_timer);
_row_buffer->reset();
int num_columns = _output_expr_ctxs.size();
int buf_ret = 0;
@@ -213,18 +212,22 @@ Status MysqlResultWriter::append_row_batch(const RowBatch* batch) {
int num_rows = batch->num_rows();
result->result_batch.rows.resize(num_rows);
- for (int i = 0; status.ok() && i < num_rows; ++i) {
- TupleRow* row = batch->get_row(i);
- status = _add_one_row(row);
+ {
+ SCOPED_TIMER(_convert_tuple_timer);
+ for (int i = 0; status.ok() && i < num_rows; ++i) {
+ TupleRow* row = batch->get_row(i);
+ status = _add_one_row(row);
- if (status.ok()) {
- result->result_batch.rows[i].assign(_row_buffer->buf(), _row_buffer->length());
- } else {
- LOG(WARNING) << "convert row to mysql result failed.";
- break;
+ if (status.ok()) {
+ result->result_batch.rows[i].assign(_row_buffer->buf(), _row_buffer->length());
+ } else {
+ LOG(WARNING) << "convert row to mysql result failed.";
+ break;
+ }
}
}
+
if (status.ok()) {
SCOPED_TIMER(_result_send_timer);
// push this batch to back
diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index d5bbd93..51b2bce 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -24,6 +24,8 @@
#include "gutil/strings/numbers.h"
#include "util/mysql_global.h"
+#include <fmt/format.h>
+
namespace doris {
// the first byte:
@@ -97,6 +99,16 @@ int MysqlRowBuffer::reserve(int size) {
return 0;
}
+template<typename T>
+static char* add_int(T data, char* pos)
+{
+ auto fi = fmt::format_int(data);
+ int length = fi.size();
+ int1store(pos++, length);
+ memcpy(pos, fi.data(), length);
+ return pos + length;
+}
+
int MysqlRowBuffer::push_tinyint(int8_t data) {
// 1 for string trail, 1 for length, 1 for sign, other for digits
int ret = reserve(3 + MAX_TINYINT_WIDTH);
@@ -106,15 +118,7 @@ int MysqlRowBuffer::push_tinyint(int8_t data) {
return ret;
}
- int length = snprintf(_pos + 1, MAX_TINYINT_WIDTH + 2, "%d", data);
-
- if (length < 0) {
- LOG(ERROR) << "snprintf failed. data = " << data;
- return length;
- }
-
- int1store(_pos, length);
- _pos += length + 1;
+ _pos = add_int(data, _pos);
return 0;
}
@@ -127,15 +131,7 @@ int MysqlRowBuffer::push_smallint(int16_t data) {
return ret;
}
- int length = snprintf(_pos + 1, MAX_SMALLINT_WIDTH + 2, "%d", data);
-
- if (length < 0) {
- LOG(ERROR) << "snprintf failed. data = " << data;
- return length;
- }
-
- int1store(_pos, length);
- _pos += length + 1;
+ _pos = add_int(data, _pos);
return 0;
}
@@ -148,15 +144,7 @@ int MysqlRowBuffer::push_int(int32_t data) {
return ret;
}
- int length = snprintf(_pos + 1, MAX_INT_WIDTH + 2, "%d", data);
-
- if (length < 0) {
- LOG(ERROR) << "snprintf failed. data = " << data;
- return length;
- }
-
- int1store(_pos, length);
- _pos += length + 1;
+ _pos = add_int(data, _pos);
return 0;
}
@@ -169,15 +157,7 @@ int MysqlRowBuffer::push_bigint(int64_t data) {
return ret;
}
- int length = snprintf(_pos + 1, MAX_BIGINT_WIDTH + 2, "%ld", data);
-
- if (length < 0) {
- LOG(ERROR) << "snprintf failed. data = " << data;
- return length;
- }
-
- int1store(_pos, length);
- _pos += length + 1;
+ _pos = add_int(data, _pos);
return 0;
}
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 98b28b7..ca0ae84 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -671,6 +671,17 @@ build_croaringbitmap() {
-DENABLE_ROARING_TESTS=OFF ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
+
+# fmt
+build_fmt() {
+ check_if_source_exist $FMT_SOURCE
+ cd $TP_SOURCE_DIR/$FMT_SOURCE
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ rm -rf CMakeCache.txt CMakeFiles/
+ $CMAKE_CMD -G "${GENERATOR}" -v -DBUILD_SHARED_LIBS=FALSE -DFMT_TEST=OFF -DFMT_DOC=OFF -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR ..
+ ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
+}
+
#orc
build_orc() {
check_if_source_exist $ORC_SOURCE
@@ -825,6 +836,7 @@ build_arrow
build_s2
build_bitshuffle
build_croaringbitmap
+build_fmt
build_orc
build_cctz
build_tsan_header
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index e8318c0..5a6ce39 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -258,6 +258,13 @@ CROARINGBITMAP_DOWNLOAD="https://github.com/RoaringBitmap/CRoaring/archive/v0.2.
CROARINGBITMAP_NAME=CRoaring-0.2.60.tar.gz
CROARINGBITMAP_SOURCE=CRoaring-0.2.60
CROARINGBITMAP_MD5SUM="29602918e6890ffdeed84cb171857046"
+
+# fmt
+FMT_DOWNLOAD="https://github.com/fmtlib/fmt/archive/7.1.3.tar.gz"
+FMT_NAME="fmt-7.1.3.tar.gz"
+FMT_SOURCE="fmt-7.1.3"
+FMT_MD5SUM="2522ec65070c0bda0ca288677ded2831"
+
# ORC
ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.5.8/orc-1.5.8.tar.gz"
ORC_NAME=orc-1.5.8.tar.gz
@@ -377,6 +384,7 @@ ZSTD
S2
BITSHUFFLE
CROARINGBITMAP
+FMT
ORC
JEMALLOC
CCTZ
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org