You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/10/15 07:29:17 UTC
[incubator-doris] 07/07: [Dependency] Upgrade thirdparty libs
(#6766)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-0.15
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit 8fa3bbf9c287c03cdd69040f3edca4b5f108db32
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Fri Oct 15 13:03:04 2021 +0800
[Dependency] Upgrade thirdparty libs (#6766)
Upgrade the following dependecies:
libevent -> 2.1.12
OpenSSL 1.0.2k -> 1.1.1l
thrift 0.9.3 -> 0.13.0
protobuf 3.5.1 -> 3.14.0
gflags 2.2.0 -> 2.2.2
glog 0.3.3 -> 0.4.0
googletest 1.8.0 -> 1.10.0
snappy 1.1.7 -> 1.1.8
gperftools 2.7 -> 2.9.1
lz4 1.7.5 -> 1.9.3
curl 7.54.1 -> 7.79.0
re2 2017-05-01 -> 2021-02-02
zstd 1.3.7 -> 1.5.0
brotli 1.0.7 -> 1.0.9
flatbuffers 1.10.0 -> 2.0.0
apache-arrow 0.15.1 -> 5.0.0
CRoaring 0.2.60 -> 0.3.4
orc 1.5.8 -> 1.6.6
libdivide 4.0.0 -> 5.0
brpc 0.97 -> 1.0.0-rc02
librdkafka 1.7.0 -> 1.8.0
after this pr compile doris should use build-env:1.4.0
---
be/CMakeLists.txt | 4 -
be/src/agent/heartbeat_server.cpp | 6 +-
be/src/agent/task_worker_pool.cpp | 5 +-
be/src/exec/odbc_connector.cpp | 11 +-
be/src/exec/parquet_reader.cpp | 53 +--
be/src/exec/parquet_reader.h | 10 +-
be/src/exec/parquet_writer.cpp | 5 +-
be/src/exec/parquet_writer.h | 11 +-
be/src/exprs/json_functions.cpp | 2 -
be/src/gutil/strings/util.cc | 9 +-
be/src/http/ev_http_server.cpp | 2 +-
be/src/olap/bloom_filter_predicate.h | 2 +-
be/src/olap/column_predicate.h | 10 +-
be/src/olap/comparison_predicate.cpp | 166 ++++-----
be/src/olap/comparison_predicate.h | 32 +-
be/src/olap/in_list_predicate.cpp | 125 +++----
be/src/olap/in_list_predicate.h | 32 +-
be/src/olap/null_predicate.cpp | 10 +-
be/src/olap/null_predicate.h | 4 +-
.../olap/rowset/segment_v2/bitmap_index_reader.cpp | 10 +-
.../olap/rowset/segment_v2/bitmap_index_reader.h | 6 +-
.../olap/rowset/segment_v2/bitmap_index_writer.cpp | 10 +-
be/src/olap/rowset/segment_v2/row_ranges.h | 4 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 6 +-
be/src/olap/rowset/segment_v2/segment_iterator.h | 5 +-
be/src/runtime/data_stream_sender.cpp | 22 +-
be/src/runtime/datetime_value.cpp | 51 +--
be/src/service/backend_service.cpp | 13 +-
be/src/service/internal_service.cpp | 2 +-
be/src/util/aes_util.cpp | 16 +-
be/src/util/arrow/row_batch.cpp | 28 +-
be/src/util/bitmap_value.h | 374 +++++++++++++--------
be/src/util/block_compression.cpp | 18 +-
be/src/util/debug_util.cpp | 35 --
be/src/util/simdutf8check.h | 121 -------
be/src/util/thread.cpp | 21 +-
be/src/util/thrift_client.h | 9 +-
be/src/util/thrift_server.cpp | 72 ++--
be/src/util/thrift_server.h | 12 +-
be/src/util/thrift_util.cpp | 9 +-
be/src/util/thrift_util.h | 17 +-
be/test/olap/file_helper_test.cpp | 15 +-
be/test/olap/file_utils_test.cpp | 8 +-
.../olap/rowset/segment_v2/bitmap_index_test.cpp | 3 +-
.../rowset/segment_v2/bitshuffle_page_test.cpp | 2 +-
be/test/olap/rowset/segment_v2/row_ranges_test.cpp | 2 +-
be/test/util/arrow/arrow_row_batch_test.cpp | 8 +-
be/test/util/arrow/arrow_row_block_test.cpp | 8 +-
be/test/util/bitmap_value_test.cpp | 1 +
be/test/util/threadpool_test.cpp | 50 ++-
extension/flink-doris-connector/pom.xml | 11 +-
.../apache/doris/flink/backend/BackendClient.java | 28 +-
.../doris/flink/datastream/ScalaValueReader.scala | 20 +-
.../doris/flink/serialization/TestRowBatch.java | 6 +-
extension/spark-doris-connector/pom.xml | 55 ++-
extension/spark-doris-connector/pom_3.0.xml | 79 ++++-
.../apache/doris/spark/backend/BackendClient.java | 28 +-
.../apache/doris/spark/rdd/ScalaValueReader.scala | 20 +-
.../doris/spark/serialization/TestRowBatch.java | 6 +-
fe/fe-common/pom.xml | 6 -
fe/fe-core/pom.xml | 2 +-
fe/pom.xml | 12 +-
fs_brokers/apache_hdfs_broker/pom.xml | 3 +-
thirdparty/CHANGELOG.md | 138 ++++++++
thirdparty/build-thirdparty.sh | 111 +++---
thirdparty/download-thirdparty.sh | 76 +----
thirdparty/patches/boost-1.64.0-gcc4.8.patch | 101 ------
thirdparty/patches/compiler-rt.patch | 11 -
thirdparty/patches/glog-0.3.3-makefile.patch | 10 -
.../glog-0.3.3-remove-unwind-dependency.patch | 31 --
.../patches/glog-0.3.3-vlog-double-lock-bug.patch | 21 --
...glog-0.3.3-for-palo2.patch => glog-0.4.0.patch} | 209 +++++-------
thirdparty/patches/incubator-brpc-0.9.7.patch | 11 -
thirdparty/patches/libevent.patch | 134 ++++++++
thirdparty/patches/libevent_on_free_cb.patch | 71 ----
thirdparty/patches/llvm-3.4.2.patch | 16 -
thirdparty/patches/lz4-1.7.5.patch | 15 -
thirdparty/patches/mysql-server-mysql-5.7.18.patch | 41 +++
thirdparty/patches/re2-2017-05-01.patch | 11 -
thirdparty/patches/s2geometry-0.9.0-uint64.patch | 94 ------
thirdparty/patches/s2geometry-0.9.0-warning.patch | 11 -
thirdparty/patches/s2geometry-0.9.0.patch | 128 ++++++-
thirdparty/patches/thrift-0.9.3-aclocal.patch | 11 -
thirdparty/vars.sh | 204 +++++------
84 files changed, 1578 insertions(+), 1610 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index ba7fc00..b9edcda 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -234,9 +234,6 @@ set_target_properties(zstd PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/
add_library(arrow STATIC IMPORTED)
set_target_properties(arrow PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libarrow.a)
-add_library(double-conversion STATIC IMPORTED)
-set_target_properties(double-conversion PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libdouble-conversion.a)
-
add_library(parquet STATIC IMPORTED)
set_target_properties(parquet PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libparquet.a)
@@ -509,7 +506,6 @@ set(COMMON_THIRDPARTY
bitshuffle
roaring
fmt
- double-conversion
jemalloc
brotlicommon
brotlidec
diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp
index 516350d..5c178a7 100644
--- a/be/src/agent/heartbeat_server.cpp
+++ b/be/src/agent/heartbeat_server.cpp
@@ -19,8 +19,8 @@
#include <thrift/TProcessor.h>
-#include <filesystem>
#include <ctime>
+#include <filesystem>
#include <fstream>
#include "common/status.h"
@@ -180,8 +180,8 @@ AgentStatus create_heartbeat_server(ExecEnv* exec_env, uint32_t server_port,
heartbeat_server->init_cluster_id();
- boost::shared_ptr<HeartbeatServer> handler(heartbeat_server);
- boost::shared_ptr<TProcessor> server_processor(new HeartbeatServiceProcessor(handler));
+ std::shared_ptr<HeartbeatServer> handler(heartbeat_server);
+ std::shared_ptr<TProcessor> server_processor(new HeartbeatServiceProcessor(handler));
string server_name("heartbeat");
*thrift_server =
new ThriftServer(server_name, server_processor, server_port, worker_thread_num);
diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index c9681a9..a4b9463 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -425,7 +425,7 @@ void TaskWorkerPool::_drop_tablet_worker_thread_callback() {
TStatus task_status;
string err;
TabletSharedPtr dropped_tablet = StorageEngine::instance()->tablet_manager()->get_tablet(
- drop_tablet_req.tablet_id, drop_tablet_req.schema_hash, &err);
+ drop_tablet_req.tablet_id, drop_tablet_req.schema_hash, false, &err);
if (dropped_tablet != nullptr) {
OLAPStatus drop_status = StorageEngine::instance()->tablet_manager()->drop_tablet(
drop_tablet_req.tablet_id, drop_tablet_req.schema_hash);
@@ -1194,7 +1194,8 @@ void TaskWorkerPool::_report_tablet_worker_thread_callback() {
// If FE create a tablet in FE meta and send CREATE task to this BE, the tablet may not be included in this
// report, and the report version has a small probability that it has not been updated in time. When FE
// receives this report, it is possible to delete the new tablet.
- LOG(WARNING) << "report version " << report_version << " change to " << _s_report_version;
+ LOG(WARNING) << "report version " << report_version << " change to "
+ << _s_report_version;
DorisMetrics::instance()->report_all_tablets_requests_skip->increment(1);
continue;
}
diff --git a/be/src/exec/odbc_connector.cpp b/be/src/exec/odbc_connector.cpp
index 173543e..7b50a61 100644
--- a/be/src/exec/odbc_connector.cpp
+++ b/be/src/exec/odbc_connector.cpp
@@ -102,9 +102,9 @@ Status ODBCConnector::open() {
// Allocate a connection handle
ODBC_DISPOSE(_env, SQL_HANDLE_ENV, SQLAllocHandle(SQL_HANDLE_DBC, _env, &_dbc), "alloc dbc");
// Set connect timeout
- unsigned int timeout = config::external_table_connect_timeout_sec;
- SQLSetConnectAttr(_dbc, SQL_LOGIN_TIMEOUT, (SQLPOINTER) timeout, 0);
- SQLSetConnectAttr(_dbc, SQL_ATTR_CONNECTION_TIMEOUT, (SQLPOINTER) timeout, 0);
+ int64_t timeout = config::external_table_connect_timeout_sec;
+ SQLSetConnectAttr(_dbc, SQL_LOGIN_TIMEOUT, (SQLPOINTER)timeout, 0);
+ SQLSetConnectAttr(_dbc, SQL_ATTR_CONNECTION_TIMEOUT, (SQLPOINTER)timeout, 0);
// Connect to the Database
ODBC_DISPOSE(_dbc, SQL_HANDLE_DBC,
SQLDriverConnect(_dbc, NULL, (SQLCHAR*)_connect_string.c_str(), SQL_NTS, NULL, 0,
@@ -152,7 +152,8 @@ Status ODBCConnector::query() {
DataBinding* column_data = new DataBinding;
column_data->target_type = SQL_C_CHAR;
auto type = _tuple_desc->slots()[i]->type().type;
- column_data->buffer_length = (type == TYPE_HLL || type == TYPE_CHAR || type == TYPE_VARCHAR || type == TYPE_STRING)
+ column_data->buffer_length = (type == TYPE_HLL || type == TYPE_CHAR ||
+ type == TYPE_VARCHAR || type == TYPE_STRING)
? BIG_COLUMN_SIZE_BUFFER
: SMALL_COLUMN_SIZE_BUFFER;
column_data->target_value_ptr = malloc(sizeof(char) * column_data->buffer_length);
@@ -286,7 +287,7 @@ Status ODBCConnector::append(const std::string& table_name, RowBatch* batch,
}
case TYPE_LARGEINT: {
fmt::format_to(_insert_stmt_buffer, "{}",
- reinterpret_cast<const PackedInt128*>(item)->value);
+ reinterpret_cast<const PackedInt128*>(item)->value);
break;
}
default: {
diff --git a/be/src/exec/parquet_reader.cpp b/be/src/exec/parquet_reader.cpp
index f994c34..0715d02 100644
--- a/be/src/exec/parquet_reader.cpp
+++ b/be/src/exec/parquet_reader.cpp
@@ -125,8 +125,13 @@ void ParquetReaderWrap::close() {
}
Status ParquetReaderWrap::size(int64_t* size) {
- _parquet->GetSize(size);
- return Status::OK();
+ arrow::Result<int64_t> result = _parquet->GetSize();
+ if (result.ok()) {
+ *size = result.ValueOrDie();
+ return Status::OK();
+ } else {
+ return Status::InternalError(result.status().ToString());
+ }
}
inline void ParquetReaderWrap::fill_slot(Tuple* tuple, SlotDescriptor* slot_desc, MemPool* mem_pool,
@@ -552,34 +557,33 @@ bool ParquetFile::closed() const {
}
}
-arrow::Status ParquetFile::Read(int64_t nbytes, int64_t* bytes_read, void* buffer) {
- return ReadAt(_pos, nbytes, bytes_read, buffer);
+arrow::Result<int64_t> ParquetFile::Read(int64_t nbytes, void* buffer) {
+ return ReadAt(_pos, nbytes, buffer);
}
-arrow::Status ParquetFile::ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read,
- void* out) {
+arrow::Result<int64_t> ParquetFile::ReadAt(int64_t position, int64_t nbytes, void* out) {
int64_t reads = 0;
+ int64_t bytes_read = 0;
_pos = position;
while (nbytes > 0) {
Status result = _file->readat(_pos, nbytes, &reads, out);
if (!result.ok()) {
- *bytes_read = 0;
+ bytes_read = 0;
return arrow::Status::IOError("Readat failed.");
}
if (reads == 0) {
break;
}
- *bytes_read += reads; // total read bytes
- nbytes -= reads; // remained bytes
+ bytes_read += reads; // total read bytes
+ nbytes -= reads; // remained bytes
_pos += reads;
out = (char*)out + reads;
}
- return arrow::Status::OK();
+ return bytes_read;
}
-arrow::Status ParquetFile::GetSize(int64_t* size) {
- *size = _file->size();
- return arrow::Status::OK();
+arrow::Result<int64_t> ParquetFile::GetSize() {
+ return _file->size();
}
arrow::Status ParquetFile::Seek(int64_t position) {
@@ -588,23 +592,22 @@ arrow::Status ParquetFile::Seek(int64_t position) {
return arrow::Status::OK();
}
-arrow::Status ParquetFile::Tell(int64_t* position) const {
- *position = _pos;
- return arrow::Status::OK();
+arrow::Result<int64_t> ParquetFile::Tell() const {
+ return _pos;
}
-arrow::Status ParquetFile::Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) {
- std::shared_ptr<arrow::Buffer> read_buf;
- ARROW_RETURN_NOT_OK(arrow::AllocateBuffer(arrow::default_memory_pool(), nbytes, &read_buf));
- int64_t bytes_read = 0;
- ARROW_RETURN_NOT_OK(ReadAt(_pos, nbytes, &bytes_read, read_buf->mutable_data()));
+arrow::Result<std::shared_ptr<arrow::Buffer>> ParquetFile::Read(int64_t nbytes) {
+ auto buffer = arrow::AllocateBuffer(nbytes, arrow::default_memory_pool());
+ ARROW_RETURN_NOT_OK(buffer);
+ std::shared_ptr<arrow::Buffer> read_buf = std::move(buffer.ValueOrDie());
+ auto bytes_read = ReadAt(_pos, nbytes, read_buf->mutable_data());
+ ARROW_RETURN_NOT_OK(bytes_read);
// If bytes_read is equal with read_buf's capacity, we just assign
- if (bytes_read == nbytes) {
- *out = std::move(read_buf);
+ if (bytes_read.ValueOrDie() == nbytes) {
+ return std::move(read_buf);
} else {
- *out = arrow::SliceBuffer(read_buf, 0, bytes_read);
+ return arrow::SliceBuffer(read_buf, 0, bytes_read.ValueOrDie());
}
- return arrow::Status::OK();
}
} // namespace doris
diff --git a/be/src/exec/parquet_reader.h b/be/src/exec/parquet_reader.h
index 7f253a0..2bd5b5a 100644
--- a/be/src/exec/parquet_reader.h
+++ b/be/src/exec/parquet_reader.h
@@ -52,12 +52,12 @@ class ParquetFile : public arrow::io::RandomAccessFile {
public:
ParquetFile(FileReader* file);
virtual ~ParquetFile();
- arrow::Status Read(int64_t nbytes, int64_t* bytes_read, void* buffer) override;
- arrow::Status ReadAt(int64_t position, int64_t nbytes, int64_t* bytes_read, void* out) override;
- arrow::Status GetSize(int64_t* size) override;
+ arrow::Result<int64_t> Read(int64_t nbytes, void* buffer) override;
+ arrow::Result<int64_t> ReadAt(int64_t position, int64_t nbytes, void* out) override;
+ arrow::Result<int64_t> GetSize() override;
arrow::Status Seek(int64_t position) override;
- arrow::Status Read(int64_t nbytes, std::shared_ptr<arrow::Buffer>* out) override;
- arrow::Status Tell(int64_t* position) const override;
+ arrow::Result<std::shared_ptr<arrow::Buffer>> Read(int64_t nbytes) override;
+ arrow::Result<int64_t> Tell() const override;
arrow::Status Close() override;
bool closed() const override;
diff --git a/be/src/exec/parquet_writer.cpp b/be/src/exec/parquet_writer.cpp
index 5a51a84..2e428ec 100644
--- a/be/src/exec/parquet_writer.cpp
+++ b/be/src/exec/parquet_writer.cpp
@@ -59,9 +59,8 @@ arrow::Status ParquetOutputStream::Write(const void* data, int64_t nbytes) {
return arrow::Status::OK();
}
-arrow::Status ParquetOutputStream::Tell(int64_t* position) const {
- *position = _cur_pos;
- return arrow::Status::OK();
+arrow::Result<int64_t> ParquetOutputStream::Tell() const {
+ return _cur_pos;
}
arrow::Status ParquetOutputStream::Close() {
diff --git a/be/src/exec/parquet_writer.h b/be/src/exec/parquet_writer.h
index 0827b02..d5c021d 100644
--- a/be/src/exec/parquet_writer.h
+++ b/be/src/exec/parquet_writer.h
@@ -33,12 +33,12 @@
#include <string>
#include "common/status.h"
+#include "exprs/expr_context.h"
#include "gen_cpp/PaloBrokerService_types.h"
#include "gen_cpp/PlanNodes_types.h"
#include "gen_cpp/Types_types.h"
-#include "runtime/tuple.h"
#include "runtime/row_batch.h"
-#include "exprs/expr_context.h"
+#include "runtime/tuple.h"
namespace doris {
class FileWriter;
@@ -51,7 +51,7 @@ public:
arrow::Status Write(const void* data, int64_t nbytes) override;
// return the current write position of the stream
- arrow::Status Tell(int64_t* position) const override;
+ arrow::Result<int64_t> Tell() const override;
arrow::Status Close() override;
bool closed() const override { return _is_closed; }
@@ -62,7 +62,7 @@ public:
private:
FileWriter* _file_writer; // not owned
- int64_t _cur_pos = 0; // current write position
+ int64_t _cur_pos = 0; // current write position
bool _is_closed = false;
int64_t _written_len = 0;
};
@@ -70,8 +70,7 @@ private:
// a wrapper of parquet output stream
class ParquetWriterWrapper {
public:
- ParquetWriterWrapper(FileWriter* file_writer,
- const std::vector<ExprContext*>& output_expr_ctxs,
+ ParquetWriterWrapper(FileWriter* file_writer, const std::vector<ExprContext*>& output_expr_ctxs,
const std::map<std::string, std::string>& properties,
const std::vector<std::vector<std::string>>& schema);
virtual ~ParquetWriterWrapper();
diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp
index 94e26ce..ed04f00 100644
--- a/be/src/exprs/json_functions.cpp
+++ b/be/src/exprs/json_functions.cpp
@@ -190,8 +190,6 @@ StringVal JsonFunctions::json_quote(FunctionContext* context, const StringVal& j
return StringVal::null();
}
rapidjson::Value array_obj(rapidjson::kObjectType);
- rapidjson::Document document;
- rapidjson::Document::AllocatorType& allocator = document.GetAllocator();
array_obj.SetString(rapidjson::StringRef((char*)json_str.ptr, json_str.len));
rapidjson::StringBuffer buf;
rapidjson::Writer<rapidjson::StringBuffer> writer(buf);
diff --git a/be/src/gutil/strings/util.cc b/be/src/gutil/strings/util.cc
index e567ed4..aa3bcd3 100644
--- a/be/src/gutil/strings/util.cc
+++ b/be/src/gutil/strings/util.cc
@@ -692,10 +692,13 @@ char* strdup_with_new(const char* the_string) {
char* strndup_with_new(const char* the_string, int max_length) {
if (the_string == nullptr) return nullptr;
-
- auto result = new char[max_length + 1];
+ size_t str_len = strlen(the_string);
+ if (str_len > max_length) {
+ str_len = max_length;
+ }
+ auto result = new char[str_len + 1];
result[max_length] = '\0'; // terminate the string because strncpy might not
- return strncpy(result, the_string, max_length);
+ return strncpy(result, the_string, str_len);
}
// ----------------------------------------------------------------------
diff --git a/be/src/http/ev_http_server.cpp b/be/src/http/ev_http_server.cpp
index 63d211c..69041ef 100644
--- a/be/src/http/ev_http_server.cpp
+++ b/be/src/http/ev_http_server.cpp
@@ -210,7 +210,7 @@ bool EvHttpServer::register_handler(const HttpMethod& method, const std::string&
if (result) {
result = root->insert(path, handler);
}
-
+
return result;
}
diff --git a/be/src/olap/bloom_filter_predicate.h b/be/src/olap/bloom_filter_predicate.h
index d14956a..b3dcbbb 100644
--- a/be/src/olap/bloom_filter_predicate.h
+++ b/be/src/olap/bloom_filter_predicate.h
@@ -55,7 +55,7 @@ public:
bool* flags) const override {};
Status evaluate(const Schema& schema, const vector<BitmapIndexIterator*>& iterators,
- uint32_t num_rows, Roaring* roaring) const override {
+ uint32_t num_rows, roaring::Roaring* roaring) const override {
return Status::OK();
}
diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h
index 1456566..21b115e 100644
--- a/be/src/olap/column_predicate.h
+++ b/be/src/olap/column_predicate.h
@@ -35,7 +35,7 @@ class RowBlockV2;
class ColumnPredicate {
public:
explicit ColumnPredicate(uint32_t column_id, bool opposite = false)
- : _column_id(column_id), _opposite(opposite) {}
+ : _column_id(column_id), _opposite(opposite) {}
virtual ~ColumnPredicate() = default;
@@ -44,13 +44,15 @@ public:
// evaluate predicate on ColumnBlock
virtual void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const = 0;
- virtual void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const = 0;
- virtual void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const = 0;
+ virtual void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size,
+ bool* flags) const = 0;
+ virtual void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size,
+ bool* flags) const = 0;
//evaluate predicate on Bitmap
virtual Status evaluate(const Schema& schema,
const std::vector<BitmapIndexIterator*>& iterators, uint32_t num_rows,
- Roaring* roaring) const = 0;
+ roaring::Roaring* roaring) const = 0;
uint32_t column_id() const { return _column_id; }
diff --git a/be/src/olap/comparison_predicate.cpp b/be/src/olap/comparison_predicate.cpp
index c097e78..7dbe7cb 100644
--- a/be/src/olap/comparison_predicate.cpp
+++ b/be/src/olap/comparison_predicate.cpp
@@ -24,9 +24,9 @@
namespace doris {
-#define COMPARISON_PRED_CONSTRUCTOR(CLASS) \
- template <class type> \
- CLASS<type>::CLASS(uint32_t column_id, const type& value, bool opposite) \
+#define COMPARISON_PRED_CONSTRUCTOR(CLASS) \
+ template <class type> \
+ CLASS<type>::CLASS(uint32_t column_id, const type& value, bool opposite) \
: ColumnPredicate(column_id, opposite), _value(value) {}
COMPARISON_PRED_CONSTRUCTOR(EqualPredicate)
@@ -36,12 +36,12 @@ COMPARISON_PRED_CONSTRUCTOR(LessEqualPredicate)
COMPARISON_PRED_CONSTRUCTOR(GreaterPredicate)
COMPARISON_PRED_CONSTRUCTOR(GreaterEqualPredicate)
-#define COMPARISON_PRED_CONSTRUCTOR_STRING(CLASS) \
- template <> \
+#define COMPARISON_PRED_CONSTRUCTOR_STRING(CLASS) \
+ template <> \
CLASS<StringValue>::CLASS(uint32_t column_id, const StringValue& value, bool opposite) \
- : ColumnPredicate(column_id, opposite) { \
- _value.len = value.len; \
- _value.ptr = value.ptr; \
+ : ColumnPredicate(column_id, opposite) { \
+ _value.len = value.len; \
+ _value.ptr = value.ptr; \
}
COMPARISON_PRED_CONSTRUCTOR_STRING(EqualPredicate)
@@ -120,7 +120,7 @@ COMPARISON_PRED_EVALUATE(GreaterEqualPredicate, >=)
const type* cell_value = \
reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
auto result = (!block->cell(idx).is_null() && (*cell_value OP _value)); \
- new_size += _opposite ? !result : result; \
+ new_size += _opposite ? !result : result; \
} \
} else { \
for (uint16_t i = 0; i < *size; ++i) { \
@@ -129,7 +129,7 @@ COMPARISON_PRED_EVALUATE(GreaterEqualPredicate, >=)
const type* cell_value = \
reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
auto result = (*cell_value OP _value); \
- new_size += _opposite ? !result : result; \
+ new_size += _opposite ? !result : result; \
} \
} \
*size = new_size; \
@@ -142,28 +142,29 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(LessEqualPredicate, <=)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterPredicate, >)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
-#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(CLASS, OP) \
- template <class type> \
- void CLASS<type>::evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const { \
- if (block->is_nullable()) { \
- for (uint16_t i = 0; i < size; ++i) { \
- if (flags[i]) continue; \
- uint16_t idx = sel[i]; \
- const type* cell_value = \
- reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
- auto result = (!block->cell(idx).is_null() && (*cell_value OP _value)); \
- flags[i] |= _opposite ? !result : result; \
- } \
- } else { \
- for (uint16_t i = 0; i < size; ++i) { \
- if (flags[i]) continue; \
- uint16_t idx = sel[i]; \
- const type* cell_value = \
- reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
- auto result = (*cell_value OP _value); \
- flags[i] |= _opposite ? !result : result; \
- } \
- } \
+#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(CLASS, OP) \
+ template <class type> \
+ void CLASS<type>::evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) \
+ const { \
+ if (block->is_nullable()) { \
+ for (uint16_t i = 0; i < size; ++i) { \
+ if (flags[i]) continue; \
+ uint16_t idx = sel[i]; \
+ const type* cell_value = \
+ reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
+ auto result = (!block->cell(idx).is_null() && (*cell_value OP _value)); \
+ flags[i] |= _opposite ? !result : result; \
+ } \
+ } else { \
+ for (uint16_t i = 0; i < size; ++i) { \
+ if (flags[i]) continue; \
+ uint16_t idx = sel[i]; \
+ const type* cell_value = \
+ reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
+ auto result = (*cell_value OP _value); \
+ flags[i] |= _opposite ? !result : result; \
+ } \
+ } \
}
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(EqualPredicate, ==)
@@ -173,28 +174,29 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(LessEqualPredicate, <=)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(GreaterPredicate, >)
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_OR(GreaterEqualPredicate, >=)
-#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(CLASS, OP) \
- template <class type> \
- void CLASS<type>::evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const { \
- if (block->is_nullable()) { \
- for (uint16_t i = 0; i < size; ++i) { \
- if (!flags[i]) continue; \
- uint16_t idx = sel[i]; \
- const type* cell_value = \
- reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
- auto result = (!block->cell(idx).is_null() && (*cell_value OP _value)); \
- flags[i] &= _opposite ? !result : result; \
- } \
- } else { \
- for (uint16_t i = 0; i < size; ++i) { \
- if (!flags[i]) continue; \
- uint16_t idx = sel[i]; \
- const type* cell_value = \
- reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
- auto result = (*cell_value OP _value); \
- flags[i] &= _opposite ? !result : result; \
- } \
- } \
+#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(CLASS, OP) \
+ template <class type> \
+ void CLASS<type>::evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) \
+ const { \
+ if (block->is_nullable()) { \
+ for (uint16_t i = 0; i < size; ++i) { \
+ if (!flags[i]) continue; \
+ uint16_t idx = sel[i]; \
+ const type* cell_value = \
+ reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
+ auto result = (!block->cell(idx).is_null() && (*cell_value OP _value)); \
+ flags[i] &= _opposite ? !result : result; \
+ } \
+ } else { \
+ for (uint16_t i = 0; i < size; ++i) { \
+ if (!flags[i]) continue; \
+ uint16_t idx = sel[i]; \
+ const type* cell_value = \
+ reinterpret_cast<const type*>(block->cell(idx).cell_ptr()); \
+ auto result = (*cell_value OP _value); \
+ flags[i] &= _opposite ? !result : result; \
+ } \
+ } \
}
COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(EqualPredicate, ==)
@@ -290,7 +292,7 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(GreaterEqualPredicate, >=)
template <class type> \
Status CLASS<type>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const { \
+ uint32_t num_rows, roaring::Roaring* bitmap) const { \
BitmapIndexIterator* iterator = iterators[_column_id]; \
if (iterator == nullptr) { \
return Status::OK(); \
@@ -298,11 +300,11 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_AND(GreaterEqualPredicate, >=)
rowid_t ordinal_limit = iterator->bitmap_nums(); \
if (iterator->has_null_bitmap()) { \
ordinal_limit--; \
- Roaring null_bitmap; \
+ roaring::Roaring null_bitmap; \
RETURN_IF_ERROR(iterator->read_null_bitmap(&null_bitmap)); \
*bitmap -= null_bitmap; \
} \
- Roaring roaring; \
+ roaring::Roaring roaring; \
bool exact_match; \
Status s = iterator->seek_dictionary(&_value, &exact_match); \
rowid_t seeked_ordinal = iterator->current_ordinal(); \
@@ -318,18 +320,20 @@ COMPARISON_PRED_BITMAP_EVALUATE(LessEqualPredicate, <=)
COMPARISON_PRED_BITMAP_EVALUATE(GreaterPredicate, >)
COMPARISON_PRED_BITMAP_EVALUATE(GreaterEqualPredicate, >=)
-#define COMPARISON_PRED_CONSTRUCTOR_DECLARATION(CLASS) \
- template CLASS<int8_t>::CLASS(uint32_t column_id, const int8_t& value, bool opposite); \
- template CLASS<int16_t>::CLASS(uint32_t column_id, const int16_t& value, bool opposite); \
- template CLASS<int32_t>::CLASS(uint32_t column_id, const int32_t& value, bool opposite); \
- template CLASS<int64_t>::CLASS(uint32_t column_id, const int64_t& value, bool opposite); \
- template CLASS<int128_t>::CLASS(uint32_t column_id, const int128_t& value, bool opposite); \
- template CLASS<float>::CLASS(uint32_t column_id, const float& value, bool opposite); \
- template CLASS<double>::CLASS(uint32_t column_id, const double& value, bool opposite); \
- template CLASS<decimal12_t>::CLASS(uint32_t column_id, const decimal12_t& value, bool opposite); \
- template CLASS<StringValue>::CLASS(uint32_t column_id, const StringValue& value, bool opposite); \
- template CLASS<uint24_t>::CLASS(uint32_t column_id, const uint24_t& value, bool opposite); \
- template CLASS<uint64_t>::CLASS(uint32_t column_id, const uint64_t& value, bool opposite); \
+#define COMPARISON_PRED_CONSTRUCTOR_DECLARATION(CLASS) \
+ template CLASS<int8_t>::CLASS(uint32_t column_id, const int8_t& value, bool opposite); \
+ template CLASS<int16_t>::CLASS(uint32_t column_id, const int16_t& value, bool opposite); \
+ template CLASS<int32_t>::CLASS(uint32_t column_id, const int32_t& value, bool opposite); \
+ template CLASS<int64_t>::CLASS(uint32_t column_id, const int64_t& value, bool opposite); \
+ template CLASS<int128_t>::CLASS(uint32_t column_id, const int128_t& value, bool opposite); \
+ template CLASS<float>::CLASS(uint32_t column_id, const float& value, bool opposite); \
+ template CLASS<double>::CLASS(uint32_t column_id, const double& value, bool opposite); \
+ template CLASS<decimal12_t>::CLASS(uint32_t column_id, const decimal12_t& value, \
+ bool opposite); \
+ template CLASS<StringValue>::CLASS(uint32_t column_id, const StringValue& value, \
+ bool opposite); \
+ template CLASS<uint24_t>::CLASS(uint32_t column_id, const uint24_t& value, bool opposite); \
+ template CLASS<uint64_t>::CLASS(uint32_t column_id, const uint64_t& value, bool opposite); \
template CLASS<bool>::CLASS(uint32_t column_id, const bool& value, bool opposite);
COMPARISON_PRED_CONSTRUCTOR_DECLARATION(EqualPredicate)
@@ -394,40 +398,40 @@ COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(GreaterEqualPredicate)
#define COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(CLASS) \
template Status CLASS<int8_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int16_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int32_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int64_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int128_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<float>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<double>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<decimal12_t>::evaluate( \
const Schema& schema, const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<StringValue>::evaluate( \
const Schema& schema, const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<uint24_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<uint64_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<bool>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const;
+ uint32_t num_rows, roaring::Roaring* bitmap) const;
COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(EqualPredicate)
COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(NotEqualPredicate)
diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h
index 542073e..d0a4049 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -26,21 +26,23 @@ namespace doris {
class VectorizedRowBatch;
-#define COMPARISON_PRED_CLASS_DEFINE(CLASS) \
- template <class type> \
- class CLASS : public ColumnPredicate { \
- public: \
- CLASS(uint32_t column_id, const type& value, bool opposite = false); \
- virtual void evaluate(VectorizedRowBatch* batch) const override; \
- void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override; \
- void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const override;\
- void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const override;\
- virtual Status evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* roaring) const override; \
- \
- private: \
- type _value; \
+#define COMPARISON_PRED_CLASS_DEFINE(CLASS) \
+ template <class type> \
+ class CLASS : public ColumnPredicate { \
+ public: \
+ CLASS(uint32_t column_id, const type& value, bool opposite = false); \
+ virtual void evaluate(VectorizedRowBatch* batch) const override; \
+ void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override; \
+ void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, \
+ bool* flags) const override; \
+ void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, \
+ bool* flags) const override; \
+ virtual Status evaluate(const Schema& schema, \
+ const std::vector<BitmapIndexIterator*>& iterators, \
+ uint32_t num_rows, roaring::Roaring* roaring) const override; \
+ \
+ private: \
+ type _value; \
};
COMPARISON_PRED_CLASS_DEFINE(EqualPredicate)
diff --git a/be/src/olap/in_list_predicate.cpp b/be/src/olap/in_list_predicate.cpp
index d7f9c69..c167a17 100644
--- a/be/src/olap/in_list_predicate.cpp
+++ b/be/src/olap/in_list_predicate.cpp
@@ -23,8 +23,8 @@
namespace doris {
-#define IN_LIST_PRED_CONSTRUCTOR(CLASS) \
- template <class type> \
+#define IN_LIST_PRED_CONSTRUCTOR(CLASS) \
+ template <class type> \
CLASS<type>::CLASS(uint32_t column_id, phmap::flat_hash_set<type>&& values, bool opposite) \
: ColumnPredicate(column_id, opposite), _values(std::move(values)) {}
@@ -173,55 +173,66 @@ IN_LIST_PRED_COLUMN_BLOCK_EVALUATE_OR(NotInListPredicate, ==)
IN_LIST_PRED_COLUMN_BLOCK_EVALUATE_AND(InListPredicate, !=)
IN_LIST_PRED_COLUMN_BLOCK_EVALUATE_AND(NotInListPredicate, ==)
-#define IN_LIST_PRED_BITMAP_EVALUATE(CLASS, OP) \
- template <class type> \
- Status CLASS<type>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* result) const { \
- BitmapIndexIterator* iterator = iterators[_column_id]; \
- if (iterator == nullptr) { \
- return Status::OK(); \
- } \
- if (iterator->has_null_bitmap()) { \
- Roaring null_bitmap; \
- RETURN_IF_ERROR(iterator->read_null_bitmap(&null_bitmap)); \
- *result -= null_bitmap; \
- } \
- Roaring indices; \
- for (auto value : _values) { \
- bool exact_match; \
- Status s = iterator->seek_dictionary(&value, &exact_match); \
- rowid_t seeked_ordinal = iterator->current_ordinal(); \
- if (!s.is_not_found()) { \
- if (!s.ok()) { \
- return s; \
- } \
- if (exact_match) { \
- Roaring index; \
- RETURN_IF_ERROR(iterator->read_bitmap(seeked_ordinal, &index)); \
- indices |= index; \
- } \
- } \
- } \
- *result OP indices; \
- return Status::OK(); \
+#define IN_LIST_PRED_BITMAP_EVALUATE(CLASS, OP) \
+ template <class type> \
+ Status CLASS<type>::evaluate(const Schema& schema, \
+ const std::vector<BitmapIndexIterator*>& iterators, \
+ uint32_t num_rows, roaring::Roaring* result) const { \
+ BitmapIndexIterator* iterator = iterators[_column_id]; \
+ if (iterator == nullptr) { \
+ return Status::OK(); \
+ } \
+ if (iterator->has_null_bitmap()) { \
+ roaring::Roaring null_bitmap; \
+ RETURN_IF_ERROR(iterator->read_null_bitmap(&null_bitmap)); \
+ *result -= null_bitmap; \
+ } \
+ roaring::Roaring indices; \
+ for (auto value : _values) { \
+ bool exact_match; \
+ Status s = iterator->seek_dictionary(&value, &exact_match); \
+ rowid_t seeked_ordinal = iterator->current_ordinal(); \
+ if (!s.is_not_found()) { \
+ if (!s.ok()) { \
+ return s; \
+ } \
+ if (exact_match) { \
+ roaring::Roaring index; \
+ RETURN_IF_ERROR(iterator->read_bitmap(seeked_ordinal, &index)); \
+ indices |= index; \
+ } \
+ } \
+ } \
+ *result OP indices; \
+ return Status::OK(); \
}
IN_LIST_PRED_BITMAP_EVALUATE(InListPredicate, &=)
IN_LIST_PRED_BITMAP_EVALUATE(NotInListPredicate, -=)
-#define IN_LIST_PRED_CONSTRUCTOR_DECLARATION(CLASS) \
- template CLASS<int8_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int8_t>&& values, bool opposite); \
- template CLASS<int16_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int16_t>&& values, bool opposite); \
- template CLASS<int32_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int32_t>&& values, bool opposite); \
- template CLASS<int64_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int64_t>&& values, bool opposite); \
- template CLASS<int128_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int128_t>&& values, bool opposite); \
- template CLASS<float>::CLASS(uint32_t column_id, phmap::flat_hash_set<float>&& values, bool opposite); \
- template CLASS<double>::CLASS(uint32_t column_id, phmap::flat_hash_set<double>&& values, bool opposite); \
- template CLASS<decimal12_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<decimal12_t>&& values, bool opposite); \
- template CLASS<StringValue>::CLASS(uint32_t column_id, phmap::flat_hash_set<StringValue>&& values, bool opposite); \
- template CLASS<uint24_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<uint24_t>&& values, bool opposite); \
- template CLASS<uint64_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<uint64_t>&& values, bool opposite);
+#define IN_LIST_PRED_CONSTRUCTOR_DECLARATION(CLASS) \
+ template CLASS<int8_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int8_t>&& values, \
+ bool opposite); \
+ template CLASS<int16_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int16_t>&& values, \
+ bool opposite); \
+ template CLASS<int32_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int32_t>&& values, \
+ bool opposite); \
+ template CLASS<int64_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int64_t>&& values, \
+ bool opposite); \
+ template CLASS<int128_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<int128_t>&& values, \
+ bool opposite); \
+ template CLASS<float>::CLASS(uint32_t column_id, phmap::flat_hash_set<float>&& values, \
+ bool opposite); \
+ template CLASS<double>::CLASS(uint32_t column_id, phmap::flat_hash_set<double>&& values, \
+ bool opposite); \
+ template CLASS<decimal12_t>::CLASS(uint32_t column_id, \
+ phmap::flat_hash_set<decimal12_t>&& values, bool opposite); \
+ template CLASS<StringValue>::CLASS(uint32_t column_id, \
+ phmap::flat_hash_set<StringValue>&& values, bool opposite); \
+ template CLASS<uint24_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<uint24_t>&& values, \
+ bool opposite); \
+ template CLASS<uint64_t>::CLASS(uint32_t column_id, phmap::flat_hash_set<uint64_t>&& values, \
+ bool opposite);
IN_LIST_PRED_CONSTRUCTOR_DECLARATION(InListPredicate)
IN_LIST_PRED_CONSTRUCTOR_DECLARATION(NotInListPredicate)
@@ -271,37 +282,37 @@ IN_LIST_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(NotInListPredicate)
#define IN_LIST_PRED_BITMAP_EVALUATE_DECLARATION(CLASS) \
template Status CLASS<int8_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int16_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int32_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int64_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<int128_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<float>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<double>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<decimal12_t>::evaluate( \
const Schema& schema, const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<StringValue>::evaluate( \
const Schema& schema, const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<uint24_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const; \
+ uint32_t num_rows, roaring::Roaring* bitmap) const; \
template Status CLASS<uint64_t>::evaluate(const Schema& schema, \
const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const;
+ uint32_t num_rows, roaring::Roaring* bitmap) const;
IN_LIST_PRED_BITMAP_EVALUATE_DECLARATION(InListPredicate)
IN_LIST_PRED_BITMAP_EVALUATE_DECLARATION(NotInListPredicate)
diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h
index e8bca6f..cf9bf61 100644
--- a/be/src/olap/in_list_predicate.h
+++ b/be/src/olap/in_list_predicate.h
@@ -18,10 +18,10 @@
#ifndef DORIS_BE_SRC_OLAP_IN_LIST_PREDICATE_H
#define DORIS_BE_SRC_OLAP_IN_LIST_PREDICATE_H
+#include <parallel_hashmap/phmap.h>
#include <stdint.h>
#include <roaring/roaring.hh>
-#include <parallel_hashmap/phmap.h>
#include "decimal12.h"
#include "olap/column_predicate.h"
@@ -76,22 +76,22 @@ namespace doris {
class VectorizedRowBatch;
-#define IN_LIST_PRED_CLASS_DEFINE(CLASS) \
- template <class type> \
- class CLASS : public ColumnPredicate { \
- public: \
+#define IN_LIST_PRED_CLASS_DEFINE(CLASS) \
+ template <class type> \
+ class CLASS : public ColumnPredicate { \
+ public: \
CLASS(uint32_t column_id, phmap::flat_hash_set<type>&& values, bool is_opposite = false); \
- virtual void evaluate(VectorizedRowBatch* batch) const override; \
- void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override; \
- void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, \
- bool* flags) const override; \
- void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, \
- bool* flags) const override; \
- virtual Status evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, Roaring* bitmap) const override; \
- \
- private: \
+ virtual void evaluate(VectorizedRowBatch* batch) const override; \
+ void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override; \
+ void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, \
+ bool* flags) const override; \
+ void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, \
+ bool* flags) const override; \
+ virtual Status evaluate(const Schema& schema, \
+ const std::vector<BitmapIndexIterator*>& iterators, \
+ uint32_t num_rows, roaring::Roaring* bitmap) const override; \
+ \
+ private: \
phmap::flat_hash_set<type> _values; \
};
diff --git a/be/src/olap/null_predicate.cpp b/be/src/olap/null_predicate.cpp
index 443c854..cc88d26 100644
--- a/be/src/olap/null_predicate.cpp
+++ b/be/src/olap/null_predicate.cpp
@@ -77,7 +77,8 @@ void NullPredicate::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size)
*size = new_size;
}
-void NullPredicate::evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const {
+void NullPredicate::evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size,
+ bool* flags) const {
if (!block->is_nullable() && _is_null) {
memset(flags, true, size);
} else {
@@ -89,7 +90,8 @@ void NullPredicate::evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size
}
}
-void NullPredicate::evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const {
+void NullPredicate::evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size,
+ bool* flags) const {
if (!block->is_nullable() && _is_null) {
return;
} else {
@@ -103,9 +105,9 @@ void NullPredicate::evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t siz
Status NullPredicate::evaluate(const Schema& schema,
const std::vector<BitmapIndexIterator*>& iterators,
- uint32_t num_rows, Roaring* roaring) const {
+ uint32_t num_rows, roaring::Roaring* roaring) const {
if (iterators[_column_id] != nullptr) {
- Roaring null_bitmap;
+ roaring::Roaring null_bitmap;
RETURN_IF_ERROR(iterators[_column_id]->read_null_bitmap(&null_bitmap));
if (_is_null) {
*roaring &= null_bitmap;
diff --git a/be/src/olap/null_predicate.h b/be/src/olap/null_predicate.h
index 9aca70c..e867d58 100644
--- a/be/src/olap/null_predicate.h
+++ b/be/src/olap/null_predicate.h
@@ -30,7 +30,7 @@ class VectorizedRowBatch;
class NullPredicate : public ColumnPredicate {
public:
- NullPredicate(uint32_t column_id, bool is_null,bool opposite = false);
+ NullPredicate(uint32_t column_id, bool is_null, bool opposite = false);
virtual void evaluate(VectorizedRowBatch* batch) const override;
@@ -41,7 +41,7 @@ public:
void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const override;
virtual Status evaluate(const Schema& schema, const vector<BitmapIndexIterator*>& iterators,
- uint32_t num_rows, Roaring* roaring) const override;
+ uint32_t num_rows, roaring::Roaring* roaring) const override;
private:
bool _is_null; //true for null, false for not null
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
index b2dfc09..43b5d68 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.cpp
@@ -45,7 +45,7 @@ Status BitmapIndexIterator::seek_dictionary(const void* value, bool* exact_match
return Status::OK();
}
-Status BitmapIndexIterator::read_bitmap(rowid_t ordinal, Roaring* result) {
+Status BitmapIndexIterator::read_bitmap(rowid_t ordinal, roaring::Roaring* result) {
DCHECK(0 <= ordinal && ordinal < _reader->bitmap_nums());
size_t num_to_read = 1;
@@ -60,16 +60,16 @@ Status BitmapIndexIterator::read_bitmap(rowid_t ordinal, Roaring* result) {
RETURN_IF_ERROR(_bitmap_column_iter.next_batch(&num_read, &column_block_view));
DCHECK(num_to_read == num_read);
- *result = Roaring::read(reinterpret_cast<const Slice*>(block.data())->data, false);
+ *result = roaring::Roaring::read(reinterpret_cast<const Slice*>(block.data())->data, false);
_pool->clear();
return Status::OK();
}
-Status BitmapIndexIterator::read_union_bitmap(rowid_t from, rowid_t to, Roaring* result) {
+Status BitmapIndexIterator::read_union_bitmap(rowid_t from, rowid_t to, roaring::Roaring* result) {
DCHECK(0 <= from && from <= to && to <= _reader->bitmap_nums());
for (rowid_t pos = from; pos < to; pos++) {
- Roaring bitmap;
+ roaring::Roaring bitmap;
RETURN_IF_ERROR(read_bitmap(pos, &bitmap));
*result |= bitmap;
}
@@ -77,4 +77,4 @@ Status BitmapIndexIterator::read_union_bitmap(rowid_t from, rowid_t to, Roaring*
}
} // namespace segment_v2
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
index c68ccc6..7e42ae9 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_reader.h
@@ -87,9 +87,9 @@ public:
Status seek_dictionary(const void* value, bool* exact_match);
// Read bitmap at the given ordinal into `result`.
- Status read_bitmap(rowid_t ordinal, Roaring* result);
+ Status read_bitmap(rowid_t ordinal, roaring::Roaring* result);
- Status read_null_bitmap(Roaring* result) {
+ Status read_null_bitmap(roaring::Roaring* result) {
if (has_null_bitmap()) {
// null bitmap is always stored at last
return read_bitmap(bitmap_nums() - 1, result);
@@ -98,7 +98,7 @@ public:
}
// Read and union all bitmaps in range [from, to) into `result`
- Status read_union_bitmap(rowid_t from, rowid_t to, Roaring* result);
+ Status read_union_bitmap(rowid_t from, rowid_t to, roaring::Roaring* result);
inline rowid_t bitmap_nums() const { return _reader->bitmap_nums(); }
diff --git a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp
index babfdb3..a81dc92 100644
--- a/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/bitmap_index_writer.cpp
@@ -37,12 +37,12 @@ namespace {
template <typename CppType>
struct BitmapIndexTraits {
- using MemoryIndexType = std::map<CppType, Roaring>;
+ using MemoryIndexType = std::map<CppType, roaring::Roaring>;
};
template <>
struct BitmapIndexTraits<Slice> {
- using MemoryIndexType = std::map<Slice, Roaring, Slice::Comparator>;
+ using MemoryIndexType = std::map<Slice, roaring::Roaring, Slice::Comparator>;
};
// Builder for bitmap index. Bitmap index is comprised of two parts
@@ -91,7 +91,7 @@ public:
// new value, copy value and insert new key->bitmap pair
CppType new_value;
_typeinfo->deep_copy(&new_value, &value, &_pool);
- _mem_index.insert({new_value, Roaring::bitmapOf(1, _rid)});
+ _mem_index.insert({new_value, roaring::Roaring::bitmapOf(1, _rid)});
it = _mem_index.find(new_value);
}
_reverted_index_size += it->second.getSizeInBytes(false) - old_size;
@@ -125,7 +125,7 @@ public:
RETURN_IF_ERROR(dict_column_writer.finish(meta->mutable_dict_column()));
}
{ // write bitmaps
- std::vector<Roaring*> bitmaps;
+ std::vector<roaring::Roaring*> bitmaps;
for (auto& it : _mem_index) {
bitmaps.push_back(&(it.second));
}
@@ -183,7 +183,7 @@ private:
uint64_t _reverted_index_size;
rowid_t _rid = 0;
// row id list for null value
- Roaring _null_bitmap;
+ roaring::Roaring _null_bitmap;
// unique value to its row id list
MemoryIndexType _mem_index;
std::shared_ptr<MemTracker> _tracker;
diff --git a/be/src/olap/rowset/segment_v2/row_ranges.h b/be/src/olap/rowset/segment_v2/row_ranges.h
index 009b890..a36c233 100644
--- a/be/src/olap/rowset/segment_v2/row_ranges.h
+++ b/be/src/olap/rowset/segment_v2/row_ranges.h
@@ -183,8 +183,8 @@ public:
*result = std::move(tmp_range);
}
- static Roaring ranges_to_roaring(const RowRanges& ranges) {
- Roaring result;
+ static roaring::Roaring ranges_to_roaring(const RowRanges& ranges) {
+ roaring::Roaring result;
for (auto it = ranges._ranges.begin(); it != ranges._ranges.end(); ++it) {
result.addRange(it->from(), it->to());
}
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 9d33ec3..24b7d74 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -43,7 +43,7 @@ namespace segment_v2 {
// output ranges: [0,2), [4,8), [10,11), [15,18), [18,20) (when max_range_size=3)
class SegmentIterator::BitmapRangeIterator {
public:
- explicit BitmapRangeIterator(const Roaring& bitmap)
+ explicit BitmapRangeIterator(const roaring::Roaring& bitmap)
: _last_val(0), _buf(new uint32_t[256]), _buf_pos(0), _buf_size(0), _eof(false) {
roaring_init_iterator(&bitmap.roaring, &_iter);
_read_next_batch();
@@ -75,14 +75,14 @@ public:
private:
void _read_next_batch() {
- uint32_t n = roaring_read_uint32_iterator(&_iter, _buf, kBatchSize);
+ uint32_t n = roaring::api::roaring_read_uint32_iterator(&_iter, _buf, kBatchSize);
_buf_pos = 0;
_buf_size = n;
_eof = n == 0;
}
static const uint32_t kBatchSize = 256;
- roaring_uint32_iterator_t _iter;
+ roaring::api::roaring_uint32_iterator_t _iter;
uint32_t _last_val;
uint32_t* _buf = nullptr;
uint32_t _buf_pos;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 62641cd..0da6ae4 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -47,7 +47,8 @@ class ColumnIterator;
class SegmentIterator : public RowwiseIterator {
public:
- SegmentIterator(std::shared_ptr<Segment> segment, const Schema& _schema, std::shared_ptr<MemTracker> parent);
+ SegmentIterator(std::shared_ptr<Segment> segment, const Schema& _schema,
+ std::shared_ptr<MemTracker> parent);
~SegmentIterator() override;
Status init(const StorageReadOptions& opts) override;
Status next_batch(RowBlockV2* row_block) override;
@@ -95,7 +96,7 @@ private:
// FIXME prefer vector<unique_ptr<BitmapIndexIterator>>
std::vector<BitmapIndexIterator*> _bitmap_index_iterators;
// after init(), `_row_bitmap` contains all rowid to scan
- Roaring _row_bitmap;
+ roaring::Roaring _row_bitmap;
// an iterator for `_row_bitmap` that can be used to extract row range to scan
std::unique_ptr<BitmapRangeIterator> _range_iter;
// the next rowid to read
diff --git a/be/src/runtime/data_stream_sender.cpp b/be/src/runtime/data_stream_sender.cpp
index 2884374..6325c56 100644
--- a/be/src/runtime/data_stream_sender.cpp
+++ b/be/src/runtime/data_stream_sender.cpp
@@ -258,30 +258,30 @@ Status DataStreamSender::Channel::close_wait(RuntimeState* state) {
}
DataStreamSender::DataStreamSender(ObjectPool* pool, int sender_id, const RowDescriptor& row_desc)
- : _pool(pool),
+ : _row_desc(row_desc),
+ _current_pb_batch(&_pb_batch1),
+ _pool(pool),
_sender_id(sender_id),
- _row_desc(row_desc),
_serialize_batch_timer(NULL),
_bytes_sent_counter(NULL),
- _local_bytes_send_counter(NULL),
- _current_pb_batch(&_pb_batch1) {}
+ _local_bytes_send_counter(NULL) {}
DataStreamSender::DataStreamSender(ObjectPool* pool, int sender_id, const RowDescriptor& row_desc,
const TDataStreamSink& sink,
const std::vector<TPlanFragmentDestination>& destinations,
int per_channel_buffer_size,
bool send_query_statistics_with_every_batch)
- : _sender_id(sender_id),
- _pool(pool),
- _row_desc(row_desc),
- _current_channel_idx(0),
- _part_type(sink.output_partition.type),
- _ignore_not_found(sink.__isset.ignore_not_found ? sink.ignore_not_found : true),
- _current_pb_batch(&_pb_batch1),
+ : _row_desc(row_desc),
_profile(NULL),
+ _current_pb_batch(&_pb_batch1),
+ _pool(pool),
+ _sender_id(sender_id),
_serialize_batch_timer(NULL),
_bytes_sent_counter(NULL),
_local_bytes_send_counter(NULL),
+ _current_channel_idx(0),
+ _part_type(sink.output_partition.type),
+ _ignore_not_found(sink.__isset.ignore_not_found ? sink.ignore_not_found : true),
_dest_node_id(sink.dest_node_id) {
DCHECK_GT(destinations.size(), 0);
DCHECK(sink.output_partition.type == TPartitionType::UNPARTITIONED ||
diff --git a/be/src/runtime/datetime_value.cpp b/be/src/runtime/datetime_value.cpp
index c2ec333..45f5374 100644
--- a/be/src/runtime/datetime_value.cpp
+++ b/be/src/runtime/datetime_value.cpp
@@ -58,9 +58,10 @@ static uint32_t calc_days_in_year(uint32_t year) {
RE2 DateTimeValue::time_zone_offset_format_reg("^[+-]{1}\\d{2}\\:\\d{2}$");
bool DateTimeValue::check_range(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
- uint32_t minute, uint32_t second, uint32_t microsecond, uint16_t type) {
+ uint32_t minute, uint32_t second, uint32_t microsecond,
+ uint16_t type) {
bool time = hour > (type == TIME_TIME ? TIME_MAX_HOUR : 23) || minute > 59 || second > 59 ||
- microsecond > 999999;
+ microsecond > 999999;
return time || check_date(year, month, day);
}
@@ -189,8 +190,8 @@ bool DateTimeValue::from_date_str(const char* date_str, int len) {
}
if (num_field < 3) return false;
- return check_range_and_set_time(date_val[0], date_val[1], date_val[2],
- date_val[3], date_val[4], date_val[5], date_val[6], _type);
+ return check_range_and_set_time(date_val[0], date_val[1], date_val[2], date_val[3], date_val[4],
+ date_val[5], date_val[6], _type);
}
// [0, 101) invalid
@@ -274,7 +275,7 @@ bool DateTimeValue::from_date_int64(int64_t value) {
uint64_t date = value / 1000000;
uint64_t time = value % 1000000;
- auto [year, month, day, hour, minute, second, microsecond] = std::tuple{0,0,0,0,0,0,0};
+ auto [year, month, day, hour, minute, second, microsecond] = std::tuple {0, 0, 0, 0, 0, 0, 0};
year = date / 10000;
date %= 10000;
month = date / 100;
@@ -299,7 +300,10 @@ void DateTimeValue::set_type(int type) {
void DateTimeValue::set_max_time(bool neg) {
set_zero(TIME_TIME);
- _hour = TIME_MAX_HOUR;
+ DCHECK(TIME_MAX_HOUR >= std::numeric_limits<uint8_t>::min() &&
+ TIME_MAX_HOUR <= std::numeric_limits<uint8_t>::max())
+ << "TIME_MAX_HOUR overflow:" << TIME_MAX_HOUR;
+ _hour = static_cast<uint8_t>(TIME_MAX_HOUR);
_minute = TIME_MAX_MINUTE;
_second = TIME_MAX_SECOND;
_neg = neg;
@@ -408,14 +412,14 @@ char* DateTimeValue::to_time_buffer(char* to) const {
int32_t DateTimeValue::to_buffer(char* buffer) const {
switch (_type) {
- case TIME_TIME:
- return to_time_buffer(buffer) - buffer;
- case TIME_DATE:
- return to_date_buffer(buffer) - buffer;
- case TIME_DATETIME:
- return to_datetime_buffer(buffer) - buffer;
- default:
- break;
+ case TIME_TIME:
+ return to_time_buffer(buffer) - buffer;
+ case TIME_DATE:
+ return to_date_buffer(buffer) - buffer;
+ case TIME_DATETIME:
+ return to_datetime_buffer(buffer) - buffer;
+ default:
+ break;
}
return 0;
}
@@ -458,7 +462,7 @@ bool DateTimeValue::get_date_from_daynr(uint64_t daynr) {
return false;
}
- auto [year, month, day] = std::tuple{0, 0, 0};
+ auto [year, month, day] = std::tuple {0, 0, 0};
year = daynr / 365;
uint32_t days_befor_year = 0;
while (daynr < (days_befor_year = calc_daynr(year, 1, 1))) {
@@ -1090,7 +1094,7 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
int strict_week_number_year = -1;
bool usa_time = false;
- auto [year, month, day, hour, minute, second, microsecond] = std::tuple{0,0,0,0,0,0,0};
+ auto [year, month, day, hour, minute, second, microsecond] = std::tuple {0, 0, 0, 0, 0, 0, 0};
while (ptr < end && val < val_end) {
// Skip space character
while (val < val_end && isspace(*val)) {
@@ -1314,14 +1318,14 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
}
val = tmp;
time_part_used = true;
- already_set_time_part = true;
+ already_set_time_part = true;
break;
case 'T':
if (!from_date_format_str("%H:%i:%S", 8, val, val_end - val, &tmp)) {
return false;
}
time_part_used = true;
- already_set_time_part = true;
+ already_set_time_part = true;
val = tmp;
break;
case '.':
@@ -1451,9 +1455,12 @@ bool DateTimeValue::from_date_format_str(const char* format, int format_len, con
// 3. if both are true, means all part of date_time be set, no need check_range_and_set_time
bool already_set_date_part = yearday > 0 || (week_num >= 0 && weekday > 0);
if (already_set_date_part && already_set_time_part) return true;
- if (already_set_date_part) return check_range_and_set_time(_year, _month, _day, hour, minute, second, microsecond, _type);
- if (already_set_time_part) return check_range_and_set_time(year, month, day,
- _hour, _minute, _second, _microsecond, _type);
+ if (already_set_date_part)
+ return check_range_and_set_time(_year, _month, _day, hour, minute, second, microsecond,
+ _type);
+ if (already_set_time_part)
+ return check_range_and_set_time(year, month, day, _hour, _minute, _second, _microsecond,
+ _type);
return check_range_and_set_time(year, month, day, hour, minute, second, microsecond, _type);
}
@@ -1613,7 +1620,7 @@ DateTimeValue DateTimeValue::local_time() {
}
void DateTimeValue::set_time(uint32_t year, uint32_t month, uint32_t day, uint32_t hour,
- uint32_t minute, uint32_t second, uint32_t microsecond) {
+ uint32_t minute, uint32_t second, uint32_t microsecond) {
_year = year;
_month = month;
_day = day;
diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp
index ecf983b..dee19c1 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -19,11 +19,10 @@
#include <arrow/record_batch.h>
#include <gperftools/heap-profiler.h>
-#include <thrift/concurrency/PosixThreadFactory.h>
+#include <thrift/concurrency/ThreadFactory.h>
#include <thrift/processor/TMultiplexedProcessor.h>
#include <thrift/protocol/TDebugProtocol.h>
-#include <boost/shared_ptr.hpp>
#include <map>
#include <memory>
@@ -65,20 +64,18 @@ using apache::thrift::TProcessor;
using apache::thrift::TMultiplexedProcessor;
using apache::thrift::transport::TTransportException;
using apache::thrift::concurrency::ThreadFactory;
-using apache::thrift::concurrency::PosixThreadFactory;
BackendService::BackendService(ExecEnv* exec_env)
- : _exec_env(exec_env), _agent_server(new AgentServer(exec_env, *exec_env->master_info())) {
-}
+ : _exec_env(exec_env), _agent_server(new AgentServer(exec_env, *exec_env->master_info())) {}
Status BackendService::create_service(ExecEnv* exec_env, int port, ThriftServer** server) {
- boost::shared_ptr<BackendService> handler(new BackendService(exec_env));
+ std::shared_ptr<BackendService> handler(new BackendService(exec_env));
// TODO: do we want a BoostThreadFactory?
// TODO: we want separate thread factories here, so that fe requests can't starve
// be requests
- boost::shared_ptr<ThreadFactory> thread_factory(new PosixThreadFactory());
+ std::shared_ptr<ThreadFactory> thread_factory(new ThreadFactory());
- boost::shared_ptr<TProcessor> be_processor(new BackendServiceProcessor(handler));
+ std::shared_ptr<TProcessor> be_processor(new BackendServiceProcessor(handler));
*server = new ThriftServer("backend", be_processor, port, config::be_service_threads);
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index cd98ea0..621c692 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -333,7 +333,7 @@ void PInternalServiceImpl<T>::send_data(google::protobuf::RpcController* control
PDataRow* row = new PDataRow();
row->CopyFrom(request->data(i));
pipe->append_and_flush(reinterpret_cast<char*>(&row), sizeof(row),
- sizeof(row) + row->ByteSize());
+ sizeof(row) + row->ByteSizeLong());
}
response->mutable_status()->set_status_code(0);
}
diff --git a/be/src/util/aes_util.cpp b/be/src/util/aes_util.cpp
index 1b4c7d1..67d9fe1 100644
--- a/be/src/util/aes_util.cpp
+++ b/be/src/util/aes_util.cpp
@@ -102,7 +102,6 @@ static int do_encrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher,
int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_length,
const unsigned char* key, uint32_t key_length, const unsigned char* iv,
bool padding, unsigned char* encrypt) {
- EVP_CIPHER_CTX aes_ctx;
const EVP_CIPHER* cipher = get_evp_type(mode);
/* The encrypt key to be used for encryption */
unsigned char encrypt_key[AES_MAX_KEY_LENGTH / 8];
@@ -111,11 +110,12 @@ int AesUtil::encrypt(AesMode mode, const unsigned char* source, uint32_t source_
if (cipher == nullptr || (EVP_CIPHER_iv_length(cipher) > 0 && !iv)) {
return AES_BAD_DATA;
}
- EVP_CIPHER_CTX_init(&aes_ctx);
+ EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new();
+ EVP_CIPHER_CTX_reset(aes_ctx);
int length = 0;
- int ret = do_encrypt(&aes_ctx, cipher, source, source_length, encrypt_key, iv, padding, encrypt,
+ int ret = do_encrypt(aes_ctx, cipher, source, source_length, encrypt_key, iv, padding, encrypt,
&length);
- EVP_CIPHER_CTX_cleanup(&aes_ctx);
+ EVP_CIPHER_CTX_free(aes_ctx);
if (ret == 0) {
ERR_clear_error();
return AES_BAD_DATA;
@@ -150,7 +150,6 @@ static int do_decrypt(EVP_CIPHER_CTX* aes_ctx, const EVP_CIPHER* cipher,
int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encrypt_length,
const unsigned char* key, uint32_t key_length, const unsigned char* iv,
bool padding, unsigned char* decrypt_content) {
- EVP_CIPHER_CTX aes_ctx;
const EVP_CIPHER* cipher = get_evp_type(mode);
/* The encrypt key to be used for decryption */
@@ -161,11 +160,12 @@ int AesUtil::decrypt(AesMode mode, const unsigned char* encrypt, uint32_t encryp
return AES_BAD_DATA;
}
- EVP_CIPHER_CTX_init(&aes_ctx);
+ EVP_CIPHER_CTX* aes_ctx = EVP_CIPHER_CTX_new();
+ EVP_CIPHER_CTX_reset(aes_ctx);
int length = 0;
- int ret = do_decrypt(&aes_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding,
+ int ret = do_decrypt(aes_ctx, cipher, encrypt, encrypt_length, encrypt_key, iv, padding,
decrypt_content, &length);
- EVP_CIPHER_CTX_cleanup(&aes_ctx);
+ EVP_CIPHER_CTX_free(aes_ctx);
if (ret > 0) {
return length;
} else {
diff --git a/be/src/util/arrow/row_batch.cpp b/be/src/util/arrow/row_batch.cpp
index 0667595..3b0e0ca 100644
--- a/be/src/util/arrow/row_batch.cpp
+++ b/be/src/util/arrow/row_batch.cpp
@@ -274,7 +274,7 @@ public:
}
return builder.Finish(&_arrays[_cur_field_idx]);
}
- // process boolean
+ // process boolean
arrow::Status Visit(const arrow::BooleanType& type) {
arrow::BooleanBuilder builder(_pool);
size_t num_rows = _batch.num_rows();
@@ -460,7 +460,6 @@ Status convert_to_row_batch(const arrow::RecordBatch& batch, const RowDescriptor
}
Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::string* result) {
- std::shared_ptr<arrow::io::BufferOutputStream> sink;
// create sink memory buffer outputstream with the computed capacity
int64_t capacity;
arrow::Status a_st = arrow::ipc::GetRecordBatchSize(record_batch, &capacity);
@@ -469,22 +468,22 @@ Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::strin
msg << "GetRecordBatchSize failure, reason: " << a_st.ToString();
return Status::InternalError(msg.str());
}
- a_st = arrow::io::BufferOutputStream::Create(capacity, arrow::default_memory_pool(), &sink);
- if (!a_st.ok()) {
+ auto sink_res = arrow::io::BufferOutputStream::Create(capacity, arrow::default_memory_pool());
+ if (!sink_res.ok()) {
std::stringstream msg;
- msg << "create BufferOutputStream failure, reason: " << a_st.ToString();
+ msg << "create BufferOutputStream failure, reason: " << sink_res.status().ToString();
return Status::InternalError(msg.str());
}
- std::shared_ptr<arrow::ipc::RecordBatchWriter> record_batch_writer;
+ std::shared_ptr<arrow::io::BufferOutputStream> sink = sink_res.ValueOrDie();
// create RecordBatch Writer
- a_st = arrow::ipc::RecordBatchStreamWriter::Open(sink.get(), record_batch.schema(),
- &record_batch_writer);
- if (!a_st.ok()) {
+ auto res = arrow::ipc::MakeStreamWriter(sink.get(), record_batch.schema());
+ if (!res.ok()) {
std::stringstream msg;
- msg << "open RecordBatchStreamWriter failure, reason: " << a_st.ToString();
+ msg << "open RecordBatchStreamWriter failure, reason: " << res.status().ToString();
return Status::InternalError(msg.str());
}
// write RecordBatch to memory buffer outputstream
+ std::shared_ptr<arrow::ipc::RecordBatchWriter> record_batch_writer = res.ValueOrDie();
a_st = record_batch_writer->WriteRecordBatch(record_batch);
if (!a_st.ok()) {
std::stringstream msg;
@@ -492,14 +491,13 @@ Status serialize_record_batch(const arrow::RecordBatch& record_batch, std::strin
return Status::InternalError(msg.str());
}
record_batch_writer->Close();
- std::shared_ptr<arrow::Buffer> buffer;
- sink->Finish(&buffer);
- if (!a_st.ok()) {
+ auto finish_res = sink->Finish();
+ if (!finish_res.ok()) {
std::stringstream msg;
- msg << "allocate result buffer failure, reason: " << a_st.ToString();
+ msg << "allocate result buffer failure, reason: " << finish_res.status().ToString();
return Status::InternalError(msg.str());
}
- *result = buffer->ToString();
+ *result = finish_res.ValueOrDie()->ToString();
// close the sink
sink->Close();
return Status::OK();
diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h
index e3a0208..d974eb3 100644
--- a/be/src/util/bitmap_value.h
+++ b/be/src/util/bitmap_value.h
@@ -75,12 +75,15 @@ struct BitmapTypeCode {
namespace detail {
class Roaring64MapSetBitForwardIterator;
+class Roaring64MapSetBitBiDirectionalIterator;
// Forked from https://github.com/RoaringBitmap/CRoaring/blob/v0.2.60/cpp/roaring64map.hh
// What we change includes
// - a custom serialization format is used inside read()/write()/getSizeInBytes()
// - added clear() and is32BitsEnough()
class Roaring64Map {
+ typedef roaring::api::roaring_bitmap_t roaring_bitmap_t;
+
public:
/**
* Create an empty bitmap
@@ -100,14 +103,29 @@ public:
/**
* Construct a 64-bit map from a 32-bit one
*/
- Roaring64Map(const Roaring& r) { emplaceOrInsert(0, r); }
+ explicit Roaring64Map(const roaring::Roaring& r) { emplaceOrInsert(0, r); }
/**
* Construct a roaring object from the C struct.
*
* Passing a NULL point is unsafe.
*/
- Roaring64Map(roaring_bitmap_t* s) { emplaceOrInsert(0, s); }
+ explicit Roaring64Map(roaring_bitmap_t* s) {
+ roaring::Roaring r(s);
+ emplaceOrInsert(0, r);
+ }
+
+ Roaring64Map(const Roaring64Map& r) : roarings(r.roarings), copyOnWrite(r.copyOnWrite) {}
+
+ Roaring64Map(Roaring64Map&& r) : roarings(r.roarings), copyOnWrite(r.copyOnWrite) {}
+
+ /**
+ * Assignment operator.
+ */
+ Roaring64Map& operator=(const Roaring64Map& r) {
+ roarings = r.roarings;
+ return *this;
+ }
/**
* Construct a bitmap from a list of integer values.
@@ -243,7 +261,7 @@ public:
if (r.roarings.count(map_entry.first) == 1)
map_entry.second &= r.roarings.at(map_entry.first);
else
- map_entry.second = Roaring();
+ map_entry.second = roaring::Roaring();
}
return *this;
}
@@ -314,11 +332,11 @@ public:
"bitmap is full, cardinality is 2^64, "
"unable to represent in a 64-bit integer");
}
- return std::accumulate(
- roarings.cbegin(), roarings.cend(), (uint64_t)0,
- [](uint64_t previous, const std::pair<const uint32_t, Roaring>& map_entry) {
- return previous + map_entry.second.cardinality();
- });
+ return std::accumulate(roarings.cbegin(), roarings.cend(), (uint64_t)0,
+ [](uint64_t previous,
+ const std::pair<const uint32_t, roaring::Roaring>& map_entry) {
+ return previous + map_entry.second.cardinality();
+ });
}
/**
@@ -326,7 +344,7 @@ public:
*/
bool isEmpty() const {
return std::all_of(roarings.cbegin(), roarings.cend(),
- [](const std::pair<const uint32_t, Roaring>& map_entry) {
+ [](const std::pair<const uint32_t, roaring::Roaring>& map_entry) {
return map_entry.second.isEmpty();
});
}
@@ -339,9 +357,10 @@ public:
//
// we put std::numeric_limits<>::max/min in parenthesis
// to avoid a clash with the Windows.h header under Windows
- return roarings.size() == ((size_t)(std::numeric_limits<uint32_t>::max)()) + 1
+ return roarings.size() == ((uint64_t)(std::numeric_limits<uint32_t>::max)()) + 1
? std::all_of(roarings.cbegin(), roarings.cend(),
- [](const std::pair<const uint32_t, Roaring>& roaring_map_entry) {
+ [](const std::pair<const uint32_t, roaring::Roaring>&
+ roaring_map_entry) {
// roarings within map are saturated if cardinality
// is uint32_t max + 1
return roaring_map_entry.second.cardinality() ==
@@ -357,7 +376,7 @@ public:
bool isSubset(const Roaring64Map& r) const {
for (const auto& map_entry : roarings) {
auto roaring_iter = r.roarings.find(map_entry.first);
- if (roaring_iter == roarings.cend())
+ if (roaring_iter == r.roarings.cend())
return false;
else if (!map_entry.second.isSubset(roaring_iter->second))
return false;
@@ -382,13 +401,13 @@ public:
*/
void toUint64Array(uint64_t* ans) const {
// Annoyingly, VS 2017 marks std::accumulate() as [[nodiscard]]
- (void)std::accumulate(
- roarings.cbegin(), roarings.cend(), ans,
- [](uint64_t* previous, const std::pair<const uint32_t, Roaring>& map_entry) {
- for (uint32_t low_bits : map_entry.second)
- *previous++ = uniteBytes(map_entry.first, low_bits);
- return previous;
- });
+ (void)std::accumulate(roarings.cbegin(), roarings.cend(), ans,
+ [](uint64_t* previous,
+ const std::pair<const uint32_t, roaring::Roaring>& map_entry) {
+ for (uint32_t low_bits : map_entry.second)
+ *previous++ = uniteBytes(map_entry.first, low_bits);
+ return previous;
+ });
}
/**
@@ -477,10 +496,11 @@ public:
* return whether a change was applied
*/
bool removeRunCompression() {
- return std::accumulate(roarings.begin(), roarings.end(), false,
- [](bool previous, std::pair<const uint32_t, Roaring>& map_entry) {
- return map_entry.second.removeRunCompression() && previous;
- });
+ return std::accumulate(
+ roarings.begin(), roarings.end(), true,
+ [](bool previous, std::pair<const uint32_t, roaring::Roaring>& map_entry) {
+ return map_entry.second.removeRunCompression() && previous;
+ });
}
/** convert array and bitmap containers to run containers when it is more
@@ -490,10 +510,11 @@ public:
* Additional savings might be possible by calling shrinkToFit().
*/
bool runOptimize() {
- return std::accumulate(roarings.begin(), roarings.end(), false,
- [](bool previous, std::pair<const uint32_t, Roaring>& map_entry) {
- return map_entry.second.runOptimize() && previous;
- });
+ return std::accumulate(
+ roarings.begin(), roarings.end(), true,
+ [](bool previous, std::pair<const uint32_t, roaring::Roaring>& map_entry) {
+ return map_entry.second.runOptimize() && previous;
+ });
}
/**
@@ -525,12 +546,14 @@ public:
* (true means that the iteration should continue while false means that it
* should stop), and takes (uint32_t,void*) as inputs.
*/
- void iterate(roaring_iterator64 iterator, void* ptr) const {
- std::for_each(roarings.begin(), roarings.cend(),
- [=](const std::pair<const uint32_t, Roaring>& map_entry) {
- roaring_iterate64(&map_entry.second.roaring, iterator,
- uint64_t(map_entry.first) << 32, ptr);
- });
+ void iterate(roaring::api::roaring_iterator64 iterator, void* ptr) const {
+ for (const auto& map_entry : roarings) {
+ bool should_continue = roaring_iterate64(&map_entry.second.roaring, iterator,
+ uint64_t(map_entry.first) << 32, ptr);
+ if (!should_continue) {
+ break;
+ }
+ }
}
/**
@@ -583,7 +606,7 @@ public:
*(buf++) = BitmapTypeCode::type::BITMAP32;
auto it = roarings.find(0);
if (it == roarings.end()) { // empty bitmap
- Roaring r;
+ roaring::Roaring r;
return r.write(buf) + 1;
}
return it->second.write(buf) + 1;
@@ -595,7 +618,7 @@ public:
// push map size
buf = (char*)encode_varint64((uint8_t*)buf, roarings.size());
std::for_each(roarings.cbegin(), roarings.cend(),
- [&buf](const std::pair<const uint32_t, Roaring>& map_entry) {
+ [&buf](const std::pair<const uint32_t, roaring::Roaring>& map_entry) {
// push map key
encode_fixed32_le((uint8_t*)buf, map_entry.first);
buf += sizeof(uint32_t);
@@ -615,7 +638,7 @@ public:
Roaring64Map result;
if (*buf == BitmapTypeCode::BITMAP32) {
- Roaring read = Roaring::read(buf + 1);
+ roaring::Roaring read = roaring::Roaring::read(buf + 1);
result.emplaceOrInsert(0, std::move(read));
return result;
}
@@ -634,10 +657,10 @@ public:
uint32_t key = decode_fixed32_le(reinterpret_cast<const uint8_t*>(buf));
buf += sizeof(uint32_t);
// read map value Roaring
- Roaring read = Roaring::read(buf);
+ roaring::Roaring read_var = roaring::Roaring::read(buf);
// forward buffer past the last Roaring Bitmap
- buf += read.getSizeInBytes();
- result.emplaceOrInsert(key, std::move(read));
+ buf += read_var.getSizeInBytes();
+ result.emplaceOrInsert(key, std::move(read_var));
}
return result;
}
@@ -649,18 +672,19 @@ public:
if (is32BitsEnough()) {
auto it = roarings.find(0);
if (it == roarings.end()) { // empty bitmap
- Roaring r;
+ roaring::Roaring r;
return r.getSizeInBytes() + 1;
}
return it->second.getSizeInBytes() + 1;
}
// start with type code, map size and size of keys for each map entry
size_t init = 1 + varint_length(roarings.size()) + roarings.size() * sizeof(uint32_t);
- return std::accumulate(roarings.cbegin(), roarings.cend(), init,
- [=](size_t previous, const std::pair<const uint32_t, Roaring>& map_entry) {
- // add in bytes used by each Roaring
- return previous + map_entry.second.getSizeInBytes();
- });
+ return std::accumulate(
+ roarings.cbegin(), roarings.cend(), init,
+ [=](size_t previous, const std::pair<const uint32_t, roaring::Roaring>& map_entry) {
+ // add in bytes used by each Roaring
+ return previous + map_entry.second.getSizeInBytes();
+ });
}
/**
@@ -704,7 +728,7 @@ public:
if (copyOnWrite == val) return;
copyOnWrite = val;
std::for_each(roarings.begin(), roarings.end(),
- [=](std::pair<const uint32_t, Roaring>& map_entry) {
+ [=](std::pair<const uint32_t, roaring::Roaring>& map_entry) {
map_entry.second.setCopyOnWrite(val);
});
}
@@ -730,16 +754,17 @@ public:
return true;
},
(void*)&outer_iter_data);
- std::for_each(
- ++map_iter, roarings.cend(), [](const std::pair<const uint32_t, Roaring>& map_entry) {
- map_entry.second.iterate(
- [](uint32_t low_bits, void* high_bits) -> bool {
- std::printf(",%llu", (long long unsigned)uniteBytes(
- *(uint32_t*)high_bits, low_bits));
- return true;
- },
- (void*)&map_entry.first);
- });
+ std::for_each(++map_iter, roarings.cend(),
+ [](const std::pair<const uint32_t, roaring::Roaring>& map_entry) {
+ map_entry.second.iterate(
+ [](uint32_t low_bits, void* high_bits) -> bool {
+ std::printf(",%llu",
+ (long long unsigned)uniteBytes(
+ *(uint32_t*)high_bits, low_bits));
+ return true;
+ },
+ (void*)&map_entry.first);
+ });
} else
std::printf("{");
std::printf("}\n");
@@ -750,9 +775,9 @@ public:
*/
std::string toString() const {
struct iter_data {
- std::string str;
+ std::string str {};
uint32_t high_bits;
- char first_char = '{';
+ char first_char {'{'};
} outer_iter_data;
if (!isEmpty()) {
auto map_iter = roarings.cbegin();
@@ -770,7 +795,8 @@ public:
(void*)&outer_iter_data);
std::for_each(
++map_iter, roarings.cend(),
- [&outer_iter_data](const std::pair<const uint32_t, Roaring>& map_entry) {
+ [&outer_iter_data](
+ const std::pair<const uint32_t, roaring::Roaring>& map_entry) {
outer_iter_data.high_bits = map_entry.first;
map_entry.second.iterate(
[](uint32_t low_bits, void* inner_iter_data) -> bool {
@@ -807,7 +833,9 @@ public:
}
friend class Roaring64MapSetBitForwardIterator;
+ friend class Roaring64MapSetBitBiDirectionalIterator;
typedef Roaring64MapSetBitForwardIterator const_iterator;
+ typedef Roaring64MapSetBitBiDirectionalIterator const_bidirectional_iterator;
/**
* Returns an iterator that can be used to access the position of the
@@ -830,8 +858,8 @@ public:
const_iterator end() const;
private:
- std::map<uint32_t, Roaring> roarings;
- bool copyOnWrite = false;
+ std::map<uint32_t, roaring::Roaring> roarings {};
+ bool copyOnWrite {false};
static uint32_t highBytes(const uint64_t in) { return uint32_t(in >> 32); }
static uint32_t lowBytes(const uint64_t in) { return uint32_t(in); }
static uint64_t uniteBytes(const uint32_t highBytes, const uint32_t lowBytes) {
@@ -839,7 +867,7 @@ private:
}
// this is needed to tolerate gcc's C++11 libstdc++ lacking emplace
// prior to version 4.8
- void emplaceOrInsert(const uint32_t key, const Roaring& value) {
+ void emplaceOrInsert(const uint32_t key, const roaring::Roaring& value) {
#if defined(__GLIBCXX__) && __GLIBCXX__ < 20130322
roarings.insert(std::make_pair(key, value));
#else
@@ -847,14 +875,14 @@ private:
#endif
}
- void emplace(const uint32_t key, Roaring&& value) {
+ void emplace(const uint32_t key, roaring::Roaring&& value) {
roarings.emplace(std::make_pair(key, std::move(value)));
}
};
// Forked from https://github.com/RoaringBitmap/CRoaring/blob/v0.2.60/cpp/roaring64map.hh
// Used to go through the set bits. Not optimally fast, but convenient.
-class Roaring64MapSetBitForwardIterator final {
+class Roaring64MapSetBitForwardIterator {
public:
typedef std::forward_iterator_tag iterator_category;
typedef uint64_t* pointer;
@@ -870,25 +898,25 @@ public:
return Roaring64Map::uniteBytes(map_iter->first, i.current_value);
}
- bool operator<(const type_of_iterator& o) {
+ bool operator<(const type_of_iterator& o) const {
if (map_iter == map_end) return false;
if (o.map_iter == o.map_end) return true;
return **this < *o;
}
- bool operator<=(const type_of_iterator& o) {
+ bool operator<=(const type_of_iterator& o) const {
if (o.map_iter == o.map_end) return true;
if (map_iter == map_end) return false;
return **this <= *o;
}
- bool operator>(const type_of_iterator& o) {
+ bool operator>(const type_of_iterator& o) const {
if (o.map_iter == o.map_end) return false;
if (map_iter == map_end) return true;
return **this > *o;
}
- bool operator>=(const type_of_iterator& o) {
+ bool operator>=(const type_of_iterator& o) const {
if (map_iter == map_end) return true;
if (o.map_iter == o.map_end) return false;
return **this >= *o;
@@ -915,6 +943,22 @@ public:
return orig;
}
+ bool move(const value_type& x) {
+ map_iter = p.lower_bound(Roaring64Map::highBytes(x));
+ if (map_iter != p.cend()) {
+ roaring_init_iterator(&map_iter->second.roaring, &i);
+ if (map_iter->first == Roaring64Map::highBytes(x)) {
+ if (roaring_move_uint32_iterator_equalorlarger(&i, Roaring64Map::lowBytes(x)))
+ return true;
+ map_iter++;
+ if (map_iter == map_end) return false;
+ roaring_init_iterator(&map_iter->second.roaring, &i);
+ }
+ return true;
+ }
+ return false;
+ }
+
bool operator==(const Roaring64MapSetBitForwardIterator& o) {
if (map_iter == map_end && o.map_iter == o.map_end) return true;
if (o.map_iter == o.map_end) return false;
@@ -926,9 +970,18 @@ public:
if (o.map_iter == o.map_end) return true;
return **this != *o;
}
+ Roaring64MapSetBitForwardIterator& operator=(const Roaring64MapSetBitForwardIterator& r) {
+ map_iter = r.map_iter;
+ map_end = r.map_end;
+ i = r.i;
+ return *this;
+ }
+
+ Roaring64MapSetBitForwardIterator(const Roaring64MapSetBitForwardIterator& r)
+ : p(r.p), map_iter(r.map_iter), map_end(r.map_end), i(r.i) {}
Roaring64MapSetBitForwardIterator(const Roaring64Map& parent, bool exhausted = false)
- : map_end(parent.roarings.cend()) {
+ : p(parent.roarings), map_end(parent.roarings.cend()) {
if (exhausted || parent.roarings.empty()) {
map_iter = parent.roarings.cend();
} else {
@@ -942,10 +995,60 @@ public:
}
}
-private:
- std::map<uint32_t, Roaring>::const_iterator map_iter;
- std::map<uint32_t, Roaring>::const_iterator map_end;
- roaring_uint32_iterator_t i;
+protected:
+ const std::map<uint32_t, roaring::Roaring>& p;
+ std::map<uint32_t, roaring::Roaring>::const_iterator map_iter {};
+ std::map<uint32_t, roaring::Roaring>::const_iterator map_end {};
+ roaring::api::roaring_uint32_iterator_t i {};
+};
+
+class Roaring64MapSetBitBiDirectionalIterator final : public Roaring64MapSetBitForwardIterator {
+public:
+ explicit Roaring64MapSetBitBiDirectionalIterator(const Roaring64Map& parent,
+ bool exhausted = false)
+ : Roaring64MapSetBitForwardIterator(parent, exhausted),
+ map_begin(parent.roarings.cbegin()) {}
+
+ Roaring64MapSetBitBiDirectionalIterator& operator=(const Roaring64MapSetBitForwardIterator& r) {
+ *(Roaring64MapSetBitForwardIterator*)this = r;
+ return *this;
+ }
+
+ Roaring64MapSetBitBiDirectionalIterator& operator--() { // --i, must return dec.value
+ if (map_iter == map_end) {
+ --map_iter;
+ roaring_init_iterator_last(&map_iter->second.roaring, &i);
+ if (i.has_value) return *this;
+ }
+
+ roaring_previous_uint32_iterator(&i);
+ while (!i.has_value) {
+ if (map_iter == map_begin) return *this;
+ map_iter--;
+ roaring_init_iterator_last(&map_iter->second.roaring, &i);
+ }
+ return *this;
+ }
+
+ Roaring64MapSetBitBiDirectionalIterator operator--(int) { // i--, must return orig. value
+ Roaring64MapSetBitBiDirectionalIterator orig(*this);
+ if (map_iter == map_end) {
+ --map_iter;
+ roaring_init_iterator_last(&map_iter->second.roaring, &i);
+ return orig;
+ }
+
+ roaring_previous_uint32_iterator(&i);
+ while (!i.has_value) {
+ if (map_iter == map_begin) return orig;
+ map_iter--;
+ roaring_init_iterator_last(&map_iter->second.roaring, &i);
+ }
+ return orig;
+ }
+
+protected:
+ std::map<uint32_t, roaring::Roaring>::const_iterator map_begin;
};
inline Roaring64MapSetBitForwardIterator Roaring64Map::begin() const {
@@ -1013,43 +1116,43 @@ public:
void remove(uint64_t value) {
switch (_type) {
- case EMPTY:
- break;
- case SINGLE:
- //there is need to convert the type if two variables are equal
- if (_sv == value) {
- _type = EMPTY;
- }
- break;
- case BITMAP:
- _bitmap.remove(value);
- _convert_to_smaller_type();
+ case EMPTY:
+ break;
+ case SINGLE:
+ //there is need to convert the type if two variables are equal
+ if (_sv == value) {
+ _type = EMPTY;
+ }
+ break;
+ case BITMAP:
+ _bitmap.remove(value);
+ _convert_to_smaller_type();
}
}
// Compute the union between the current bitmap and the provided bitmap.
BitmapValue& operator-=(const BitmapValue& rhs) {
switch (rhs._type) {
+ case EMPTY:
+ break;
+ case SINGLE:
+ remove(rhs._sv);
+ break;
+ case BITMAP:
+ switch (_type) {
case EMPTY:
break;
case SINGLE:
- remove(rhs._sv);
+ if (rhs._bitmap.contains(_sv)) {
+ _type = EMPTY;
+ }
break;
case BITMAP:
- switch (_type) {
- case EMPTY:
- break;
- case SINGLE:
- if (rhs._bitmap.contains(_sv)) {
- _type = EMPTY;
- }
- break;
- case BITMAP:
- _bitmap -= rhs._bitmap;
- _convert_to_smaller_type();
- break;
- }
+ _bitmap -= rhs._bitmap;
+ _convert_to_smaller_type();
break;
+ }
+ break;
}
return *this;
}
@@ -1142,56 +1245,55 @@ public:
// BITMAP -> SINGLE
BitmapValue& operator^=(const BitmapValue& rhs) {
switch (rhs._type) {
+ case EMPTY:
+ break;
+ case SINGLE:
+ switch (_type) {
case EMPTY:
+ add(rhs._sv);
break;
case SINGLE:
- switch (_type) {
- case EMPTY:
- add(rhs._sv);
- break;
- case SINGLE:
- if (_sv == rhs._sv) {
- _type = EMPTY;
- _bitmap.clear();
- } else {
- add(rhs._sv);
- }
- break;
- case BITMAP:
- if (!_bitmap.contains(rhs._sv)) {
- add(rhs._sv);
- } else {
- _bitmap.remove(rhs._sv);
- }
- break;
+ if (_sv == rhs._sv) {
+ _type = EMPTY;
+ _bitmap.clear();
+ } else {
+ add(rhs._sv);
}
break;
case BITMAP:
- switch (_type) {
- case EMPTY:
- _bitmap = rhs._bitmap;
- _type = BITMAP;
- break;
- case SINGLE:
- _bitmap = rhs._bitmap;
- _type = BITMAP;
- if (!rhs._bitmap.contains(_sv)) {
- _bitmap.add(_sv);
- } else {
- _bitmap.remove(_sv);
- }
- break;
- case BITMAP:
- _bitmap ^= rhs._bitmap;
- _convert_to_smaller_type();
- break;
+ if (!_bitmap.contains(rhs._sv)) {
+ add(rhs._sv);
+ } else {
+ _bitmap.remove(rhs._sv);
+ }
+ break;
+ }
+ break;
+ case BITMAP:
+ switch (_type) {
+ case EMPTY:
+ _bitmap = rhs._bitmap;
+ _type = BITMAP;
+ break;
+ case SINGLE:
+ _bitmap = rhs._bitmap;
+ _type = BITMAP;
+ if (!rhs._bitmap.contains(_sv)) {
+ _bitmap.add(_sv);
+ } else {
+ _bitmap.remove(_sv);
}
break;
+ case BITMAP:
+ _bitmap ^= rhs._bitmap;
+ _convert_to_smaller_type();
+ break;
+ }
+ break;
}
return *this;
}
-
// check if value x is present
bool contains(uint64_t x) const {
switch (_type) {
diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp
index 8ba1e4f..1b0f814 100644
--- a/be/src/util/block_compression.cpp
+++ b/be/src/util/block_compression.cpp
@@ -190,18 +190,12 @@ private:
};
LZ4F_preferences_t Lz4fBlockCompression::_s_preferences = {
- {
- LZ4F_max256KB,
- LZ4F_blockLinked,
- LZ4F_noContentChecksum,
- LZ4F_frame,
- 0, // unknown content size,
- {0, 0} // reserved, must be set to 0
- },
- 0, // compression level; 0 == default
- 0, // autoflush
- {0, 0, 0, 0}, // reserved, must be set to 0
-};
+ {LZ4F_max256KB, LZ4F_blockLinked, LZ4F_noContentChecksum, LZ4F_frame, 0ULL, 0U,
+ LZ4F_noBlockChecksum},
+ 0,
+ 0u,
+ 0u,
+ {0u, 0u, 0u}};
class SnappySlicesSource : public snappy::Source {
public:
diff --git a/be/src/util/debug_util.cpp b/be/src/util/debug_util.cpp
index fd6856a..854d40f 100644
--- a/be/src/util/debug_util.cpp
+++ b/be/src/util/debug_util.cpp
@@ -41,41 +41,6 @@
namespace doris {
-#define THRIFT_ENUM_OUTPUT_FN_IMPL(E, MAP) \
- std::ostream& operator<<(std::ostream& os, const E::type& e) { \
- std::map<int, const char*>::const_iterator i; \
- i = MAP.find(e); \
- if (i != MAP.end()) { \
- os << i->second; \
- } \
- return os; \
- }
-
-// Macro to stamp out operator<< for thrift enums. Why doesn't thrift do this?
-#define THRIFT_ENUM_OUTPUT_FN(E) THRIFT_ENUM_OUTPUT_FN_IMPL(E, _##E##_VALUES_TO_NAMES)
-
-// Macro to implement Print function that returns string for thrift enums
-#define THRIFT_ENUM_PRINT_FN(E) \
- std::string Print##E(const E::type& e) { \
- std::stringstream ss; \
- ss << e; \
- return ss.str(); \
- }
-
-THRIFT_ENUM_OUTPUT_FN(TExprOpcode);
-THRIFT_ENUM_OUTPUT_FN(TAggregationOp);
-THRIFT_ENUM_OUTPUT_FN(THdfsFileFormat);
-THRIFT_ENUM_OUTPUT_FN(THdfsCompression);
-THRIFT_ENUM_OUTPUT_FN(TStmtType);
-THRIFT_ENUM_OUTPUT_FN(QueryState);
-THRIFT_ENUM_OUTPUT_FN(TAgentServiceVersion);
-
-THRIFT_ENUM_PRINT_FN(TStmtType);
-THRIFT_ENUM_PRINT_FN(QueryState);
-
-THRIFT_ENUM_PRINT_FN(TMetricKind);
-THRIFT_ENUM_PRINT_FN(TUnit);
-
std::string print_plan_node_type(const TPlanNodeType::type& type) {
std::map<int, const char*>::const_iterator i;
i = _TPlanNodeType_VALUES_TO_NAMES.find(type);
diff --git a/be/src/util/simdutf8check.h b/be/src/util/simdutf8check.h
index 01303ee..9b0105b 100644
--- a/be/src/util/simdutf8check.h
+++ b/be/src/util/simdutf8check.h
@@ -310,126 +310,5 @@ static inline void avx_count_nibbles(__m256i bytes, struct avx_processed_utf_byt
answer->high_nibbles = _mm256_and_si256(_mm256_srli_epi16(bytes, 4), _mm256_set1_epi8(0x0F));
}
-// check whether the current bytes are valid UTF-8
-// at the end of the function, previous gets updated
-static struct avx_processed_utf_bytes avxcheckUTF8Bytes(__m256i current_bytes,
- struct avx_processed_utf_bytes* previous,
- __m256i* has_error) {
- struct avx_processed_utf_bytes pb;
- avx_count_nibbles(current_bytes, &pb);
-
- avxcheckSmallerThan0xF4(current_bytes, has_error);
-
- __m256i initial_lengths = avxcontinuationLengths(pb.high_nibbles);
-
- pb.carried_continuations =
- avxcarryContinuations(initial_lengths, previous->carried_continuations);
-
- avxcheckContinuations(initial_lengths, pb.carried_continuations, has_error);
-
- __m256i off1_current_bytes = push_last_byte_of_a_to_b(previous->rawbytes, pb.rawbytes);
- avxcheckFirstContinuationMax(current_bytes, off1_current_bytes, has_error);
-
- avxcheckOverlong(current_bytes, off1_current_bytes, pb.high_nibbles, previous->high_nibbles,
- has_error);
- return pb;
-}
-
-// check whether the current bytes are valid UTF-8
-// at the end of the function, previous gets updated
-static struct avx_processed_utf_bytes avxcheckUTF8Bytes_asciipath(
- __m256i current_bytes, struct avx_processed_utf_bytes* previous, __m256i* has_error) {
- if (_mm256_testz_si256(current_bytes,
- _mm256_set1_epi8(0x80))) { // fast ascii path
- *has_error = _mm256_or_si256(
- _mm256_cmpgt_epi8(previous->carried_continuations,
- _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)),
- *has_error);
- return *previous;
- }
-
- struct avx_processed_utf_bytes pb;
- avx_count_nibbles(current_bytes, &pb);
-
- avxcheckSmallerThan0xF4(current_bytes, has_error);
-
- __m256i initial_lengths = avxcontinuationLengths(pb.high_nibbles);
-
- pb.carried_continuations =
- avxcarryContinuations(initial_lengths, previous->carried_continuations);
-
- avxcheckContinuations(initial_lengths, pb.carried_continuations, has_error);
-
- __m256i off1_current_bytes = push_last_byte_of_a_to_b(previous->rawbytes, pb.rawbytes);
- avxcheckFirstContinuationMax(current_bytes, off1_current_bytes, has_error);
-
- avxcheckOverlong(current_bytes, off1_current_bytes, pb.high_nibbles, previous->high_nibbles,
- has_error);
- return pb;
-}
-
-static bool validate_utf8_fast_avx_asciipath(const char* src, size_t len) {
- size_t i = 0;
- __m256i has_error = _mm256_setzero_si256();
- struct avx_processed_utf_bytes previous = {.rawbytes = _mm256_setzero_si256(),
- .high_nibbles = _mm256_setzero_si256(),
- .carried_continuations = _mm256_setzero_si256()};
- if (len >= 32) {
- for (; i <= len - 32; i += 32) {
- __m256i current_bytes = _mm256_loadu_si256((const __m256i*)(src + i));
- previous = avxcheckUTF8Bytes_asciipath(current_bytes, &previous, &has_error);
- }
- }
-
- // last part
- if (i < len) {
- char buffer[32];
- memset(buffer, 0, 32);
- memcpy(buffer, src + i, len - i);
- __m256i current_bytes = _mm256_loadu_si256((const __m256i*)(buffer));
- previous = avxcheckUTF8Bytes(current_bytes, &previous, &has_error);
- } else {
- has_error = _mm256_or_si256(
- _mm256_cmpgt_epi8(previous.carried_continuations,
- _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)),
- has_error);
- }
-
- return _mm256_testz_si256(has_error, has_error);
-}
-
-static bool validate_utf8_fast_avx(const char* src, size_t len) {
- size_t i = 0;
- __m256i has_error = _mm256_setzero_si256();
- struct avx_processed_utf_bytes previous = {.rawbytes = _mm256_setzero_si256(),
- .high_nibbles = _mm256_setzero_si256(),
- .carried_continuations = _mm256_setzero_si256()};
- if (len >= 32) {
- for (; i <= len - 32; i += 32) {
- __m256i current_bytes = _mm256_loadu_si256((const __m256i*)(src + i));
- previous = avxcheckUTF8Bytes(current_bytes, &previous, &has_error);
- }
- }
-
- // last part
- if (i < len) {
- char buffer[32];
- memset(buffer, 0, 32);
- memcpy(buffer, src + i, len - i);
- __m256i current_bytes = _mm256_loadu_si256((const __m256i*)(buffer));
- previous = avxcheckUTF8Bytes(current_bytes, &previous, &has_error);
- } else {
- has_error = _mm256_or_si256(
- _mm256_cmpgt_epi8(previous.carried_continuations,
- _mm256_setr_epi8(9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9,
- 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 1)),
- has_error);
- }
-
- return _mm256_testz_si256(has_error, has_error);
-}
-
#endif // __AVX2__
#endif
diff --git a/be/src/util/thread.cpp b/be/src/util/thread.cpp
index acf8268..60cb267 100644
--- a/be/src/util/thread.cpp
+++ b/be/src/util/thread.cpp
@@ -301,8 +301,25 @@ int64_t Thread::wait_for_tid() const {
int loop_count = 0;
while (true) {
int64_t t = Acquire_Load(&_tid);
- if (t != PARENT_WAITING_TID) return t;
- boost::detail::yield(loop_count++);
+ if (t != PARENT_WAITING_TID) {
+ return t;
+ }
+ // copied from boost::detail::yield
+ int k = loop_count++;
+ if (k < 32 || k & 1) {
+ sched_yield();
+ } else {
+ // g++ -Wextra warns on {} or {0}
+ struct timespec rqtp = {0, 0};
+
+ // POSIX says that timespec has tv_sec and tv_nsec
+ // But it doesn't guarantee order or placement
+
+ rqtp.tv_sec = 0;
+ rqtp.tv_nsec = 1000;
+
+ nanosleep(&rqtp, 0);
+ }
}
}
diff --git a/be/src/util/thrift_client.h b/be/src/util/thrift_client.h
index d9a8034..920dfd8 100644
--- a/be/src/util/thrift_client.h
+++ b/be/src/util/thrift_client.h
@@ -27,7 +27,6 @@
#include <thrift/transport/TBufferTransports.h>
#include <thrift/transport/TSocket.h>
-#include <boost/shared_ptr.hpp>
#include <ostream>
#include <sstream>
#include <string>
@@ -76,9 +75,9 @@ protected:
int _port;
// All shared pointers, because Thrift requires them to be
- boost::shared_ptr<apache::thrift::transport::TSocket> _socket;
- boost::shared_ptr<apache::thrift::transport::TTransport> _transport;
- boost::shared_ptr<apache::thrift::protocol::TBinaryProtocol> _protocol;
+ std::shared_ptr<apache::thrift::transport::TSocket> _socket;
+ std::shared_ptr<apache::thrift::transport::TTransport> _transport;
+ std::shared_ptr<apache::thrift::protocol::TBinaryProtocol> _protocol;
};
// Utility client to a Thrift server. The parameter type is the
@@ -94,7 +93,7 @@ public:
InterfaceType* iface() { return _iface.get(); }
private:
- boost::shared_ptr<InterfaceType> _iface;
+ std::shared_ptr<InterfaceType> _iface;
};
template <class InterfaceType>
diff --git a/be/src/util/thrift_server.cpp b/be/src/util/thrift_server.cpp
index 5257498..ca048f7 100644
--- a/be/src/util/thrift_server.cpp
+++ b/be/src/util/thrift_server.cpp
@@ -17,13 +17,14 @@
#include "util/thrift_server.h"
-#include <thrift/concurrency/PosixThreadFactory.h>
#include <thrift/concurrency/Thread.h>
+#include <thrift/concurrency/ThreadFactory.h>
#include <thrift/concurrency/ThreadManager.h>
#include <thrift/protocol/TBinaryProtocol.h>
#include <thrift/server/TNonblockingServer.h>
#include <thrift/server/TThreadPoolServer.h>
#include <thrift/server/TThreadedServer.h>
+#include <thrift/transport/TNonblockingServerSocket.h>
#include <thrift/transport/TServerSocket.h>
#include <thrift/transport/TSocket.h>
@@ -60,17 +61,17 @@ public:
// Called when a client connects; we create per-client state and call any
// SessionHandlerIf handler.
- virtual void* createContext(boost::shared_ptr<apache::thrift::protocol::TProtocol> input,
- boost::shared_ptr<apache::thrift::protocol::TProtocol> output);
+ virtual void* createContext(std::shared_ptr<apache::thrift::protocol::TProtocol> input,
+ std::shared_ptr<apache::thrift::protocol::TProtocol> output);
// Called when a client starts an RPC; we set the thread-local session key.
virtual void processContext(void* context,
- boost::shared_ptr<apache::thrift::transport::TTransport> output);
+ std::shared_ptr<apache::thrift::transport::TTransport> output);
// Called when a client disconnects; we call any SessionHandlerIf handler.
virtual void deleteContext(void* serverContext,
- boost::shared_ptr<apache::thrift::protocol::TProtocol> input,
- boost::shared_ptr<apache::thrift::protocol::TProtocol> output);
+ std::shared_ptr<apache::thrift::protocol::TProtocol> input,
+ std::shared_ptr<apache::thrift::protocol::TProtocol> output);
// Waits for a timeout of TIMEOUT_MS for a server to signal that it has started
// correctly.
@@ -140,7 +141,7 @@ Status ThriftServer::ThriftServerEventProcessor::start_and_wait_for_server() {
}
void ThriftServer::ThriftServerEventProcessor::supervise() {
- DCHECK(_thrift_server->_server.get() != NULL);
+ DCHECK(_thrift_server->_server.get() != nullptr);
try {
_thrift_server->_server->serve();
@@ -188,11 +189,11 @@ ThriftServer::SessionKey* ThriftServer::get_thread_session_key() {
}
void* ThriftServer::ThriftServerEventProcessor::createContext(
- boost::shared_ptr<apache::thrift::protocol::TProtocol> input,
- boost::shared_ptr<apache::thrift::protocol::TProtocol> output) {
+ std::shared_ptr<apache::thrift::protocol::TProtocol> input,
+ std::shared_ptr<apache::thrift::protocol::TProtocol> output) {
std::stringstream ss;
- apache::thrift::transport::TSocket* socket = NULL;
+ apache::thrift::transport::TSocket* socket = nullptr;
apache::thrift::transport::TTransport* transport = input->getTransport().get();
{
switch (_thrift_server->_server_type) {
@@ -221,13 +222,13 @@ void* ThriftServer::ThriftServerEventProcessor::createContext(
{
std::lock_guard<std::mutex> _l(_thrift_server->_session_keys_lock);
- boost::shared_ptr<SessionKey> key_ptr(new std::string(ss.str()));
+ std::shared_ptr<SessionKey> key_ptr(new std::string(ss.str()));
_session_key = key_ptr.get();
_thrift_server->_session_keys[key_ptr.get()] = key_ptr;
}
- if (_thrift_server->_session_handler != NULL) {
+ if (_thrift_server->_session_handler != nullptr) {
_thrift_server->_session_handler->session_start(*_session_key);
}
@@ -241,16 +242,16 @@ void* ThriftServer::ThriftServerEventProcessor::createContext(
}
void ThriftServer::ThriftServerEventProcessor::processContext(
- void* context, boost::shared_ptr<apache::thrift::transport::TTransport> transport) {
+ void* context, std::shared_ptr<apache::thrift::transport::TTransport> transport) {
_session_key = reinterpret_cast<SessionKey*>(context);
}
void ThriftServer::ThriftServerEventProcessor::deleteContext(
- void* serverContext, boost::shared_ptr<apache::thrift::protocol::TProtocol> input,
- boost::shared_ptr<apache::thrift::protocol::TProtocol> output) {
+ void* serverContext, std::shared_ptr<apache::thrift::protocol::TProtocol> input,
+ std::shared_ptr<apache::thrift::protocol::TProtocol> output) {
_session_key = (SessionKey*)serverContext;
- if (_thrift_server->_session_handler != NULL) {
+ if (_thrift_server->_session_handler != nullptr) {
_thrift_server->_session_handler->session_end(*_session_key);
}
@@ -263,17 +264,17 @@ void ThriftServer::ThriftServerEventProcessor::deleteContext(
}
ThriftServer::ThriftServer(const std::string& name,
- const boost::shared_ptr<apache::thrift::TProcessor>& processor, int port,
+ const std::shared_ptr<apache::thrift::TProcessor>& processor, int port,
int num_worker_threads, ServerType server_type)
: _started(false),
_port(port),
_num_worker_threads(num_worker_threads),
_server_type(server_type),
_name(name),
- _server_thread(NULL),
- _server(NULL),
+ _server_thread(nullptr),
+ _server(nullptr),
_processor(processor),
- _session_handler(NULL) {
+ _session_handler(nullptr) {
_thrift_server_metric_entity = DorisMetrics::instance()->metric_registry()->register_entity(
std::string("thrift_server.") + name, {{"name", name}});
INT_GAUGE_METRIC_REGISTER(_thrift_server_metric_entity, thrift_current_connections);
@@ -282,14 +283,15 @@ ThriftServer::ThriftServer(const std::string& name,
Status ThriftServer::start() {
DCHECK(!_started);
- boost::shared_ptr<apache::thrift::protocol::TProtocolFactory> protocol_factory(
+ std::shared_ptr<apache::thrift::protocol::TProtocolFactory> protocol_factory(
new apache::thrift::protocol::TBinaryProtocolFactory());
- boost::shared_ptr<apache::thrift::concurrency::ThreadManager> thread_mgr;
- boost::shared_ptr<apache::thrift::concurrency::ThreadFactory> thread_factory(
- new apache::thrift::concurrency::PosixThreadFactory());
- boost::shared_ptr<apache::thrift::transport::TServerTransport> fe_server_transport;
- boost::shared_ptr<apache::thrift::transport::TTransportFactory> transport_factory;
-
+ std::shared_ptr<apache::thrift::concurrency::ThreadManager> thread_mgr;
+ std::shared_ptr<apache::thrift::concurrency::ThreadFactory> thread_factory =
+ std::make_shared<apache::thrift::concurrency::ThreadFactory>();
+ std::shared_ptr<apache::thrift::transport::TServerTransport> fe_server_transport;
+ std::shared_ptr<apache::thrift::transport::TTransportFactory> transport_factory;
+ std::shared_ptr<apache::thrift::transport::TNonblockingServerSocket> socket =
+ std::make_shared<apache::thrift::transport::TNonblockingServerSocket>(_port);
if (_server_type != THREADED) {
thread_mgr = apache::thrift::concurrency::ThreadManager::newSimpleThreadManager(
_num_worker_threads);
@@ -299,23 +301,23 @@ Status ThriftServer::start() {
// Note - if you change the transport types here, you must check that the
// logic in createContext is still accurate.
- apache::thrift::transport::TServerSocket* server_socket = NULL;
+ apache::thrift::transport::TServerSocket* server_socket = nullptr;
switch (_server_type) {
case NON_BLOCKING:
- if (transport_factory.get() == NULL) {
+ if (transport_factory.get() == nullptr) {
transport_factory.reset(new apache::thrift::transport::TTransportFactory());
}
_server.reset(new apache::thrift::server::TNonblockingServer(
_processor, transport_factory, transport_factory, protocol_factory,
- protocol_factory, _port, thread_mgr));
+ protocol_factory, socket, thread_mgr));
break;
case THREAD_POOL:
fe_server_transport.reset(new apache::thrift::transport::TServerSocket(_port));
- if (transport_factory.get() == NULL) {
+ if (transport_factory.get() == nullptr) {
transport_factory.reset(new apache::thrift::transport::TBufferedTransportFactory());
}
@@ -328,7 +330,7 @@ Status ThriftServer::start() {
// server_socket->setAcceptTimeout(500);
fe_server_transport.reset(server_socket);
- if (transport_factory.get() == NULL) {
+ if (transport_factory.get() == nullptr) {
transport_factory.reset(new apache::thrift::transport::TBufferedTransportFactory());
}
@@ -344,7 +346,7 @@ Status ThriftServer::start() {
return Status::InternalError(error_msg.str());
}
- boost::shared_ptr<ThriftServer::ThriftServerEventProcessor> event_processor(
+ std::shared_ptr<ThriftServer::ThriftServerEventProcessor> event_processor(
new ThriftServer::ThriftServerEventProcessor(this));
_server->setServerEventHandler(event_processor);
@@ -361,13 +363,13 @@ void ThriftServer::stop() {
}
void ThriftServer::join() {
- DCHECK(_server_thread != NULL);
+ DCHECK(_server_thread != nullptr);
DCHECK(_started);
_server_thread->join();
}
void ThriftServer::stop_for_testing() {
- DCHECK(_server_thread != NULL);
+ DCHECK(_server_thread != nullptr);
DCHECK(_server);
DCHECK_EQ(_server_type, THREADED);
_server->stop();
diff --git a/be/src/util/thrift_server.h b/be/src/util/thrift_server.h
index 0c4150b..197b0b0 100644
--- a/be/src/util/thrift_server.h
+++ b/be/src/util/thrift_server.h
@@ -21,8 +21,6 @@
#include <thrift/TProcessor.h>
#include <thrift/server/TServer.h>
-#include <boost/scoped_ptr.hpp>
-#include <boost/shared_ptr.hpp>
#include <boost/thread.hpp>
#include <mutex>
#include <unordered_map>
@@ -71,7 +69,7 @@ public:
// - num_worker_threads: the number of worker threads to use in any thread pool
// - server_type: the type of IO strategy this server should employ
ThriftServer(const std::string& name,
- const boost::shared_ptr<apache::thrift::TProcessor>& processor, int port,
+ const std::shared_ptr<apache::thrift::TProcessor>& processor, int port,
int num_worker_threads = DEFAULT_WORKER_THREADS,
ServerType server_type = THREADED);
@@ -120,11 +118,11 @@ private:
const std::string _name;
// Thread that runs the TNonblockingServer::serve loop
- boost::scoped_ptr<boost::thread> _server_thread;
+ std::unique_ptr<boost::thread> _server_thread;
// Thrift housekeeping
- boost::scoped_ptr<apache::thrift::server::TServer> _server;
- boost::shared_ptr<apache::thrift::TProcessor> _processor;
+ std::unique_ptr<apache::thrift::server::TServer> _server;
+ std::shared_ptr<apache::thrift::TProcessor> _processor;
// If not NULL, called when session events happen. Not owned by us.
SessionHandlerIf* _session_handler;
@@ -134,7 +132,7 @@ private:
// Map of active session keys to shared_ptr containing that key; when a key is
// removed it is automatically freed.
- typedef std::unordered_map<SessionKey*, boost::shared_ptr<SessionKey>> SessionKeySet;
+ typedef std::unordered_map<SessionKey*, std::shared_ptr<SessionKey>> SessionKeySet;
SessionKeySet _session_keys;
// Helper class which monitors starting servers. Needs access to internal members, and
diff --git a/be/src/util/thrift_util.cpp b/be/src/util/thrift_util.cpp
index ac59a07..1e57ee8 100644
--- a/be/src/util/thrift_util.cpp
+++ b/be/src/util/thrift_util.cpp
@@ -18,15 +18,12 @@
#include "util/thrift_util.h"
#include <thrift/Thrift.h>
-#include <thrift/concurrency/PosixThreadFactory.h>
+#include <thrift/concurrency/ThreadFactory.h>
#include <thrift/concurrency/ThreadManager.h>
#include <thrift/server/TNonblockingServer.h>
#include <thrift/transport/TServerSocket.h>
#include <thrift/transport/TSocket.h>
-#include <boost/shared_ptr.hpp>
-#include <boost/thread.hpp>
-
#include "gen_cpp/Data_types.h"
#include "gen_cpp/Types_types.h"
#include "util/hash_util.hpp"
@@ -58,8 +55,8 @@ ThriftSerializer::ThriftSerializer(bool compact, int initial_buffer_size)
}
}
-boost::shared_ptr<apache::thrift::protocol::TProtocol> create_deserialize_protocol(
- boost::shared_ptr<apache::thrift::transport::TMemoryBuffer> mem, bool compact) {
+std::shared_ptr<apache::thrift::protocol::TProtocol> create_deserialize_protocol(
+ std::shared_ptr<apache::thrift::transport::TMemoryBuffer> mem, bool compact) {
if (compact) {
apache::thrift::protocol::TCompactProtocolFactoryT<apache::thrift::transport::TMemoryBuffer>
tproto_factory;
diff --git a/be/src/util/thrift_util.h b/be/src/util/thrift_util.h
index 9bdfd40..ab69ae8 100644
--- a/be/src/util/thrift_util.h
+++ b/be/src/util/thrift_util.h
@@ -23,7 +23,6 @@
#include <thrift/protocol/TDebugProtocol.h>
#include <thrift/transport/TBufferTransports.h>
-#include <boost/shared_ptr.hpp>
#include <sstream>
#include <vector>
@@ -106,8 +105,8 @@ public:
void get_buffer(uint8_t** buffer, uint32_t* length) { _mem_buffer->getBuffer(buffer, length); }
private:
- boost::shared_ptr<apache::thrift::transport::TMemoryBuffer> _mem_buffer;
- boost::shared_ptr<apache::thrift::protocol::TProtocol> _protocol;
+ std::shared_ptr<apache::thrift::transport::TMemoryBuffer> _mem_buffer;
+ std::shared_ptr<apache::thrift::protocol::TProtocol> _protocol;
};
class ThriftDeserializer {
@@ -115,13 +114,13 @@ public:
ThriftDeserializer(bool compact);
private:
- boost::shared_ptr<apache::thrift::protocol::TProtocolFactory> _factory;
- boost::shared_ptr<apache::thrift::protocol::TProtocol> _tproto;
+ std::shared_ptr<apache::thrift::protocol::TProtocolFactory> _factory;
+ std::shared_ptr<apache::thrift::protocol::TProtocol> _tproto;
};
// Utility to create a protocol (deserialization) object for 'mem'.
-boost::shared_ptr<apache::thrift::protocol::TProtocol> create_deserialize_protocol(
- boost::shared_ptr<apache::thrift::transport::TMemoryBuffer> mem, bool compact);
+std::shared_ptr<apache::thrift::protocol::TProtocol> create_deserialize_protocol(
+ std::shared_ptr<apache::thrift::transport::TMemoryBuffer> mem, bool compact);
// Deserialize a thrift message from buf/len. buf/len must at least contain
// all the bytes needed to store the thrift message. On return, len will be
@@ -132,9 +131,9 @@ Status deserialize_thrift_msg(const uint8_t* buf, uint32_t* len, bool compact,
// Deserialize msg bytes into c++ thrift msg using memory
// transport. TMemoryBuffer is not const-safe, although we use it in
// a const-safe way, so we have to explicitly cast away the const.
- boost::shared_ptr<apache::thrift::transport::TMemoryBuffer> tmem_transport(
+ std::shared_ptr<apache::thrift::transport::TMemoryBuffer> tmem_transport(
new apache::thrift::transport::TMemoryBuffer(const_cast<uint8_t*>(buf), *len));
- boost::shared_ptr<apache::thrift::protocol::TProtocol> tproto =
+ std::shared_ptr<apache::thrift::protocol::TProtocol> tproto =
create_deserialize_protocol(tmem_transport, compact);
try {
diff --git a/be/test/olap/file_helper_test.cpp b/be/test/olap/file_helper_test.cpp
index c015e64..33e9a3f 100644
--- a/be/test/olap/file_helper_test.cpp
+++ b/be/test/olap/file_helper_test.cpp
@@ -26,8 +26,8 @@
#include "gmock/gmock.h"
#include "gtest/gtest.h"
#include "olap/olap_define.h"
-#include "util/logging.h"
#include "test_util/test_util.h"
+#include "util/logging.h"
#ifndef BE_TEST
#define BE_TEST
@@ -44,6 +44,7 @@ class FileHandlerTest : public testing::Test {
public:
// create a mock cgroup folder
virtual void SetUp() {
+ std::filesystem::remove_all(_s_test_data_path);
ASSERT_FALSE(std::filesystem::exists(_s_test_data_path));
// create a mock cgroup path
ASSERT_TRUE(std::filesystem::create_directory(_s_test_data_path));
@@ -83,19 +84,19 @@ TEST_F(FileHandlerTest, TestWrite) {
ASSERT_EQ(10, cur_offset);
// write 12 bytes to disk
- char* ten_bytes[12];
- memset(ten_bytes, 0, sizeof(char) * 12);
- file_handler.write(ten_bytes, 12);
+ char ten_bytes[12];
+ memset(&ten_bytes, 0, sizeof(ten_bytes));
+ file_handler.write(&ten_bytes, sizeof(ten_bytes));
cur_offset = file_handler.tell();
ASSERT_EQ(22, cur_offset);
length = file_handler.length();
ASSERT_EQ(22, length);
- char* large_bytes2[(1 << 10)];
- memset(large_bytes2, 0, sizeof(char) * ((1 << 12)));
+ char large_bytes2[(1 << 10)];
+ memset(&large_bytes2, 0, sizeof(large_bytes2));
int i = 1;
while (i < LOOP_LESS_OR_MORE(1 << 10, 1 << 17)) {
- file_handler.write(large_bytes2, ((1 << 12)));
+ file_handler.write(&large_bytes2, sizeof(large_bytes2));
++i;
}
}
diff --git a/be/test/olap/file_utils_test.cpp b/be/test/olap/file_utils_test.cpp
index a7ee9cd..d2c744e 100644
--- a/be/test/olap/file_utils_test.cpp
+++ b/be/test/olap/file_utils_test.cpp
@@ -76,14 +76,14 @@ TEST_F(FileUtilsTest, TestCopyFile) {
ASSERT_EQ(OLAPStatus::OLAP_SUCCESS, op_status);
ASSERT_TRUE(std::filesystem::exists(src_file_name));
- char* large_bytes2[(1 << 12)];
- memset(large_bytes2, 0, sizeof(char) * ((1 << 12)));
+ char large_bytes2[(1 << 12)];
+ memset(&large_bytes2, 0, sizeof(large_bytes2));
int i = 0;
while (i < 1 << 10) {
- src_file_handler.write(large_bytes2, ((1 << 12)));
+ src_file_handler.write(&large_bytes2, sizeof(large_bytes2));
++i;
}
- src_file_handler.write(large_bytes2, 13);
+ src_file_handler.write(&large_bytes2, 13);
src_file_handler.close();
std::string dst_file_name = _s_test_data_path + "/abcd123456.txt";
diff --git a/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp b/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp
index 8603e4c..6211997 100644
--- a/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp
+++ b/be/test/olap/rowset/segment_v2/bitmap_index_test.cpp
@@ -30,11 +30,12 @@
#include "olap/types.h"
#include "runtime/mem_pool.h"
#include "runtime/mem_tracker.h"
-#include "util/file_utils.h"
#include "test_util/test_util.h"
+#include "util/file_utils.h"
namespace doris {
namespace segment_v2 {
+using roaring::Roaring;
class BitmapIndexTest : public testing::Test {
public:
diff --git a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp
index 9c9450e..0668799 100644
--- a/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp
+++ b/be/test/olap/rowset/segment_v2/bitshuffle_page_test.cpp
@@ -336,7 +336,7 @@ TEST_F(BitShufflePageTest, TestBitShuffleDecimal12BlockEncoderSeekValue) {
const uint32_t size = 1000;
std::unique_ptr<decimal12_t[]> decimals(new decimal12_t[size]);
for (int i = 0; i < size; i++) {
- decimals.get()[i] = {i + 100, random()};
+ decimals.get()[i] = {i + 100, std::rand()};
}
decimal12_t small_than_smallest = {99, 9};
diff --git a/be/test/olap/rowset/segment_v2/row_ranges_test.cpp b/be/test/olap/rowset/segment_v2/row_ranges_test.cpp
index 1c6b142..800ee21 100644
--- a/be/test/olap/rowset/segment_v2/row_ranges_test.cpp
+++ b/be/test/olap/rowset/segment_v2/row_ranges_test.cpp
@@ -112,7 +112,7 @@ TEST_F(RowRangesTest, TestRangesToRoaring) {
RowRanges row_ranges3 = RowRanges::create_single(15, 30);
RowRanges row_ranges4 = RowRanges::create_single(40, 50);
- Roaring row_bitmap = RowRanges::ranges_to_roaring(row_ranges1);
+ roaring::Roaring row_bitmap = RowRanges::ranges_to_roaring(row_ranges1);
ASSERT_EQ(row_ranges1.count(), row_bitmap.cardinality());
row_bitmap = RowRanges::ranges_to_roaring(row_ranges3);
diff --git a/be/test/util/arrow/arrow_row_batch_test.cpp b/be/test/util/arrow/arrow_row_batch_test.cpp
index 2d59110..c629249 100644
--- a/be/test/util/arrow/arrow_row_batch_test.cpp
+++ b/be/test/util/arrow/arrow_row_batch_test.cpp
@@ -28,6 +28,7 @@
#include <arrow/json/api.h>
#include <arrow/json/test_common.h>
#include <arrow/pretty_print.h>
+#include <arrow/result.h>
#include "common/object_pool.h"
#include "runtime/mem_tracker.h"
@@ -47,7 +48,8 @@ std::string test_str() {
}
void MakeBuffer(const std::string& data, std::shared_ptr<arrow::Buffer>* out) {
- arrow::AllocateBuffer(arrow::default_memory_pool(), data.size(), out);
+ auto res = arrow::AllocateBuffer(data.size(), arrow::default_memory_pool());
+ *out = std::move(res.ValueOrDie());
std::copy(std::begin(data), std::end(data), (*out)->mutable_data());
}
@@ -60,9 +62,9 @@ TEST_F(ArrowRowBatchTest, PrettyPrint) {
arrow::field("c1", arrow::int64()),
});
- std::shared_ptr<arrow::RecordBatch> record_batch;
- auto arrow_st = arrow::json::ParseOne(parse_opts, buffer, &record_batch);
+ auto arrow_st = arrow::json::ParseOne(parse_opts, buffer);
ASSERT_TRUE(arrow_st.ok());
+ std::shared_ptr<arrow::RecordBatch> record_batch = arrow_st.ValueOrDie();
ObjectPool obj_pool;
RowDescriptor* row_desc;
diff --git a/be/test/util/arrow/arrow_row_block_test.cpp b/be/test/util/arrow/arrow_row_block_test.cpp
index 9b17000..450ae4c 100644
--- a/be/test/util/arrow/arrow_row_block_test.cpp
+++ b/be/test/util/arrow/arrow_row_block_test.cpp
@@ -29,6 +29,7 @@
#include <arrow/memory_pool.h>
#include <arrow/pretty_print.h>
#include <arrow/record_batch.h>
+#include <arrow/result.h>
#include "olap/row_block2.h"
#include "olap/schema.h"
@@ -51,7 +52,8 @@ std::string test_str() {
}
void MakeBuffer(const std::string& data, std::shared_ptr<arrow::Buffer>* out) {
- arrow::AllocateBuffer(arrow::default_memory_pool(), data.size(), out);
+ auto buffer_res = arrow::AllocateBuffer(data.size(), arrow::default_memory_pool());
+ *out = std::move(buffer_res.ValueOrDie());
std::copy(std::begin(data), std::end(data), (*out)->mutable_data());
}
@@ -64,9 +66,9 @@ TEST_F(ArrowRowBlockTest, Normal) {
arrow::field("c1", arrow::int64()),
});
- std::shared_ptr<arrow::RecordBatch> record_batch;
- auto arrow_st = arrow::json::ParseOne(parse_opts, buffer, &record_batch);
+ auto arrow_st = arrow::json::ParseOne(parse_opts, buffer);
ASSERT_TRUE(arrow_st.ok());
+ std::shared_ptr<arrow::RecordBatch> record_batch = arrow_st.ValueOrDie();
std::shared_ptr<Schema> schema;
auto doris_st = convert_to_doris_schema(*record_batch->schema(), &schema);
diff --git a/be/test/util/bitmap_value_test.cpp b/be/test/util/bitmap_value_test.cpp
index 5518a64..dc4a0f3 100644
--- a/be/test/util/bitmap_value_test.cpp
+++ b/be/test/util/bitmap_value_test.cpp
@@ -25,6 +25,7 @@
#include "util/bitmap_value.h"
namespace doris {
+using roaring::Roaring;
TEST(BitmapValueTest, bitmap_union) {
BitmapValue empty;
diff --git a/be/test/util/threadpool_test.cpp b/be/test/util/threadpool_test.cpp
index ef3d908..75ceefc 100644
--- a/be/test/util/threadpool_test.cpp
+++ b/be/test/util/threadpool_test.cpp
@@ -94,7 +94,20 @@ TEST_F(ThreadPoolTest, TestNoTaskOpenClose) {
static void simple_task_method(int n, std::atomic<int32_t>* counter) {
while (n--) {
(*counter)++;
- boost::detail::yield(n);
+ if (n < 32 || n & 1) {
+ sched_yield();
+ } else {
+ // g++ -Wextra warns on {} or {0}
+ struct timespec rqtp = {0, 0};
+
+ // POSIX says that timespec has tv_sec and tv_nsec
+ // But it doesn't guarantee order or placement
+
+ rqtp.tv_sec = 0;
+ rqtp.tv_nsec = 1000;
+
+ nanosleep(&rqtp, 0);
+ }
}
}
@@ -740,21 +753,24 @@ TEST_F(ThreadPoolTest, TestTokenConcurrency) {
}
static void MyFunc(int idx, int n) {
- std::cout << idx << ", " << std::this_thread::get_id() << " before sleep " << n << " seconds" << std::endl;
+ std::cout << idx << ", " << std::this_thread::get_id() << " before sleep " << n << " seconds"
+ << std::endl;
sleep(n);
- std::cout << idx << ", " << std::this_thread::get_id() << " after sleep " << n << " seconds" << std::endl;
+ std::cout << idx << ", " << std::this_thread::get_id() << " after sleep " << n << " seconds"
+ << std::endl;
}
TEST_F(ThreadPoolTest, TestNormal) {
std::unique_ptr<ThreadPool> thread_pool;
ThreadPoolBuilder("my_pool")
- .set_min_threads(0)
- .set_max_threads(5)
- .set_max_queue_size(10)
- .set_idle_timeout(MonoDelta::FromMilliseconds(2000))
- .build(&thread_pool);
-
- std::unique_ptr<ThreadPoolToken> token1 = thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 2);
+ .set_min_threads(0)
+ .set_max_threads(5)
+ .set_max_queue_size(10)
+ .set_idle_timeout(MonoDelta::FromMilliseconds(2000))
+ .build(&thread_pool);
+
+ std::unique_ptr<ThreadPoolToken> token1 =
+ thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 2);
for (int i = 0; i < 10; i++) {
token1->submit_func(std::bind(&MyFunc, i, 1));
}
@@ -762,7 +778,8 @@ TEST_F(ThreadPoolTest, TestNormal) {
token1->wait();
ASSERT_EQ(0, token1->num_tasks());
- std::unique_ptr<ThreadPoolToken> token2 = thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 20);
+ std::unique_ptr<ThreadPoolToken> token2 =
+ thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 20);
for (int i = 0; i < 10; i++) {
token2->submit_func(std::bind(&MyFunc, i, 1));
}
@@ -770,7 +787,8 @@ TEST_F(ThreadPoolTest, TestNormal) {
token2->wait();
ASSERT_EQ(0, token2->num_tasks());
- std::unique_ptr<ThreadPoolToken> token3 = thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 1);
+ std::unique_ptr<ThreadPoolToken> token3 =
+ thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 1);
for (int i = 0; i < 10; i++) {
token3->submit_func(std::bind(&MyFunc, i, 1));
}
@@ -778,7 +796,8 @@ TEST_F(ThreadPoolTest, TestNormal) {
token3->wait();
ASSERT_EQ(0, token3->num_tasks());
- std::unique_ptr<ThreadPoolToken> token4 = thread_pool->new_token(ThreadPool::ExecutionMode::SERIAL);
+ std::unique_ptr<ThreadPoolToken> token4 =
+ thread_pool->new_token(ThreadPool::ExecutionMode::SERIAL);
for (int i = 0; i < 10; i++) {
token4->submit_func(std::bind(&MyFunc, i, 1));
}
@@ -786,7 +805,8 @@ TEST_F(ThreadPoolTest, TestNormal) {
token4->wait();
ASSERT_EQ(0, token4->num_tasks());
- std::unique_ptr<ThreadPoolToken> token5 = thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 20);
+ std::unique_ptr<ThreadPoolToken> token5 =
+ thread_pool->new_token(ThreadPool::ExecutionMode::CONCURRENT, 20);
for (int i = 0; i < 10; i++) {
token5->submit_func(std::bind(&MyFunc, i, 1));
}
@@ -800,7 +820,7 @@ TEST_F(ThreadPoolTest, TestThreadPoolDynamicAdjustMaximumMinimum) {
.set_min_threads(3)
.set_max_threads(3)
.set_idle_timeout(MonoDelta::FromMilliseconds(1)))
- .ok());
+ .ok());
ASSERT_EQ(3, _pool->min_threads());
ASSERT_EQ(3, _pool->max_threads());
diff --git a/extension/flink-doris-connector/pom.xml b/extension/flink-doris-connector/pom.xml
index a0d10f4..ffe6784 100644
--- a/extension/flink-doris-connector/pom.xml
+++ b/extension/flink-doris-connector/pom.xml
@@ -11,8 +11,8 @@
<properties>
<scala.version>2.12</scala.version>
<flink.version>1.11.2</flink.version>
- <libthrift.version>0.9.3</libthrift.version>
- <arrow.version>0.15.1</arrow.version>
+ <libthrift.version>0.13.0</libthrift.version>
+ <arrow.version>5.0.0</arrow.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
<maven-javadoc-plugin.version>3.3.0</maven-javadoc-plugin.version>
<maven-source-plugin.version>3.2.1</maven-source-plugin.version>
@@ -140,6 +140,12 @@
<version>${arrow.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ <version>${arrow.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
@@ -196,6 +202,7 @@
<version>0.1.11</version>
<configuration>
<thriftExecutable>${doris.thirdparty}/installed/bin/thrift</thriftExecutable>
+ <generator>java:fullcamel</generator>
</configuration>
<executions>
<execution>
diff --git a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java
index 40bb5c9..9b8d955 100644
--- a/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java
+++ b/extension/flink-doris-connector/src/main/java/org/apache/doris/flink/backend/BackendClient.java
@@ -126,14 +126,14 @@ public class BackendClient {
for (int attempt = 0; attempt < retries; ++attempt) {
logger.debug("Attempt {} to openScanner {}.", attempt, routing);
try {
- TScanOpenResult result = client.open_scanner(openParams);
+ TScanOpenResult result = client.openScanner(openParams);
if (result == null) {
logger.warn("Open scanner result from {} is null.", routing);
continue;
}
- if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+ if (!TStatusCode.OK.equals(result.getStatus().getStatusCode())) {
logger.warn("The status of open scanner result from {} is '{}', error message is: {}.",
- routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+ routing, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs());
continue;
}
return result;
@@ -163,14 +163,14 @@ public class BackendClient {
for (int attempt = 0; attempt < retries; ++attempt) {
logger.debug("Attempt {} to getNext {}.", attempt, routing);
try {
- result = client.get_next(nextBatchParams);
+ result = client.getNext(nextBatchParams);
if (result == null) {
logger.warn("GetNext result from {} is null.", routing);
continue;
}
- if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+ if (!TStatusCode.OK.equals(result.getStatus().getStatusCode())) {
logger.warn("The status of get next result from {} is '{}', error message is: {}.",
- routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+ routing, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs());
continue;
}
return result;
@@ -179,11 +179,11 @@ public class BackendClient {
ex = e;
}
}
- if (result != null && (TStatusCode.OK != (result.getStatus().getStatus_code()))) {
- logger.error(ErrorMessages.DORIS_INTERNAL_FAIL_MESSAGE, routing, result.getStatus().getStatus_code(),
- result.getStatus().getError_msgs());
- throw new DorisInternalException(routing.toString(), result.getStatus().getStatus_code(),
- result.getStatus().getError_msgs());
+ if (result != null && (TStatusCode.OK != (result.getStatus().getStatusCode()))) {
+ logger.error(ErrorMessages.DORIS_INTERNAL_FAIL_MESSAGE, routing, result.getStatus().getStatusCode(),
+ result.getStatus().getErrorMsgs());
+ throw new DorisInternalException(routing.toString(), result.getStatus().getStatusCode(),
+ result.getStatus().getErrorMsgs());
}
logger.error(ErrorMessages.CONNECT_FAILED_MESSAGE, routing);
throw new ConnectedFailedException(routing.toString(), ex);
@@ -199,14 +199,14 @@ public class BackendClient {
for (int attempt = 0; attempt < retries; ++attempt) {
logger.debug("Attempt {} to closeScanner {}.", attempt, routing);
try {
- TScanCloseResult result = client.close_scanner(closeParams);
+ TScanCloseResult result = client.closeScanner(closeParams);
if (result == null) {
logger.warn("CloseScanner result from {} is null.", routing);
continue;
}
- if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+ if (!TStatusCode.OK.equals(result.getStatus().getStatusCode())) {
logger.warn("The status of get next result from {} is '{}', error message is: {}.",
- routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+ routing, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs());
continue;
}
break;
diff --git a/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala b/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
index e69a86f..093390d 100644
--- a/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
+++ b/extension/flink-doris-connector/src/main/scala/org/apache/doris/flink/datastream/ScalaValueReader.scala
@@ -102,9 +102,9 @@ class ScalaValueReader(partition: PartitionDefinition, options: DorisOptions, re
DORIS_EXEC_MEM_LIMIT_DEFAULT
}
- params.setBatch_size(batchSize)
- params.setQuery_timeout(queryDorisTimeout)
- params.setMem_limit(execMemLimit)
+ params.setBatchSize(batchSize)
+ params.setQueryTimeout(queryDorisTimeout)
+ params.setMemLimit(execMemLimit)
params.setUser(options.getUsername)
params.setPasswd(options.getPassword)
@@ -112,25 +112,25 @@ class ScalaValueReader(partition: PartitionDefinition, options: DorisOptions, re
s"cluster: ${params.getCluster}, " +
s"database: ${params.getDatabase}, " +
s"table: ${params.getTable}, " +
- s"tabletId: ${params.getTablet_ids}, " +
+ s"tabletId: ${params.getTabletIds}, " +
s"batch size: $batchSize, " +
s"query timeout: $queryDorisTimeout, " +
s"execution memory limit: $execMemLimit, " +
s"user: ${params.getUser}, " +
- s"query plan: ${params.opaqued_query_plan}")
+ s"query plan: ${params.getOpaquedQueryPlan}")
params
}
protected val openResult: TScanOpenResult = client.openScanner(openParams)
- protected val contextId: String = openResult.getContext_id
+ protected val contextId: String = openResult.getContextId
protected val schema: Schema =
- SchemaUtils.convertToSchema(openResult.getSelected_columns)
+ SchemaUtils.convertToSchema(openResult.getSelectedColumns)
protected val asyncThread: Thread = new Thread {
override def run {
val nextBatchParams = new TScanNextBatchParams
- nextBatchParams.setContext_id(contextId)
+ nextBatchParams.setContextId(contextId)
while (!eos.get) {
nextBatchParams.setOffset(offset)
val nextResult = client.getNext(nextBatchParams)
@@ -189,7 +189,7 @@ class ScalaValueReader(partition: PartitionDefinition, options: DorisOptions, re
rowBatch.close
}
val nextBatchParams = new TScanNextBatchParams
- nextBatchParams.setContext_id(contextId)
+ nextBatchParams.setContextId(contextId)
nextBatchParams.setOffset(offset)
val nextResult = client.getNext(nextBatchParams)
eos.set(nextResult.isEos)
@@ -216,7 +216,7 @@ class ScalaValueReader(partition: PartitionDefinition, options: DorisOptions, re
def close(): Unit = {
val closeParams = new TScanCloseParams
- closeParams.context_id = contextId
+ closeParams.setContextId(contextId)
client.closeScanner(closeParams)
}
diff --git a/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java b/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
index a9d2492..ac19066 100644
--- a/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
+++ b/extension/flink-doris-connector/src/test/java/org/apache/doris/flink/serialization/TestRowBatch.java
@@ -220,7 +220,7 @@ public class TestRowBatch {
arrowStreamWriter.close();
TStatus status = new TStatus();
- status.setStatus_code(TStatusCode.OK);
+ status.setStatusCode(TStatusCode.OK);
TScanBatchResult scanBatchResult = new TScanBatchResult();
scanBatchResult.setStatus(status);
scanBatchResult.setEos(false);
@@ -342,7 +342,7 @@ public class TestRowBatch {
arrowStreamWriter.close();
TStatus status = new TStatus();
- status.setStatus_code(TStatusCode.OK);
+ status.setStatusCode(TStatusCode.OK);
TScanBatchResult scanBatchResult = new TScanBatchResult();
scanBatchResult.setStatus(status);
scanBatchResult.setEos(false);
@@ -404,7 +404,7 @@ public class TestRowBatch {
arrowStreamWriter.close();
TStatus status = new TStatus();
- status.setStatus_code(TStatusCode.OK);
+ status.setStatusCode(TStatusCode.OK);
TScanBatchResult scanBatchResult = new TScanBatchResult();
scanBatchResult.setStatus(status);
scanBatchResult.setEos(false);
diff --git a/extension/spark-doris-connector/pom.xml b/extension/spark-doris-connector/pom.xml
index e015f06..d8ebfe7 100644
--- a/extension/spark-doris-connector/pom.xml
+++ b/extension/spark-doris-connector/pom.xml
@@ -31,8 +31,8 @@
<properties>
<scala.version>2.11</scala.version>
<spark.version>2.3.4</spark.version>
- <libthrift.version>0.9.3</libthrift.version>
- <arrow.version>0.15.1</arrow.version>
+ <libthrift.version>0.13.0</libthrift.version>
+ <arrow.version>5.0.0</arrow.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
<maven-javadoc-plugin.version>3.3.0</maven-javadoc-plugin.version>
<maven-source-plugin.version>3.2.1</maven-source-plugin.version>
@@ -107,7 +107,12 @@
<artifactId>arrow-vector</artifactId>
<version>${arrow.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ <version>${arrow.version}</version>
+ <scope>runtime</scope>
+ </dependency>
<!--Test-->
<dependency>
<groupId>org.hamcrest</groupId>
@@ -145,14 +150,55 @@
<version>${spark.version}</version>
<scope>test</scope>
</dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.10.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ <version>2.10.0</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <version>4.1.27.Final</version>
+ <scope>provided</scope>
+ </dependency>
+
</dependencies>
<build>
<plugins>
+ <!-- add gensrc java build src dir -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>3.2.0</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <!-- add arbitrary num of src dirs here -->
+ <source>${project.build.directory}/generated-sources/thrift/</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.thrift.tools</groupId>
<artifactId>maven-thrift-plugin</artifactId>
<version>0.1.11</version>
+ <configuration>
+ <generator>java:fullcamel</generator>
+ </configuration>
<executions>
<execution>
<id>thrift-sources</id>
@@ -242,7 +288,8 @@
<version>0.7.8</version>
<configuration>
<excludes>
- <exclude>**/thrift/**</exclude></excludes>
+ <exclude>**/thrift/**</exclude>
+ </excludes>
</configuration>
<executions>
<execution>
diff --git a/extension/spark-doris-connector/pom_3.0.xml b/extension/spark-doris-connector/pom_3.0.xml
index 4973ff8..6c8eee5 100644
--- a/extension/spark-doris-connector/pom_3.0.xml
+++ b/extension/spark-doris-connector/pom_3.0.xml
@@ -31,9 +31,13 @@
<properties>
<scala.version>2.12</scala.version>
<spark.version>3.1.2</spark.version>
- <libthrift.version>0.9.3</libthrift.version>
- <arrow.version>1.0.1</arrow.version>
+ <libthrift.version>0.13.0</libthrift.version>
+ <arrow.version>5.0.0</arrow.version>
+ <maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
+ <maven-javadoc-plugin.version>3.3.0</maven-javadoc-plugin.version>
+ <maven-source-plugin.version>3.2.1</maven-source-plugin.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <doris.thirdparty>${basedir}/../../thirdparty</doris.thirdparty>
</properties>
<profiles>
@@ -103,7 +107,13 @@
<artifactId>arrow-vector</artifactId>
<version>${arrow.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ <version>${arrow.version}</version>
+ <scope>runtime</scope>
+ </dependency>
+ <!--Test-->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-core</artifactId>
@@ -154,12 +164,36 @@
</dependencies>
- <build>
+ <build>
<plugins>
+ <!-- add gensrc java build src dir -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>3.2.0</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <!-- add arbitrary num of src dirs here -->
+ <source>${project.build.directory}/generated-sources/thrift/</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.thrift.tools</groupId>
<artifactId>maven-thrift-plugin</artifactId>
<version>0.1.11</version>
+ <configuration>
+ <generator>java:fullcamel</generator>
+ </configuration>
<executions>
<execution>
<id>thrift-sources</id>
@@ -277,14 +311,47 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
- <version>3.8.1</version>
+ <version>${maven-compiler-plugin.version}</version>
<configuration>
<source>8</source>
<target>8</target>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>${maven-javadoc-plugin.version}</version>
+ <configuration>
+ <source>8</source>
+ <failOnError>false</failOnError>
+ <aggregate>true</aggregate>
+ </configuration>
+ <executions>
+ <execution>
+ <id>attach-javadocs</id>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>${maven-source-plugin.version}</version>
+ <configuration>
+ <attach>true</attach>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>compile</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
-
diff --git a/extension/spark-doris-connector/src/main/java/org/apache/doris/spark/backend/BackendClient.java b/extension/spark-doris-connector/src/main/java/org/apache/doris/spark/backend/BackendClient.java
index 90baf79..b5f6112 100644
--- a/extension/spark-doris-connector/src/main/java/org/apache/doris/spark/backend/BackendClient.java
+++ b/extension/spark-doris-connector/src/main/java/org/apache/doris/spark/backend/BackendClient.java
@@ -127,14 +127,14 @@ public class BackendClient {
for (int attempt = 0; attempt < retries; ++attempt) {
logger.debug("Attempt {} to openScanner {}.", attempt, routing);
try {
- TScanOpenResult result = client.open_scanner(openParams);
+ TScanOpenResult result = client.openScanner(openParams);
if (result == null) {
logger.warn("Open scanner result from {} is null.", routing);
continue;
}
- if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+ if (!TStatusCode.OK.equals(result.getStatus().getStatusCode())) {
logger.warn("The status of open scanner result from {} is '{}', error message is: {}.",
- routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+ routing, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs());
continue;
}
return result;
@@ -163,14 +163,14 @@ public class BackendClient {
for (int attempt = 0; attempt < retries; ++attempt) {
logger.debug("Attempt {} to getNext {}.", attempt, routing);
try {
- result = client.get_next(nextBatchParams);
+ result = client.getNext(nextBatchParams);
if (result == null) {
logger.warn("GetNext result from {} is null.", routing);
continue;
}
- if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+ if (!TStatusCode.OK.equals(result.getStatus().getStatusCode())) {
logger.warn("The status of get next result from {} is '{}', error message is: {}.",
- routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+ routing, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs());
continue;
}
return result;
@@ -179,11 +179,11 @@ public class BackendClient {
ex = e;
}
}
- if (result != null && (TStatusCode.OK != (result.getStatus().getStatus_code()))) {
- logger.error(ErrorMessages.DORIS_INTERNAL_FAIL_MESSAGE, routing, result.getStatus().getStatus_code(),
- result.getStatus().getError_msgs());
- throw new DorisInternalException(routing.toString(), result.getStatus().getStatus_code(),
- result.getStatus().getError_msgs());
+ if (result != null && (TStatusCode.OK != (result.getStatus().getStatusCode()))) {
+ logger.error(ErrorMessages.DORIS_INTERNAL_FAIL_MESSAGE, routing, result.getStatus().getStatusCode(),
+ result.getStatus().getErrorMsgs());
+ throw new DorisInternalException(routing.toString(), result.getStatus().getStatusCode(),
+ result.getStatus().getErrorMsgs());
}
logger.error(ErrorMessages.CONNECT_FAILED_MESSAGE, routing);
throw new ConnectedFailedException(routing.toString(), ex);
@@ -206,14 +206,14 @@ public class BackendClient {
for (int attempt = 0; attempt < retries; ++attempt) {
logger.debug("Attempt {} to closeScanner {}.", attempt, routing);
try {
- TScanCloseResult result = client.close_scanner(closeParams);
+ TScanCloseResult result = client.closeScanner(closeParams);
if (result == null) {
logger.warn("CloseScanner result from {} is null.", routing);
continue;
}
- if (!TStatusCode.OK.equals(result.getStatus().getStatus_code())) {
+ if (!TStatusCode.OK.equals(result.getStatus().getStatusCode())) {
logger.warn("The status of get next result from {} is '{}', error message is: {}.",
- routing, result.getStatus().getStatus_code(), result.getStatus().getError_msgs());
+ routing, result.getStatus().getStatusCode(), result.getStatus().getErrorMsgs());
continue;
}
break;
diff --git a/extension/spark-doris-connector/src/main/scala/org/apache/doris/spark/rdd/ScalaValueReader.scala b/extension/spark-doris-connector/src/main/scala/org/apache/doris/spark/rdd/ScalaValueReader.scala
index f3334b9..a1b26e4 100644
--- a/extension/spark-doris-connector/src/main/scala/org/apache/doris/spark/rdd/ScalaValueReader.scala
+++ b/extension/spark-doris-connector/src/main/scala/org/apache/doris/spark/rdd/ScalaValueReader.scala
@@ -107,9 +107,9 @@ class ScalaValueReader(partition: PartitionDefinition, settings: Settings) {
DORIS_EXEC_MEM_LIMIT_DEFAULT
}
- params.setBatch_size(batchSize)
- params.setQuery_timeout(queryDorisTimeout)
- params.setMem_limit(execMemLimit)
+ params.setBatchSize(batchSize)
+ params.setQueryTimeout(queryDorisTimeout)
+ params.setMemLimit(execMemLimit)
params.setUser(settings.getProperty(DORIS_REQUEST_AUTH_USER, ""))
params.setPasswd(settings.getProperty(DORIS_REQUEST_AUTH_PASSWORD, ""))
@@ -117,25 +117,25 @@ class ScalaValueReader(partition: PartitionDefinition, settings: Settings) {
s"cluster: ${params.getCluster}, " +
s"database: ${params.getDatabase}, " +
s"table: ${params.getTable}, " +
- s"tabletId: ${params.getTablet_ids}, " +
+ s"tabletId: ${params.getTabletIds}, " +
s"batch size: $batchSize, " +
s"query timeout: $queryDorisTimeout, " +
s"execution memory limit: $execMemLimit, " +
s"user: ${params.getUser}, " +
- s"query plan: ${params.opaqued_query_plan}")
+ s"query plan: ${params.getOpaquedQueryPlan}")
params
}
protected val openResult: TScanOpenResult = lockClient(_.openScanner(openParams))
- protected val contextId: String = openResult.getContext_id
+ protected val contextId: String = openResult.getContextId
protected val schema: Schema =
- SchemaUtils.convertToSchema(openResult.getSelected_columns)
+ SchemaUtils.convertToSchema(openResult.getSelectedColumns)
protected val asyncThread: Thread = new Thread {
override def run {
val nextBatchParams = new TScanNextBatchParams
- nextBatchParams.setContext_id(contextId)
+ nextBatchParams.setContextId(contextId)
while (!eos.get) {
nextBatchParams.setOffset(offset)
val nextResult = lockClient(_.getNext(nextBatchParams))
@@ -194,7 +194,7 @@ class ScalaValueReader(partition: PartitionDefinition, settings: Settings) {
rowBatch.close
}
val nextBatchParams = new TScanNextBatchParams
- nextBatchParams.setContext_id(contextId)
+ nextBatchParams.setContextId(contextId)
nextBatchParams.setOffset(offset)
val nextResult = lockClient(_.getNext(nextBatchParams))
eos.set(nextResult.isEos)
@@ -221,7 +221,7 @@ class ScalaValueReader(partition: PartitionDefinition, settings: Settings) {
def close(): Unit = {
val closeParams = new TScanCloseParams
- closeParams.context_id = contextId
+ closeParams.setContextId(contextId)
lockClient(_.closeScanner(closeParams))
}
diff --git a/extension/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java b/extension/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java
index 64eaf63..e5f5ee7 100644
--- a/extension/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java
+++ b/extension/spark-doris-connector/src/test/java/org/apache/doris/spark/serialization/TestRowBatch.java
@@ -222,7 +222,7 @@ public class TestRowBatch {
arrowStreamWriter.close();
TStatus status = new TStatus();
- status.setStatus_code(TStatusCode.OK);
+ status.setStatusCode(TStatusCode.OK);
TScanBatchResult scanBatchResult = new TScanBatchResult();
scanBatchResult.setStatus(status);
scanBatchResult.setEos(false);
@@ -344,7 +344,7 @@ public class TestRowBatch {
arrowStreamWriter.close();
TStatus status = new TStatus();
- status.setStatus_code(TStatusCode.OK);
+ status.setStatusCode(TStatusCode.OK);
TScanBatchResult scanBatchResult = new TScanBatchResult();
scanBatchResult.setStatus(status);
scanBatchResult.setEos(false);
@@ -406,7 +406,7 @@ public class TestRowBatch {
arrowStreamWriter.close();
TStatus status = new TStatus();
- status.setStatus_code(TStatusCode.OK);
+ status.setStatusCode(TStatusCode.OK);
TScanBatchResult scanBatchResult = new TScanBatchResult();
scanBatchResult.setStatus(status);
scanBatchResult.setEos(false);
diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml
index 576598f..ea7669e 100644
--- a/fe/fe-common/pom.xml
+++ b/fe/fe-common/pom.xml
@@ -66,12 +66,6 @@ under the License.
<artifactId>guava</artifactId>
</dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.thrift/libfb303 -->
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libfb303</artifactId>
- <type>pom</type>
- </dependency>
<!-- https://mvnrepository.com/artifact/org.apache.thrift/libthrift -->
<dependency>
<groupId>org.apache.thrift</groupId>
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index f84abbc..1ebbba5 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -779,7 +779,7 @@ under the License.
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
- <version>1.7</version>
+ <version>3.2.0</version>
<executions>
<execution>
<id>add-source</id>
diff --git a/fe/pom.xml b/fe/pom.xml
index 687d633..bf8b049 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -49,7 +49,7 @@ under the License.
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<grpc.version>1.30.0</grpc.version>
- <protobuf.version>3.5.1</protobuf.version>
+ <protobuf.version>3.14.0</protobuf.version>
<skip.plugin>false</skip.plugin>
</properties>
@@ -295,19 +295,11 @@ under the License.
<scope>test</scope>
</dependency>
- <!-- https://mvnrepository.com/artifact/org.apache.thrift/libfb303 -->
- <dependency>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libfb303</artifactId>
- <version>0.9.3</version>
- <type>pom</type>
- </dependency>
-
<!-- https://mvnrepository.com/artifact/org.apache.thrift/libthrift -->
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.9.3</version>
+ <version>0.13.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.logging.log4j/log4j-api -->
diff --git a/fs_brokers/apache_hdfs_broker/pom.xml b/fs_brokers/apache_hdfs_broker/pom.xml
index 1db9c68..71e1893 100644
--- a/fs_brokers/apache_hdfs_broker/pom.xml
+++ b/fs_brokers/apache_hdfs_broker/pom.xml
@@ -213,7 +213,7 @@ under the License.
<dependency>
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
- <version>0.9.3</version>
+ <version>0.13.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/log4j/log4j -->
@@ -328,6 +328,7 @@ under the License.
<thriftExecutable>${env.DORIS_THIRDPARTY}/installed/bin/thrift</thriftExecutable>
<thriftSourceRoot>${basedir}/src/main/resources/thrift/</thriftSourceRoot>
<outputDirectory>${basedir}/src/main/thrift/</outputDirectory>
+ <generator>java:fullcamel</generator>
<skip>${skip.plugin}</skip>
</configuration>
<executions>
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
new file mode 100644
index 0000000..81cde3c
--- /dev/null
+++ b/thirdparty/CHANGELOG.md
@@ -0,0 +1,138 @@
+# Changelog
+
+This file contains version of the third-party dependency libraries in the build-env image. The docker build-env image is apache/incubator-doris, and the tag is `build-env-${version}`
+
+## build-env-1.4.0 [2021-10-13]
+
+### Changes
+
+- libevent -> 2.1.12
+- OpenSSL 1.0.2k -> 1.1.1l
+- thrift 0.9.3 -> 0.13.0
+- protobuf 3.5.1 -> 3.14.0
+- gflags 2.2.0 -> 2.2.2
+- glog 0.3.3 -> 0.4.0
+- googletest 1.8.0 -> 1.10.0
+- snappy 1.1.7 -> 1.1.8
+- gperftools 2.7 -> 2.9.1
+- lz4 1.7.5 -> 1.9.3
+- curl 7.54.1 -> 7.79.0
+- re2 2017-05-01 -> 2021-02-02
+- zstd 1.3.7 -> 1.5.0
+- brotli 1.0.7 -> 1.0.9
+- flatbuffers 1.10.0 -> 2.0.0
+- apache-arrow 0.15.1 -> 5.0.0
+- CRoaring 0.2.60 -> 0.3.4
+- orc 1.5.8 -> 1.6.6
+- libdivide 4.0.0 -> 5.0
+- brpc 0.97 -> 1.0.0-rc02
+- librdkafka 1.6.5 -> 1.8.0
+
+### versions
+
+- libevent-release: 2.1.12
+- openssl: 1.1.1l
+- thrift: 0.13.0
+- protobuf: 3.14.0
+- gflags: 2.2.2
+- glog: 0.4.0
+- googletest: 1.10.0
+- snappy: 1.1.8
+- gperftools: 2.9.1
+- zlib: 1.2.11
+- lz4: 1.9.3
+- bzip2: 1.0.8
+- lzo: 2.10
+- rapidjson: 1a803826f1197b5e30703afe4b9c0e7dd48074f5
+- curl: 7.79.0
+- re2: 2021-02-02
+- boost: 1.73.0
+- mysql-server: 5.7.18
+- unixODBC: 2.3.7
+- leveldb: 1.20
+- incubator-brpc: 1.0.0-rc02
+- rocksdb: 5.14.2
+- librdkafka: 1.8.0
+- zstd: 1.5.0
+- brotli: 1.0.9
+- flatbuffers: 2.0.0
+- apache-arrow: 5.0.0
+- s2geometry: 0.9.0
+- bitshuffle: 0.3.5
+- CRoaring: 0.3.4
+- fmt: 7.1.3
+- parallel-hashmap: 1.33
+- orc: 1.6.6
+- jemalloc: 5.2.1
+- cctz: 2.3
+- DataTables: 1.10.25
+- aws-c-common: 0.4.63
+- aws-c-event-stream: 0.2.6
+- aws-checksums: 0.1.10
+- aws-c-io-0.7.0
+- aws-s2n: 0.10.0
+- aws-c-cal: 0.4.5
+- aws-sdk-cpp: 1.8.108
+- liblzma: master
+- libxml2: v2.9.10
+- libgsasl: 1.10.0
+- libhdfs3: master
+- libdivide: 5.0
+- pdqsort: git20180419
+- benchmark: 1.5.6
+
+## build_env-1.3.1
+
+### versions
+- libevent: 24236aed01798303745470e6c498bf606e88724a
+- openssl: 1.0.2k
+- thrift: 0.9.3
+- cfe: 3.4.2.src
+- protobuf: 3.5.1
+- gflags: 2.2.0
+- glog: 0.3.3
+- googletest: 1.8.0
+- snappy: 1.1.7
+- gperftools: 2.7
+- zlib: 1.2.11
+- lz4: 1.7.5
+- bzip2: 1.0.8
+- lzo: 2.10
+- rapidjson: 1a803826f1197b5e30703afe4b9c0e7dd48074f5
+- curl: 7.54.1
+- re2: 2017-05-01
+- boost: 1.73.0
+- mysql-server: 5.7.18
+- unixODBC: 2.3.7
+- leveldb: 1.20
+- incubator-brpc: 0.9.7
+- rocksdb: 5.14.2
+- librdkafka: 1.6.1-RC3
+- zstd: 1.3.7
+- double-conversion: 3.1.1
+- brotli: 1.0.7
+- flatbuffers: 1.10.0
+- arrow-apache: 0.15.1
+- s2geometry: 0.9.0
+- bitshuffle: 0.3.5
+- CRoaring: 0.2.60
+- fmt: 7.1.3
+- parallel-hashmap: 1.33
+- orc: 1.5.8
+- jemalloc: 5.2.1
+- cctz: 2.3
+- DataTables: 1.10.25
+- aws-c-common: 0.4.63
+- aws-c-event-stream: 0.2.6
+- aws-checksums: 0.1.10
+- aws-c-io: 0.7.0
+- s2n-tls: 0.10.0
+- aws-c-cal: 0.4.5
+- aws-sdk-cpp: 1.8.108
+- liblzma: master
+- libxml2: 2.9.10
+- libgsasl: 1.10.0
+- libhdfs3: master
+- libdivide: 4.0.0
+- pdqsort: git20180419
+- benchmark: 1.5.6
\ No newline at end of file
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 0637b95..cf210e7 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -201,14 +201,14 @@ check_if_archieve_exist() {
build_libevent() {
check_if_source_exist $LIBEVENT_SOURCE
cd $TP_SOURCE_DIR/$LIBEVENT_SOURCE
- if [ ! -f configure ]; then
- ./autogen.sh
- fi
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
CFLAGS="-std=c99 -fPIC -D_BSD_SOURCE -fno-omit-frame-pointer -g -ggdb -O2 -I${TP_INCLUDE_DIR}" \
+ CPPLAGS="-I${TP_INCLUDE_DIR}" \
LDFLAGS="-L${TP_LIB_DIR}" \
- ./configure --prefix=$TP_INSTALL_DIR --enable-shared=no --disable-samples --disable-libevent-regress
- make -j $PARALLEL && make install
+ ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DEVENT__DISABLE_TESTS=ON \
+ -DEVENT__DISABLE_OPENSSL=ON -DEVENT__DISABLE_SAMPLES=ON -DEVENT__DISABLE_REGRESS=ON ..
+ ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
build_openssl() {
@@ -243,10 +243,6 @@ build_thrift() {
check_if_source_exist $THRIFT_SOURCE
cd $TP_SOURCE_DIR/$THRIFT_SOURCE
- if [ ! -f configure ]; then
- ./bootstrap.sh
- fi
-
echo ${TP_LIB_DIR}
./configure CPPFLAGS="-I${TP_INCLUDE_DIR}" LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" LIBS="-lcrypto -ldl -lssl" CFLAGS="-fPIC" \
--prefix=$TP_INSTALL_DIR --docdir=$TP_INSTALL_DIR/doc --enable-static --disable-shared --disable-tests \
@@ -262,41 +258,13 @@ build_thrift() {
make -j $PARALLEL && make install
}
-# llvm
-build_llvm() {
- MACHINE_TYPE=$(uname -m)
- LLVM_TARGET="X86"
- if [[ "${MACHINE_TYPE}" == "aarch64" ]]; then
- LLVM_TARGET="AArch64"
- fi
-
- check_if_source_exist $LLVM_SOURCE
- check_if_source_exist $CLANG_SOURCE
- check_if_source_exist $COMPILER_RT_SOURCE
-
- if [ ! -d $TP_SOURCE_DIR/$LLVM_SOURCE/tools/clang ]; then
- cp -rf $TP_SOURCE_DIR/$CLANG_SOURCE $TP_SOURCE_DIR/$LLVM_SOURCE/tools/clang
- fi
-
- if [ ! -d $TP_SOURCE_DIR/$LLVM_SOURCE/projects/compiler-rt ]; then
- cp -rf $TP_SOURCE_DIR/$COMPILER_RT_SOURCE $TP_SOURCE_DIR/$LLVM_SOURCE/projects/compiler-rt
- fi
-
- cd $TP_SOURCE_DIR
- mkdir -p llvm-build && cd llvm-build
- rm -rf CMakeCache.txt CMakeFiles/
- LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
- ${CMAKE_CMD} -G "${GENERATOR}" -DLLVM_REQUIRES_RTTI:Bool=True -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGET} -DLLVM_ENABLE_TERMINFO=OFF LLVM_BUILD_LLVM_DYLIB:BOOL=OFF -DLLVM_ENABLE_PIC=true -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/llvm ../$LLVM_SOURCE
- ${BUILD_SYSTEM} -j $PARALLEL REQUIRES_RTTI=1 && ${BUILD_SYSTEM} install
-}
-
# protobuf
build_protobuf() {
check_if_source_exist $PROTOBUF_SOURCE
cd $TP_SOURCE_DIR/$PROTOBUF_SOURCE
rm -fr gmock
- mkdir gmock && cd gmock && tar xf ${TP_SOURCE_DIR}/googletest-release-1.8.0.tar.gz \
- && mv googletest-release-1.8.0 gtest && cd $TP_SOURCE_DIR/$PROTOBUF_SOURCE && ./autogen.sh
+ mkdir gmock && cd gmock && tar xf ${TP_SOURCE_DIR}/${GTEST_NAME} \
+ && mv ${GTEST_SOURCE} gtest && cd $TP_SOURCE_DIR/$PROTOBUF_SOURCE && ./autogen.sh
CXXFLAGS="-fPIC -O2 -I ${TP_INCLUDE_DIR}" \
LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
./configure --prefix=${TP_INSTALL_DIR} --disable-shared --enable-static --with-zlib=${TP_INSTALL_DIR}/include
@@ -416,10 +384,20 @@ build_lz4() {
check_if_source_exist $LZ4_SOURCE
cd $TP_SOURCE_DIR/$LZ4_SOURCE
- make -j $PARALLEL install PREFIX=$TP_INSTALL_DIR \
+ make -j $PARALLEL install PREFIX=$TP_INSTALL_DIR BUILD_SHARED=no\
INCLUDEDIR=$TP_INCLUDE_DIR/lz4/
}
+# zstd
+build_zstd() {
+ check_if_source_exist $ZSTD_SOURCE
+ cd $TP_SOURCE_DIR/$ZSTD_SOURCE/build/cmake
+ mkdir -p $BUILD_DIR && cd $BUILD_DIR
+ ${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_TESTING=OFF -DZSTD_BUILD_TESTS=OFF -ZSTD_BUILD_STATIC=ON \
+ -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_SHARED=OFF -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR ..
+ ${BUILD_SYSTEM} -j $PARALLEL install
+}
+
# bzip
build_bzip() {
check_if_source_exist $BZIP_SOURCE
@@ -588,36 +566,39 @@ build_arrow() {
check_if_source_exist $ARROW_SOURCE
cd $TP_SOURCE_DIR/$ARROW_SOURCE/cpp && mkdir -p release && cd release
export ARROW_BROTLI_URL=${TP_SOURCE_DIR}/${BROTLI_NAME}
- export ARROW_DOUBLE_CONVERSION_URL=${TP_SOURCE_DIR}/${DOUBLE_CONVERSION_NAME}
export ARROW_GLOG_URL=${TP_SOURCE_DIR}/${GLOG_NAME}
export ARROW_LZ4_URL=${TP_SOURCE_DIR}/${LZ4_NAME}
export ARROW_FLATBUFFERS_URL=${TP_SOURCE_DIR}/${FLATBUFFERS_NAME}
export ARROW_ZSTD_URL=${TP_SOURCE_DIR}/${ZSTD_NAME}
export ARROW_JEMALLOC_URL=${TP_SOURCE_DIR}/${JEMALLOC_NAME}
export ARROW_Thrift_URL=${TP_SOURCE_DIR}/${THRIFT_NAME}
+ export ARROW_SNAPPY_URL=${TP_SOURCE_DIR}/${SNAPPY_NAME}
+ export ARROW_ZLIB_URL=${TP_SOURCE_DIR}/${ZLIB_NAME}
export LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc"
- #--trace-expand \
- ${CMAKE_CMD} -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_USE_GLOG=off \
- -DARROW_BUILD_SHARED=OFF -DARROW_BUILD_STATIC=ON -DARROW_WITH_ZSTD=ON \
+ ${CMAKE_CMD} -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_BUILD_SHARED=OFF \
+ -DARROW_BUILD_STATIC=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_LZ4=ON \
+ -DARROW_WITH_SNAPPY=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_JSON=ON \
+ -DARROW_WITH_UTF8PROC=OFF -DARROW_WITH_RE2=OFF \
-DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR \
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DARROW_BOOST_USE_SHARED=OFF -DARROW_GFLAGS_USE_SHARED=OFF \
-DBoost_NO_BOOST_CMAKE=ON -DBOOST_ROOT=$TP_INSTALL_DIR \
-DPARQUET_ARROW_LINKAGE=static \
- -Dgflags_ROOT=$TP_INSTALL_DIR/ \
- -DSnappy_ROOT=$TP_INSTALL_DIR/ \
- -DGLOG_ROOT=$TP_INSTALL_DIR/ \
+ -Dgflags_ROOT=$TP_INSTALL_DIR \
+ -DSnappy_ROOT=$TP_INSTALL_DIR \
+ -DGLOG_ROOT=$TP_INSTALL_DIR \
-DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a \
-DGLOG_LIB=$TP_INSTALL_DIR/lib/libglog.a \
- -DLZ4_ROOT=$TP_INSTALL_DIR/ \
+ -DLZ4_ROOT=$TP_INSTALL_DIR \
-DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \
- -DZSTD_SOURCE=BUNDLED \
+ -DZSTD_ROOT=$TP_INSTALL_DIR \
-DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a -DZLIB_INCLUDE_DIR=$TP_INSTALL_DIR/include \
-DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a \
- -Ddouble-conversion_SOURCE=BUNDLED \
- -DThrift_ROOT=$TP_INSTALL_DIR/ ..
+ -DBoost_NO_BOOST_CMAKE=ON \
+ -DBOOST_ROOT=$TP_INSTALL_DIR \
+ -DThrift_ROOT=$TP_INSTALL_DIR ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
@@ -626,12 +607,6 @@ build_arrow() {
cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a $TP_INSTALL_DIR/lib64/libbrotlienc.a
cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a $TP_INSTALL_DIR/lib64/libbrotlidec.a
cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a $TP_INSTALL_DIR/lib64/libbrotlicommon.a
- if [ -f ./zstd_ep-install/lib64/libzstd.a ]; then
- cp -rf ./zstd_ep-install/lib64/libzstd.a $TP_INSTALL_DIR/lib64/libzstd.a
- else
- cp -rf ./zstd_ep-install/lib/libzstd.a $TP_INSTALL_DIR/lib64/libzstd.a
- fi
- cp -rf ./double-conversion_ep/src/double-conversion_ep/lib/libdouble-conversion.a $TP_INSTALL_DIR/lib64/libdouble-conversion.a
}
# s2
@@ -648,9 +623,9 @@ build_s2() {
-DGFLAGS_ROOT_DIR="$TP_INSTALL_DIR/include" \
-DWITH_GFLAGS=ON \
-DGLOG_ROOT_DIR="$TP_INSTALL_DIR/include" \
- -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib" \
- -DOPENSSL_ROOT_DIR=$TP_INSTALL_DIR/include \
- -DWITH_GLOG=ON ..
+ -DCMAKE_LIBRARY_PATH="$TP_INSTALL_DIR/lib64" \
+ -DOPENSSL_ROOT_DIR="$TP_INSTALL_DIR/include" \
+ -DWITH_GLOG=ON ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -714,8 +689,7 @@ build_croaringbitmap() {
CXXFLAGS="-O3" \
LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \
${CMAKE_CMD} -G "${GENERATOR}" -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
- -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \
- -DENABLE_ROARING_TESTS=OFF ..
+ -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" -DENABLE_ROARING_TESTS=OFF ..
${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install
}
@@ -763,7 +737,7 @@ build_orc() {
-DGTEST_HOME=$TP_INSTALL_DIR \
-DLZ4_HOME=$TP_INSTALL_DIR \
-DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \
- -DZLIB_HOME=$TP_INSTALL_DIR\
+ -DZLIB_HOME=$TP_INSTALL_DIR \
-DBUILD_LIBHDFSPP=OFF \
-DBUILD_CPP_TESTS=OFF \
-DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR
@@ -929,20 +903,14 @@ build_benchmark() {
cp $TP_SOURCE_DIR/$BENCHMARK_SOURCE/build/src/libbenchmark.a $TP_LIB_DIR/
}
-# See https://github.com/apache/incubator-doris/issues/2910
-# LLVM related codes have already be removed in master, so there is
-# no need to build llvm tool here.
-# Currently, some old release of Doris may still need it, so for now
-# we just comment it, instead of remove it.
-# build_llvm
-
build_libunixodbc
+build_openssl
build_libevent
build_zlib
build_lz4
build_bzip
build_lzo2
-build_openssl
+build_zstd
build_boost # must before thrift
build_protobuf
build_gflags
@@ -986,3 +954,4 @@ build_hdfs3
build_benchmark
echo "Finished to build all thirdparties"
+
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
index 75a7728..8f118f0 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -23,7 +23,6 @@
# You can run this script multi-times.
# Things will only be downloaded, unpacked and patched once.
################################################################
-
curdir=`dirname "$0"`
curdir=`cd "$curdir"; pwd`
@@ -225,33 +224,19 @@ echo "===== Patching thirdparty archives..."
###################################################################################
PATCHED_MARK="patched_mark"
-# glog patch
-cd $TP_SOURCE_DIR/$GLOG_SOURCE
-if [ ! -f $PATCHED_MARK ]; then
- patch -p1 < $TP_PATCH_DIR/glog-0.3.3-vlog-double-lock-bug.patch
- patch -p1 < $TP_PATCH_DIR/glog-0.3.3-for-palo2.patch
- patch -p1 < $TP_PATCH_DIR/glog-0.3.3-remove-unwind-dependency.patch
- # patch Makefile.am to make autoreconf work
- patch -p0 < $TP_PATCH_DIR/glog-0.3.3-makefile.patch
- touch $PATCHED_MARK
-fi
-cd -
-echo "Finished patching $GLOG_SOURCE"
-
-# re2 patch
-cd $TP_SOURCE_DIR/$RE2_SOURCE
-if [ ! -f $PATCHED_MARK ]; then
- patch -p0 < $TP_PATCH_DIR/re2-2017-05-01.patch
- touch $PATCHED_MARK
-fi
-cd -
-echo "Finished patching $RE2_SOURCE"
+ # glog patch
+ cd $TP_SOURCE_DIR/$GLOG_SOURCE
+ if [ ! -f $PATCHED_MARK ]; then
+ patch -p1 < $TP_PATCH_DIR/glog-0.4.0.patch
+ touch $PATCHED_MARK
+ fi
+ cd -
+ echo "Finished patching $GLOG_SOURCE"
# mysql patch
cd $TP_SOURCE_DIR/$MYSQL_SOURCE
if [ ! -f $PATCHED_MARK ]; then
- patch -p0 < $TP_PATCH_DIR/mysql-5.7.18.patch
- patch -Rp0 < $TP_PATCH_DIR/mysql-5.7.18-boost.patch
+ patch -p1 < $TP_PATCH_DIR/mysql-server-mysql-5.7.18.patch
touch $PATCHED_MARK
fi
cd -
@@ -260,56 +245,16 @@ echo "Finished patching $MYSQL_SOURCE"
# libevent patch
cd $TP_SOURCE_DIR/$LIBEVENT_SOURCE
if [ ! -f $PATCHED_MARK ]; then
- patch -p1 < $TP_PATCH_DIR/libevent_on_free_cb.patch
+ patch -p1 < $TP_PATCH_DIR/libevent.patch
touch $PATCHED_MARK
fi
cd -
echo "Finished patching $LIBEVENT_SOURCE"
-# thrift patch
-# cd $TP_SOURCE_DIR/$THRIFT_SOURCE
-# if [ ! -f $PATCHED_MARK ]; then
-# patch -p0 < $TP_PATCH_DIR/thrift-0.9.3-aclocal.patch
-# touch $PATCHED_MARK
-# fi
-# cd -
-# echo "Finished patching $THRIFT_SOURCE"
-
-# this patch is only used inside Baidu
-if test "x$PATCH_COMPILER_RT" == "xtrue"; then
- cd $TP_SOURCE_DIR/$COMPILER_RT_SOURCE
- if [ ! -f $PATCHED_MARK ]; then
- patch -p0 < $TP_PATCH_DIR/compiler-rt.patch
- touch $PATCHED_MARK
- fi
- cd -
- echo "Finished patching $COMPILER_RT_SOURCE"
-fi
-
-# patch to llvm to support aarch64 platform
-cd $TP_SOURCE_DIR/$LLVM_SOURCE
-if [ ! -f $PATCHED_MARK ]; then
- patch -p0 < $TP_PATCH_DIR/llvm-3.4.2.patch
- touch $PATCHED_MARK
-fi
-cd -
-echo "Finished patching $LLVM_SOURCE"
-
-# lz4 patch to disable shared library
-cd $TP_SOURCE_DIR/$LZ4_SOURCE
-if [ ! -f $PATCHED_MARK ]; then
- patch -p0 < $TP_PATCH_DIR/lz4-1.7.5.patch
- touch $PATCHED_MARK
-fi
-cd -
-echo "Finished patching $LZ4_SOURCE"
-
# s2 patch to disable shared library
cd $TP_SOURCE_DIR/$S2_SOURCE
if [ ! -f $PATCHED_MARK ]; then
patch -p1 < $TP_PATCH_DIR/s2geometry-0.9.0.patch
- # replace uint64 with uint64_t to make compiler happy
- patch -p0 < $TP_PATCH_DIR/s2geometry-0.9.0-uint64.patch
touch $PATCHED_MARK
fi
cd -
@@ -323,3 +268,4 @@ if [ ! -f $PATCHED_MARK ]; then
fi
cd -
echo "Finished patching $HDFS3_SOURCE"
+
diff --git a/thirdparty/patches/boost-1.64.0-gcc4.8.patch b/thirdparty/patches/boost-1.64.0-gcc4.8.patch
deleted file mode 100644
index d336da9..0000000
--- a/thirdparty/patches/boost-1.64.0-gcc4.8.patch
+++ /dev/null
@@ -1,101 +0,0 @@
---- boost/context/continuation.hpp
-+++ boost/context/continuation.hpp
-@@ -299,6 +299,24 @@
- friend continuation
- callcc( std::allocator_arg_t, preallocated, StackAlloc, Fn &&);
-
-+#if defined(BOOST_GCC) && BOOST_GCC < 50000
-+ template< typename StackAlloc, typename Fn, typename ... Arg >
-+ friend continuation
-+ callcc_with_args( std::allocator_arg_t, StackAlloc, Fn &&, Arg ...);
-+
-+ template< typename StackAlloc, typename Fn, typename ... Arg >
-+ friend continuation
-+ callcc_with_args_palloc( std::allocator_arg_t, preallocated, StackAlloc, Fn &&, Arg ...);
-+
-+ template< typename StackAlloc, typename Fn >
-+ friend continuation
-+ callcc_no_args( std::allocator_arg_t, StackAlloc, Fn &&);
-+
-+ template< typename StackAlloc, typename Fn >
-+ friend continuation
-+ callcc_no_args_palloc( std::allocator_arg_t, preallocated, StackAlloc, Fn &&);
-+#endif
-+
- detail::transfer_t t_{ nullptr, nullptr };
-
- continuation( detail::fcontext_t fctx) noexcept :
-@@ -518,6 +536,56 @@
- palloc, salloc, std::forward< Fn >( fn) ) }.resume();
- }
-
-+#if defined(BOOST_GCC) && BOOST_GCC < 50000
-+
-+template<
-+ typename StackAlloc,
-+ typename Fn,
-+ typename ... Arg
-+>
-+continuation
-+callcc_with_args( std::allocator_arg_t, StackAlloc salloc, Fn && fn, Arg ... arg) {
-+ using Record = detail::record< continuation, StackAlloc, Fn >;
-+ return continuation{
-+ detail::context_create< Record >(
-+ salloc, std::forward< Fn >( fn) ) }.resume(
-+ std::forward< Arg >( arg) ... );
-+}
-+
-+template<
-+ typename StackAlloc,
-+ typename Fn,
-+ typename ... Arg
-+>
-+continuation
-+callcc_with_args_palloc( std::allocator_arg_t, preallocated palloc, StackAlloc salloc, Fn && fn, Arg ... arg) {
-+ using Record = detail::record< continuation, StackAlloc, Fn >;
-+ return continuation{
-+ detail::context_create< Record >(
-+ palloc, salloc, std::forward< Fn >( fn) ) }.resume(
-+ std::forward< Arg >( arg) ... );
-+}
-+
-+template< typename StackAlloc, typename Fn >
-+continuation
-+callcc_no_args( std::allocator_arg_t, StackAlloc salloc, Fn && fn) {
-+ using Record = detail::record< continuation, StackAlloc, Fn >;
-+ return continuation{
-+ detail::context_create< Record >(
-+ salloc, std::forward< Fn >( fn) ) }.resume();
-+}
-+
-+template< typename StackAlloc, typename Fn >
-+continuation
-+callcc_no_args_palloc( std::allocator_arg_t, preallocated palloc, StackAlloc salloc, Fn && fn) {
-+ using Record = detail::record< continuation, StackAlloc, Fn >;
-+ return continuation{
-+ detail::context_create< Record >(
-+ palloc, salloc, std::forward< Fn >( fn) ) }.resume();
-+}
-+
-+#endif
-+
- #if defined(BOOST_USE_SEGMENTED_STACKS)
- template<
- typename Fn,
-
---- libs/fiber/src/context.cpp
-+++ libs/fiber/src/context.cpp
-@@ -219,7 +219,11 @@
- type_{ type::dispatcher_context },
- policy_{ launch::post }
- {
-+#if defined(BOOST_GCC) && BOOST_GCC < 50000
-+ c_ = boost::context::callcc_no_args_palloc(
-+#else
- c_ = boost::context::callcc(
-+#endif
- std::allocator_arg, palloc, salloc,
- [this,sched](boost::context::continuation && c) noexcept {
- c = c.resume();
-
-
diff --git a/thirdparty/patches/compiler-rt.patch b/thirdparty/patches/compiler-rt.patch
deleted file mode 100644
index 3549e7b..0000000
--- a/thirdparty/patches/compiler-rt.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- lib/sanitizer_common/sanitizer_platform_limits_posix.h
-+++ lib/sanitizer_common/sanitizer_platform_limits_posix.h
-@@ -507,7 +507,7 @@ namespace __sanitizer {
-
- typedef long long __sanitizer___kernel_loff_t;
- typedef struct {
-- unsigned long fds_bits[1024 / (8 * sizeof(long))];
-+ unsigned long fds_bits[51200 / (8 * sizeof(long))];
- } __sanitizer___kernel_fd_set;
- #endif
-
diff --git a/thirdparty/patches/glog-0.3.3-makefile.patch b/thirdparty/patches/glog-0.3.3-makefile.patch
deleted file mode 100644
index cb99d4f..0000000
--- a/thirdparty/patches/glog-0.3.3-makefile.patch
+++ /dev/null
@@ -1,10 +0,0 @@
---- Makefile.am.orig 2019-12-25 09:42:04.990000000 +0800
-+++ Makefile.am 2019-12-25 09:42:26.300000000 +0800
-@@ -1,5 +1,7 @@
- ## Process this file with automake to produce Makefile.in
-
-+AUTOMAKE_OPTIONS = subdir-objects foreign
-+
- # Make sure that when we re-make ./configure, we get the macros we need
- ACLOCAL_AMFLAGS = -I m4
-
diff --git a/thirdparty/patches/glog-0.3.3-remove-unwind-dependency.patch b/thirdparty/patches/glog-0.3.3-remove-unwind-dependency.patch
deleted file mode 100644
index d0bdf84..0000000
--- a/thirdparty/patches/glog-0.3.3-remove-unwind-dependency.patch
+++ /dev/null
@@ -1,31 +0,0 @@
---- glog-0.3.3/src/utilities.h.orig 2017-08-31 15:38:59.050406303 +0800
-+++ glog-0.3.3/src/utilities.h 2017-08-31 15:40:02.015350365 +0800
-@@ -101,17 +101,17 @@
- // correctly when GetStackTrace() is called with max_depth == 0.
- // Some code may do that.
-
--#if defined(HAVE_LIB_UNWIND)
--# define STACKTRACE_H "stacktrace_libunwind-inl.h"
--#elif !defined(NO_FRAME_POINTER)
--# if defined(__i386__) && __GNUC__ >= 2
--# define STACKTRACE_H "stacktrace_x86-inl.h"
--# elif defined(__x86_64__) && __GNUC__ >= 2 && HAVE_UNWIND_H
--# define STACKTRACE_H "stacktrace_x86_64-inl.h"
--# elif (defined(__ppc__) || defined(__PPC__)) && __GNUC__ >= 2
--# define STACKTRACE_H "stacktrace_powerpc-inl.h"
--# endif
--#endif
-+// #if defined(HAVE_LIB_UNWIND)
-+// # define STACKTRACE_H "stacktrace_libunwind-inl.h"
-+// #elif !defined(NO_FRAME_POINTER)
-+// # if defined(__i386__) && __GNUC__ >= 2
-+// # define STACKTRACE_H "stacktrace_x86-inl.h"
-+// # elif defined(__x86_64__) && __GNUC__ >= 2 && HAVE_UNWIND_H
-+// # define STACKTRACE_H "stacktrace_x86_64-inl.h"
-+// # elif (defined(__ppc__) || defined(__PPC__)) && __GNUC__ >= 2
-+// # define STACKTRACE_H "stacktrace_powerpc-inl.h"
-+// # endif
-+// #endif
-
- #if !defined(STACKTRACE_H) && defined(HAVE_EXECINFO_H)
- # define STACKTRACE_H "stacktrace_generic-inl.h"
diff --git a/thirdparty/patches/glog-0.3.3-vlog-double-lock-bug.patch b/thirdparty/patches/glog-0.3.3-vlog-double-lock-bug.patch
deleted file mode 100644
index b3dd769..0000000
--- a/thirdparty/patches/glog-0.3.3-vlog-double-lock-bug.patch
+++ /dev/null
@@ -1,21 +0,0 @@
---- glog-0.3.3/src/vlog_is_on.cc 2012-01-12 16:40:58.000000000 +0800
-+++ glog-0.3.3/src/vlog_is_on.cc 2015-03-22 16:48:50.759037426 +0800
-@@ -165,7 +165,8 @@
- int result = FLAGS_v;
- int const pattern_len = strlen(module_pattern);
- bool found = false;
-- MutexLock l(&vmodule_lock); // protect whole read-modify-write
-+ // MutexLock l(&vmodule_lock); // protect whole read-modify-write
-+ vmodule_lock.Lock(); // protect whole read-modify-write
- for (const VModuleInfo* info = vmodule_list;
- info != NULL; info = info->next) {
- if (info->module_pattern == module_pattern) {
-@@ -189,6 +189,8 @@
- info->next = vmodule_list;
- vmodule_list = info;
- }
-+ vmodule_lock.Unlock();
-+
- RAW_VLOG(1, "Set VLOG level for \"%s\" to %d", module_pattern, log_level);
- return result;
- }
diff --git a/thirdparty/patches/glog-0.3.3-for-palo2.patch b/thirdparty/patches/glog-0.4.0.patch
similarity index 63%
rename from thirdparty/patches/glog-0.3.3-for-palo2.patch
rename to thirdparty/patches/glog-0.4.0.patch
index 33cdc38..12b6ea5 100644
--- a/thirdparty/patches/glog-0.3.3-for-palo2.patch
+++ b/thirdparty/patches/glog-0.4.0.patch
@@ -1,20 +1,31 @@
---- glog-0.3.3/src/glog/logging.h.in 2013-02-01 14:20:46.000000000 +0800
-+++ glog-0.3.3/src/glog/logging.h.in 2016-03-18 10:32:00.127125712 +0800
-@@ -316,6 +316,11 @@
+diff -uprN a/src/glog/logging.h.in b/src/glog/logging.h.in
+--- a/src/glog/logging.h.in 2021-09-24 18:29:44.631818715 +0800
++++ b/src/glog/logging.h.in 2021-09-24 18:25:17.592767387 +0800
+@@ -330,6 +330,12 @@ typedef unsigned __int64 uint64;
using fLS::FLAGS_##name
#endif
++// Set max log file num
+DECLARE_int32(log_filenum_quota);
+
++// Set log file split method
+DECLARE_string(log_split_method);
+
-+
// Set whether log messages go to stderr instead of logfiles
DECLARE_bool(logtostderr);
---- glog-0.3.3/src/logging.cc 2013-02-01 14:20:46.000000000 +0800
-+++ glog-0.3.3/src/logging.cc 2016-03-18 10:37:18.331126494 +0800
-@@ -48,13 +48,17 @@
+diff -uprN a/src/logging.cc b/src/logging.cc
+--- a/src/logging.cc 2021-09-24 18:29:21.619263109 +0800
++++ b/src/logging.cc 2021-09-24 18:25:17.600767927 +0800
+@@ -34,6 +34,7 @@
+ #include <algorithm>
+ #include <assert.h>
+ #include <iomanip>
++#include <list>
+ #include <string>
+ #ifdef HAVE_UNISTD_H
+ # include <unistd.h> // For _exit.
+@@ -49,6 +50,7 @@
#include <iostream>
#include <stdarg.h>
#include <stdlib.h>
@@ -22,30 +33,20 @@
#ifdef HAVE_PWD_H
# include <pwd.h>
#endif
- #ifdef HAVE_SYSLOG_H
- # include <syslog.h>
- #endif
-+#include <algorithm>
- #include <vector>
-+#include <list>
-+#include <string>
- #include <errno.h> // for errno
- #include <sstream>
- #include "base/commandlineflags.h" // to get the program name
-@@ -170,6 +174,12 @@
- "approx. maximum log file size (in MB). A value of 0 will "
- "be silently overridden to 1.");
+@@ -178,6 +180,12 @@ GLOG_DEFINE_bool(stop_logging_if_full_di
+ GLOG_DEFINE_string(log_backtrace_at, "",
+ "Emit a backtrace when logging at file:linenum.");
+GLOG_DEFINE_string(log_split_method, "day",
-+ "split log by size, day, hour");
++ "split log by size, day, hour");
+
+GLOG_DEFINE_int32(log_filenum_quota, 10,
-+ "max log file num in log dir");
++ "max log file num in log dir");
+
- GLOG_DEFINE_bool(stop_logging_if_full_disk, false,
- "Stop attempting to log to disk if the disk is full.");
+ // TODO(hamaji): consider windows
+ #define PATH_SEPARATOR '/'
-@@ -351,6 +361,15 @@
+@@ -394,6 +402,15 @@ base::Logger::~Logger() {
namespace {
@@ -56,21 +57,20 @@
+ bool operator < (const struct filetime& o) const {
+ return o.time > time;
+ }
-+}Filetime;
++} Filetime;
+
// Encapsulates all file-system related state
class LogFileObject : public base::Logger {
public:
-@@ -382,6 +401,8 @@
+@@ -424,6 +441,7 @@ class LogFileObject : public base::Logge
+ // can avoid grabbing a lock. Usually Flush() calls it after
// acquiring lock_.
void FlushUnlocked();
-
+ void CheckFileNumQuota();
-+
+
private:
static const uint32 kRolloverAttemptFrequency = 0x20;
-
-@@ -396,6 +417,9 @@
+@@ -440,6 +458,9 @@ class LogFileObject : public base::Logge
uint32 file_length_;
unsigned int rollover_attempt_;
int64 next_flush_time_; // cycle count at which to flush log
@@ -80,75 +80,16 @@
// Actually create a logfile using the value of base_filename_ and the
// supplied argument time_pid_string
-@@ -414,7 +438,7 @@
-
- // These methods are just forwarded to by their global versions.
- static void SetLogDestination(LogSeverity severity,
-- const char* base_filename);
-+ const char* base_filename);
- static void SetLogSymlink(LogSeverity severity,
- const char* symlink_basename);
- static void AddLogSink(LogSink *destination);
-@@ -446,17 +470,17 @@
- // Take a log message of a particular severity and log it to stderr
- // iff it's of a high enough severity to deserve it.
- static void MaybeLogToStderr(LogSeverity severity, const char* message,
-- size_t len);
-+ size_t len);
-
- // Take a log message of a particular severity and log it to email
- // iff it's of a high enough severity to deserve it.
- static void MaybeLogToEmail(LogSeverity severity, const char* message,
-- size_t len);
-+ size_t len);
- // Take a log message of a particular severity and log it to a file
- // iff the base filename is not "" (which means "don't log to me")
- static void MaybeLogToLogfile(LogSeverity severity,
- time_t timestamp,
-- const char* message, size_t len);
-+ const char* message, size_t len);
- // Take a log message of a particular severity and log it to the file
- // for that severity and also for all files with severity less than
- // this severity.
-@@ -531,7 +555,7 @@
- // assume we have the log_mutex or we simply don't care
- // about it
+@@ -589,7 +610,7 @@ inline void LogDestination::FlushLogFile
+ // all this stuff.
+ MutexLock l(&log_mutex);
for (int i = min_severity; i < NUM_SEVERITIES; i++) {
- LogDestination* log = log_destination(i);
+ LogDestination* log = log_destinations_[i];
if (log != NULL) {
- // Flush the base fileobject_ logger directly instead of going
- // through any wrappers to reduce chance of deadlock.
-@@ -620,7 +644,7 @@
- }
-
- inline void LogDestination::SetEmailLogging(LogSeverity min_severity,
-- const char* addresses) {
-+ const char* addresses) {
- assert(min_severity >= 0 && min_severity < NUM_SEVERITIES);
- // Prevent any subtle race conditions by wrapping a mutex lock around
- // all this stuff.
-@@ -673,7 +697,7 @@
- }
-
- inline void LogDestination::MaybeLogToStderr(LogSeverity severity,
-- const char* message, size_t len) {
-+ const char* message, size_t len) {
- if ((severity >= FLAGS_stderrthreshold) || FLAGS_alsologtostderr) {
- ColoredWriteToStderr(severity, message, len);
- #ifdef OS_WINDOWS
-@@ -712,8 +736,8 @@
-
- inline void LogDestination::MaybeLogToLogfile(LogSeverity severity,
- time_t timestamp,
-- const char* message,
-- size_t len) {
-+ const char* message,
-+ size_t len) {
- const bool should_flush = severity > FLAGS_logbuflevel;
- LogDestination* destination = log_destination(severity);
- destination->logger_->Write(should_flush, timestamp, message, len);
-@@ -727,8 +751,12 @@
+ log->logger_->Flush();
+ }
+@@ -771,8 +792,12 @@ inline void LogDestination::LogToAllLogf
if ( FLAGS_logtostderr ) { // global flag: never log to file
ColoredWriteToStderr(severity, message, len);
} else {
@@ -156,15 +97,15 @@
- LogDestination::MaybeLogToLogfile(i, timestamp, message, len);
+ if (severity >= 1) {
+ LogDestination::MaybeLogToLogfile(1, timestamp, message, len);
-+ LogDestination::MaybeLogToLogfile(0, timestamp, message, len);
++ LogDestination::MaybeLogToLogfile(0, timestamp, message, len);
+ } else if (severity == 0) {
+ LogDestination::MaybeLogToLogfile(0, timestamp, message, len);
+ } else {}
}
}
-@@ -793,7 +821,8 @@
- bytes_since_flush_(0),
+@@ -841,7 +866,8 @@ LogFileObject::LogFileObject(LogSeverity
+ dropped_mem_length_(0),
file_length_(0),
rollover_attempt_(kRolloverAttemptFrequency-1),
- next_flush_time_(0) {
@@ -173,16 +114,16 @@
assert(severity >= 0);
assert(severity < NUM_SEVERITIES);
}
-@@ -858,7 +887,7 @@
+@@ -906,7 +932,7 @@ bool LogFileObject::CreateLogfile(const
string string_filename = base_filename_+filename_extension_+
time_pid_string;
const char* filename = string_filename.c_str();
-- int fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, 0664);
-+ int fd = open(filename, O_WRONLY | O_CREAT /* | O_EXCL */ | O_APPEND, 0664);
+- int fd = open(filename, O_WRONLY | O_CREAT | O_EXCL, FLAGS_logfile_mode);
++ int fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, FLAGS_logfile_mode);
if (fd == -1) return false;
#ifdef HAVE_FCNTL
// Mark the file close-on-exec. We don't really care if this fails
-@@ -872,6 +901,10 @@
+@@ -920,6 +946,10 @@ bool LogFileObject::CreateLogfile(const
return false;
}
@@ -193,7 +134,7 @@
// We try to create a symlink called <program_name>.<severity>,
// which is easier to use. (Every time we create a new logfile,
// we destroy the old symlink and create a new one, so it always
-@@ -911,6 +944,59 @@
+@@ -961,6 +991,59 @@ bool LogFileObject::CreateLogfile(const
return true; // Everything worked
}
@@ -222,7 +163,7 @@
+ std::string filename = std::string(entry->d_name);
+ //fprintf(stderr, "filename: %s\n", filename.c_str());
+
-+ if (filename.find(symlink_basename_ + '.' + LogSeverityNames[severity_]) == 0) {
++ if (filename.find(symlink_basename_ + '.' + LogSeverityNames[severity_]) == 0) {
+ std::string filepath = log_dirs[0] + "/" + filename;
+
+ struct stat fstat;
@@ -253,7 +194,7 @@
void LogFileObject::Write(bool force_flush,
time_t timestamp,
const char* message,
-@@ -922,14 +1008,55 @@
+@@ -972,14 +1055,55 @@ void LogFileObject::Write(bool force_flu
return;
}
@@ -285,14 +226,14 @@
+ if (is_split) {
if (file_ != NULL) fclose(file_);
file_ = NULL;
- file_length_ = bytes_since_flush_ = 0;
+ file_length_ = bytes_since_flush_ = dropped_mem_length_ = 0;
rollover_attempt_ = kRolloverAttemptFrequency-1;
}
-
+
+ if ((file_ == NULL) && (!inited_) && (FLAGS_log_split_method == "size")) {
+ CheckFileNumQuota();
+ const char* filename = file_list_.back().name.c_str();
-+ int fd = open(filename, O_WRONLY | O_CREAT /* | O_EXCL */ | O_APPEND, 0664);
++ int fd = open(filename, O_WRONLY | O_CREAT | O_APPEND, FLAGS_logfile_mode);
+ if (fd != -1) {
+#ifdef HAVE_FCNTL
+ // Mark the file close-on-exec. We don't really care if this fails
@@ -307,16 +248,15 @@
+ inited_ = true;
+ }
+ }
-+
++
// If there's no destination file, make one before outputting
if (file_ == NULL) {
// Try to rollover the log file every 32 log messages. The only time
-@@ -938,21 +1065,35 @@
+@@ -988,7 +1112,16 @@ void LogFileObject::Write(bool force_flu
if (++rollover_attempt_ != kRolloverAttemptFrequency) return;
rollover_attempt_ = 0;
- struct ::tm tm_time;
-- localtime_r(×tamp, &tm_time);
+ if (!inited_) {
+ CheckFileNumQuota();
+ inited_ = true;
@@ -326,10 +266,11 @@
+ file_list_.pop_front();
+ }
+ }
++
+ localtime_r(×tamp, &tm_time);
-+ localtime_r(×tamp, &tm_time);
// The logfile's filename will have the date/time & pid in it
- ostringstream time_pid_stream;
+@@ -996,13 +1129,19 @@ void LogFileObject::Write(bool force_flu
time_pid_stream.fill('0');
time_pid_stream << 1900+tm_time.tm_year
<< setw(2) << 1+tm_time.tm_mon
@@ -350,16 +291,17 @@
+ << setw(2) << tm_time.tm_min
+ << setw(2) << tm_time.tm_sec;
+ }
-+
++
+ tm_time_ = tm_time;
+
const string& time_pid_string = time_pid_stream.str();
if (base_filename_selected_) {
-@@ -986,9 +1127,7 @@
+@@ -1035,10 +1174,7 @@ void LogFileObject::Write(bool force_flu
+ // attempt to hold on to the same mutex, and get into a
// deadlock. Simply use a name like invalid-user.
if (uidname.empty()) uidname = "invalid-user";
-
+-
- stripped_filename = stripped_filename+'.'+hostname+'.'
- +uidname+".log."
- +LogSeverityNames[severity_]+'.';
@@ -367,21 +309,32 @@
// We're going to (potentially) try to put logs in several different dirs
const vector<string> & log_dirs = GetLoggingDirectories();
-@@ -1012,7 +1151,8 @@
+@@ -1062,28 +1198,6 @@ void LogFileObject::Write(bool force_flu
return;
}
}
-
-+
-+ /*
- // Write a header message into the log file
- ostringstream file_header_stream;
- file_header_stream.fill('0');
-@@ -1034,6 +1174,7 @@
- fwrite(file_header_string.data(), 1, header_len, file_);
- file_length_ += header_len;
- bytes_since_flush_ += header_len;
-+ */
+- // Write a header message into the log file
+- ostringstream file_header_stream;
+- file_header_stream.fill('0');
+- file_header_stream << "Log file created at: "
+- << 1900+tm_time.tm_year << '/'
+- << setw(2) << 1+tm_time.tm_mon << '/'
+- << setw(2) << tm_time.tm_mday
+- << ' '
+- << setw(2) << tm_time.tm_hour << ':'
+- << setw(2) << tm_time.tm_min << ':'
+- << setw(2) << tm_time.tm_sec << '\n'
+- << "Running on machine: "
+- << LogDestination::hostname() << '\n'
+- << "Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu "
+- << "threadid file:line] msg" << '\n';
+- const string& file_header_string = file_header_stream.str();
+-
+- const int header_len = file_header_string.size();
+- fwrite(file_header_string.data(), 1, header_len, file_);
+- file_length_ += header_len;
+- bytes_since_flush_ += header_len;
}
// Write to LOG file
diff --git a/thirdparty/patches/incubator-brpc-0.9.7.patch b/thirdparty/patches/incubator-brpc-0.9.7.patch
deleted file mode 100644
index 57e7c4f..0000000
--- a/thirdparty/patches/incubator-brpc-0.9.7.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- src/bvar/detail/series.h
-+++ src/bvar/detail/series.h
-@@ -128,7 +128,7 @@ private:
- // is_pod does not work for gcc 3.4
- if (butil::is_integral<T>::value ||
- butil::is_floating_point<T>::value) {
-- memset(_array, 0, sizeof(_array));
-+ memset(static_cast<void*>(_array), 0, sizeof(_array));
- }
- }
-
diff --git a/thirdparty/patches/libevent.patch b/thirdparty/patches/libevent.patch
new file mode 100644
index 0000000..43c4b5b
--- /dev/null
+++ b/thirdparty/patches/libevent.patch
@@ -0,0 +1,134 @@
+diff -uprN a/http.c b/http.c
+--- a/http.c 2020-07-05 20:02:46.000000000 +0800
++++ b/http.c 2021-09-28 13:56:14.045159153 +0800
+@@ -3975,6 +3975,14 @@ evhttp_set_bevcb(struct evhttp *http,
+ http->bevcbarg = cbarg;
+ }
+
++void
++evhttp_set_newreqcb(struct evhttp *http,
++ int (*cb)(struct evhttp_request *, void *), void *cbarg)
++{
++ http->newreqcb = cb;
++ http->newreqcbarg = cbarg;
++}
++
+ /*
+ * Request related functions
+ */
+@@ -4036,6 +4044,8 @@ evhttp_request_free(struct evhttp_reques
+ req->flags |= EVHTTP_REQ_NEEDS_FREE;
+ return;
+ }
++ if (req->on_free_cb)
++ (*req->on_free_cb)(req, req->on_free_cb_arg);
+
+ if (req->remote_host != NULL)
+ mm_free(req->remote_host);
+@@ -4116,6 +4126,15 @@ evhttp_request_set_on_complete_cb(struct
+ req->on_complete_cb_arg = cb_arg;
+ }
+
++void
++evhttp_request_set_on_free_cb(struct evhttp_request *req,
++ void (*cb)(struct evhttp_request *, void *), void *cb_arg)
++{
++ req->on_free_cb = cb;
++ req->on_free_cb_arg = cb_arg;
++}
++
++
+ /*
+ * Allows for inspection of the request URI
+ */
+@@ -4307,10 +4326,15 @@ evhttp_associate_new_request_with_connec
+ */
+ req->userdone = 1;
+
+- TAILQ_INSERT_TAIL(&evcon->requests, req, next);
+-
+ req->kind = EVHTTP_REQUEST;
+
++ if (http->newreqcb && http->newreqcb(req, http->newreqcbarg) == -1) {
++ evhttp_request_free(req);
++ return (-1);
++ }
++
++ TAILQ_INSERT_TAIL(&evcon->requests, req, next);
++
+
+ evhttp_start_read_(evcon);
+
+diff -uprN a/http-internal.h b/http-internal.h
+--- a/http-internal.h 2020-07-05 20:02:46.000000000 +0800
++++ b/http-internal.h 2021-09-28 13:56:13.925151028 +0800
+@@ -167,6 +167,8 @@ struct evhttp {
+ void *gencbarg;
+ struct bufferevent* (*bevcb)(struct event_base *, void *);
+ void *bevcbarg;
++ int (*newreqcb)(struct evhttp_request *req, void *);
++ void *newreqcbarg;
+
+ struct event_base *base;
+ };
+diff -uprN a/include/event2/http.h b/include/event2/http.h
+--- a/include/event2/http.h 2020-07-05 20:02:46.000000000 +0800
++++ b/include/event2/http.h 2021-09-28 13:56:13.928151231 +0800
+@@ -299,6 +299,20 @@ void evhttp_set_bevcb(struct evhttp *htt
+ struct bufferevent *(*cb)(struct event_base *, void *), void *arg);
+
+ /**
++ Set a callback which allows the user to note or throttle incoming requests.
++ The requests are not populated with HTTP level information. They
++ are just associated to a connection.
++ If the callback returns -1, the associated connection is terminated
++ and the request is closed.
++ @param http the evhttp server object for which to set the callback
++ @param cb the callback to invoke for incoming connections
++ @param arg an context argument for the callback
++ */
++EVENT2_EXPORT_SYMBOL
++void evhttp_set_newreqcb(struct evhttp *http,
++ int (*cb)(struct evhttp_request*, void *), void *arg);
++
++/**
+ Adds a virtual host to the http server.
+
+ A virtual host is a newly initialized evhttp object that has request
+@@ -624,6 +638,20 @@ EVENT2_EXPORT_SYMBOL
+ void evhttp_request_set_on_complete_cb(struct evhttp_request *req,
+ void (*cb)(struct evhttp_request *, void *), void *cb_arg);
+
++/**
++ * Set a callback to be called on request free.
++ *
++ * The callback function will be called just before the evhttp_request object
++ * is destroyed.
++ *
++ * @param req a request object
++ * @param cb callback function that will be called before request free
++ * @param cb_arg an additional context argument for the callback
++ */
++EVENT2_EXPORT_SYMBOL
++void evhttp_request_set_on_free_cb(struct evhttp_request *req,
++ void (*cb)(struct evhttp_request *, void *), void *cb_arg);
++
+ /** Frees the request object and removes associated events. */
+ EVENT2_EXPORT_SYMBOL
+ void evhttp_request_free(struct evhttp_request *req);
+diff -uprN a/include/event2/http_struct.h b/include/event2/http_struct.h
+--- a/include/event2/http_struct.h 2020-07-05 20:02:46.000000000 +0800
++++ b/include/event2/http_struct.h 2021-09-28 13:56:13.928151231 +0800
+@@ -142,6 +142,12 @@ struct {
+ */
+ void (*on_complete_cb)(struct evhttp_request *, void *);
+ void *on_complete_cb_arg;
++
++ /*
++ * Free callback - called just before the request is freed.
++ */
++ void (*on_free_cb)(struct evhttp_request *, void *);
++ void *on_free_cb_arg;
+ };
+
+ #ifdef __cplusplus
diff --git a/thirdparty/patches/libevent_on_free_cb.patch b/thirdparty/patches/libevent_on_free_cb.patch
deleted file mode 100644
index 7e5cffa..0000000
--- a/thirdparty/patches/libevent_on_free_cb.patch
+++ /dev/null
@@ -1,71 +0,0 @@
-diff --git a/http.c b/http.c
-index b04af9a..5183a11 100644
---- a/http.c
-+++ b/http.c
-@@ -3995,6 +3995,9 @@ evhttp_request_free(struct evhttp_request *req)
- return;
- }
-
-+ if (req->on_free_cb)
-+ (*req->on_free_cb)(req, req->on_free_cb_arg);
-+
- if (req->remote_host != NULL)
- mm_free(req->remote_host);
- if (req->uri != NULL)
-@@ -4074,6 +4077,14 @@ evhttp_request_set_on_complete_cb(struct evhttp_request *req,
- req->on_complete_cb_arg = cb_arg;
- }
-
-+void
-+evhttp_request_set_on_free_cb(struct evhttp_request *req,
-+ void (*cb)(struct evhttp_request *, void *), void *cb_arg)
-+{
-+ req->on_free_cb = cb;
-+ req->on_free_cb_arg = cb_arg;
-+}
-+
- /*
- * Allows for inspection of the request URI
- */
-diff --git a/include/event2/http.h b/include/event2/http.h
-index ed9acf4..3a73c6c 100644
---- a/include/event2/http.h
-+++ b/include/event2/http.h
-@@ -642,6 +642,20 @@ EVENT2_EXPORT_SYMBOL
- void evhttp_request_set_on_complete_cb(struct evhttp_request *req,
- void (*cb)(struct evhttp_request *, void *), void *cb_arg);
-
-+/**
-+ * Set a callback to be called on request free.
-+ *
-+ * The callback function will be called just before the evhttp_request object
-+ * is destroyed.
-+ *
-+ * @param req a request object
-+ * @param cb callback function that will be called before request free
-+ * @param cb_arg an additional context argument for the callback
-+ */
-+EVENT2_EXPORT_SYMBOL
-+void evhttp_request_set_on_free_cb(struct evhttp_request *req,
-+ void (*cb)(struct evhttp_request *, void *), void *cb_arg);
-+
- /** Frees the request object and removes associated events. */
- EVENT2_EXPORT_SYMBOL
- void evhttp_request_free(struct evhttp_request *req);
-diff --git a/include/event2/http_struct.h b/include/event2/http_struct.h
-index 4bf5b1f..0762cab 100644
---- a/include/event2/http_struct.h
-+++ b/include/event2/http_struct.h
-@@ -142,6 +142,12 @@ struct {
- */
- void (*on_complete_cb)(struct evhttp_request *, void *);
- void *on_complete_cb_arg;
-+
-+ /*
-+ * Free callback - called just before the request is freed.
-+ */
-+ void (*on_free_cb)(struct evhttp_request *, void *);
-+ void *on_free_cb_arg;
- };
-
- #ifdef __cplusplus
diff --git a/thirdparty/patches/llvm-3.4.2.patch b/thirdparty/patches/llvm-3.4.2.patch
deleted file mode 100644
index 97ffec6..0000000
--- a/thirdparty/patches/llvm-3.4.2.patch
+++ /dev/null
@@ -1,16 +0,0 @@
---- autoconf/config.guess 2013-07-26 09:35:43.000000000 +0800
-+++ autoconf/config.guess 2019-12-24 16:00:25.940000000 +0800
-@@ -864,6 +864,13 @@
- i*86:Minix:*:*)
- echo ${UNAME_MACHINE}-pc-minix
- exit ;;
-+ aarch64:Linux:*:*)
-+ echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
-+ exit ;;
-+ aarch64_be:Linux:*:*)
-+ UNAME_MACHINE=aarch64_be
-+ echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"
-+ exit ;;
- alpha:Linux:*:*)
- case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in
- EV5) UNAME_MACHINE=alphaev5 ;;
diff --git a/thirdparty/patches/lz4-1.7.5.patch b/thirdparty/patches/lz4-1.7.5.patch
deleted file mode 100644
index 5598797..0000000
--- a/thirdparty/patches/lz4-1.7.5.patch
+++ /dev/null
@@ -1,15 +0,0 @@
---- lib/Makefile
-+++ lib/Makefile
-@@ -147,9 +147,9 @@ install: lib liblz4.pc
- ifeq ($(BUILD_STATIC),yes)
- @$(INSTALL_LIB) liblz4.a $(DESTDIR)$(LIBDIR)/liblz4.a
- endif
-- @$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-- @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-- @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
-+# @$(INSTALL_LIB) liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-+# @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT_MAJOR)
-+# @ln -sf liblz4.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/liblz4.$(SHARED_EXT)
- @echo Installing includes
- @$(INSTALL_DATA) lz4.h $(DESTDIR)$(INCLUDEDIR)/lz4.h
- @$(INSTALL_DATA) lz4hc.h $(DESTDIR)$(INCLUDEDIR)/lz4hc.h
diff --git a/thirdparty/patches/mysql-server-mysql-5.7.18.patch b/thirdparty/patches/mysql-server-mysql-5.7.18.patch
new file mode 100644
index 0000000..c179754
--- /dev/null
+++ b/thirdparty/patches/mysql-server-mysql-5.7.18.patch
@@ -0,0 +1,41 @@
+diff -uprN a/cmake/boost.cmake b/cmake/boost.cmake
+--- a/cmake/boost.cmake 2021-09-28 17:56:01.656714647 +0800
++++ b/cmake/boost.cmake 2021-09-28 17:58:09.156339908 +0800
+@@ -31,7 +31,7 @@
+ # we assume that the correct version (see below)
+ # is installed on the compile host in the standard location.
+
+-SET(BOOST_PACKAGE_NAME "boost_1_59_0")
++SET(BOOST_PACKAGE_NAME "boost_1_73_0")
+ SET(BOOST_TARBALL "${BOOST_PACKAGE_NAME}.tar.gz")
+ SET(BOOST_DOWNLOAD_URL
+ "http://sourceforge.net/projects/boost/files/boost/1.59.0/${BOOST_TARBALL}"
+@@ -262,7 +262,7 @@ IF(NOT BOOST_MAJOR_VERSION EQUAL 10)
+ COULD_NOT_FIND_BOOST()
+ ENDIF()
+
+-IF(NOT BOOST_MINOR_VERSION EQUAL 59)
++IF(NOT BOOST_MINOR_VERSION EQUAL 73)
+ MESSAGE(WARNING "Boost minor version found is ${BOOST_MINOR_VERSION} "
+ "we need 59"
+ )
+diff -uprN a/cmd-line-utils/libedit/terminal.c b/cmd-line-utils/libedit/terminal.c
+--- a/cmd-line-utils/libedit/terminal.c 2021-09-28 17:56:01.661714985 +0800
++++ b/cmd-line-utils/libedit/terminal.c 2021-09-28 17:56:52.794174061 +0800
+@@ -866,7 +866,6 @@ protected int
+ terminal_set(EditLine *el, const char *term)
+ {
+ int i;
+- char buf[TC_BUFSIZE];
+ char *area;
+ const struct termcapstr *t;
+ sigset_t oset, nset;
+@@ -876,7 +875,7 @@ terminal_set(EditLine *el, const char *t
+ (void) sigaddset(&nset, SIGWINCH);
+ (void) sigprocmask(SIG_BLOCK, &nset, &oset);
+
+- area = buf;
++ area = NULL;
+
+
+ if (term == NULL)
diff --git a/thirdparty/patches/re2-2017-05-01.patch b/thirdparty/patches/re2-2017-05-01.patch
deleted file mode 100644
index 9844e66..0000000
--- a/thirdparty/patches/re2-2017-05-01.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- Makefile.orig 2017-05-24 18:39:51.016085062 -0700
-+++ Makefile 2017-05-24 18:40:03.100085592 -0700
-@@ -26,7 +26,7 @@
-
- # Variables mandated by GNU, the arbiter of all good taste on the internet.
- # http://www.gnu.org/prep/standards/standards.html
--prefix=/usr/local
-+prefix=/
- exec_prefix=$(prefix)
- includedir=$(prefix)/include
- libdir=$(exec_prefix)/lib
diff --git a/thirdparty/patches/s2geometry-0.9.0-uint64.patch b/thirdparty/patches/s2geometry-0.9.0-uint64.patch
deleted file mode 100644
index 940a322..0000000
--- a/thirdparty/patches/s2geometry-0.9.0-uint64.patch
+++ /dev/null
@@ -1,94 +0,0 @@
---- src/s2/third_party/absl/base/internal/unaligned_access.h.orig 2019-12-25 11:13:05.290000000 +0800
-+++ src/s2/third_party/absl/base/internal/unaligned_access.h 2019-12-25 11:15:06.650000000 +0800
-@@ -80,7 +80,7 @@
- return __sanitizer_unaligned_load32(p);
- }
-
--inline uint64 UnalignedLoad64(const void *p) {
-+inline uint64_t UnalignedLoad64(const void *p) {
- return __sanitizer_unaligned_load64(p);
- }
-
-@@ -92,7 +92,7 @@
- __sanitizer_unaligned_store32(p, v);
- }
-
--inline void UnalignedStore64(void *p, uint64 v) {
-+inline void UnalignedStore64(void *p, uint64_t v) {
- __sanitizer_unaligned_store64(p, v);
- }
-
-@@ -130,8 +130,8 @@
- return t;
- }
-
--inline uint64 UnalignedLoad64(const void *p) {
-- uint64 t;
-+inline uint64_t UnalignedLoad64(const void *p) {
-+ uint64_t t;
- memcpy(&t, p, sizeof t);
- return t;
- }
-@@ -140,7 +140,7 @@
-
- inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); }
-
--inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
-+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
-
- } // namespace base_internal
- } // namespace absl
-@@ -172,14 +172,14 @@
- #define ABSL_INTERNAL_UNALIGNED_LOAD32(_p) \
- (*reinterpret_cast<const uint32_t *>(_p))
- #define ABSL_INTERNAL_UNALIGNED_LOAD64(_p) \
-- (*reinterpret_cast<const uint64 *>(_p))
-+ (*reinterpret_cast<const uint64_t *>(_p))
-
- #define ABSL_INTERNAL_UNALIGNED_STORE16(_p, _val) \
- (*reinterpret_cast<uint16_t *>(_p) = (_val))
- #define ABSL_INTERNAL_UNALIGNED_STORE32(_p, _val) \
- (*reinterpret_cast<uint32_t *>(_p) = (_val))
- #define ABSL_INTERNAL_UNALIGNED_STORE64(_p, _val) \
-- (*reinterpret_cast<uint64 *>(_p) = (_val))
-+ (*reinterpret_cast<uint64_t *>(_p) = (_val))
-
- #elif defined(__arm__) && \
- !defined(__ARM_ARCH_5__) && \
-@@ -246,13 +246,13 @@
- namespace absl {
- namespace base_internal {
-
--inline uint64 UnalignedLoad64(const void *p) {
-- uint64 t;
-+inline uint64_t UnalignedLoad64(const void *p) {
-+ uint64_t t;
- memcpy(&t, p, sizeof t);
- return t;
- }
-
--inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
-+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
-
- } // namespace base_internal
- } // namespace absl
-@@ -286,8 +286,8 @@
- return t;
- }
-
--inline uint64 UnalignedLoad64(const void *p) {
-- uint64 t;
-+inline uint64_t UnalignedLoad64(const void *p) {
-+ uint64_t t;
- memcpy(&t, p, sizeof t);
- return t;
- }
-@@ -296,7 +296,7 @@
-
- inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); }
-
--inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
-+inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
-
- } // namespace base_internal
- } // namespace absl
diff --git a/thirdparty/patches/s2geometry-0.9.0-warning.patch b/thirdparty/patches/s2geometry-0.9.0-warning.patch
deleted file mode 100644
index c28186b..0000000
--- a/thirdparty/patches/s2geometry-0.9.0-warning.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- src/s2/util/coding/coder.h
-+++ src/s2/util/coding/coder.h
-@@ -155,7 +155,7 @@ class Encoder {
- class Decoder {
- public:
- // Empty constructor to create uninitialized decoder
-- inline Decoder() { }
-+ inline Decoder() = default;
-
- // NOTE: for efficiency reasons, this is not virtual. so don't add
- // any members that really need to be destructed, and be careful about
diff --git a/thirdparty/patches/s2geometry-0.9.0.patch b/thirdparty/patches/s2geometry-0.9.0.patch
index 7f7a646..5d7618d 100644
--- a/thirdparty/patches/s2geometry-0.9.0.patch
+++ b/thirdparty/patches/s2geometry-0.9.0.patch
@@ -1,6 +1,6 @@
-diff -rupN s2geometry-0.9.0/CMakeLists.txt s2geometry-0.9.0.new/CMakeLists.txt
---- s2geometry-0.9.0/CMakeLists.txt 2019-03-05 00:53:16.000000000 +0800
-+++ s2geometry-0.9.0.new/CMakeLists.txt 2019-05-29 07:12:28.672879024 +0800
+diff -uprN a/CMakeLists.txt b/CMakeLists.txt
+--- a/CMakeLists.txt 2021-09-28 17:39:01.747718492 +0800
++++ b/CMakeLists.txt 2021-09-28 17:44:47.934137768 +0800
@@ -531,6 +531,6 @@ if (BUILD_EXAMPLES)
add_subdirectory("doc/examples" examples)
endif()
@@ -8,6 +8,122 @@ diff -rupN s2geometry-0.9.0/CMakeLists.txt s2geometry-0.9.0.new/CMakeLists.txt
-if (${SWIG_FOUND} AND ${PYTHONLIBS_FOUND})
- add_subdirectory("src/python" python)
-endif()
-+# if (${SWIG_FOUND} AND ${PYTHONLIBS_FOUND})
-+# add_subdirectory("src/python" python)
-+# endif()
++#if (${SWIG_FOUND} AND ${PYTHONLIBS_FOUND})
++# add_subdirectory("src/python" python)
++#endif()
+diff -uprN a/src/s2/third_party/absl/base/internal/unaligned_access.h b/src/s2/third_party/absl/base/internal/unaligned_access.h
+--- a/src/s2/third_party/absl/base/internal/unaligned_access.h 2021-09-28 17:39:01.782720861 +0800
++++ b/src/s2/third_party/absl/base/internal/unaligned_access.h 2021-09-28 17:42:50.863217996 +0800
+@@ -80,7 +80,7 @@ inline uint32_t UnalignedLoad32(const vo
+ return __sanitizer_unaligned_load32(p);
+ }
+
+-inline uint64 UnalignedLoad64(const void *p) {
++inline uint64_t UnalignedLoad64(const void *p) {
+ return __sanitizer_unaligned_load64(p);
+ }
+
+@@ -92,7 +92,7 @@ inline void UnalignedStore32(void *p, ui
+ __sanitizer_unaligned_store32(p, v);
+ }
+
+-inline void UnalignedStore64(void *p, uint64 v) {
++inline void UnalignedStore64(void *p, uint64_t v) {
+ __sanitizer_unaligned_store64(p, v);
+ }
+
+@@ -130,8 +130,8 @@ inline uint32_t UnalignedLoad32(const vo
+ return t;
+ }
+
+-inline uint64 UnalignedLoad64(const void *p) {
+- uint64 t;
++inline uint64_t UnalignedLoad64(const void *p) {
++ uint64_t t;
+ memcpy(&t, p, sizeof t);
+ return t;
+ }
+@@ -140,7 +140,7 @@ inline void UnalignedStore16(void *p, ui
+
+ inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); }
+
+-inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
++inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
+
+ } // namespace base_internal
+ } // namespace absl
+@@ -172,14 +172,14 @@ inline void UnalignedStore64(void *p, ui
+ #define ABSL_INTERNAL_UNALIGNED_LOAD32(_p) \
+ (*reinterpret_cast<const uint32_t *>(_p))
+ #define ABSL_INTERNAL_UNALIGNED_LOAD64(_p) \
+- (*reinterpret_cast<const uint64 *>(_p))
++ (*reinterpret_cast<const uint64_t *>(_p))
+
+ #define ABSL_INTERNAL_UNALIGNED_STORE16(_p, _val) \
+ (*reinterpret_cast<uint16_t *>(_p) = (_val))
+ #define ABSL_INTERNAL_UNALIGNED_STORE32(_p, _val) \
+ (*reinterpret_cast<uint32_t *>(_p) = (_val))
+ #define ABSL_INTERNAL_UNALIGNED_STORE64(_p, _val) \
+- (*reinterpret_cast<uint64 *>(_p) = (_val))
++ (*reinterpret_cast<uint64_t *>(_p) = (_val))
+
+ #elif defined(__arm__) && \
+ !defined(__ARM_ARCH_5__) && \
+@@ -246,13 +246,13 @@ struct Unaligned32Struct {
+ namespace absl {
+ namespace base_internal {
+
+-inline uint64 UnalignedLoad64(const void *p) {
+- uint64 t;
++inline uint64_t UnalignedLoad64(const void *p) {
++ uint64_t t;
+ memcpy(&t, p, sizeof t);
+ return t;
+ }
+
+-inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
++inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
+
+ } // namespace base_internal
+ } // namespace absl
+@@ -286,8 +286,8 @@ inline uint32_t UnalignedLoad32(const vo
+ return t;
+ }
+
+-inline uint64 UnalignedLoad64(const void *p) {
+- uint64 t;
++inline uint64_t UnalignedLoad64(const void *p) {
++ uint64_t t;
+ memcpy(&t, p, sizeof t);
+ return t;
+ }
+@@ -296,7 +296,7 @@ inline void UnalignedStore16(void *p, ui
+
+ inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); }
+
+-inline void UnalignedStore64(void *p, uint64 v) { memcpy(p, &v, sizeof v); }
++inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); }
+
+ } // namespace base_internal
+ } // namespace absl
+diff -uprN a/src/s2/util/coding/coder.h b/src/s2/util/coding/coder.h
+--- a/src/s2/util/coding/coder.h 2021-09-28 17:39:01.762719508 +0800
++++ b/src/s2/util/coding/coder.h 2021-09-28 17:44:11.903700328 +0800
+@@ -155,7 +155,7 @@ class Encoder {
+ class Decoder {
+ public:
+ // Empty constructor to create uninitialized decoder
+- inline Decoder() { }
++ inline Decoder() = default;
+
+ // NOTE: for efficiency reasons, this is not virtual. so don't add
+ // any members that really need to be destructed, and be careful about
+@@ -461,7 +461,7 @@ inline void DecoderExtensions::FillArray
+ "Decoder must be trivially copy-assignable");
+ static_assert(absl::is_trivially_destructible<Decoder>::value,
+ "Decoder must be trivially destructible");
+- std::memset(array, 0, num_decoders * sizeof(Decoder));
++ std::memset(static_cast<void*>(array), 0, num_decoders * sizeof(Decoder));
+ }
+
+ inline void Encoder::put8(unsigned char v) {
diff --git a/thirdparty/patches/thrift-0.9.3-aclocal.patch b/thirdparty/patches/thrift-0.9.3-aclocal.patch
deleted file mode 100644
index f35663c..0000000
--- a/thirdparty/patches/thrift-0.9.3-aclocal.patch
+++ /dev/null
@@ -1,11 +0,0 @@
---- bootstrap.sh
-+++ bootstrap.sh
-@@ -48,7 +48,7 @@ fi
-
- autoscan
- $LIBTOOLIZE --copy --automake
--aclocal -I ./aclocal
-+aclocal -I ./aclocal -I /usr/share/aclocal
- autoheader
- autoconf
- automake --copy --add-missing --foreign
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 5aa6efa..5edd167 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -52,76 +52,58 @@ export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
# libevent
# the last release version of libevent is 2.1.8, which was released on 26 Jan 2017, that is too old.
# so we use the master version of libevent, which is downloaded on 22 Jun 2018, with commit 24236aed01798303745470e6c498bf606e88724a
-LIBEVENT_DOWNLOAD="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip"
-LIBEVENT_NAME=libevent-20180622-24236aed01798303745470e6c498bf606e88724a.zip
-LIBEVENT_SOURCE=libevent-master
-LIBEVENT_MD5SUM="e8b9ba50270ba3b520aec8ff1089f9d7"
+LIBEVENT_DOWNLOAD="https://github.com/libevent/libevent/archive/release-2.1.12-stable.tar.gz"
+LIBEVENT_NAME=libevent-release-2.1.12-stable.tar.gz
+LIBEVENT_SOURCE=libevent-release-2.1.12-stable
+LIBEVENT_MD5SUM="0d5a27436bf7ff8253420c8cf09f47ca"
# openssl
-OPENSSL_DOWNLOAD="https://www.openssl.org/source/old/1.0.2/openssl-1.0.2k.tar.gz"
-OPENSSL_NAME=openssl-1.0.2k.tar.gz
-OPENSSL_SOURCE=openssl-1.0.2k
-OPENSSL_MD5SUM="f965fc0bf01bf882b31314b61391ae65"
+OPENSSL_DOWNLOAD="https://github.com/openssl/openssl/archive/OpenSSL_1_1_1l.tar.gz"
+OPENSSL_NAME=openssl-OpenSSL_1_1_1l.tar.gz
+OPENSSL_SOURCE=openssl-OpenSSL_1_1_1l
+OPENSSL_MD5SUM="2ff02cd10f96671162df3efd60470cfe"
# thrift
-THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.9.3/thrift-0.9.3.tar.gz"
-THRIFT_NAME=thrift-0.9.3.tar.gz
-THRIFT_SOURCE=thrift-0.9.3
-THRIFT_MD5SUM="88d667a8ae870d5adeca8cb7d6795442"
-
-# llvm
-LLVM_DOWNLOAD="http://releases.llvm.org/3.4.2/llvm-3.4.2.src.tar.gz"
-LLVM_NAME=llvm-3.4.2.src.tar.gz
-LLVM_SOURCE=llvm-3.4.2.src
-LLVM_MD5SUM="a20669f75967440de949ac3b1bad439c"
-
-# clang
-CLANG_DOWNLOAD="http://releases.llvm.org/3.4.2/cfe-3.4.2.src.tar.gz"
-CLANG_NAME=cfe-3.4.2.src.tar.gz
-CLANG_SOURCE=cfe-3.4.2.src
-CLANG_MD5SUM="87945973b7c73038871c5f849a818588"
-
-# compiler-rt
-COMPILER_RT_DOWNLOAD="http://releases.llvm.org/5.0.0/compiler-rt-5.0.0.src.tar.xz"
-COMPILER_RT_NAME=compiler-rt-5.0.0.src.tar.xz
-COMPILER_RT_SOURCE=compiler-rt-5.0.0.src
-COMPILER_RT_MD5SUM="da735894133589cbc6052c8ef06b1230"
+THRIFT_DOWNLOAD="http://archive.apache.org/dist/thrift/0.13.0/thrift-0.13.0.tar.gz"
+THRIFT_NAME=thrift-0.13.0.tar.gz
+THRIFT_SOURCE=thrift-0.13.0
+THRIFT_MD5SUM="38a27d391a2b03214b444cb13d5664f1"
# protobuf
-PROTOBUF_DOWNLOAD="https://github.com/google/protobuf/archive/v3.5.1.tar.gz"
-PROTOBUF_NAME=protobuf-3.5.1.tar.gz
-PROTOBUF_SOURCE=protobuf-3.5.1
-PROTOBUF_MD5SUM="710f1a75983092c9b45ecef207236104"
+PROTOBUF_DOWNLOAD="https://github.com/google/protobuf/archive/v3.14.0.tar.gz"
+PROTOBUF_NAME=protobuf-3.14.0.tar.gz
+PROTOBUF_SOURCE=protobuf-3.14.0
+PROTOBUF_MD5SUM="0c9d2a96f3656ba7ef3b23b533fb6170"
# gflags
-GFLAGS_DOWNLOAD="https://github.com/gflags/gflags/archive/v2.2.0.tar.gz"
-GFLAGS_NAME=gflags-2.2.0.tar.gz
-GFLAGS_SOURCE=gflags-2.2.0
-GFLAGS_MD5SUM="b99048d9ab82d8c56e876fb1456c285e"
+GFLAGS_DOWNLOAD="https://github.com/gflags/gflags/archive/v2.2.2.tar.gz"
+GFLAGS_NAME=gflags-2.2.2.tar.gz
+GFLAGS_SOURCE=gflags-2.2.2
+GFLAGS_MD5SUM="1a865b93bacfa963201af3f75b7bd64c"
# glog
-GLOG_DOWNLOAD="https://github.com/google/glog/archive/v0.3.3.tar.gz"
-GLOG_NAME=glog-0.3.3.tar.gz
-GLOG_SOURCE=glog-0.3.3
-GLOG_MD5SUM="c1f86af27bd9c73186730aa957607ed0"
+GLOG_DOWNLOAD="https://github.com/google/glog/archive/v0.4.0.tar.gz"
+GLOG_NAME=glog-0.4.0.tar.gz
+GLOG_SOURCE=glog-0.4.0
+GLOG_MD5SUM="0daea8785e6df922d7887755c3d100d0"
# gtest
-GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.8.0.tar.gz"
-GTEST_NAME=googletest-release-1.8.0.tar.gz
-GTEST_SOURCE=googletest-release-1.8.0
-GTEST_MD5SUM="16877098823401d1bf2ed7891d7dce36"
+GTEST_DOWNLOAD="https://github.com/google/googletest/archive/release-1.10.0.tar.gz"
+GTEST_NAME=googletest-release-1.10.0.tar.gz
+GTEST_SOURCE=googletest-release-1.10.0
+GTEST_MD5SUM="ecd1fa65e7de707cd5c00bdac56022cd"
# snappy
-SNAPPY_DOWNLOAD="https://github.com/google/snappy/archive/1.1.7.tar.gz"
-SNAPPY_NAME=snappy-1.1.7.tar.gz
-SNAPPY_SOURCE=snappy-1.1.7
-SNAPPY_MD5SUM="ee9086291c9ae8deb4dac5e0b85bf54a"
+SNAPPY_DOWNLOAD="https://github.com/google/snappy/archive/1.1.8.tar.gz"
+SNAPPY_NAME=snappy-1.1.8.tar.gz
+SNAPPY_SOURCE=snappy-1.1.8
+SNAPPY_MD5SUM="70e48cba7fecf289153d009791c9977f"
# gperftools
-GPERFTOOLS_DOWNLOAD="https://github.com/gperftools/gperftools/archive/gperftools-2.7.tar.gz"
-GPERFTOOLS_NAME=gperftools-2.7.tar.gz
-GPERFTOOLS_SOURCE=gperftools-gperftools-2.7
-GPERFTOOLS_MD5SUM="797e7b7f6663288e2b90ab664861c61a"
+GPERFTOOLS_DOWNLOAD="https://github.com/gperftools/gperftools/archive/gperftools-2.9.1.tar.gz"
+GPERFTOOLS_NAME=gperftools-2.9.1.tar.gz
+GPERFTOOLS_SOURCE=gperftools-gperftools-2.9.1
+GPERFTOOLS_MD5SUM="e340f1b247ff512119a2db98c1538dc1"
# zlib
ZLIB_DOWNLOAD="https://sourceforge.net/projects/libpng/files/zlib/1.2.11/zlib-1.2.11.tar.gz"
@@ -130,10 +112,10 @@ ZLIB_SOURCE=zlib-1.2.11
ZLIB_MD5SUM="1c9f62f0778697a09d36121ead88e08e"
# lz4
-LZ4_DOWNLOAD="https://github.com/lz4/lz4/archive/v1.7.5.tar.gz"
-LZ4_NAME=lz4-1.7.5.tar.gz
-LZ4_SOURCE=lz4-1.7.5
-LZ4_MD5SUM="c9610c5ce97eb431dddddf0073d919b9"
+LZ4_DOWNLOAD="https://github.com/lz4/lz4/archive/v1.9.3.tar.gz"
+LZ4_NAME=lz4-1.9.3.tar.gz
+LZ4_SOURCE=lz4-1.9.3
+LZ4_MD5SUM="3a1ab1684e14fc1afc66228ce61b2db3"
# bzip
BZIP_DOWNLOAD="https://fossies.org/linux/misc/bzip2-1.0.8.tar.gz"
@@ -155,16 +137,16 @@ RAPIDJSON_SOURCE=rapidjson-1a803826f1197b5e30703afe4b9c0e7dd48074f5
RAPIDJSON_MD5SUM="f2212a77e055a15501477f1e390007ea"
# curl
-CURL_DOWNLOAD="https://curl.haxx.se/download/curl-7.54.1.tar.gz"
-CURL_NAME=curl-7.54.1.tar.gz
-CURL_SOURCE=curl-7.54.1
-CURL_MD5SUM="21a6e5658fd55103a90b11de7b2a8a8c"
+CURL_DOWNLOAD="https://curl.se/download/curl-7.79.0.tar.gz"
+CURL_NAME=curl-7.79.0.tar.gz
+CURL_SOURCE=curl-7.79.0
+CURL_MD5SUM="b40e4dc4bbc9e109c330556cd58c8ec8"
# RE2
-RE2_DOWNLOAD="https://github.com/google/re2/archive/2017-05-01.tar.gz"
-RE2_NAME=re2-2017-05-01.tar.gz
-RE2_SOURCE=re2-2017-05-01
-RE2_MD5SUM="4aa65a0b22edacb7ddcd7e4aec038dcf"
+RE2_DOWNLOAD="https://github.com/google/re2/archive/2021-02-02.tar.gz"
+RE2_NAME=re2-2021-02-02.tar.gz
+RE2_SOURCE=re2-2021-02-02
+RE2_MD5SUM="48bc665463a86f68243c5af1bac75cd0"
# boost
BOOST_DOWNLOAD="https://boostorg.jfrog.io/artifactory/main/release/1.73.0/source/boost_1_73_0.tar.gz"
@@ -191,10 +173,12 @@ LEVELDB_SOURCE=leveldb-1.20
LEVELDB_MD5SUM="298b5bddf12c675d6345784261302252"
# brpc
-BRPC_DOWNLOAD="https://github.com/apache/incubator-brpc/archive/0.9.7.tar.gz"
-BRPC_NAME=incubator-brpc-0.9.7.tar.gz
-BRPC_SOURCE=incubator-brpc-0.9.7
-BRPC_MD5SUM="a5b79339d139d1c55d39689c0a69bcef"
+# incubator-brpc-1.0.0-rc02 is a stable version。 Due to the issue of the version release process,
+# there may not be an official version
+BRPC_DOWNLOAD="https://github.com/apache/incubator-brpc/archive/1.0.0-rc02.tar.gz"
+BRPC_NAME=incubator-brpc-1.0.0-rc02.tar.gz
+BRPC_SOURCE=incubator-brpc-1.0.0-rc02
+BRPC_MD5SUM="863b9e53f3642fd31eeb438dbfc4f55c"
# rocksdb
ROCKSDB_DOWNLOAD="https://github.com/facebook/rocksdb/archive/v5.14.2.tar.gz"
@@ -202,41 +186,35 @@ ROCKSDB_NAME=rocksdb-5.14.2.tar.gz
ROCKSDB_SOURCE=rocksdb-5.14.2
ROCKSDB_MD5SUM="b72720ea3b1e9ca9e4ed0febfef65b14"
-# librdkafka-1.7.0
-LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/refs/tags/v1.7.0.tar.gz"
-LIBRDKAFKA_NAME=v1.7.0.tar.gz
-LIBRDKAFKA_SOURCE=librdkafka-1.7.0
-LIBRDKAFKA_MD5SUM="fe3c45deb182bd9c644b6bc6375bffc3"
+# librdkafka-1.8.0
+LIBRDKAFKA_DOWNLOAD="https://github.com/edenhill/librdkafka/archive/v1.8.0.tar.gz"
+LIBRDKAFKA_NAME=librdkafka-1.8.0.tar.gz
+LIBRDKAFKA_SOURCE=librdkafka-1.8.0
+LIBRDKAFKA_MD5SUM="f31bd3b7a91a486d65b740a720b925dc"
# zstd
-ZSTD_DOWNLOAD="https://github.com/facebook/zstd/archive/v1.3.7.tar.gz"
-ZSTD_NAME=zstd-1.3.7.tar.gz
-ZSTD_SOURCE=zstd-1.3.7
-ZSTD_MD5SUM="9b89923a360ac85a3b8076fdf495318d"
-
-# double-conversion
-DOUBLE_CONVERSION_DOWNLOAD="https://github.com/google/double-conversion/archive/v3.1.1.tar.gz"
-DOUBLE_CONVERSION_NAME=double-conversion-3.1.1.tar.gz
-DOUBLE_CONVERSION_SOURCE=double-conversion-3.1.1
-DOUBLE_CONVERSION_MD5SUM="befd431c3de3f3ed7926ba2845ee609d"
+ZSTD_DOWNLOAD="https://github.com/facebook/zstd/archive/v1.5.0.tar.gz"
+ZSTD_NAME=zstd-1.5.0.tar.gz
+ZSTD_SOURCE=zstd-1.5.0
+ZSTD_MD5SUM="d5ac89d5df9e81243ce40d0c6a66691d"
# brotli
-BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.7.tar.gz"
-BROTLI_NAME="brotli-1.0.7.tar.gz"
-BROTLI_SOURCE="brotli-1.0.7"
-BROTLI_MD5SUM="7b6edd4f2128f22794d0ca28c53898a5"
+BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz"
+BROTLI_NAME="brotli-1.0.9.tar.gz"
+BROTLI_SOURCE="brotli-1.0.9"
+BROTLI_MD5SUM="c2274f0c7af8470ad514637c35bcee7d"
# flatbuffers
-FLATBUFFERS_DOWNLOAD="https://github.com/google/flatbuffers/archive/v1.10.0.tar.gz"
-FLATBUFFERS_NAME=flatbuffers-v1.10.0.tar.gz
-FLATBUFFERS_SOURCE=flatbuffers-1.10.0
-FLATBUFFERS_MD5SUM="f7d19a3f021d93422b0bc287d7148cd2"
+FLATBUFFERS_DOWNLOAD="https://github.com/google/flatbuffers/archive/2.0.0.tar.gz"
+FLATBUFFERS_NAME=flatbuffers-2.0.0.tar.gz
+FLATBUFFERS_SOURCE=flatbuffers-2.0.0
+FLATBUFFERS_MD5SUM="a27992324c3cbf86dd888268a23d17bd"
# arrow
-ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/apache-arrow-0.15.1.tar.gz"
-ARROW_NAME="arrow-apache-arrow-0.15.1.tar.gz"
-ARROW_SOURCE="arrow-apache-arrow-0.15.1"
-ARROW_MD5SUM="7d822863bbbe409794c6cdec3dcb8e6c"
+ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/apache-arrow-5.0.0.tar.gz"
+ARROW_NAME="arrow-apache-arrow-5.0.0.tar.gz"
+ARROW_SOURCE="arrow-apache-arrow-5.0.0"
+ARROW_MD5SUM="9caf5dbd36ef4972c3a591bcfeaf59c8"
# S2
S2_DOWNLOAD="https://github.com/google/s2geometry/archive/v0.9.0.tar.gz"
@@ -251,10 +229,10 @@ BITSHUFFLE_SOURCE=bitshuffle-0.3.5
BITSHUFFLE_MD5SUM="2648ec7ccd0b896595c6636d926fc867"
# CROARINGBITMAP
-CROARINGBITMAP_DOWNLOAD="https://github.com/RoaringBitmap/CRoaring/archive/v0.2.60.tar.gz"
-CROARINGBITMAP_NAME=CRoaring-0.2.60.tar.gz
-CROARINGBITMAP_SOURCE=CRoaring-0.2.60
-CROARINGBITMAP_MD5SUM="29602918e6890ffdeed84cb171857046"
+CROARINGBITMAP_DOWNLOAD="https://github.com/RoaringBitmap/CRoaring/archive/v0.3.4.tar.gz"
+CROARINGBITMAP_NAME=CRoaring-0.3.4.tar.gz
+CROARINGBITMAP_SOURCE=CRoaring-0.3.4
+CROARINGBITMAP_MD5SUM="29d1fa59f86b46915c6dd92c1a9a6c75"
# fmt
FMT_DOWNLOAD="https://github.com/fmtlib/fmt/archive/7.1.3.tar.gz"
@@ -263,16 +241,16 @@ FMT_SOURCE="fmt-7.1.3"
FMT_MD5SUM="2522ec65070c0bda0ca288677ded2831"
# parallel-hashmap
-PARALLEL_HASHMAP_DOWNLOAD="https://github.com/greg7mdp/parallel-hashmap/archive/refs/tags/1.33.tar.gz"
+PARALLEL_HASHMAP_DOWNLOAD="https://github.com/greg7mdp/parallel-hashmap/archive/1.33.tar.gz"
PARALLEL_HASHMAP_NAME="parallel-hashmap-1.33.tar.gz"
PARALLEL_HASHMAP_SOURCE="parallel-hashmap-1.33"
PARALLEL_HASHMAP_MD5SUM="7626b5215f745c4ce59b5a4e41d16235"
# ORC
-ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.5.8/orc-1.5.8.tar.gz"
-ORC_NAME=orc-1.5.8.tar.gz
-ORC_SOURCE=orc-1.5.8
-ORC_MD5SUM="2318b0a8233c8833b3a6cfd771c60883"
+ORC_DOWNLOAD="https://archive.apache.org/dist/orc/orc-1.6.6/orc-1.6.6.tar.gz"
+ORC_NAME=orc-1.6.6.tar.gz
+ORC_SOURCE=orc-1.6.6
+ORC_MD5SUM="26c94135111d312fb1ea4fc80d776c5f"
# jemalloc
JEMALLOC_DOWNLOAD="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
@@ -378,13 +356,12 @@ HDFS3_DOWNLOAD="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/libhdfs3-
HDFS3_NAME="libhdfs3-master.zip"
HDFS3_SOURCE="libhdfs3-master"
HDFS3_MD5SUM="8c071fd2e7b0b1ccc1ec9c0d073d4146"
-# commit id: 22781c4ebd31aee918c38b3af7b2e8da5adec93e
#libdivide
-LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/refs/tags/v4.0.0.tar.gz"
-LIBDIVIDE_NAME="libdivide-4.0.0.tar.gz"
-LIBDIVIDE_SOURCE="libdivide-4.0.0"
-LIBDIVIDE_MD5SUM="71b99d499b9a5effd0fb2e249622b017"
+LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/5.0.tar.gz"
+LIBDIVIDE_NAME="libdivide-5.0.tar.gz"
+LIBDIVIDE_SOURCE="libdivide-5.0"
+LIBDIVIDE_MD5SUM="7fd16b0bb4ab6812b2e2fdc7bfb81641"
#pdqsort
PDQSORT_DOWNLOAD="http://ftp.cise.ufl.edu/ubuntu/pool/universe/p/pdqsort/pdqsort_0.0.0+git20180419.orig.tar.gz"
@@ -402,9 +379,6 @@ BENCHMARK_MD5SUM="668b9e10d8b0795e5d461894db18db3c"
export TP_ARCHIVES="LIBEVENT
OPENSSL
THRIFT
-LLVM
-CLANG
-COMPILER_RT
PROTOBUF
GFLAGS
GLOG
@@ -428,7 +402,6 @@ LIBRDKAFKA
FLATBUFFERS
ARROW
BROTLI
-DOUBLE_CONVERSION
ZSTD
S2
BITSHUFFLE
@@ -458,3 +431,4 @@ LIBDIVIDE
PDQSORT
BENCHMARK"
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org