You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2024/02/28 11:39:05 UTC
(doris) branch master updated: [cloud](Variant) support variant in cloud mode (#31483)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 76a0edb2bff [cloud](Variant) support variant in cloud mode (#31483)
76a0edb2bff is described below
commit 76a0edb2bff2f84c47024fcc3aa637015271d17b
Author: lihangyu <15...@163.com>
AuthorDate: Wed Feb 28 19:38:57 2024 +0800
[cloud](Variant) support variant in cloud mode (#31483)
* [cloud](Variant) support variant in cloud mode
seperate variant tablet schema kv since it's schema is volatile
---
be/src/cloud/cloud_schema_change_job.cpp | 3 ++
be/src/cloud/pb_convert.cpp | 2 +
be/src/olap/rowset/rowset_meta.cpp | 5 ++
be/src/olap/rowset/rowset_meta.h | 2 +
be/test/olap/test_data/rowset_meta.json | 3 +-
cloud/src/meta-service/keys.cpp | 11 +++-
cloud/src/meta-service/keys.h | 5 ++
cloud/src/meta-service/meta_service.cpp | 60 +++++++++++++++-------
cloud/src/recycler/recycler.cpp | 32 ++++++++++--
gensrc/proto/olap_file.proto | 2 +
.../cloud_p0/conf/regression-conf-custom.groovy | 2 +-
regression-test/pipeline/cloud_p0/deploy.sh | 2 +
.../suites/variant_github_events_p0/load.groovy | 1 -
.../suites/variant_p0/complexjson.groovy | 1 -
regression-test/suites/variant_p0/desc.groovy | 3 ++
regression-test/suites/variant_p0/sql/gh_data.sql | 2 +-
.../suites/variant_p0/sql/implicit_cast.sql | 2 +-
17 files changed, 110 insertions(+), 28 deletions(-)
diff --git a/be/src/cloud/cloud_schema_change_job.cpp b/be/src/cloud/cloud_schema_change_job.cpp
index e74c21a5501..2099f22f1cd 100644
--- a/be/src/cloud/cloud_schema_change_job.cpp
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -25,6 +25,7 @@
#include "cloud/cloud_tablet_mgr.h"
#include "common/status.h"
#include "olap/delete_handler.h"
+#include "olap/olap_define.h"
#include "olap/rowset/beta_rowset.h"
#include "olap/rowset/rowset_factory.h"
#include "olap/rowset/segment_v2/inverted_index_desc.h"
@@ -243,8 +244,10 @@ Status CloudSchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParam
context.rowset_state = VISIBLE;
context.segments_overlap = rs_reader->rowset()->rowset_meta()->segments_overlap();
context.tablet_schema = _new_tablet->tablet_schema();
+ context.original_tablet_schema = _new_tablet->tablet_schema();
context.newest_write_timestamp = rs_reader->newest_write_timestamp();
context.fs = _cloud_storage_engine.latest_fs();
+ context.write_type = DataWriteType::TYPE_SCHEMA_CHANGE;
auto rowset_writer = DORIS_TRY(_new_tablet->create_rowset_writer(context, false));
RowsetMetaSharedPtr existed_rs_meta;
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 402b78449c2..3fdc3678a55 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -81,6 +81,7 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
out->set_schema_version(in.schema_version());
}
out->set_enable_segments_file_size(in.enable_segments_file_size());
+ out->set_has_variant_type_in_schema(in.has_has_variant_type_in_schema());
}
void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
@@ -130,6 +131,7 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
out->set_schema_version(in.schema_version());
}
out->set_enable_segments_file_size(in.enable_segments_file_size());
+ out->set_has_variant_type_in_schema(in.has_variant_type_in_schema());
}
RowsetMetaPB cloud_rowset_meta_to_doris(const RowsetMetaCloudPB& in) {
diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp
index 125089a2862..412bf56e6f5 100644
--- a/be/src/olap/rowset/rowset_meta.cpp
+++ b/be/src/olap/rowset/rowset_meta.cpp
@@ -103,6 +103,10 @@ void RowsetMeta::set_fs(io::FileSystemSPtr fs) {
_fs = std::move(fs);
}
+bool RowsetMeta::has_variant_type_in_schema() const {
+ return _schema && _schema->num_variant_columns() > 0;
+}
+
void RowsetMeta::to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema) const {
*rs_meta_pb = _rowset_meta_pb;
if (_schema) [[likely]] {
@@ -112,6 +116,7 @@ void RowsetMeta::to_rowset_pb(RowsetMetaPB* rs_meta_pb, bool skip_schema) const
_schema->to_schema_pb(rs_meta_pb->mutable_tablet_schema());
}
}
+ rs_meta_pb->set_has_variant_type_in_schema(has_variant_type_in_schema());
}
RowsetMetaPB RowsetMeta::get_rowset_pb(bool skip_schema) const {
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index dd278aac4ad..f5cae0fcd4e 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -58,6 +58,8 @@ public:
bool is_local() const { return !_rowset_meta_pb.has_resource_id(); }
+ bool has_variant_type_in_schema() const;
+
RowsetId rowset_id() const { return _rowset_id; }
void set_rowset_id(const RowsetId& rowset_id) {
diff --git a/be/test/olap/test_data/rowset_meta.json b/be/test/olap/test_data/rowset_meta.json
index e2ac4bb8082..d446e2a34e9 100644
--- a/be/test/olap/test_data/rowset_meta.json
+++ b/be/test/olap/test_data/rowset_meta.json
@@ -17,5 +17,6 @@
"hi": 10,
"lo": 10
},
- "num_segments": 1
+ "num_segments": 1,
+ "has_variant_type_in_schema": false
}
diff --git a/cloud/src/meta-service/keys.cpp b/cloud/src/meta-service/keys.cpp
index 2ccd1e6f2d1..7307a8d5eb1 100644
--- a/cloud/src/meta-service/keys.cpp
+++ b/cloud/src/meta-service/keys.cpp
@@ -50,6 +50,7 @@ namespace doris::cloud {
[[maybe_unused]] static const char* META_KEY_INFIX_TABLET = "tablet";
[[maybe_unused]] static const char* META_KEY_INFIX_TABLET_IDX = "tablet_index";
[[maybe_unused]] static const char* META_KEY_INFIX_SCHEMA = "schema";
+[[maybe_unused]] static const char* META_KEY_INFIX_ROWSET_SCHEMA = "rowset_schema";
[[maybe_unused]] static const char* META_KEY_INFIX_DELETE_BITMAP = "delete_bitmap";
[[maybe_unused]] static const char* META_KEY_INFIX_DELETE_BITMAP_LOCK = "delete_bitmap_lock";
[[maybe_unused]] static const char* META_KEY_INFIX_DELETE_BITMAP_PENDING = "delete_bitmap_pending";
@@ -115,7 +116,7 @@ static void encode_prefix(const T& t, std::string* key) {
RecycleIndexKeyInfo, RecyclePartKeyInfo, RecycleRowsetKeyInfo, RecycleTxnKeyInfo, RecycleStageKeyInfo,
StatsTabletKeyInfo,
JobTabletKeyInfo, JobRecycleKeyInfo, RLJobProgressKeyInfo,
- CopyJobKeyInfo, CopyFileKeyInfo>);
+ CopyJobKeyInfo, CopyFileKeyInfo, MetaRowsetSchemaKeyInfo>);
key->push_back(CLOUD_USER_KEY_SPACE01);
// Prefixes for key families
@@ -131,6 +132,7 @@ static void encode_prefix(const T& t, std::string* key) {
|| std::is_same_v<T, MetaTabletKeyInfo>
|| std::is_same_v<T, MetaTabletIdxKeyInfo>
|| std::is_same_v<T, MetaSchemaKeyInfo>
+ || std::is_same_v<T, MetaRowsetSchemaKeyInfo>
|| std::is_same_v<T, MetaDeleteBitmapInfo>
|| std::is_same_v<T, MetaDeleteBitmapUpdateLockInfo>
|| std::is_same_v<T, MetaPendingDeleteBitmapInfo>) {
@@ -265,6 +267,13 @@ void meta_schema_key(const MetaSchemaKeyInfo& in, std::string* out) {
encode_int64(std::get<2>(in), out); // schema_version
}
+void meta_rowset_schema_key(const MetaRowsetSchemaKeyInfo& in, std::string* out) {
+ encode_prefix(in, out); // 0x01 "meta" ${instance_id}
+ encode_bytes(META_KEY_INFIX_ROWSET_SCHEMA, out); // "rowset_schema"
+ encode_int64(std::get<1>(in), out); // tablet_id
+ encode_bytes(std::get<2>(in), out); // rowset_id
+}
+
void meta_delete_bitmap_key(const MetaDeleteBitmapInfo& in, std::string* out) {
encode_prefix(in, out); // 0x01 "meta" ${instance_id}
encode_bytes(META_KEY_INFIX_DELETE_BITMAP, out); // "delete_bitmap"
diff --git a/cloud/src/meta-service/keys.h b/cloud/src/meta-service/keys.h
index 8786d669158..3093a906f77 100644
--- a/cloud/src/meta-service/keys.h
+++ b/cloud/src/meta-service/keys.h
@@ -162,6 +162,9 @@ using JobRecycleKeyInfo = BasicKeyInfo<20 , std::tuple<std::string>>;
// 0:instance_id 1:index_id 2:schema_version
using MetaSchemaKeyInfo = BasicKeyInfo<21, std::tuple<std::string, int64_t, int64_t>>;
+// 0:instance_id 1:tablet_id 2:rowset_id
+using MetaRowsetSchemaKeyInfo = BasicKeyInfo<21, std::tuple<std::string, int64_t, std::string>>;
+
// 0:instance_id 1:tablet_id 2:rowest_id 3:version 4:seg_id
using MetaDeleteBitmapInfo = BasicKeyInfo<22 , std::tuple<std::string, int64_t, std::string, int64_t, int64_t>>;
@@ -197,6 +200,7 @@ void meta_rowset_tmp_key(const MetaRowsetTmpKeyInfo& in, std::string* out);
void meta_tablet_idx_key(const MetaTabletIdxKeyInfo& in, std::string* out);
void meta_tablet_key(const MetaTabletKeyInfo& in, std::string* out);
void meta_schema_key(const MetaSchemaKeyInfo& in, std::string* out);
+void meta_rowset_schema_key(const MetaRowsetSchemaKeyInfo& in, std::string* out);
void meta_delete_bitmap_key(const MetaDeleteBitmapInfo& in, std::string* out);
void meta_delete_bitmap_update_lock_key(const MetaDeleteBitmapUpdateLockInfo& in, std::string* out);
void meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in, std::string* out);
@@ -205,6 +209,7 @@ static inline std::string meta_rowset_tmp_key(const MetaRowsetTmpKeyInfo& in) {
static inline std::string meta_tablet_idx_key(const MetaTabletIdxKeyInfo& in) { std::string s; meta_tablet_idx_key(in, &s); return s; }
static inline std::string meta_tablet_key(const MetaTabletKeyInfo& in) { std::string s; meta_tablet_key(in, &s); return s; }
static inline std::string meta_schema_key(const MetaSchemaKeyInfo& in) { std::string s; meta_schema_key(in, &s); return s; }
+static inline std::string meta_rowset_schema_key(const MetaRowsetSchemaKeyInfo& in) { std::string s; meta_rowset_schema_key(in, &s); return s; }
static inline std::string meta_delete_bitmap_key(const MetaDeleteBitmapInfo& in) { std::string s; meta_delete_bitmap_key(in, &s); return s; }
static inline std::string meta_delete_bitmap_update_lock_key(const MetaDeleteBitmapUpdateLockInfo& in) { std::string s; meta_delete_bitmap_update_lock_key(in, &s); return s; }
static inline std::string meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in) { std::string s; meta_pending_delete_bitmap_key(in, &s); return s; }
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 780b05ae38d..2b9ba01cc2d 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -50,6 +50,7 @@
#include "common/string_util.h"
#include "common/sync_point.h"
#include "common/util.h"
+#include "keys.h"
#include "meta-service/codec.h"
#include "meta-service/doris_txn.h"
#include "meta-service/keys.h"
@@ -1008,8 +1009,15 @@ void MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
DCHECK(rowset_meta.tablet_schema().has_schema_version());
DCHECK_GE(rowset_meta.tablet_schema().schema_version(), 0);
rowset_meta.set_schema_version(rowset_meta.tablet_schema().schema_version());
- auto schema_key = meta_schema_key(
- {instance_id, rowset_meta.index_id(), rowset_meta.schema_version()});
+ std::string schema_key;
+ if (rowset_meta.has_variant_type_in_schema()) {
+ // encodes schema in a seperate kv, since variant schema is volatile
+ schema_key = meta_rowset_schema_key({instance_id,
+ rowset_meta.tablet_id(), rowset_meta.rowset_id_v2()});
+ } else {
+ schema_key = meta_schema_key(
+ {instance_id, rowset_meta.index_id(), rowset_meta.schema_version()});
+ }
put_schema_kv(code, msg, txn.get(), schema_key, rowset_meta.tablet_schema());
if (code != MetaServiceCode::OK) return;
rowset_meta.set_allocated_tablet_schema(nullptr);
@@ -1248,6 +1256,28 @@ std::vector<std::pair<int64_t, int64_t>> calc_sync_versions(int64_t req_bc_cnt,
return versions;
}
+static bool try_fetch_and_parse_schema(
+ Transaction* txn, RowsetMetaCloudPB& rowset_meta,
+ const std::string& key, MetaServiceCode& code, std::string& msg) {
+ ValueBuf val_buf;
+ TxnErrorCode err = cloud::get(txn, key, &val_buf);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::READ>(err);
+ msg = fmt::format("failed to get schema, schema_version={}, rowset_version=[{}-{}]: {}",
+ rowset_meta.schema_version(), rowset_meta.start_version(),
+ rowset_meta.end_version(),
+ err == TxnErrorCode::TXN_KEY_NOT_FOUND ? "not found" : "internal error");
+ return false;
+ }
+ auto schema = rowset_meta.mutable_tablet_schema();
+ if (!parse_schema_value(val_buf, schema)) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ msg = fmt::format("malformed schema value, key={}", key);
+ return false;
+ }
+ return true;
+}
+
void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller,
const GetRowsetRequest* request, GetRowsetResponse* response,
::google::protobuf::Closure* done) {
@@ -1352,6 +1382,14 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller,
rowset_meta.start_version(), rowset_meta.end_version());
return;
}
+ if (rowset_meta.has_variant_type_in_schema()) {
+ // get rowset schema kv
+ auto key = meta_rowset_schema_key({instance_id, idx.tablet_id(), rowset_meta.rowset_id_v2()});
+ if (!try_fetch_and_parse_schema(txn.get(), rowset_meta, key, code, msg)) {
+ return;
+ }
+ continue;
+ }
if (auto it = version_to_schema.find(rowset_meta.schema_version());
it != version_to_schema.end()) {
if (arena != nullptr) {
@@ -1361,24 +1399,10 @@ void MetaServiceImpl::get_rowset(::google::protobuf::RpcController* controller,
}
} else {
auto key = meta_schema_key({instance_id, idx.index_id(), rowset_meta.schema_version()});
- ValueBuf val_buf;
- TxnErrorCode err = cloud::get(txn.get(), key, &val_buf);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::READ>(err);
- msg = fmt::format(
- "failed to get schema, schema_version={}, rowset_version=[{}-{}]: {}",
- rowset_meta.schema_version(), rowset_meta.start_version(),
- rowset_meta.end_version(),
- err == TxnErrorCode::TXN_KEY_NOT_FOUND ? "not found" : "internal error");
- return;
- }
- auto schema = rowset_meta.mutable_tablet_schema();
- if (!parse_schema_value(val_buf, schema)) {
- code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- msg = fmt::format("malformed schema value, key={}", key);
+ if (!try_fetch_and_parse_schema(txn.get(), rowset_meta, key, code, msg)) {
return;
}
- version_to_schema.emplace(rowset_meta.schema_version(), schema);
+ version_to_schema.emplace(rowset_meta.schema_version(), rowset_meta.mutable_tablet_schema());
}
}
}
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 288089334f6..a45bdc1c8ff 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -1226,6 +1226,11 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id) {
std::string delete_bitmap_end = meta_delete_bitmap_key({instance_id_, tablet_id + 1, "", 0, 0});
txn->remove(delete_bitmap_start, delete_bitmap_end);
+ // remove rowset schema
+ std::string rowset_schema_start = meta_rowset_schema_key({instance_id_, tablet_id, ""});
+ std::string rowset_schema_end = meta_rowset_schema_key({instance_id_, tablet_id + 1, ""});
+ txn->remove(rowset_schema_start, rowset_schema_end);
+
TxnErrorCode err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
LOG(WARNING) << "failed to delete rowset kv of tablet " << tablet_id << ", err=" << err;
@@ -1283,6 +1288,7 @@ int InstanceRecycler::recycle_rowsets() {
});
std::vector<std::string> rowset_keys;
+ std::vector<std::string> rowset_schema_keys;
std::vector<doris::RowsetMetaCloudPB> rowsets;
// Store keys of rowset recycled by background workers
@@ -1408,6 +1414,8 @@ int InstanceRecycler::recycle_rowsets() {
return -1;
}
} else {
+ auto schema_key = meta_rowset_schema_key({instance_id_, rowset_meta->tablet_id(), rowset_meta->rowset_id_v2()});
+ rowset_schema_keys.push_back(std::move(schema_key));
rowset_keys.push_back(std::string(k));
if (rowset_meta->num_segments() > 0) { // Skip empty rowset
rowsets.push_back(std::move(*rowset_meta));
@@ -1418,15 +1426,22 @@ int InstanceRecycler::recycle_rowsets() {
auto loop_done = [&]() -> int {
std::vector<std::string> rowset_keys_to_delete;
+ std::vector<std::string> rowset_schema_keys_to_delete;
std::vector<doris::RowsetMetaCloudPB> rowsets_to_delete;
rowset_keys_to_delete.swap(rowset_keys);
+ rowset_schema_keys_to_delete.swap(rowset_schema_keys);
rowsets_to_delete.swap(rowsets);
worker_pool->submit([&, rowset_keys_to_delete = std::move(rowset_keys_to_delete),
- rowsets_to_delete = std::move(rowsets_to_delete)]() {
+ rowsets_to_delete = std::move(rowsets_to_delete),
+ rowset_schema_keys_to_delete = std::move(rowset_schema_keys_to_delete)]() {
if (delete_rowset_data(rowsets_to_delete) != 0) {
LOG(WARNING) << "failed to delete rowset data, instance_id=" << instance_id_;
return;
}
+ if (txn_remove(txn_kv_.get(), rowset_schema_keys_to_delete) != 0) {
+ LOG(WARNING) << "failed to delete recycle rowset kv, instance_id=" << instance_id_;
+ return;
+ }
if (txn_remove(txn_kv_.get(), rowset_keys_to_delete) != 0) {
LOG(WARNING) << "failed to delete recycle rowset kv, instance_id=" << instance_id_;
return;
@@ -1483,10 +1498,11 @@ int InstanceRecycler::recycle_tmp_rowsets() {
// Elements in `tmp_rowset_keys` has the same lifetime as `it`
std::vector<std::string_view> tmp_rowset_keys;
+ std::vector<std::string> tmp_rowset_schema_keys;
std::vector<doris::RowsetMetaCloudPB> tmp_rowsets;
auto handle_rowset_kv = [&num_scanned, &num_expired, &tmp_rowset_keys, &tmp_rowsets,
- &expired_rowset_size, &total_rowset_size,
+ &expired_rowset_size, &total_rowset_size, &tmp_rowset_schema_keys,
this](std::string_view k, std::string_view v) -> int {
++num_scanned;
total_rowset_size += v.size();
@@ -1516,6 +1532,11 @@ int InstanceRecycler::recycle_tmp_rowsets() {
tmp_rowset_keys.push_back(k);
return 0;
}
+ if (rowset.has_variant_type_in_schema()) {
+ auto schema_key = meta_rowset_schema_key({instance_id_,
+ rowset.tablet_id(), rowset.rowset_id_v2()});
+ tmp_rowset_schema_keys.push_back(std::move(schema_key));
+ }
// TODO(plat1ko): check rowset not referenced
LOG(INFO) << "delete rowset data, instance_id=" << instance_id_
<< " tablet_id=" << rowset.tablet_id() << " rowset_id=" << rowset.rowset_id_v2()
@@ -1529,7 +1550,8 @@ int InstanceRecycler::recycle_tmp_rowsets() {
return 0;
};
- auto loop_done = [&tmp_rowset_keys, &tmp_rowsets, &num_recycled, this]() -> int {
+ auto loop_done = [&tmp_rowset_keys, &tmp_rowsets,
+ &num_recycled, &tmp_rowset_schema_keys, this]() -> int {
std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01, [&](int*) {
tmp_rowset_keys.clear();
tmp_rowsets.clear();
@@ -1538,6 +1560,10 @@ int InstanceRecycler::recycle_tmp_rowsets() {
LOG(WARNING) << "failed to delete tmp rowset data, instance_id=" << instance_id_;
return -1;
}
+ if (txn_remove(txn_kv_.get(), tmp_rowset_schema_keys) != 0) {
+ LOG(WARNING) << "failed to delete tmp rowset schema kv, instance_id=" << instance_id_;
+ return -1;
+ }
if (txn_remove(txn_kv_.get(), tmp_rowset_keys) != 0) {
LOG(WARNING) << "failed to delete tmp rowset kv, instance_id=" << instance_id_;
return -1;
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index eacc1ec9cb1..98acd24e387 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -129,6 +129,7 @@ message RowsetMetaPB {
// If enable_segments_file_size is false,
// the segments_file_size maybe is empty or error
optional bool enable_segments_file_size = 1004;
+ optional bool has_variant_type_in_schema = 1005;
}
message RowsetMetaCloudPB {
@@ -198,6 +199,7 @@ message RowsetMetaCloudPB {
// If enable_segments_file_size is false,
// the segments_file_size maybe is empty or error
optional bool enable_segments_file_size = 103;
+ optional bool has_variant_type_in_schema = 104;
}
message SegmentStatisticsPB {
diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
index 0af6e798f1e..1518def58c1 100644
--- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
+++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
@@ -16,7 +16,7 @@
// under the License.
testGroups = "p0"
-testDirectories = "ddl_p0,database_p0,load,load_p0,query_p0,table_p0,account_p0,autobucket,bitmap_functions,bloom_filter_p0,cast_decimal_to_boolean,cast_double_to_decimal,compression_p0,connector_p0,correctness,correctness_p0,csv_header_p0,data_model_p0,database_p0,datatype_p0,delete_p0,demo_p0,empty_relation,export_p0,external_table_p0,fault_injection_p0,flink_connector_p0,insert_overwrite_p0,insert_p0,internal_schema_p0,javaudf_p0,job_p0,json_p0,jsonb_p0,meta_action_p0,metrics_p0,mtmv_ [...]
+testDirectories = "ddl_p0,database_p0,load,load_p0,query_p0,table_p0,account_p0,autobucket,bitmap_functions,bloom_filter_p0,cast_decimal_to_boolean,cast_double_to_decimal,compression_p0,connector_p0,correctness,correctness_p0,csv_header_p0,data_model_p0,database_p0,datatype_p0,delete_p0,demo_p0,empty_relation,export_p0,external_table_p0,fault_injection_p0,flink_connector_p0,insert_overwrite_p0,insert_p0,internal_schema_p0,javaudf_p0,job_p0,json_p0,jsonb_p0,meta_action_p0,metrics_p0,mtmv_ [...]
//exclude groups and exclude suites is more prior than include groups and include suites.
excludeSuites = "test_index_failure_injection,test_dump_image,test_profile,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_information_schema_external"
excludeDirectories = "workload_manager_p1,nereids_rules_p0/subquery"
diff --git a/regression-test/pipeline/cloud_p0/deploy.sh b/regression-test/pipeline/cloud_p0/deploy.sh
index 2cfe9f96f48..69e39187e3a 100644
--- a/regression-test/pipeline/cloud_p0/deploy.sh
+++ b/regression-test/pipeline/cloud_p0/deploy.sh
@@ -61,6 +61,8 @@ exit_flag=0
cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/cloud_p0/conf/be_custom.conf "${DORIS_HOME}"/be/conf/
fdb_cluster="$(cat /etc/foundationdb/fdb.cluster)"
sed -i "s/^fdb_cluster = .*/fdb_cluster = ${fdb_cluster}/" "${DORIS_HOME}"/ms/conf/doris_cloud.conf
+ # this is a temporary config, need to replace in cloud_p0/conf/doris_cloud.conf
+ echo "meta_schema_value_version = 1" >>"${DORIS_HOME}"/ms/conf/doris_cloud.conf
sed -i "s/^fdb_cluster = .*/fdb_cluster = ${fdb_cluster}/" "${DORIS_HOME}"/recycler/conf/doris_cloud.conf
sed -i "s/^brpc_listen_port = .*/fbrpc_listen_port = 6000/" "${DORIS_HOME}"/recycler/conf/doris_cloud.conf
print_doris_conf
diff --git a/regression-test/suites/variant_github_events_p0/load.groovy b/regression-test/suites/variant_github_events_p0/load.groovy
index 4bde4400f44..695046961a2 100644
--- a/regression-test/suites/variant_github_events_p0/load.groovy
+++ b/regression-test/suites/variant_github_events_p0/load.groovy
@@ -25,7 +25,6 @@ suite("regression_test_variant_github_events_p0", "nonConcurrent"){
logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
}
}
- sql "set enable_memtable_on_sink_node = true"
def load_json_data = {table_name, file_name ->
// load the json data
streamLoad {
diff --git a/regression-test/suites/variant_p0/complexjson.groovy b/regression-test/suites/variant_p0/complexjson.groovy
index cd6a0716fda..0ebb038bb34 100644
--- a/regression-test/suites/variant_p0/complexjson.groovy
+++ b/regression-test/suites/variant_p0/complexjson.groovy
@@ -16,7 +16,6 @@
// under the License.
suite("regression_test_variant_complexjson", "variant_type_complex_json") {
- sql "set enable_memtable_on_sink_node = true"
def create_table = { table_name ->
sql "DROP TABLE IF EXISTS ${table_name}"
sql """
diff --git a/regression-test/suites/variant_p0/desc.groovy b/regression-test/suites/variant_p0/desc.groovy
index 78c9f078c9f..644a03fe715 100644
--- a/regression-test/suites/variant_p0/desc.groovy
+++ b/regression-test/suites/variant_p0/desc.groovy
@@ -16,6 +16,9 @@
// under the License.
suite("regression_test_variant_desc", "nonConcurrent"){
+ if (isCloudCluster()) {
+ return
+ }
def load_json_data = {table_name, file_name ->
// load the json data
diff --git a/regression-test/suites/variant_p0/sql/gh_data.sql b/regression-test/suites/variant_p0/sql/gh_data.sql
index 9daf28fdce6..91cb7dc0bcc 100644
--- a/regression-test/suites/variant_p0/sql/gh_data.sql
+++ b/regression-test/suites/variant_p0/sql/gh_data.sql
@@ -1,5 +1,5 @@
set exec_mem_limit=8G;
-set enable_two_phase_read_opt = true;
+set exec_mem_limit = 8G;
set topn_opt_limit_threshold = 1024;
SELECT count() from ghdata;
SELECT cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM ghdata WHERE cast(v["type"] as string) = 'WatchEvent' GROUP BY repo_name ORDER BY stars DESC, repo_name LIMIT 5;
diff --git a/regression-test/suites/variant_p0/sql/implicit_cast.sql b/regression-test/suites/variant_p0/sql/implicit_cast.sql
index 0653a52eed7..45acfd38513 100644
--- a/regression-test/suites/variant_p0/sql/implicit_cast.sql
+++ b/regression-test/suites/variant_p0/sql/implicit_cast.sql
@@ -1,5 +1,5 @@
set exec_mem_limit=8G;
-set enable_two_phase_read_opt = true;
+set exec_mem_limit = 8G;
set topn_opt_limit_threshold = 1024;
SELECT count() from ghdata;
SELECT cast(v["repo"]["name"] as string) as repo_name, count() AS stars FROM ghdata WHERE v["type"] = 'WatchEvent' GROUP BY repo_name ORDER BY stars DESC, repo_name LIMIT 5;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org